From d74a6100a4a28d7d385bfda59e70164d3a4252f4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 12:12:37 +0000 Subject: [PATCH 1/2] feat(lockfile): warn and keep packages when an optional field is unknown An unknown optional key on a lockfile that already resolves packages now prints one stable warning per file per key and leaves the dependency set in place. Truncated or invalid lockfiles still fail closed. Signed-off-by: Cursor Agent Co-authored-by: Crowers --- CHANGELOG.md | 2 + DOCS.md | 13 + src/engine/discover.ts | 24 +- src/engine/lockfile-optional.test.ts | 394 ++++++++++++ src/engine/lockfile-optional.ts | 673 +++++++++++++++++++++ src/engine/lockfile.ts | 8 +- src/reporting/scanners/dependency-graph.ts | 4 + 7 files changed, 1116 insertions(+), 2 deletions(-) create mode 100644 src/engine/lockfile-optional.test.ts create mode 100644 src/engine/lockfile-optional.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 72af8f05..08861362 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ backward compatible. ### Added +- **Unknown optional lockfile fields warn and keep the dependency set.** A key `vg` does not model leaves that file's packages in the graph and SBOM. The command prints one stable warning per file per field name and continues. A truncated or invalid lockfile still exits non-zero. + - **`vg build --attest` and `vg build --verify` are documented.** The docs cover the DSSE in-toto statement written to `.vibgrate/attestation.intoto.jsonl`, the Ed25519 key diff --git a/DOCS.md b/DOCS.md index 5b1d5626..2f2d5408 100644 --- a/DOCS.md +++ b/DOCS.md @@ -9,6 +9,7 @@ For a quick overview, see the [README](./README.md). This document covers everyt ## Table of Contents - [How It Works](#how-it-works) +- [Lockfiles with unknown fields](#lockfiles-with-unknown-fields) - [Choosing a rollout model: one-off vs CI](#choosing-a-rollout-model-one-off-vs-ci) - [Commands Reference](#commands-reference) - [vg baseline](#vg-baseline) @@ -179,6 +180,18 @@ Vibgrate evaluates **upgrade drift** in depth for: **Known-vulnerability detection** (`--vulns`) and **dependency attribution** (`vg why`, exposure windows) additionally cover npm / pnpm / yarn, pip / poetry / pipenv, cargo, composer, bundler, pub, hex, NuGet, and Maven/Gradle, read from each project's lockfile. Go is matched from direct `require` lines in `go.mod`. Pseudo-versions and `+incompatible` tags follow [Go modules: pseudo-versions and +incompatible](#go-modules-pseudo-versions-and-incompatible). +### Lockfiles with unknown fields + +Lockfile formats grow optional fields. A pnpm `catalogs` block and a package `devEngines` entry are two examples. When the file still resolves a dependency set, `vg` keeps every package it understands and writes one warning per unknown field name: + +```text +warning: pnpm-lock.yaml: unknown optional lockfile field "catalogs"; kept the dependencies this file already resolved +``` + +The warning line contains the field name and the lockfile's path relative to the project root. The same file and the same field produce the same line on every run. Several unknown fields on one file are sorted by field name, and a field that appears many times is warned once. Field values stay off that line, including registry tokens and authorization header values. + +`vg build` prints the warning while it discovers the tree. `vg sbom export` prints it while it reads the lockfile graph. A lockfile that is truncated, unreadable, or missing the structure that dialect requires still exits non-zero. The error names the file and the format and leaves the file's contents out. `go.sum` and `gradle.lockfile` have no optional-key slot: a line that is not valid for that dialect is this structural failure. + ### End-to-end workflow (recommended) 1. Run an initial scan. diff --git a/src/engine/discover.ts b/src/engine/discover.ts index 2f523416..4ef2ac10 100644 --- a/src/engine/discover.ts +++ b/src/engine/discover.ts @@ -5,6 +5,7 @@ import { langForExtension, langById, type LanguageDef } from './languages.js'; import { requireDataConfig } from '../core-open/config.js'; import { dropBlankPatterns, gitignoreWithoutBlankLines } from '../core-open/utils/glob.js'; import { assertLockfileFile, lockfileKind } from '../core-open/utils/lockfile-parse.js'; +import { warnUnknownOptionalLockfile } from './lockfile-optional.js'; import { assertSafeWalkRoot, createWalkBudget, noteWalkEntry } from '../core-open/utils/root-safety.js'; /** @@ -248,6 +249,7 @@ export function discover(options: DiscoverOptions): DiscoveredFile[] { const found = new Map(); const budget = createWalkBudget(root, options.maxEntries); + const lockfiles: string[] = []; const considerFile = (abs: string): void => { const rel = toPosix(path.relative(root, abs)); @@ -256,7 +258,12 @@ export function discover(options: DiscoverOptions): DiscoveredFile[] { if (SKIP_FILES.has(path.basename(abs).toLowerCase())) { // Lockfiles are not source, but a truncated one must fail the build // here — before the parse pool starts — rather than being skipped. - if (lockfileKind(path.basename(abs))) assertLockfileFile(abs); + // A file that parses keeps its packages; unknown optional fields warn + // after the walk, in path order. + if (lockfileKind(path.basename(abs))) { + assertLockfileFile(abs); + lockfiles.push(abs); + } return; } const lang = langForExtension(path.extname(abs)); @@ -297,6 +304,21 @@ export function discover(options: DiscoverOptions): DiscoveredFile[] { } else if (stat.isFile()) considerFile(scope); } + lockfiles.sort((a, b) => { + const ra = toPosix(path.relative(root, a)); + const rb = toPosix(path.relative(root, b)); + return ra < rb ? -1 : ra > rb ? 1 : 0; + }); + for (const abs of lockfiles) { + let text: string; + try { + text = fs.readFileSync(abs, 'utf8'); + } catch { + continue; + } + warnUnknownOptionalLockfile(root, abs, text); + } + return [...found.values()].sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0)); } diff --git a/src/engine/lockfile-optional.test.ts b/src/engine/lockfile-optional.test.ts new file mode 100644 index 00000000..3a40b01b --- /dev/null +++ b/src/engine/lockfile-optional.test.ts @@ -0,0 +1,394 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { LockfileParseError } from '../core-open/utils/lockfile-parse.js'; +import { parseCargoLock } from '../core-open/scanners/cargo-lockfile.js'; +import { parseGemfileLock } from '../core-open/scanners/gemfile-lock.js'; +import { parseGradleLockfile } from '../core-open/scanners/gradle-lockfile.js'; +import { parsePackageLock, parsePnpmLock, parseYarnLock } from '../core-open/scanners/npm-lockfile.js'; +import { parsePipfileLock, parsePyTomlLock } from '../core-open/scanners/python-lockfile.js'; +import { + parseComposerLock, + parseMixLock, + parseNpmLockfile, + parseNugetLock, + parsePipfileLock as parseTimelinePipfile, + parsePnpmLockfile, + parsePubspecLock, + parseTomlPackages, + parseYarnLockfile, +} from '../core-open/utils/version-timeline.js'; +import { discover } from './discover.js'; +import { fullDependencyGraph, lockfileVersion } from './lockfile.js'; +import { + formatUnknownOptionalFieldWarnings, + unknownOptionalFieldWarning, + unknownOptionalLockfileKeys, +} from './lockfile-optional.js'; + +const pkgRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const SECRET = 'Bearer npm_AAAAAAAAAAAAAAAAAAAA'; + +function captureStderr(run: () => void): { stderr: string; error?: unknown } { + let stderr = ''; + const write = process.stderr.write; + process.stderr.write = ((chunk: string | Uint8Array) => { + stderr += typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'); + return true; + }) as typeof process.stderr.write; + try { + run(); + return { stderr }; + } catch (error) { + return { stderr, error }; + } finally { + process.stderr.write = write; + } +} + +describe('unknown optional lockfile fields', () => { + let root: string; + const token = 'super-secret-token'; + const previousToken = process.env.NPM_TOKEN; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'vg-lock-optional-')); + process.env.NPM_TOKEN = token; + }); + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + if (previousToken === undefined) delete process.env.NPM_TOKEN; + else process.env.NPM_TOKEN = previousToken; + }); + + const write = (rel: string, body: string): void => { + const abs = path.join(root, rel); + fs.mkdirSync(path.dirname(abs), { recursive: true }); + fs.writeFileSync(abs, body); + }; + + it('formats one sorted warning per key and leaves values out of the line', () => { + const line = unknownOptionalFieldWarning('pnpm-lock.yaml', 'catalogs'); + expect(line).toBe( + 'warning: pnpm-lock.yaml: unknown optional lockfile field "catalogs"; kept the dependencies this file already resolved', + ); + expect(formatUnknownOptionalFieldWarnings('package-lock.json', ['zebraOptional', 'alphaOptional', 'alphaOptional'])).toEqual([ + unknownOptionalFieldWarning('package-lock.json', 'alphaOptional'), + unknownOptionalFieldWarning('package-lock.json', 'zebraOptional'), + ]); + expect(formatUnknownOptionalFieldWarnings('package-lock.json', ['ok', 'Bearer secret', 'a b'])).toEqual([ + unknownOptionalFieldWarning('package-lock.json', 'ok'), + ]); + }); + + it('keeps an npm dependency set and warns once for a repeated unknown key', () => { + write( + 'package-lock.json', + JSON.stringify({ + lockfileVersion: 3, + futureOptional: SECRET, + packages: { + '': { name: 'app' }, + 'node_modules/left-pad': { version: '1.3.0', devEngines: { runtime: { name: 'node' } }, integrity: SECRET }, + 'node_modules/chalk': { version: '5.3.0', devEngines: { cpu: 'x64' } }, + }, + }), + ); + const first = captureStderr(() => { + expect(fullDependencyGraph(root)?.components).toEqual([ + { package: 'chalk', version: '5.3.0' }, + { package: 'left-pad', version: '1.3.0' }, + ]); + }); + expect(first.error).toBeUndefined(); + expect(first.stderr).toBe( + [ + unknownOptionalFieldWarning('package-lock.json', 'devEngines'), + unknownOptionalFieldWarning('package-lock.json', 'futureOptional'), + '', + ].join('\n'), + ); + expect(first.stderr).not.toContain(SECRET); + expect(first.stderr).not.toContain(token); + expect(first.stderr).not.toContain(root); + expect(first.stderr).not.toContain('left-pad'); + const second = captureStderr(() => fullDependencyGraph(root)); + expect(second.stderr).toBe(''); + }); + + it('keeps a pnpm catalog lockfile and warns for the unknown catalogs key', () => { + const body = [ + "lockfileVersion: '9.0'", + 'catalogs:', + ' default:', + ' left-pad:', + " specifier: '^1.3.0'", + ' version: 1.3.0', + 'importers:', + ' .:', + ' dependencies:', + ' left-pad:', + " specifier: 'catalog:'", + ' version: 1.3.0', + 'packages:', + ' left-pad@1.3.0:', + ' resolution: {integrity: sha512-abc}', + '', + ].join('\n'); + write('pnpm-lock.yaml', body); + const captured = captureStderr(() => { + expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'left-pad', version: '1.3.0' }]); + expect(lockfileVersion(root, 'npm', 'left-pad')).toBe('1.3.0'); + }); + expect(captured.error).toBeUndefined(); + expect(captured.stderr).toBe(`${unknownOptionalFieldWarning('pnpm-lock.yaml', 'catalogs')}\n`); + expect(captured.stderr).not.toContain(SECRET); + }); + + it('keeps yarn, cargo, and poetry rows when one optional key is unknown', () => { + write('yarn.lock', ['left-pad@^1.3.0:', ' version "1.3.0"', ' futureOptional true', ''].join('\n')); + const yarn = captureStderr(() => { + expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'left-pad', version: '1.3.0' }]); + }); + expect(yarn.stderr).toBe(`${unknownOptionalFieldWarning('yarn.lock', 'futureOptional')}\n`); + + fs.rmSync(path.join(root, 'yarn.lock')); + write('Cargo.lock', ['[[package]]', 'name = "memchr"', 'version = "2.7.4"', 'futureOptional = "kept"', ''].join('\n')); + const cargo = captureStderr(() => { + expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'memchr', version: '2.7.4' }]); + }); + expect(cargo.stderr).toBe(`${unknownOptionalFieldWarning('Cargo.lock', 'futureOptional')}\n`); + + fs.rmSync(path.join(root, 'Cargo.lock')); + write('poetry.lock', ['[[package]]', 'name = "click"', 'version = "8.1.3"', 'futureOptional = "kept"', ''].join('\n')); + const poetry = captureStderr(() => { + expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'click', version: '8.1.3' }]); + }); + expect(poetry.stderr).toBe(`${unknownOptionalFieldWarning('poetry.lock', 'futureOptional')}\n`); + expect(poetry.stderr).not.toContain('click'); + }); + + it('warns for other ecosystems and still returns the locked version', () => { + const cases: Array<{ file: string; body: string; ecosystem: 'php' | 'pypi' | 'dart' | 'dotnet' | 'swift' | 'ruby'; name: string; version: string }> = [ + { + file: 'composer.lock', + ecosystem: 'php', + name: 'monolog/monolog', + version: '3.5.0', + body: JSON.stringify({ + futureOptional: SECRET, + packages: [{ name: 'monolog/monolog', version: '3.5.0' }], + }), + }, + { + file: 'Pipfile.lock', + ecosystem: 'pypi', + name: 'requests', + version: '2.31.0', + body: JSON.stringify({ + futureOptional: true, + default: { requests: { version: '==2.31.0' } }, + }), + }, + { + file: 'pubspec.lock', + ecosystem: 'dart', + name: 'http', + version: '1.2.0', + body: ['packages:', ' http:', ' version: "1.2.0"', ' futureOptional: true', ''].join('\n'), + }, + { + file: 'packages.lock.json', + ecosystem: 'dotnet', + name: 'Newtonsoft.Json', + version: '13.0.3', + body: JSON.stringify({ + version: 1, + futureOptional: true, + dependencies: { 'net8.0': { 'Newtonsoft.Json': { resolved: '13.0.3' } } }, + }), + }, + { + file: 'Package.resolved', + ecosystem: 'swift', + name: 'vapor', + version: '4.89.0', + body: JSON.stringify({ + version: 3, + futureOptional: true, + pins: [{ identity: 'vapor', state: { version: '4.89.0' } }], + }), + }, + { + file: 'Gemfile.lock', + ecosystem: 'ruby', + name: 'rake', + version: '13.0.6', + body: ['GEM', ' remote: https://rubygems.org/', ' specs:', ' rake (13.0.6)', 'FUTURE', ''].join('\n'), + }, + ]; + for (const item of cases) { + write(item.file, item.body); + const captured = captureStderr(() => { + expect(lockfileVersion(root, item.ecosystem, item.name)).toBe(item.version); + }); + const key = item.file === 'Gemfile.lock' ? 'FUTURE' : 'futureOptional'; + expect(captured.stderr, item.file).toBe(`${unknownOptionalFieldWarning(item.file, key)}\n`); + expect(captured.stderr, item.file).not.toContain(SECRET); + expect(captured.stderr, item.file).not.toContain(token); + fs.rmSync(path.join(root, item.file)); + } + }); + + it('does not warn when the lockfile has no resolved package', () => { + write('package-lock.json', JSON.stringify({ lockfileVersion: 3, futureOptional: true, packages: {} })); + const captured = captureStderr(() => { + expect(fullDependencyGraph(root)).toBeUndefined(); + }); + expect(captured.stderr).toBe(''); + expect(unknownOptionalLockfileKeys('package-lock.json', fs.readFileSync(path.join(root, 'package-lock.json'), 'utf8'))).toEqual([]); + }); + + it('fails closed on broken required structure and does not warn', () => { + write('package-lock.json', `{"lockfileVersion":3,"futureOptional":true,"packages":{"node_modules/left-pad":{"version":"1.0.0","integrity":"${SECRET}"`); + const npm = captureStderr(() => { + fullDependencyGraph(root); + }); + expect(npm.error).toBeInstanceOf(LockfileParseError); + expect((npm.error as Error).message).toContain('truncated or invalid JSON'); + expect(npm.stderr).toBe(''); + expect(npm.stderr).not.toContain(SECRET); + + fs.rmSync(path.join(root, 'package-lock.json')); + write('gradle.lockfile', 'com.example:lib:1.2.3=runtimeClasspath\nthis is not a lock line\n'); + const gradle = captureStderr(() => { + lockfileVersion(root, 'java', 'com.example:lib'); + }); + expect(gradle.error).toBeInstanceOf(LockfileParseError); + expect((gradle.error as Error).message).toContain('truncated or invalid gradle.lockfile'); + expect(gradle.stderr).not.toContain('unknown optional'); + + fs.rmSync(path.join(root, 'gradle.lockfile')); + write('go.sum', 'github.com/gin-gonic/gin v1.9.0 h1:abc=\nnot-a-sum-line\n'); + const go = captureStderr(() => { + fullDependencyGraph(root); + }); + expect(go.error).toBeInstanceOf(LockfileParseError); + expect((go.error as Error).message).toContain('truncated or invalid go.sum'); + expect(go.stderr).not.toContain('unknown optional'); + }); + + it('keeps a valid gradle pin with no optional-key warning', () => { + write('gradle.lockfile', '# generated\ncom.example:lib:1.2.3=runtimeClasspath\nempty=\n'); + const captured = captureStderr(() => { + expect(lockfileVersion(root, 'java', 'com.example:lib')).toBe('1.2.3'); + }); + expect(captured.stderr).toBe(''); + }); + + it('discover keeps source files, warns in path order, and still rejects a truncated lockfile', () => { + write('app.ts', 'export const n = 1;\n'); + const lock = JSON.stringify({ + lockfileVersion: 3, + packages: { 'node_modules/left-pad': { version: '1.3.0' } }, + futureOptional: true, + }); + write('a/package-lock.json', lock); + write('b/package-lock.json', lock); + const ok = captureStderr(() => { + const files = discover({ root }); + expect(files.map((file) => file.rel)).toEqual(['app.ts']); + }); + expect(ok.stderr).toBe( + [ + unknownOptionalFieldWarning('a/package-lock.json', 'futureOptional'), + unknownOptionalFieldWarning('b/package-lock.json', 'futureOptional'), + '', + ].join('\n'), + ); + + const broken = fs.mkdtempSync(path.join(os.tmpdir(), 'vg-lock-optional-')); + try { + fs.writeFileSync(path.join(broken, 'package-lock.json'), '{"packages":{'); + fs.writeFileSync(path.join(broken, 'app.ts'), 'export const n = 1;\n'); + const failed = captureStderr(() => { + discover({ root: broken }); + }); + expect(failed.error).toBeInstanceOf(LockfileParseError); + expect(failed.stderr).toBe(''); + } finally { + fs.rmSync(broken, { recursive: true, force: true }); + } + }); + + it('does not treat current lockfiles in this repository as unknown', () => { + const pnpm = fs.readFileSync(path.join(pkgRoot, 'pnpm-lock.yaml'), 'utf8'); + expect(unknownOptionalLockfileKeys('pnpm-lock.yaml', pnpm)).toEqual([]); + const npmLock = fs.readFileSync(path.join(pkgRoot, 'packages/test-solutions/package-lock.json'), 'utf8'); + expect(unknownOptionalLockfileKeys('package-lock.json', npmLock)).toEqual([]); + }); + + it('scanner readers keep the resolved row when an optional key is unknown', () => { + const npm = JSON.stringify({ + lockfileVersion: 3, + futureOptional: true, + packages: { 'node_modules/left-pad': { version: '1.3.0', devEngines: { runtime: 'node' } } }, + }); + expect(parsePackageLock(JSON.parse(npm)).get('left-pad')).toBe('1.3.0'); + expect(parseNpmLockfile(npm).get('left-pad')).toBe('1.3.0'); + + const pnpmDoc = { + lockfileVersion: '9.0', + catalogs: { default: { 'left-pad': { version: '1.3.0' } } }, + importers: { '.': { dependencies: { 'left-pad': { specifier: 'catalog:', version: '1.3.0' } } } }, + }; + expect(parsePnpmLock(pnpmDoc).get('left-pad')).toBe('1.3.0'); + const pnpmText = [ + "lockfileVersion: '9.0'", + 'catalogs:', + ' default:', + ' left-pad:', + ' version: 1.3.0', + 'packages:', + ' left-pad@1.3.0:', + ' resolution: {integrity: sha512-abc}', + '', + ].join('\n'); + expect(parsePnpmLockfile(pnpmText).get('left-pad')).toBe('1.3.0'); + + const yarn = ['left-pad@^1.3.0:', ' version "1.3.0"', ' futureOptional true', ''].join('\n'); + expect(parseYarnLock(yarn).byName.get('left-pad')).toBe('1.3.0'); + expect(parseYarnLockfile(yarn).get('left-pad')).toBe('1.3.0'); + + const cargo = ['[[package]]', 'name = "memchr"', 'version = "2.7.4"', 'futureOptional = "kept"', ''].join('\n'); + expect(parseCargoLock(cargo).get('memchr')).toEqual(['2.7.4']); + expect(parseTomlPackages(cargo).get('memchr')).toBe('2.7.4'); + expect(parsePyTomlLock(cargo).get('memchr')).toBe('2.7.4'); + + const pip = JSON.stringify({ futureOptional: true, default: { requests: { version: '==2.31.0', hashes: ['abc'] } } }); + expect(parsePipfileLock(JSON.parse(pip)).get('requests')).toBe('2.31.0'); + expect(parseTimelinePipfile(pip).get('requests')).toBe('2.31.0'); + + expect(parseGradleLockfile('# comment\ncom.example:lib:1.2.3=runtimeClasspath\nempty=\n').get('com.example:lib')).toBe('1.2.3'); + const gem = ['GEM', ' specs:', ' rake (13.0.6)', 'FUTURE', ' extra: 1', ''].join('\n'); + expect(parseGemfileLock(gem).get('rake')).toBe('13.0.6'); + + const composer = JSON.stringify({ futureOptional: true, packages: [{ name: 'monolog/monolog', version: 'v3.5.0', extra: {} }] }); + expect(parseComposerLock(composer).get('monolog/monolog')).toBe('3.5.0'); + + const pub = ['packages:', ' http:', ' futureOptional: true', ' version: "1.2.0"', ''].join('\n'); + expect(parsePubspecLock(pub).get('http')).toBe('1.2.0'); + + const nuget = JSON.stringify({ + version: 2, + futureOptional: true, + dependencies: { 'net8.0': { 'Newtonsoft.Json': { resolved: '13.0.3', contentHash: 'abc' } } }, + }); + expect(parseNugetLock(nuget).get('Newtonsoft.Json')).toBe('13.0.3'); + + const mix = '"decimal" => {:hex, :decimal, "2.1.1", "extra-field", [:mix], "hexpm", "abc"},\n'; + expect(parseMixLock(mix).get('decimal')).toBe('2.1.1'); + }); +}); diff --git a/src/engine/lockfile-optional.ts b/src/engine/lockfile-optional.ts new file mode 100644 index 00000000..46b0c61e --- /dev/null +++ b/src/engine/lockfile-optional.ts @@ -0,0 +1,673 @@ +/** + * Unknown optional lockfile fields. + * + * A key this tool does not model, on a lockfile that already resolves a + * dependency set, must not discard that set. The warning names the key and + * the lockfile's root-relative path. It never includes a value, so a registry + * token, an environment variable, or an authorization header stored in the + * file cannot leak. At most one warning is emitted per file per key name, + * in sorted order. + * + * Line-oriented lockfiles with no optional-key slot (`go.sum`, + * `gradle.lockfile`) are not warned here. A line that is not valid for that + * dialect fails closed in the syntax check before this runs. + */ +import * as path from 'node:path'; +import { parse as parseToml } from 'smol-toml'; +import { parse as parseYaml } from 'yaml'; + +/** Field names we will print. Anything else is omitted rather than echoed. */ +const SAFE_KEY = /^[A-Za-z_][A-Za-z0-9_-]{0,64}$/; + +const seenContent = new Set(); +const emittedKeys = new Set(); + +function setOf(keys: readonly string[]): ReadonlySet { + return new Set(keys); +} + +function asRecord(value: unknown): Record | undefined { + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; + return value as Record; +} + +function ownKeys(value: unknown): string[] { + const rec = asRecord(value); + return rec ? Object.keys(rec) : []; +} + +function addUnknown(keys: readonly string[], known: ReadonlySet, into: Set): void { + for (const key of keys) { + if (!known.has(key) && SAFE_KEY.test(key)) into.add(key); + } +} + +function sorted(keys: Set): string[] { + return [...keys].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); +} + +function parseJson(text: string): unknown { + try { + return JSON.parse(text); + } catch { + return undefined; + } +} + +function parseYamlDoc(text: string): unknown { + try { + return parseYaml(text, { uniqueKeys: false, logLevel: 'silent' }); + } catch { + return undefined; + } +} + +function parseTomlDoc(text: string): Record | undefined { + try { + const doc = parseToml(text); + return asRecord(doc); + } catch { + return undefined; + } +} + +// ── npm package-lock / npm-shrinkwrap ──────────────────────────────────────── + +const NPM_ROOT = setOf(['name', 'version', 'lockfileVersion', 'requires', 'packages', 'dependencies']); +const NPM_ENTRY = setOf([ + 'name', + 'version', + 'dependencies', + 'devDependencies', + 'optionalDependencies', + 'peerDependencies', + 'peerDependenciesMeta', + 'bundledDependencies', + 'bin', + 'license', + 'engines', + 'funding', + 'os', + 'cpu', + 'libc', + 'resolved', + 'integrity', + 'link', + 'dev', + 'optional', + 'devOptional', + 'inBundle', + 'hasInstallScript', + 'hasShrinkwrap', + 'extraneous', + 'deprecated', + 'workspaces', + 'requires', + 'bundled', + 'from', + 'peer', + 'overrides', +]); + +function npmEntryHasVersion(entry: unknown): boolean { + const rec = asRecord(entry); + return typeof rec?.version === 'string' && rec.version.length > 0; +} + +function npmTreeHas(deps: unknown): boolean { + const rec = asRecord(deps); + if (!rec) return false; + for (const entry of Object.values(rec)) { + if (npmEntryHasVersion(entry)) return true; + if (npmTreeHas(asRecord(entry)?.dependencies)) return true; + } + return false; +} + +function walkNpmEntry(entry: unknown, into: Set): void { + addUnknown(ownKeys(entry), NPM_ENTRY, into); + const nested = asRecord(entry)?.dependencies; + const rec = asRecord(nested); + if (!rec) return; + for (const child of Object.values(rec)) { + if (asRecord(child)) walkNpmEntry(child, into); + } +} + +function npmKeys(text: string): string[] { + const root = asRecord(parseJson(text)); + if (!root) return []; + const packages = asRecord(root.packages); + let usable = false; + if (packages) { + for (const [key, entry] of Object.entries(packages)) { + if (key && npmEntryHasVersion(entry)) usable = true; + } + } + if (!usable && npmTreeHas(root.dependencies)) usable = true; + if (!usable) return []; + const into = new Set(); + addUnknown(ownKeys(root), NPM_ROOT, into); + if (packages) { + for (const entry of Object.values(packages)) walkNpmEntry(entry, into); + } + if (asRecord(root.dependencies)) walkNpmEntry({ dependencies: root.dependencies }, into); + return sorted(into); +} + +// ── pnpm-lock.yaml ─────────────────────────────────────────────────────────── + +const PNPM_ROOT = setOf([ + 'lockfileVersion', + 'settings', + 'importers', + 'packages', + 'snapshots', + 'time', + 'neverBuiltDependencies', + 'onlyBuiltDependencies', + 'overrides', + 'packageExtensionsChecksum', + 'patchedDependencies', + 'pnpmfileChecksum', + 'ignoredOptionalDependencies', + 'dependencies', + 'devDependencies', + 'optionalDependencies', + 'peerDependencies', + 'specifiers', +]); +const PNPM_SETTINGS = setOf([ + 'autoInstallPeers', + 'excludeLinksFromLockfile', + 'peersSuffixMaxLength', + 'injectWorkspacePackages', +]); +const PNPM_PACKAGE = setOf([ + 'resolution', + 'engines', + 'cpu', + 'os', + 'libc', + 'deprecated', + 'hasBin', + 'optional', + 'dev', + 'requiresBuild', + 'prepare', + 'patched', + 'id', + 'name', + 'version', + 'dependencies', + 'optionalDependencies', + 'peerDependencies', + 'peerDependenciesMeta', + 'transitivePeerDependencies', + 'bundledDependencies', +]); +const PNPM_SNAPSHOT = setOf([ + 'dependencies', + 'optionalDependencies', + 'transitivePeerDependencies', + 'optional', + 'dev', + 'id', + 'patched', +]); +const PNPM_IMPORTER = setOf([ + 'dependencies', + 'devDependencies', + 'optionalDependencies', + 'peerDependencies', + 'dependenciesMeta', + 'optionalDependenciesMeta', + 'specifiers', +]); +const PNPM_DEP = setOf(['specifier', 'version']); +const PNPM_DEP_SECTIONS = ['dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies'] as const; + +function pnpmDepSectionHas(section: unknown): boolean { + const deps = asRecord(section); + if (!deps) return false; + for (const dep of Object.values(deps)) { + if (typeof dep === 'string' && dep.length > 0) return true; + const rec = asRecord(dep); + if (typeof rec?.version === 'string' && rec.version.length > 0) return true; + } + return false; +} + +function notePnpmDepSection(section: unknown, into: Set): void { + const deps = asRecord(section); + if (!deps) return; + for (const dep of Object.values(deps)) addUnknown(ownKeys(dep), PNPM_DEP, into); +} + +function pnpmKeys(text: string): string[] { + const root = asRecord(parseYamlDoc(text)); + if (!root) return []; + const packages = asRecord(root.packages); + const snapshots = asRecord(root.snapshots); + const importers = asRecord(root.importers); + let usable = Boolean(packages && Object.keys(packages).length > 0); + if (!usable && snapshots && Object.keys(snapshots).length > 0) usable = true; + if (!usable && importers) { + for (const imp of Object.values(importers)) { + const rec = asRecord(imp); + if (!rec) continue; + if (PNPM_DEP_SECTIONS.some((section) => pnpmDepSectionHas(rec[section]))) usable = true; + } + } + if (!usable && PNPM_DEP_SECTIONS.some((section) => pnpmDepSectionHas(root[section]))) usable = true; + if (!usable) return []; + const into = new Set(); + addUnknown(ownKeys(root), PNPM_ROOT, into); + addUnknown(ownKeys(root.settings), PNPM_SETTINGS, into); + if (packages) { + for (const entry of Object.values(packages)) addUnknown(ownKeys(entry), PNPM_PACKAGE, into); + } + if (snapshots) { + for (const entry of Object.values(snapshots)) addUnknown(ownKeys(entry), PNPM_SNAPSHOT, into); + } + if (importers) { + for (const imp of Object.values(importers)) { + addUnknown(ownKeys(imp), PNPM_IMPORTER, into); + const rec = asRecord(imp); + if (!rec) continue; + for (const section of PNPM_DEP_SECTIONS) notePnpmDepSection(rec[section], into); + } + } + for (const section of PNPM_DEP_SECTIONS) notePnpmDepSection(root[section], into); + return sorted(into); +} + +// ── yarn.lock ──────────────────────────────────────────────────────────────── + +const YARN_FIELDS = setOf([ + 'version', + 'resolved', + 'integrity', + 'dependencies', + 'optionalDependencies', + 'peerDependencies', + 'dependenciesMeta', + 'peerDependenciesMeta', + 'checksum', + 'languageName', + 'linkType', + 'bin', + 'conditions', + 'resolution', + 'cacheKey', +]); + +function yarnKeys(text: string): string[] { + let descriptor = false; + let version = false; + const into = new Set(); + for (const raw of text.split('\n')) { + if (!raw.trim() || raw.trimStart().startsWith('#')) continue; + if (!/^\s/.test(raw)) { + if (raw.includes('@')) descriptor = true; + continue; + } + const field = /^ {2}([A-Za-z_][A-Za-z0-9_-]*)\b/.exec(raw); + if (!field) continue; + if (field[1] === 'version' && /^ {2}version:?\s+\S/.test(raw)) version = true; + addUnknown([field[1]!], YARN_FIELDS, into); + } + return descriptor && version ? sorted(into) : []; +} + +// ── TOML locks: Cargo, poetry, uv, pdm ─────────────────────────────────────── + +const TOML_ROOT = setOf([ + 'version', + 'package', + 'metadata', + 'patch', + 'replace', + 'requires-python', + 'revision', + 'resolution-markers', + 'manifest', + 'conflicts', + 'supported-markers', + 'options', + 'dependency-groups', + 'extras', +]); +const TOML_PACKAGE = setOf([ + 'name', + 'version', + 'source', + 'checksum', + 'dependencies', + 'replace', + 'optional', + 'description', + 'python-versions', + 'files', + 'extras', + 'category', + 'groups', + 'markers', + 'develop', + 'dev-dependencies', + 'optional-dependencies', + 'metadata', + 'wheels', + 'sdist', + 'dev', + 'resolution-markers', + 'environments', + 'summary', + 'marker', + 'index', +]); + +function tomlPackages(doc: Record): unknown[] { + const list = doc.package; + if (Array.isArray(list)) return list; + return list ? [list] : []; +} + +function tomlKeys(text: string): string[] { + const doc = parseTomlDoc(text); + if (!doc) return []; + const packages = tomlPackages(doc); + const usable = packages.some((entry) => { + const rec = asRecord(entry); + return typeof rec?.name === 'string' && typeof rec.version === 'string' && rec.version.length > 0; + }); + if (!usable) return []; + const into = new Set(); + addUnknown(ownKeys(doc), TOML_ROOT, into); + for (const entry of packages) addUnknown(ownKeys(entry), TOML_PACKAGE, into); + return sorted(into); +} + +// ── Gemfile.lock ───────────────────────────────────────────────────────────── + +const GEM_SECTIONS = setOf([ + 'GEM', + 'GIT', + 'PATH', + 'PLATFORMS', + 'DEPENDENCIES', + 'BUNDLED WITH', + 'CHECKSUMS', + 'PLUGIN SOURCE', +]); + +function gemfileKeys(text: string): string[] { + if (!/^ {4}\S+ \(\d/m.test(text)) return []; + const into = new Set(); + for (const raw of text.split('\n')) { + if (!raw || /^[ \t]/.test(raw)) continue; + const header = raw.trim(); + if (!/^[A-Z][A-Z ]+$/.test(header)) continue; + addUnknown([header], GEM_SECTIONS, into); + } + return sorted(into); +} + +// ── composer.lock ──────────────────────────────────────────────────────────── + +const COMPOSER_ROOT = setOf([ + '_readme', + 'content-hash', + 'packages', + 'packages-dev', + 'aliases', + 'minimum-stability', + 'stability-flags', + 'prefer-stable', + 'prefer-lowest', + 'platform', + 'platform-dev', + 'plugin-api-version', +]); +const COMPOSER_PKG = setOf([ + 'name', + 'version', + 'version_normalized', + 'source', + 'dist', + 'require', + 'require-dev', + 'conflict', + 'replace', + 'provide', + 'suggest', + 'bin', + 'type', + 'extra', + 'autoload', + 'autoload-dev', + 'notification-url', + 'license', + 'authors', + 'description', + 'homepage', + 'keywords', + 'support', + 'funding', + 'time', + 'abandoned', + 'default-branch', + 'transport-options', + 'scripts', + 'target-dir', +]); + +function composerListHas(list: unknown): boolean { + if (!Array.isArray(list)) return false; + return list.some((entry) => { + const rec = asRecord(entry); + return typeof rec?.name === 'string' && typeof rec.version === 'string' && rec.version.length > 0; + }); +} + +function composerKeys(text: string): string[] { + const root = asRecord(parseJson(text)); + if (!root) return []; + if (!composerListHas(root.packages) && !composerListHas(root['packages-dev'])) return []; + const into = new Set(); + addUnknown(ownKeys(root), COMPOSER_ROOT, into); + for (const section of ['packages', 'packages-dev']) { + const list = root[section]; + if (!Array.isArray(list)) continue; + for (const entry of list) addUnknown(ownKeys(entry), COMPOSER_PKG, into); + } + return sorted(into); +} + +// ── Pipfile.lock ───────────────────────────────────────────────────────────── + +const PIPFILE_ROOT = setOf(['_meta', 'default', 'develop']); +const PIPFILE_ENTRY = setOf(['version', 'markers', 'hashes', 'index', 'extras']); + +function pipfileSectionHas(section: unknown): boolean { + const deps = asRecord(section); + if (!deps) return false; + return Object.values(deps).some((entry) => typeof asRecord(entry)?.version === 'string'); +} + +function pipfileKeys(text: string): string[] { + const root = asRecord(parseJson(text)); + if (!root) return []; + if (!pipfileSectionHas(root.default) && !pipfileSectionHas(root.develop)) return []; + const into = new Set(); + addUnknown(ownKeys(root), PIPFILE_ROOT, into); + for (const section of ['default', 'develop']) { + const deps = asRecord(root[section]); + if (!deps) continue; + for (const entry of Object.values(deps)) addUnknown(ownKeys(entry), PIPFILE_ENTRY, into); + } + return sorted(into); +} + +// ── packages.lock.json ─────────────────────────────────────────────────────── + +const NUGET_ROOT = setOf(['version', 'dependencies']); +const NUGET_ENTRY = setOf(['type', 'requested', 'resolved', 'contentHash', 'dependencies', 'framework']); + +function nugetKeys(text: string): string[] { + const root = asRecord(parseJson(text)); + const frameworks = asRecord(root?.dependencies); + if (!root || !frameworks) return []; + let usable = false; + const into = new Set(); + for (const framework of Object.values(frameworks)) { + const packages = asRecord(framework); + if (!packages) continue; + for (const entry of Object.values(packages)) { + const rec = asRecord(entry); + if (typeof rec?.resolved === 'string' && rec.resolved.length > 0) usable = true; + addUnknown(ownKeys(entry), NUGET_ENTRY, into); + } + } + if (!usable) return []; + addUnknown(ownKeys(root), NUGET_ROOT, into); + return sorted(into); +} + +// ── Package.resolved ───────────────────────────────────────────────────────── + +const SWIFT_ROOT = setOf(['version', 'originHash', 'pins', 'object']); +const SWIFT_OBJECT = setOf(['pins']); +const SWIFT_PIN = setOf(['identity', 'kind', 'location', 'state', 'package', 'repositoryURL']); +const SWIFT_STATE = setOf(['version', 'revision', 'branch']); + +function swiftPins(root: Record): unknown[] { + if (Array.isArray(root.pins)) return root.pins; + const object = asRecord(root.object); + return Array.isArray(object?.pins) ? object.pins : []; +} + +function swiftKeys(text: string): string[] { + const root = asRecord(parseJson(text)); + if (!root) return []; + const pins = swiftPins(root); + const usable = pins.some((pin) => typeof asRecord(asRecord(pin)?.state)?.version === 'string'); + if (!usable) return []; + const into = new Set(); + addUnknown(ownKeys(root), SWIFT_ROOT, into); + addUnknown(ownKeys(root.object), SWIFT_OBJECT, into); + for (const pin of pins) { + addUnknown(ownKeys(pin), SWIFT_PIN, into); + addUnknown(ownKeys(asRecord(pin)?.state), SWIFT_STATE, into); + } + return sorted(into); +} + +// ── pubspec.lock ───────────────────────────────────────────────────────────── + +const PUB_ROOT = setOf(['packages', 'sdks']); +const PUB_ENTRY = setOf(['dependency', 'description', 'source', 'version']); + +function pubspecKeys(text: string): string[] { + const root = asRecord(parseYamlDoc(text)); + const packages = asRecord(root?.packages); + if (!root || !packages) return []; + let usable = false; + const into = new Set(); + for (const entry of Object.values(packages)) { + const rec = asRecord(entry); + if (typeof rec?.version === 'string' && rec.version.length > 0) usable = true; + addUnknown(ownKeys(entry), PUB_ENTRY, into); + } + if (!usable) return []; + addUnknown(ownKeys(root), PUB_ROOT, into); + return sorted(into); +} + +/** + * Sorted unique unknown optional keys on a lockfile that already resolves at + * least one package. Empty when the file has no resolved package, is not a + * known lockfile, or does not parse. Never throws. + */ +export function unknownOptionalLockfileKeys(basename: string, text: string): string[] { + switch (basename.toLowerCase()) { + case 'package-lock.json': + case 'npm-shrinkwrap.json': + return npmKeys(text); + case 'pnpm-lock.yaml': + return pnpmKeys(text); + case 'yarn.lock': + return yarnKeys(text); + case 'cargo.lock': + case 'poetry.lock': + case 'uv.lock': + case 'pdm.lock': + return tomlKeys(text); + case 'gemfile.lock': + return gemfileKeys(text); + case 'composer.lock': + return composerKeys(text); + case 'pipfile.lock': + return pipfileKeys(text); + case 'packages.lock.json': + return nugetKeys(text); + case 'package.resolved': + return swiftKeys(text); + case 'pubspec.lock': + return pubspecKeys(text); + default: + return []; + } +} + +/** Stable warning line. `relPath` is the lockfile path relative to the project root. */ +export function unknownOptionalFieldWarning(relPath: string, key: string): string { + return `warning: ${relPath}: unknown optional lockfile field "${key}"; kept the dependencies this file already resolved`; +} + +/** One line per key, sorted by key name. Unsafe key names are dropped. */ +export function formatUnknownOptionalFieldWarnings(relPath: string, keys: readonly string[]): string[] { + const unique = new Set(); + for (const key of keys) { + if (SAFE_KEY.test(key)) unique.add(key); + } + return [...unique] + .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) + .map((key) => unknownOptionalFieldWarning(relPath, key)); +} + +function relativeLockfilePath(root: string, absPath: string): string { + const rel = path.relative(root, absPath); + if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) return path.basename(absPath); + return rel.split(path.sep).join('/'); +} + +function contentKey(absPath: string, text: string): string { + let hash = 0x811c9dc5; + for (let i = 0; i < text.length; i++) { + hash ^= text.charCodeAt(i); + hash = Math.imul(hash, 0x01000193); + } + return `${absPath}\0${(hash >>> 0).toString(16)}:${text.length}`; +} + +/** + * Write the unknown-optional-field warnings for one lockfile. A repeated read + * of the same bytes, or the same key on a later read, does not write again. + */ +export function warnUnknownOptionalLockfile(root: string, absPath: string, text: string): void { + const id = contentKey(absPath, text); + if (seenContent.has(id)) return; + seenContent.add(id); + const rel = relativeLockfilePath(root, absPath); + const fresh: string[] = []; + for (const key of unknownOptionalLockfileKeys(path.basename(absPath), text)) { + const mark = `${absPath}\0${key}`; + if (emittedKeys.has(mark)) continue; + emittedKeys.add(mark); + fresh.push(key); + } + for (const line of formatUnknownOptionalFieldWarnings(rel, fresh)) { + process.stderr.write(`${line}\n`); + } +} diff --git a/src/engine/lockfile.ts b/src/engine/lockfile.ts index 94ec0a67..6774929c 100644 --- a/src/engine/lockfile.ts +++ b/src/engine/lockfile.ts @@ -7,6 +7,7 @@ import { parseLockfileJson, } from '../core-open/utils/lockfile-parse.js'; import type { DepRecord } from './drift.js'; +import { warnUnknownOptionalLockfile } from './lockfile-optional.js'; /** * Read a lockfile that may be absent. A missing file is `undefined` (the @@ -29,6 +30,9 @@ function readCheckedText(root: string, file: string): string | undefined { if (text === undefined) return undefined; const kind = lockfileKind(file); if (kind) assertLockfileText(abs, text, kind); + // Syntax already passed. An unknown optional field warns and the text is + // still returned, so the dependency set is kept. + warnUnknownOptionalLockfile(root, abs, text); return text; } @@ -36,7 +40,9 @@ function readCheckedJson(root: string, file: string): unknown | undefined { const abs = path.join(root, file); const text = readOptionalText(abs); if (text === undefined) return undefined; - return parseLockfileJson(abs, text); + const parsed = parseLockfileJson(abs, text); + warnUnknownOptionalLockfile(root, abs, text); + return parsed; } /** diff --git a/src/reporting/scanners/dependency-graph.ts b/src/reporting/scanners/dependency-graph.ts index a446026a..c7c23109 100644 --- a/src/reporting/scanners/dependency-graph.ts +++ b/src/reporting/scanners/dependency-graph.ts @@ -1,6 +1,7 @@ import * as path from 'node:path'; import { readTextFile, pathExists, findPackageJsonFiles, readJsonFile, FileCache } from '../../core-open/index.js'; import { assertLockfileText, parseLockfileJson } from '../../core-open/utils/lockfile-parse.js'; +import { warnUnknownOptionalLockfile } from '../../engine/lockfile-optional.js'; import type { PackageJson, DependencyGraphResult, DuplicatedPackage, PhantomDependency } from '../../core-open/index.js'; interface LockEntry { @@ -101,15 +102,18 @@ export async function scanDependencyGraph(rootDir: string, cache?: FileCache): P result.lockfileType = 'pnpm'; const content = await _readTextFile(pnpmLock); assertLockfileText(pnpmLock, content, 'YAML'); + warnUnknownOptionalLockfile(rootDir, pnpmLock, content); entries = parsePnpmLock(content); } else if (await _pathExists(npmLock)) { result.lockfileType = 'npm'; const content = await _readTextFile(npmLock); entries = parseNpmLock(content, npmLock); + warnUnknownOptionalLockfile(rootDir, npmLock, content); } else if (await _pathExists(yarnLock)) { result.lockfileType = 'yarn'; const content = await _readTextFile(yarnLock); assertLockfileText(yarnLock, content, 'yarn.lock'); + warnUnknownOptionalLockfile(rootDir, yarnLock, content); entries = parseYarnLock(content); } From 47823258be246eec104d7dcef917afba1dfbd738 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 8 Oct 2026 21:27:27 +0000 Subject: [PATCH 2/2] feat(lockfile): warn and continue when an optional field is unknown Unknown optional lockfile fields keep the dependency set. vg prints one warning per path and field: unknown optional lockfile field ""; continuing with the fields this version understands. A field name that is not a plain identifier is reported as (name omitted). Signed-off-by: Cursor Agent Co-authored-by: Crowers --- CHANGELOG.md | 2 +- DOCS.md | 30 +- src/core-open/scanners/cargo-lockfile.ts | 18 +- src/core-open/scanners/dart-scanner.ts | 5 + src/core-open/scanners/npm-lockfile.ts | 52 +- src/core-open/scanners/php-scanner.ts | 27 +- src/core-open/scanners/python-lockfile.ts | 40 +- src/core-open/scanners/swift-scanner.ts | 6 + src/core-open/utils/lockfile-unknown.ts | 448 ++++++++++++++ src/engine/discover.ts | 24 +- src/engine/lockfile-optional.test.ts | 394 ------------ src/engine/lockfile-optional.ts | 673 --------------------- src/engine/lockfile-unknown.test.ts | 302 +++++++++ src/engine/lockfile.ts | 54 +- src/reporting/scanners/dependency-graph.ts | 23 +- 15 files changed, 958 insertions(+), 1140 deletions(-) create mode 100644 src/core-open/utils/lockfile-unknown.ts delete mode 100644 src/engine/lockfile-optional.test.ts delete mode 100644 src/engine/lockfile-optional.ts create mode 100644 src/engine/lockfile-unknown.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ea3e0f64..ab198d26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ backward compatible. ### Added -- **Unknown optional lockfile fields warn and keep the dependency set.** A key `vg` does not model leaves that file's packages in the graph and SBOM. The command prints one stable warning per file per field name and continues. A truncated or invalid lockfile still exits non-zero. +- **Unknown optional lockfile fields warn and continue.** When a lockfile still lists dependencies vg can read, an extra field does not drop that graph or the SBOM rows built from it. The command prints one warning per lockfile path and field name, sorted by path then field: `pnpm-lock.yaml: unknown optional lockfile field ""; continuing with the fields this version understands.` A field name that is not a plain identifier is reported as `(name omitted)`. A truncated lockfile, or one missing the structure its format requires, still exits non-zero. - **`vg build --attest` and `vg build --verify` are documented.** The docs cover the DSSE in-toto statement written to diff --git a/DOCS.md b/DOCS.md index 8e57c5de..6c86163c 100644 --- a/DOCS.md +++ b/DOCS.md @@ -9,7 +9,6 @@ For a quick overview, see the [README](./README.md). This document covers everyt ## Table of Contents - [How It Works](#how-it-works) -- [Lockfiles with unknown fields](#lockfiles-with-unknown-fields) - [Choosing a rollout model: one-off vs CI](#choosing-a-rollout-model-one-off-vs-ci) - [Commands Reference](#commands-reference) - [vg baseline](#vg-baseline) @@ -28,6 +27,7 @@ For a quick overview, see the [README](./README.md). This document covers everyt - [CycloneDX type and SPDX primaryPackagePurpose](#cyclonedx-type-and-spdx-primarypackagepurpose) - [Production, development, and optional scope](./docs/sbom-dependency-scope.md) - [vg scan](#vg-scan) + - [Unknown lockfile fields](#unknown-lockfile-fields) - [Offline scan with a package-version manifest](#offline-scan-with-a-package-version-manifest) - [Vulnerabilities and exposure attribution](#vulnerabilities-and-exposure-attribution) - [Go modules: pseudo-versions and +incompatible](#go-modules-pseudo-versions-and-incompatible) @@ -187,18 +187,6 @@ Vibgrate evaluates **upgrade drift** in depth for: **Known-vulnerability detection** (`--vulns`) and **dependency attribution** (`vg why`, exposure windows) additionally cover npm / pnpm / yarn, pip / poetry / pipenv, cargo, composer, bundler, pub, hex, NuGet, and Maven/Gradle, read from each project's lockfile. Go is matched from direct `require` lines in `go.mod`. Pseudo-versions and `+incompatible` tags follow [Go modules: pseudo-versions and +incompatible](#go-modules-pseudo-versions-and-incompatible). -### Lockfiles with unknown fields - -Lockfile formats grow optional fields. A pnpm `catalogs` block and a package `devEngines` entry are two examples. When the file still resolves a dependency set, `vg` keeps every package it understands and writes one warning per unknown field name: - -```text -warning: pnpm-lock.yaml: unknown optional lockfile field "catalogs"; kept the dependencies this file already resolved -``` - -The warning line contains the field name and the lockfile's path relative to the project root. The same file and the same field produce the same line on every run. Several unknown fields on one file are sorted by field name, and a field that appears many times is warned once. Field values stay off that line, including registry tokens and authorization header values. - -`vg build` prints the warning while it discovers the tree. `vg sbom export` prints it while it reads the lockfile graph. A lockfile that is truncated, unreadable, or missing the structure that dialect requires still exits non-zero. The error names the file and the format and leaves the file's contents out. `go.sum` and `gradle.lockfile` have no optional-key slot: a line that is not valid for that dialect is this structural failure. - ### End-to-end workflow (recommended) 1. Run an initial scan. @@ -2141,6 +2129,22 @@ By default, the scan writes `.vibgrate/scan_result.json`. Use `--no-local-artifa For offline drift scoring, pass `--package-manifest ` with a downloaded manifest bundle such as `https://github.com/vibgrate/manifests/latest-packages.zip`. The manifest shape, the fail-closed errors, and what offline mode skips are in [Offline scan with a package-version manifest](#offline-scan-with-a-package-version-manifest). +### Unknown lockfile fields + +Package managers add fields to lockfiles as they release. When the file still lists dependencies vg can read, an extra field does not erase that graph or the SBOM rows built from it. vg keeps the package names, versions, and dependency edges it understands. + +vg prints one warning on stderr for each unknown field name in that file. Lines are sorted by lockfile path, then by field name. The same path and field name is printed once. A warning looks like this: + +```text +pnpm-lock.yaml: unknown optional lockfile field "futureOptional"; continuing with the fields this version understands. +``` + +The path is relative to the directory you ran the command from, when the lockfile is under that directory. Otherwise the warning uses the file name only. The warning names the field. It does not print the field's value, an environment value, or an authorization header. A field name that is not a plain identifier is reported as `(name omitted)`. + +A lockfile that is truncated, empty, or missing the structure its format requires still fails the command. The error names the file and the format, tells you to regenerate the file with your package manager, and the process exits non-zero. The error does not include the file's contents. + +`vg scan` and `vg sbom export` follow this for npm, pnpm, yarn, poetry, uv, pdm, Pipfile, Cargo, Composer, NuGet, Swift package pins, and pub. A Gradle lock line, a Gemfile spec, and a `go.sum` line do not have optional field names. A line that does not match that format is invalid, and the command fails. + ### Offline scan with a package-version manifest `--offline` scores drift, and with `--vulns` it matches known vulnerabilities, from files on the machine. `--package-manifest` is the local file those lookups read. diff --git a/src/core-open/scanners/cargo-lockfile.ts b/src/core-open/scanners/cargo-lockfile.ts index 1d04ed90..dcb59894 100644 --- a/src/core-open/scanners/cargo-lockfile.ts +++ b/src/core-open/scanners/cargo-lockfile.ts @@ -15,6 +15,7 @@ import * as path from 'node:path'; import * as semver from 'semver'; import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { type LockfileUnknownFields, noteTomlLockDoc, withLockfileNotes } from '../utils/lockfile-unknown.js'; import { parseToml } from '../utils/toml.js'; import type { LockfileIo } from './npm-lockfile.js'; export type { LockfileIo } from './npm-lockfile.js'; @@ -30,9 +31,20 @@ export interface CargoLockIndex { * Parse a Cargo.lock into name → locked versions. A crate may appear more than once (multiple * incompatible majors in the tree), so versions accumulate into a list per name. */ -export function parseCargoLock(text: string): Map { +export function parseCargoLock( + text: string, + relPath = 'Cargo.lock', + notes?: LockfileUnknownFields, +): Map { + return withLockfileNotes(notes, (bucket) => { + const doc = parseToml(text); + noteTomlLockDoc(doc, relPath, bucket, 'cargo'); + return parseCargoLockBody(doc); + }); +} + +function parseCargoLockBody(doc: Record | null): Map { const out = new Map(); - const doc = parseToml(text); const packages = doc?.package; if (!Array.isArray(packages)) return out; for (const entry of packages) { @@ -70,7 +82,7 @@ export async function loadCargoLockIndex(dir: string, io: LockfileIo): Promise pickLockedVersion(map.get(name), spec) }; } catch (err) { diff --git a/src/core-open/scanners/dart-scanner.ts b/src/core-open/scanners/dart-scanner.ts index f8a4d67b..ff6c4291 100644 --- a/src/core-open/scanners/dart-scanner.ts +++ b/src/core-open/scanners/dart-scanner.ts @@ -5,6 +5,7 @@ import * as path from 'node:path'; import * as semver from 'semver'; import { readTextFile, FileCache } from '../utils/fs.js'; import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { lockfileWarningPath, notePubspecLockText, withLockfileNotes } from '../utils/lockfile-unknown.js'; import { withTimeout } from '../utils/timeout.js'; import { PubCache } from './pub-cache.js'; import type { ProjectScan, DependencyRow, DetectedFramework } from '../types.js'; @@ -188,6 +189,10 @@ async function parsePubspecLock(filePath: string, cache?: FileCache): Promise { + notePubspecLockText(content, lockfileWarningPath(filePath), bucket); + return undefined; + }); let currentPackage: string | null = null; for (const line of content.split(/\r?\n/)) { diff --git a/src/core-open/scanners/npm-lockfile.ts b/src/core-open/scanners/npm-lockfile.ts index 03e98764..fcfd2efd 100644 --- a/src/core-open/scanners/npm-lockfile.ts +++ b/src/core-open/scanners/npm-lockfile.ts @@ -17,6 +17,13 @@ import * as path from 'node:path'; import { parse as parseYaml } from 'yaml'; import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { + type LockfileUnknownFields, + notePackageLock, + notePnpmLockDoc, + noteYarnLockText, + withLockfileNotes, +} from '../utils/lockfile-unknown.js'; export type NpmLockSource = 'package-lock' | 'yarn' | 'pnpm'; @@ -37,7 +44,18 @@ interface PackageLockShape { const TOP_LEVEL_NODE_MODULES = /^node_modules\/((?:@[^/]+\/)?[^/]+)$/; /** name → top-level installed version from a package-lock.json (v2/v3 `packages`, falling back to v1 `dependencies`). */ -export function parsePackageLock(json: unknown): Map { +export function parsePackageLock( + json: unknown, + relPath = 'package-lock.json', + notes?: LockfileUnknownFields, +): Map { + return withLockfileNotes(notes, (bucket) => { + notePackageLock(json, relPath, bucket); + return parsePackageLockBody(json); + }); +} + +function parsePackageLockBody(json: unknown): Map { const out = new Map(); if (!json || typeof json !== 'object') return out; const lock = json as PackageLockShape; @@ -79,7 +97,18 @@ function specName(spec: string): string | null { * specs ending in `:`, followed by an indented `version: x.y.z`. Returns both a precise `name@range` * map and a name fallback. */ -export function parseYarnLock(text: string): YarnLockIndex { +export function parseYarnLock( + text: string, + relPath = 'yarn.lock', + notes?: LockfileUnknownFields, +): YarnLockIndex { + return withLockfileNotes(notes, (bucket) => { + noteYarnLockText(text, relPath, bucket); + return parseYarnLockBody(text); + }); +} + +function parseYarnLockBody(text: string): YarnLockIndex { const bySpec = new Map(); const byName = new Map(); let currentSpecs: string[] = []; @@ -123,7 +152,18 @@ function cleanPnpmVersion(v: string): string | null { * and the top-level `dependencies`/`devDependencies`/`optionalDependencies` (v5/v6), whose values are * either a version string or `{ specifier, version }`. First occurrence wins (the root importer). */ -export function parsePnpmLock(doc: unknown): Map { +export function parsePnpmLock( + doc: unknown, + relPath = 'pnpm-lock.yaml', + notes?: LockfileUnknownFields, +): Map { + return withLockfileNotes(notes, (bucket) => { + notePnpmLockDoc(doc, relPath, bucket); + return parsePnpmLockBody(doc); + }); +} + +function parsePnpmLockBody(doc: unknown): Map { const out = new Map(); if (!doc || typeof doc !== 'object') return out; const d = doc as Record; @@ -180,7 +220,7 @@ export async function loadNpmLockIndex(dir: string, io: LockfileIo): Promise map.get(name) ?? null }; } // package-lock next: structured JSON and unambiguous. @@ -192,7 +232,7 @@ export async function loadNpmLockIndex(dir: string, io: LockfileIo): Promise map.get(name) ?? null }; } if (await io.exists(yarnPath).catch(() => false)) { @@ -200,7 +240,7 @@ export async function loadNpmLockIndex(dir: string, io: LockfileIo): Promise; try { assertLockfileText(yarnPath, text, 'yarn.lock'); - parsed = parseYarnLock(text); + parsed = parseYarnLock(text, yarnPath); } catch (err) { rethrowLockfileParseError(err); throw new LockfileParseError(yarnPath, 'yarn.lock'); diff --git a/src/core-open/scanners/php-scanner.ts b/src/core-open/scanners/php-scanner.ts index e5993ca3..618e95c7 100644 --- a/src/core-open/scanners/php-scanner.ts +++ b/src/core-open/scanners/php-scanner.ts @@ -5,6 +5,7 @@ import * as path from 'node:path'; import * as semver from 'semver'; import { FileCache, readJsonFile } from '../utils/fs.js'; import { LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { lockfileWarningPath, noteComposerLock, withLockfileNotes } from '../utils/lockfile-unknown.js'; import { withTimeout } from '../utils/timeout.js'; import { ComposerCache } from './composer-cache.js'; import type { ProjectScan, DependencyRow, DetectedFramework } from '../types.js'; @@ -146,19 +147,23 @@ async function parseComposerLock(filePath: string, cache?: FileCache): Promise) { - if (pkg.name && pkg.version) { - resolved.set(pkg.name, pkg.version); + withLockfileNotes(undefined, (bucket) => { + noteComposerLock(data, lockfileWarningPath(filePath), bucket); + // Parse packages + for (const pkg of (data.packages ?? []) as Array<{ name?: string; version?: string }>) { + if (pkg.name && pkg.version) { + resolved.set(pkg.name, pkg.version); + } } - } - - // Parse dev packages - for (const pkg of (data['packages-dev'] ?? []) as Array<{ name?: string; version?: string }>) { - if (pkg.name && pkg.version) { - resolved.set(pkg.name, pkg.version); + + // Parse dev packages + for (const pkg of (data['packages-dev'] ?? []) as Array<{ name?: string; version?: string }>) { + if (pkg.name && pkg.version) { + resolved.set(pkg.name, pkg.version); + } } - } + return resolved; + }); } catch (err) { rethrowLockfileParseError(err); throw new LockfileParseError(filePath, 'JSON'); diff --git a/src/core-open/scanners/python-lockfile.ts b/src/core-open/scanners/python-lockfile.ts index 8a9f0204..deced737 100644 --- a/src/core-open/scanners/python-lockfile.ts +++ b/src/core-open/scanners/python-lockfile.ts @@ -21,6 +21,13 @@ */ import * as path from 'node:path'; import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { + type LockfileUnknownFields, + type TomlLockKind, + notePipfileLock, + noteTomlLockDoc, + withLockfileNotes, +} from '../utils/lockfile-unknown.js'; import { parseToml } from '../utils/toml.js'; import type { LockfileIo } from './npm-lockfile.js'; export type { LockfileIo } from './npm-lockfile.js'; @@ -48,9 +55,21 @@ function bareVersion(v: string): string | null { * name → version from a TOML lockfile of `[[package]]` tables (poetry / uv / pdm). First occurrence * wins. Keys are PEP 503 normalized. */ -export function parsePyTomlLock(text: string): Map { +export function parsePyTomlLock( + text: string, + relPath = 'poetry.lock', + notes?: LockfileUnknownFields, + kind: TomlLockKind = 'poetry', +): Map { + return withLockfileNotes(notes, (bucket) => { + const doc = parseToml(text); + noteTomlLockDoc(doc, relPath, bucket, kind); + return parsePyTomlLockBody(doc); + }); +} + +function parsePyTomlLockBody(doc: Record | null): Map { const out = new Map(); - const doc = parseToml(text); const packages = doc?.package; if (!Array.isArray(packages)) return out; for (const entry of packages) { @@ -64,7 +83,18 @@ export function parsePyTomlLock(text: string): Map { } /** name → version from a Pipfile.lock (JSON `default` + `develop` maps). Keys are PEP 503 normalized. */ -export function parsePipfileLock(json: unknown): Map { +export function parsePipfileLock( + json: unknown, + relPath = 'Pipfile.lock', + notes?: LockfileUnknownFields, +): Map { + return withLockfileNotes(notes, (bucket) => { + notePipfileLock(json, relPath, bucket); + return parsePipfileLockBody(json); + }); +} + +function parsePipfileLockBody(json: unknown): Map { const out = new Map(); if (!json || typeof json !== 'object') return out; const doc = json as Record; @@ -96,7 +126,7 @@ export async function loadPythonLockIndex(dir: string, io: LockfileIo): Promise< try { const text = await io.readText(p); assertLockfileText(p, text, 'TOML'); - const map = parsePyTomlLock(text); + const map = parsePyTomlLock(text, p, undefined, source); if (map.size) return { source, size: map.size, resolve: (n) => map.get(normalizePyName(n)) ?? null }; } catch (err) { rethrowLockfileParseError(err); @@ -112,7 +142,7 @@ export async function loadPythonLockIndex(dir: string, io: LockfileIo): Promise< rethrowLockfileParseError(err); throw new LockfileParseError(pipfile, 'JSON'); } - const map = parsePipfileLock(json); + const map = parsePipfileLock(json, pipfile); if (map.size) return { source: 'pipfile', size: map.size, resolve: (n) => map.get(normalizePyName(n)) ?? null }; } return null; diff --git a/src/core-open/scanners/swift-scanner.ts b/src/core-open/scanners/swift-scanner.ts index 9c4aec6f..54a1d3be 100644 --- a/src/core-open/scanners/swift-scanner.ts +++ b/src/core-open/scanners/swift-scanner.ts @@ -5,6 +5,7 @@ import * as path from 'node:path'; import * as semver from 'semver'; import { readTextFile, FileCache } from '../utils/fs.js'; import { parseLockfileJson, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { lockfileWarningPath, notePackageResolved, withLockfileNotes } from '../utils/lockfile-unknown.js'; import { withTimeout } from '../utils/timeout.js'; import { SwiftCache } from './swift-cache.js'; import type { ProjectScan, DependencyRow, DetectedFramework } from '../types.js'; @@ -135,6 +136,11 @@ function parsePackageResolved(filePath: string, content: string): Map; } | null; + withLockfileNotes(undefined, (bucket) => { + notePackageResolved(data, lockfileWarningPath(filePath), bucket); + return undefined; + }); + if (!data || typeof data !== 'object' || !data.pins) return resolved; for (const pin of data.pins) { if (!pin.identity || !pin.state?.version) continue; diff --git a/src/core-open/utils/lockfile-unknown.ts b/src/core-open/utils/lockfile-unknown.ts new file mode 100644 index 00000000..024c2be7 --- /dev/null +++ b/src/core-open/utils/lockfile-unknown.ts @@ -0,0 +1,448 @@ +// VENDORED from @vibgrate/core-open (packages/vibgrate-core-open) by +// scripts/vendor-core-open.mjs. Do not edit here — change the source package +// and re-run the vendor script. Apache-2.0. +/** + * Unknown optional lockfile fields. + * + * A key this version does not model is not a reason to drop a dependency + * set that is otherwise readable. Callers record the key, keep parsing, and + * emit one warning per file per key name. The warning names the key and the + * lockfile path. It never includes the key's value, an environment value, or + * an absolute path (those can carry home directories or registry tokens). + * + * Corrupt or missing required structure stays fail-closed in the syntax + * checker. This module does not accept a broken document. + */ +import * as path from 'node:path'; + +/** Identifier-like key. Anything else can be a header or a pasted secret. */ +const SAFE_KEY = /^[A-Za-z_][A-Za-z0-9_-]{0,63}$/; +const SAFE_REL = /^[A-Za-z0-9_@.+-][A-Za-z0-9_@.+/-]{0,180}$/; + +const emitted = new Set(); + +export function resetUnknownOptionalLockfileWarnings(): void { + emitted.clear(); +} + +/** + * Path safe to print: root-relative when the file sits under the process + * working directory, otherwise the file name alone. + */ +export function lockfileWarningPath(filePath: string, root?: string): string { + const abs = path.resolve(root ?? process.cwd(), filePath); + const fromCwd = safeRelative(path.relative(process.cwd(), abs)); + if (fromCwd) return fromCwd; + if (root) { + const fromRoot = safeRelative(path.relative(root, abs)); + if (fromRoot) return fromRoot; + } + const base = path.basename(abs); + return safeRelative(base) ?? 'lockfile'; +} + +function safeRelative(rel: string): string | undefined { + const posix = rel.replace(/\\/g, '/').replace(/^\.\//, ''); + if (!posix || posix.startsWith('..') || posix.split('/').includes('..')) return undefined; + if (!SAFE_REL.test(posix)) return undefined; + return posix; +} + +export function formatUnknownOptionalLockfileField(relPath: string, key: string): string { + return `${relPath}: unknown optional lockfile field "${key}"; continuing with the fields this version understands.`; +} + +export function formatOmittedUnknownOptionalLockfileField(relPath: string): string { + return `${relPath}: unknown optional lockfile field (name omitted); continuing with the fields this version understands.`; +} + +export class LockfileUnknownFields { + private readonly byFile = new Map>(); + private readonly omitted = new Set(); + + note(relPath: string, key: string): void { + const file = lockfileWarningPath(relPath); + if (!SAFE_KEY.test(key)) { + this.omitted.add(file); + return; + } + let set = this.byFile.get(file); + if (!set) { + set = new Set(); + this.byFile.set(file, set); + } + set.add(key); + } + + noteObject(relPath: string, value: unknown, known: ReadonlySet): void { + if (!value || typeof value !== 'object' || Array.isArray(value)) return; + for (const key of Object.keys(value as object)) { + if (known.has(key)) continue; + this.note(relPath, key); + } + } + + /** Sorted by path, then by key name. One line per pair. */ + messages(): string[] { + const files = new Set([...this.byFile.keys(), ...this.omitted]); + const out: string[] = []; + for (const file of [...files].sort()) { + const keys = [...(this.byFile.get(file) ?? [])].sort(); + for (const key of keys) out.push(formatUnknownOptionalLockfileField(file, key)); + if (this.omitted.has(file)) out.push(formatOmittedUnknownOptionalLockfileField(file)); + } + return out; + } +} + +export function emitUnknownOptionalLockfileWarnings( + notes: LockfileUnknownFields, + write: (line: string) => void = (line) => { + process.stderr.write(`${line}\n`); + }, +): string[] { + const fresh: string[] = []; + for (const message of notes.messages()) { + if (emitted.has(message)) continue; + emitted.add(message); + fresh.push(message); + write(message); + } + return fresh; +} + +/** Fill `notes` when the caller owns them; otherwise warn on stderr. */ +export function withLockfileNotes( + notes: LockfileUnknownFields | undefined, + fill: (bucket: LockfileUnknownFields) => T, +): T { + const bucket = notes ?? new LockfileUnknownFields(); + const result = fill(bucket); + if (!notes) emitUnknownOptionalLockfileWarnings(bucket); + return result; +} + +// ── Known keys ────────────────────────────────────────────────────────────── +// A key in these sets is part of the documented lockfile shape. It is not an +// "unknown" field, even when this reader does not copy the value (integrity, +// checksum, and similar). Missing a real key here would warn on every project. + +export const NPM_LOCK_TOP: ReadonlySet = new Set([ + 'name', 'version', 'lockfileVersion', 'requires', 'packages', 'dependencies', +]); + +export const NPM_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'resolved', 'integrity', 'link', 'dev', 'optional', 'devOptional', + 'inBundle', 'hasInstallScript', 'hasShrinkwrap', 'bin', 'license', 'engines', + 'dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies', + 'peerDependenciesMeta', 'bundledDependencies', 'bundleDependencies', 'os', 'cpu', + 'libc', 'funding', 'deprecated', 'extraneous', 'workspaces', 'requires', 'from', + 'bundled', 'peer', 'licenseFile', 'scripts', +]); + +export const PNPM_LOCK_TOP: ReadonlySet = new Set([ + 'lockfileVersion', 'settings', 'importers', 'packages', 'snapshots', 'time', + 'neverBuiltDependencies', 'onlyBuiltDependencies', 'overrides', 'packageExtensions', + 'packageExtensionsChecksum', 'patchedDependencies', 'catalogs', 'catalog', + 'ignoredOptionalDependencies', 'configDependencies', 'pnpmfileChecksum', + // pnpm v5/v6 put these maps at the top of the file. v9 moved them under importers. + 'dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies', + 'specifiers', 'dependenciesMeta', +]); + +export const PNPM_LOCK_IMPORTER: ReadonlySet = new Set([ + 'dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies', + 'specifiers', 'dependenciesMeta', 'publishDirectory', +]); + +export const PNPM_LOCK_DEP: ReadonlySet = new Set(['specifier', 'version', 'optional']); + +export const PNPM_LOCK_BLOCK: ReadonlySet = new Set([ + 'bundledDependencies', 'cpu', 'dependencies', 'deprecated', 'dev', 'devDependencies', + 'engines', 'hasBin', 'hash', 'libc', 'name', 'optional', 'optionalDependencies', 'os', + 'path', 'peerDependencies', 'peerDependenciesMeta', 'requiresBuild', 'resolution', + 'transitivePeerDependencies', 'version', +]); + +export const YARN_LOCK_FIELDS: ReadonlySet = new Set([ + 'version', 'resolved', 'integrity', 'dependencies', 'optionalDependencies', + 'dependenciesMeta', 'peerDependencies', 'peerDependenciesMeta', 'checksum', + 'languageName', 'linkType', 'resolution', 'bin', 'conditions', 'cacheKey', +]); + +export const CARGO_LOCK_TOP: ReadonlySet = new Set(['version', 'package', 'metadata', 'patch']); +export const CARGO_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'source', 'checksum', 'dependencies', 'replace', 'explicit', +]); + +export const POETRY_LOCK_TOP: ReadonlySet = new Set(['package', 'metadata']); +export const POETRY_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'description', 'optional', 'python-versions', 'files', + 'dependencies', 'extras', 'category', 'groups', 'markers', 'source', 'develop', + 'dev-dependencies', +]); +export const POETRY_LOCK_METADATA: ReadonlySet = new Set([ + 'lock-version', 'python-versions', 'content-hash', +]); + +export const UV_LOCK_TOP: ReadonlySet = new Set([ + 'version', 'requires-python', 'package', 'manifest', 'resolution-markers', +]); +export const UV_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'source', 'dependencies', 'optional-dependencies', + 'dev-dependencies', 'wheels', 'sdist', 'metadata', 'provides-extras', + 'resolution-markers', +]); + +export const PDM_LOCK_TOP: ReadonlySet = new Set(['metadata', 'package']); +export const PDM_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'requires_python', 'summary', 'dependencies', 'files', + 'marker', 'extras', 'groups', +]); +export const PDM_LOCK_METADATA: ReadonlySet = new Set([ + 'groups', 'strategy', 'lock_version', 'content_hash', +]); + +export type TomlLockKind = 'cargo' | 'poetry' | 'uv' | 'pdm'; + +export const PIPFILE_TOP: ReadonlySet = new Set(['_meta', 'default', 'develop']); +export const PIPFILE_META: ReadonlySet = new Set(['hash', 'pipfile-spec', 'requires', 'sources']); +export const PIPFILE_ENTRY: ReadonlySet = new Set([ + 'hashes', 'markers', 'version', 'index', 'extras', 'subdirectory', + 'editable', 'git', 'ref', 'path', 'file', 'branch', 'tag', 'revision', +]); + +export const COMPOSER_LOCK_TOP: ReadonlySet = new Set([ + 'content-hash', 'packages', 'packages-dev', 'aliases', 'minimum-stability', + 'stability-flags', 'prefer-stable', 'prefer-lowest', 'platform', 'platform-dev', + 'platform-overrides', 'plugin-api-version', +]); +export const COMPOSER_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'version_normalized', 'source', 'dist', 'require', 'require-dev', + 'conflict', 'replace', 'provide', 'suggest', 'type', 'extra', 'autoload', 'autoload-dev', + 'notification-url', 'license', 'authors', 'description', 'homepage', 'keywords', 'support', + 'funding', 'time', 'bin', 'archive', 'abandoned', 'transport-options', 'installation-source', + 'target-dir', 'include-path', 'scripts', 'repositories', 'config', 'minimum-stability', + 'prefer-stable', 'default-branch', 'aliases', +]); + +export const NUGET_LOCK_TOP: ReadonlySet = new Set(['version', 'dependencies']); +export const NUGET_LOCK_ENTRY: ReadonlySet = new Set([ + 'type', 'requested', 'resolved', 'contentHash', 'dependencies', +]); + +export const SWIFT_RESOLVED_TOP: ReadonlySet = new Set(['version', 'originHash', 'pins', 'object']); +export const SWIFT_PIN: ReadonlySet = new Set(['identity', 'kind', 'location', 'state', 'package']); +export const SWIFT_PIN_STATE: ReadonlySet = new Set(['version', 'revision', 'branch']); + +export const PUBSPEC_LOCK_TOP: ReadonlySet = new Set(['packages', 'sdks']); +export const PUBSPEC_LOCK_FIELDS: ReadonlySet = new Set([ + 'dependency', 'description', 'source', 'version', +]); + +function tomlSets(kind: TomlLockKind): { + top: ReadonlySet; + pkg: ReadonlySet; + meta?: ReadonlySet; +} { + switch (kind) { + case 'cargo': + return { top: CARGO_LOCK_TOP, pkg: CARGO_LOCK_PACKAGE }; + case 'poetry': + return { top: POETRY_LOCK_TOP, pkg: POETRY_LOCK_PACKAGE, meta: POETRY_LOCK_METADATA }; + case 'uv': + return { top: UV_LOCK_TOP, pkg: UV_LOCK_PACKAGE }; + case 'pdm': + return { top: PDM_LOCK_TOP, pkg: PDM_LOCK_PACKAGE, meta: PDM_LOCK_METADATA }; + default: { + const _exhaustive: never = kind; + return _exhaustive; + } + } +} + +export function notePackageLock(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, NPM_LOCK_TOP); + if (!doc || typeof doc !== 'object') return; + const data = doc as { packages?: unknown; dependencies?: unknown }; + if (data.packages && typeof data.packages === 'object' && !Array.isArray(data.packages)) { + for (const entry of Object.values(data.packages as object)) { + notes.noteObject(relPath, entry, NPM_LOCK_PACKAGE); + } + } + noteNpmDependencyTree(data.dependencies, relPath, notes); +} + +function noteNpmDependencyTree(deps: unknown, relPath: string, notes: LockfileUnknownFields): void { + if (!deps || typeof deps !== 'object' || Array.isArray(deps)) return; + for (const entry of Object.values(deps as object)) { + notes.noteObject(relPath, entry, NPM_LOCK_PACKAGE); + if (entry && typeof entry === 'object') { + noteNpmDependencyTree((entry as { dependencies?: unknown }).dependencies, relPath, notes); + } + } +} + +export function notePnpmLockDoc(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, PNPM_LOCK_TOP); + if (!doc || typeof doc !== 'object') return; + const data = doc as Record; + notePnpmDependencyBlocks(data, relPath, notes); + const importers = data.importers; + if (!importers || typeof importers !== 'object' || Array.isArray(importers)) return; + for (const imp of Object.values(importers as object)) { + notes.noteObject(relPath, imp, PNPM_LOCK_IMPORTER); + if (imp && typeof imp === 'object') notePnpmDependencyBlocks(imp as Record, relPath, notes); + } +} + +function notePnpmDependencyBlocks(data: Record, relPath: string, notes: LockfileUnknownFields): void { + for (const section of ['dependencies', 'devDependencies', 'optionalDependencies'] as const) { + const deps = data[section]; + if (!deps || typeof deps !== 'object' || Array.isArray(deps)) continue; + for (const entry of Object.values(deps as object)) { + notes.noteObject(relPath, entry, PNPM_LOCK_DEP); + } + } +} + +const PNPM_TOP_LINE = /^([A-Za-z_][A-Za-z0-9_-]*):/; +const PNPM_BLOCK_LINE = /^ {4}([A-Za-z_][A-Za-z0-9_-]*):/; +const PNPM_DEP_LINE = /^ {8}([A-Za-z_][A-Za-z0-9_-]*):/; + +/** Text form of pnpm-lock.yaml. Package names sit at other indents and are not keys. */ +export function notePnpmLockText(text: string, relPath: string, notes: LockfileUnknownFields): void { + for (const line of text.split('\n')) { + const top = PNPM_TOP_LINE.exec(line); + if (top) { + if (!PNPM_LOCK_TOP.has(top[1]!)) notes.note(relPath, top[1]!); + continue; + } + const block = PNPM_BLOCK_LINE.exec(line); + if (block) { + if (!PNPM_LOCK_BLOCK.has(block[1]!)) notes.note(relPath, block[1]!); + continue; + } + const dep = PNPM_DEP_LINE.exec(line); + if (dep && !PNPM_LOCK_DEP.has(dep[1]!)) notes.note(relPath, dep[1]!); + } +} + +const YARN_FIELD_LINE = /^ {2}([A-Za-z_][A-Za-z0-9_-]*)\b/; + +export function noteYarnLockText(text: string, relPath: string, notes: LockfileUnknownFields): void { + for (const line of text.split('\n')) { + const field = YARN_FIELD_LINE.exec(line); + if (field && !YARN_LOCK_FIELDS.has(field[1]!)) notes.note(relPath, field[1]!); + } +} + +const TOML_ASSIGN = /^([A-Za-z_][A-Za-z0-9_-]*)\s*=/; + +function tomlAssignmentKeys(block: string): string[] { + const keys: string[] = []; + for (const line of block.split('\n')) { + const match = TOML_ASSIGN.exec(line); + if (match) keys.push(match[1]!); + } + return keys; +} + +export function noteTomlLockDoc(doc: unknown, relPath: string, notes: LockfileUnknownFields, kind: TomlLockKind): void { + const sets = tomlSets(kind); + notes.noteObject(relPath, doc, sets.top); + if (!doc || typeof doc !== 'object') return; + const data = doc as { package?: unknown; metadata?: unknown }; + if (Array.isArray(data.package)) { + for (const entry of data.package) notes.noteObject(relPath, entry, sets.pkg); + } + if (sets.meta) notes.noteObject(relPath, data.metadata, sets.meta); +} + +export function noteTomlLockText(text: string, relPath: string, notes: LockfileUnknownFields, kind: TomlLockKind): void { + const sets = tomlSets(kind); + // Preamble assignments only. A file that opens with `[[package]]` has no + // top-level keys; package names inside later tables are not fields. + const tableAt = text.search(/^\s*\[/m); + const head = tableAt === -1 ? text : text.slice(0, tableAt); + for (const key of tomlAssignmentKeys(head)) { + if (!sets.top.has(key)) notes.note(relPath, key); + } + for (const block of text.split(/\[\[package\]\]/).slice(1)) { + const body = block.split(/\n\[/)[0] ?? block; + for (const key of tomlAssignmentKeys(body)) { + if (!sets.pkg.has(key)) notes.note(relPath, key); + } + } + if (!sets.meta) return; + const meta = /\[metadata\][^\n]*\n([\s\S]*?)(?=\n\[|$)/.exec(text); + if (!meta) return; + for (const key of tomlAssignmentKeys(meta[1]!)) { + if (!sets.meta.has(key)) notes.note(relPath, key); + } +} + +export function notePipfileLock(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, PIPFILE_TOP); + if (!doc || typeof doc !== 'object') return; + const data = doc as Record; + notes.noteObject(relPath, data._meta, PIPFILE_META); + for (const section of ['default', 'develop']) { + const deps = data[section]; + if (!deps || typeof deps !== 'object' || Array.isArray(deps)) continue; + for (const entry of Object.values(deps as object)) notes.noteObject(relPath, entry, PIPFILE_ENTRY); + } +} + +export function noteComposerLock(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, COMPOSER_LOCK_TOP); + if (!doc || typeof doc !== 'object') return; + const data = doc as Record; + for (const section of ['packages', 'packages-dev']) { + const arr = data[section]; + if (!Array.isArray(arr)) continue; + for (const entry of arr) notes.noteObject(relPath, entry, COMPOSER_LOCK_PACKAGE); + } +} + +export function noteNugetLock(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, NUGET_LOCK_TOP); + if (!doc || typeof doc !== 'object') return; + const deps = (doc as { dependencies?: unknown }).dependencies; + if (!deps || typeof deps !== 'object' || Array.isArray(deps)) return; + for (const framework of Object.values(deps as object)) { + if (!framework || typeof framework !== 'object' || Array.isArray(framework)) continue; + for (const entry of Object.values(framework as object)) { + notes.noteObject(relPath, entry, NUGET_LOCK_ENTRY); + } + } +} + +export function notePackageResolved(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, SWIFT_RESOLVED_TOP); + if (!doc || typeof doc !== 'object') return; + const data = doc as { pins?: unknown; object?: { pins?: unknown } }; + const pins = Array.isArray(data.pins) ? data.pins : data.object?.pins; + if (!Array.isArray(pins)) return; + for (const pin of pins) { + notes.noteObject(relPath, pin, SWIFT_PIN); + if (pin && typeof pin === 'object') { + notes.noteObject(relPath, (pin as { state?: unknown }).state, SWIFT_PIN_STATE); + } + } +} + +const PUBSPEC_TOP_LINE = /^([A-Za-z_][A-Za-z0-9_-]*):/; +const PUBSPEC_FIELD_LINE = /^ {4}([A-Za-z_][A-Za-z0-9_-]*):/; + +export function notePubspecLockText(text: string, relPath: string, notes: LockfileUnknownFields): void { + for (const line of text.split('\n')) { + const top = PUBSPEC_TOP_LINE.exec(line); + if (top) { + if (!PUBSPEC_LOCK_TOP.has(top[1]!)) notes.note(relPath, top[1]!); + continue; + } + const field = PUBSPEC_FIELD_LINE.exec(line); + if (field && !PUBSPEC_LOCK_FIELDS.has(field[1]!)) notes.note(relPath, field[1]!); + } +} diff --git a/src/engine/discover.ts b/src/engine/discover.ts index 4ef2ac10..2f523416 100644 --- a/src/engine/discover.ts +++ b/src/engine/discover.ts @@ -5,7 +5,6 @@ import { langForExtension, langById, type LanguageDef } from './languages.js'; import { requireDataConfig } from '../core-open/config.js'; import { dropBlankPatterns, gitignoreWithoutBlankLines } from '../core-open/utils/glob.js'; import { assertLockfileFile, lockfileKind } from '../core-open/utils/lockfile-parse.js'; -import { warnUnknownOptionalLockfile } from './lockfile-optional.js'; import { assertSafeWalkRoot, createWalkBudget, noteWalkEntry } from '../core-open/utils/root-safety.js'; /** @@ -249,7 +248,6 @@ export function discover(options: DiscoverOptions): DiscoveredFile[] { const found = new Map(); const budget = createWalkBudget(root, options.maxEntries); - const lockfiles: string[] = []; const considerFile = (abs: string): void => { const rel = toPosix(path.relative(root, abs)); @@ -258,12 +256,7 @@ export function discover(options: DiscoverOptions): DiscoveredFile[] { if (SKIP_FILES.has(path.basename(abs).toLowerCase())) { // Lockfiles are not source, but a truncated one must fail the build // here — before the parse pool starts — rather than being skipped. - // A file that parses keeps its packages; unknown optional fields warn - // after the walk, in path order. - if (lockfileKind(path.basename(abs))) { - assertLockfileFile(abs); - lockfiles.push(abs); - } + if (lockfileKind(path.basename(abs))) assertLockfileFile(abs); return; } const lang = langForExtension(path.extname(abs)); @@ -304,21 +297,6 @@ export function discover(options: DiscoverOptions): DiscoveredFile[] { } else if (stat.isFile()) considerFile(scope); } - lockfiles.sort((a, b) => { - const ra = toPosix(path.relative(root, a)); - const rb = toPosix(path.relative(root, b)); - return ra < rb ? -1 : ra > rb ? 1 : 0; - }); - for (const abs of lockfiles) { - let text: string; - try { - text = fs.readFileSync(abs, 'utf8'); - } catch { - continue; - } - warnUnknownOptionalLockfile(root, abs, text); - } - return [...found.values()].sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0)); } diff --git a/src/engine/lockfile-optional.test.ts b/src/engine/lockfile-optional.test.ts deleted file mode 100644 index 3a40b01b..00000000 --- a/src/engine/lockfile-optional.test.ts +++ /dev/null @@ -1,394 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import * as fs from 'node:fs'; -import * as os from 'node:os'; -import * as path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { LockfileParseError } from '../core-open/utils/lockfile-parse.js'; -import { parseCargoLock } from '../core-open/scanners/cargo-lockfile.js'; -import { parseGemfileLock } from '../core-open/scanners/gemfile-lock.js'; -import { parseGradleLockfile } from '../core-open/scanners/gradle-lockfile.js'; -import { parsePackageLock, parsePnpmLock, parseYarnLock } from '../core-open/scanners/npm-lockfile.js'; -import { parsePipfileLock, parsePyTomlLock } from '../core-open/scanners/python-lockfile.js'; -import { - parseComposerLock, - parseMixLock, - parseNpmLockfile, - parseNugetLock, - parsePipfileLock as parseTimelinePipfile, - parsePnpmLockfile, - parsePubspecLock, - parseTomlPackages, - parseYarnLockfile, -} from '../core-open/utils/version-timeline.js'; -import { discover } from './discover.js'; -import { fullDependencyGraph, lockfileVersion } from './lockfile.js'; -import { - formatUnknownOptionalFieldWarnings, - unknownOptionalFieldWarning, - unknownOptionalLockfileKeys, -} from './lockfile-optional.js'; - -const pkgRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); -const SECRET = 'Bearer npm_AAAAAAAAAAAAAAAAAAAA'; - -function captureStderr(run: () => void): { stderr: string; error?: unknown } { - let stderr = ''; - const write = process.stderr.write; - process.stderr.write = ((chunk: string | Uint8Array) => { - stderr += typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8'); - return true; - }) as typeof process.stderr.write; - try { - run(); - return { stderr }; - } catch (error) { - return { stderr, error }; - } finally { - process.stderr.write = write; - } -} - -describe('unknown optional lockfile fields', () => { - let root: string; - const token = 'super-secret-token'; - const previousToken = process.env.NPM_TOKEN; - - beforeEach(() => { - root = fs.mkdtempSync(path.join(os.tmpdir(), 'vg-lock-optional-')); - process.env.NPM_TOKEN = token; - }); - afterEach(() => { - fs.rmSync(root, { recursive: true, force: true }); - if (previousToken === undefined) delete process.env.NPM_TOKEN; - else process.env.NPM_TOKEN = previousToken; - }); - - const write = (rel: string, body: string): void => { - const abs = path.join(root, rel); - fs.mkdirSync(path.dirname(abs), { recursive: true }); - fs.writeFileSync(abs, body); - }; - - it('formats one sorted warning per key and leaves values out of the line', () => { - const line = unknownOptionalFieldWarning('pnpm-lock.yaml', 'catalogs'); - expect(line).toBe( - 'warning: pnpm-lock.yaml: unknown optional lockfile field "catalogs"; kept the dependencies this file already resolved', - ); - expect(formatUnknownOptionalFieldWarnings('package-lock.json', ['zebraOptional', 'alphaOptional', 'alphaOptional'])).toEqual([ - unknownOptionalFieldWarning('package-lock.json', 'alphaOptional'), - unknownOptionalFieldWarning('package-lock.json', 'zebraOptional'), - ]); - expect(formatUnknownOptionalFieldWarnings('package-lock.json', ['ok', 'Bearer secret', 'a b'])).toEqual([ - unknownOptionalFieldWarning('package-lock.json', 'ok'), - ]); - }); - - it('keeps an npm dependency set and warns once for a repeated unknown key', () => { - write( - 'package-lock.json', - JSON.stringify({ - lockfileVersion: 3, - futureOptional: SECRET, - packages: { - '': { name: 'app' }, - 'node_modules/left-pad': { version: '1.3.0', devEngines: { runtime: { name: 'node' } }, integrity: SECRET }, - 'node_modules/chalk': { version: '5.3.0', devEngines: { cpu: 'x64' } }, - }, - }), - ); - const first = captureStderr(() => { - expect(fullDependencyGraph(root)?.components).toEqual([ - { package: 'chalk', version: '5.3.0' }, - { package: 'left-pad', version: '1.3.0' }, - ]); - }); - expect(first.error).toBeUndefined(); - expect(first.stderr).toBe( - [ - unknownOptionalFieldWarning('package-lock.json', 'devEngines'), - unknownOptionalFieldWarning('package-lock.json', 'futureOptional'), - '', - ].join('\n'), - ); - expect(first.stderr).not.toContain(SECRET); - expect(first.stderr).not.toContain(token); - expect(first.stderr).not.toContain(root); - expect(first.stderr).not.toContain('left-pad'); - const second = captureStderr(() => fullDependencyGraph(root)); - expect(second.stderr).toBe(''); - }); - - it('keeps a pnpm catalog lockfile and warns for the unknown catalogs key', () => { - const body = [ - "lockfileVersion: '9.0'", - 'catalogs:', - ' default:', - ' left-pad:', - " specifier: '^1.3.0'", - ' version: 1.3.0', - 'importers:', - ' .:', - ' dependencies:', - ' left-pad:', - " specifier: 'catalog:'", - ' version: 1.3.0', - 'packages:', - ' left-pad@1.3.0:', - ' resolution: {integrity: sha512-abc}', - '', - ].join('\n'); - write('pnpm-lock.yaml', body); - const captured = captureStderr(() => { - expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'left-pad', version: '1.3.0' }]); - expect(lockfileVersion(root, 'npm', 'left-pad')).toBe('1.3.0'); - }); - expect(captured.error).toBeUndefined(); - expect(captured.stderr).toBe(`${unknownOptionalFieldWarning('pnpm-lock.yaml', 'catalogs')}\n`); - expect(captured.stderr).not.toContain(SECRET); - }); - - it('keeps yarn, cargo, and poetry rows when one optional key is unknown', () => { - write('yarn.lock', ['left-pad@^1.3.0:', ' version "1.3.0"', ' futureOptional true', ''].join('\n')); - const yarn = captureStderr(() => { - expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'left-pad', version: '1.3.0' }]); - }); - expect(yarn.stderr).toBe(`${unknownOptionalFieldWarning('yarn.lock', 'futureOptional')}\n`); - - fs.rmSync(path.join(root, 'yarn.lock')); - write('Cargo.lock', ['[[package]]', 'name = "memchr"', 'version = "2.7.4"', 'futureOptional = "kept"', ''].join('\n')); - const cargo = captureStderr(() => { - expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'memchr', version: '2.7.4' }]); - }); - expect(cargo.stderr).toBe(`${unknownOptionalFieldWarning('Cargo.lock', 'futureOptional')}\n`); - - fs.rmSync(path.join(root, 'Cargo.lock')); - write('poetry.lock', ['[[package]]', 'name = "click"', 'version = "8.1.3"', 'futureOptional = "kept"', ''].join('\n')); - const poetry = captureStderr(() => { - expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'click', version: '8.1.3' }]); - }); - expect(poetry.stderr).toBe(`${unknownOptionalFieldWarning('poetry.lock', 'futureOptional')}\n`); - expect(poetry.stderr).not.toContain('click'); - }); - - it('warns for other ecosystems and still returns the locked version', () => { - const cases: Array<{ file: string; body: string; ecosystem: 'php' | 'pypi' | 'dart' | 'dotnet' | 'swift' | 'ruby'; name: string; version: string }> = [ - { - file: 'composer.lock', - ecosystem: 'php', - name: 'monolog/monolog', - version: '3.5.0', - body: JSON.stringify({ - futureOptional: SECRET, - packages: [{ name: 'monolog/monolog', version: '3.5.0' }], - }), - }, - { - file: 'Pipfile.lock', - ecosystem: 'pypi', - name: 'requests', - version: '2.31.0', - body: JSON.stringify({ - futureOptional: true, - default: { requests: { version: '==2.31.0' } }, - }), - }, - { - file: 'pubspec.lock', - ecosystem: 'dart', - name: 'http', - version: '1.2.0', - body: ['packages:', ' http:', ' version: "1.2.0"', ' futureOptional: true', ''].join('\n'), - }, - { - file: 'packages.lock.json', - ecosystem: 'dotnet', - name: 'Newtonsoft.Json', - version: '13.0.3', - body: JSON.stringify({ - version: 1, - futureOptional: true, - dependencies: { 'net8.0': { 'Newtonsoft.Json': { resolved: '13.0.3' } } }, - }), - }, - { - file: 'Package.resolved', - ecosystem: 'swift', - name: 'vapor', - version: '4.89.0', - body: JSON.stringify({ - version: 3, - futureOptional: true, - pins: [{ identity: 'vapor', state: { version: '4.89.0' } }], - }), - }, - { - file: 'Gemfile.lock', - ecosystem: 'ruby', - name: 'rake', - version: '13.0.6', - body: ['GEM', ' remote: https://rubygems.org/', ' specs:', ' rake (13.0.6)', 'FUTURE', ''].join('\n'), - }, - ]; - for (const item of cases) { - write(item.file, item.body); - const captured = captureStderr(() => { - expect(lockfileVersion(root, item.ecosystem, item.name)).toBe(item.version); - }); - const key = item.file === 'Gemfile.lock' ? 'FUTURE' : 'futureOptional'; - expect(captured.stderr, item.file).toBe(`${unknownOptionalFieldWarning(item.file, key)}\n`); - expect(captured.stderr, item.file).not.toContain(SECRET); - expect(captured.stderr, item.file).not.toContain(token); - fs.rmSync(path.join(root, item.file)); - } - }); - - it('does not warn when the lockfile has no resolved package', () => { - write('package-lock.json', JSON.stringify({ lockfileVersion: 3, futureOptional: true, packages: {} })); - const captured = captureStderr(() => { - expect(fullDependencyGraph(root)).toBeUndefined(); - }); - expect(captured.stderr).toBe(''); - expect(unknownOptionalLockfileKeys('package-lock.json', fs.readFileSync(path.join(root, 'package-lock.json'), 'utf8'))).toEqual([]); - }); - - it('fails closed on broken required structure and does not warn', () => { - write('package-lock.json', `{"lockfileVersion":3,"futureOptional":true,"packages":{"node_modules/left-pad":{"version":"1.0.0","integrity":"${SECRET}"`); - const npm = captureStderr(() => { - fullDependencyGraph(root); - }); - expect(npm.error).toBeInstanceOf(LockfileParseError); - expect((npm.error as Error).message).toContain('truncated or invalid JSON'); - expect(npm.stderr).toBe(''); - expect(npm.stderr).not.toContain(SECRET); - - fs.rmSync(path.join(root, 'package-lock.json')); - write('gradle.lockfile', 'com.example:lib:1.2.3=runtimeClasspath\nthis is not a lock line\n'); - const gradle = captureStderr(() => { - lockfileVersion(root, 'java', 'com.example:lib'); - }); - expect(gradle.error).toBeInstanceOf(LockfileParseError); - expect((gradle.error as Error).message).toContain('truncated or invalid gradle.lockfile'); - expect(gradle.stderr).not.toContain('unknown optional'); - - fs.rmSync(path.join(root, 'gradle.lockfile')); - write('go.sum', 'github.com/gin-gonic/gin v1.9.0 h1:abc=\nnot-a-sum-line\n'); - const go = captureStderr(() => { - fullDependencyGraph(root); - }); - expect(go.error).toBeInstanceOf(LockfileParseError); - expect((go.error as Error).message).toContain('truncated or invalid go.sum'); - expect(go.stderr).not.toContain('unknown optional'); - }); - - it('keeps a valid gradle pin with no optional-key warning', () => { - write('gradle.lockfile', '# generated\ncom.example:lib:1.2.3=runtimeClasspath\nempty=\n'); - const captured = captureStderr(() => { - expect(lockfileVersion(root, 'java', 'com.example:lib')).toBe('1.2.3'); - }); - expect(captured.stderr).toBe(''); - }); - - it('discover keeps source files, warns in path order, and still rejects a truncated lockfile', () => { - write('app.ts', 'export const n = 1;\n'); - const lock = JSON.stringify({ - lockfileVersion: 3, - packages: { 'node_modules/left-pad': { version: '1.3.0' } }, - futureOptional: true, - }); - write('a/package-lock.json', lock); - write('b/package-lock.json', lock); - const ok = captureStderr(() => { - const files = discover({ root }); - expect(files.map((file) => file.rel)).toEqual(['app.ts']); - }); - expect(ok.stderr).toBe( - [ - unknownOptionalFieldWarning('a/package-lock.json', 'futureOptional'), - unknownOptionalFieldWarning('b/package-lock.json', 'futureOptional'), - '', - ].join('\n'), - ); - - const broken = fs.mkdtempSync(path.join(os.tmpdir(), 'vg-lock-optional-')); - try { - fs.writeFileSync(path.join(broken, 'package-lock.json'), '{"packages":{'); - fs.writeFileSync(path.join(broken, 'app.ts'), 'export const n = 1;\n'); - const failed = captureStderr(() => { - discover({ root: broken }); - }); - expect(failed.error).toBeInstanceOf(LockfileParseError); - expect(failed.stderr).toBe(''); - } finally { - fs.rmSync(broken, { recursive: true, force: true }); - } - }); - - it('does not treat current lockfiles in this repository as unknown', () => { - const pnpm = fs.readFileSync(path.join(pkgRoot, 'pnpm-lock.yaml'), 'utf8'); - expect(unknownOptionalLockfileKeys('pnpm-lock.yaml', pnpm)).toEqual([]); - const npmLock = fs.readFileSync(path.join(pkgRoot, 'packages/test-solutions/package-lock.json'), 'utf8'); - expect(unknownOptionalLockfileKeys('package-lock.json', npmLock)).toEqual([]); - }); - - it('scanner readers keep the resolved row when an optional key is unknown', () => { - const npm = JSON.stringify({ - lockfileVersion: 3, - futureOptional: true, - packages: { 'node_modules/left-pad': { version: '1.3.0', devEngines: { runtime: 'node' } } }, - }); - expect(parsePackageLock(JSON.parse(npm)).get('left-pad')).toBe('1.3.0'); - expect(parseNpmLockfile(npm).get('left-pad')).toBe('1.3.0'); - - const pnpmDoc = { - lockfileVersion: '9.0', - catalogs: { default: { 'left-pad': { version: '1.3.0' } } }, - importers: { '.': { dependencies: { 'left-pad': { specifier: 'catalog:', version: '1.3.0' } } } }, - }; - expect(parsePnpmLock(pnpmDoc).get('left-pad')).toBe('1.3.0'); - const pnpmText = [ - "lockfileVersion: '9.0'", - 'catalogs:', - ' default:', - ' left-pad:', - ' version: 1.3.0', - 'packages:', - ' left-pad@1.3.0:', - ' resolution: {integrity: sha512-abc}', - '', - ].join('\n'); - expect(parsePnpmLockfile(pnpmText).get('left-pad')).toBe('1.3.0'); - - const yarn = ['left-pad@^1.3.0:', ' version "1.3.0"', ' futureOptional true', ''].join('\n'); - expect(parseYarnLock(yarn).byName.get('left-pad')).toBe('1.3.0'); - expect(parseYarnLockfile(yarn).get('left-pad')).toBe('1.3.0'); - - const cargo = ['[[package]]', 'name = "memchr"', 'version = "2.7.4"', 'futureOptional = "kept"', ''].join('\n'); - expect(parseCargoLock(cargo).get('memchr')).toEqual(['2.7.4']); - expect(parseTomlPackages(cargo).get('memchr')).toBe('2.7.4'); - expect(parsePyTomlLock(cargo).get('memchr')).toBe('2.7.4'); - - const pip = JSON.stringify({ futureOptional: true, default: { requests: { version: '==2.31.0', hashes: ['abc'] } } }); - expect(parsePipfileLock(JSON.parse(pip)).get('requests')).toBe('2.31.0'); - expect(parseTimelinePipfile(pip).get('requests')).toBe('2.31.0'); - - expect(parseGradleLockfile('# comment\ncom.example:lib:1.2.3=runtimeClasspath\nempty=\n').get('com.example:lib')).toBe('1.2.3'); - const gem = ['GEM', ' specs:', ' rake (13.0.6)', 'FUTURE', ' extra: 1', ''].join('\n'); - expect(parseGemfileLock(gem).get('rake')).toBe('13.0.6'); - - const composer = JSON.stringify({ futureOptional: true, packages: [{ name: 'monolog/monolog', version: 'v3.5.0', extra: {} }] }); - expect(parseComposerLock(composer).get('monolog/monolog')).toBe('3.5.0'); - - const pub = ['packages:', ' http:', ' futureOptional: true', ' version: "1.2.0"', ''].join('\n'); - expect(parsePubspecLock(pub).get('http')).toBe('1.2.0'); - - const nuget = JSON.stringify({ - version: 2, - futureOptional: true, - dependencies: { 'net8.0': { 'Newtonsoft.Json': { resolved: '13.0.3', contentHash: 'abc' } } }, - }); - expect(parseNugetLock(nuget).get('Newtonsoft.Json')).toBe('13.0.3'); - - const mix = '"decimal" => {:hex, :decimal, "2.1.1", "extra-field", [:mix], "hexpm", "abc"},\n'; - expect(parseMixLock(mix).get('decimal')).toBe('2.1.1'); - }); -}); diff --git a/src/engine/lockfile-optional.ts b/src/engine/lockfile-optional.ts deleted file mode 100644 index 46b0c61e..00000000 --- a/src/engine/lockfile-optional.ts +++ /dev/null @@ -1,673 +0,0 @@ -/** - * Unknown optional lockfile fields. - * - * A key this tool does not model, on a lockfile that already resolves a - * dependency set, must not discard that set. The warning names the key and - * the lockfile's root-relative path. It never includes a value, so a registry - * token, an environment variable, or an authorization header stored in the - * file cannot leak. At most one warning is emitted per file per key name, - * in sorted order. - * - * Line-oriented lockfiles with no optional-key slot (`go.sum`, - * `gradle.lockfile`) are not warned here. A line that is not valid for that - * dialect fails closed in the syntax check before this runs. - */ -import * as path from 'node:path'; -import { parse as parseToml } from 'smol-toml'; -import { parse as parseYaml } from 'yaml'; - -/** Field names we will print. Anything else is omitted rather than echoed. */ -const SAFE_KEY = /^[A-Za-z_][A-Za-z0-9_-]{0,64}$/; - -const seenContent = new Set(); -const emittedKeys = new Set(); - -function setOf(keys: readonly string[]): ReadonlySet { - return new Set(keys); -} - -function asRecord(value: unknown): Record | undefined { - if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; - return value as Record; -} - -function ownKeys(value: unknown): string[] { - const rec = asRecord(value); - return rec ? Object.keys(rec) : []; -} - -function addUnknown(keys: readonly string[], known: ReadonlySet, into: Set): void { - for (const key of keys) { - if (!known.has(key) && SAFE_KEY.test(key)) into.add(key); - } -} - -function sorted(keys: Set): string[] { - return [...keys].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); -} - -function parseJson(text: string): unknown { - try { - return JSON.parse(text); - } catch { - return undefined; - } -} - -function parseYamlDoc(text: string): unknown { - try { - return parseYaml(text, { uniqueKeys: false, logLevel: 'silent' }); - } catch { - return undefined; - } -} - -function parseTomlDoc(text: string): Record | undefined { - try { - const doc = parseToml(text); - return asRecord(doc); - } catch { - return undefined; - } -} - -// ── npm package-lock / npm-shrinkwrap ──────────────────────────────────────── - -const NPM_ROOT = setOf(['name', 'version', 'lockfileVersion', 'requires', 'packages', 'dependencies']); -const NPM_ENTRY = setOf([ - 'name', - 'version', - 'dependencies', - 'devDependencies', - 'optionalDependencies', - 'peerDependencies', - 'peerDependenciesMeta', - 'bundledDependencies', - 'bin', - 'license', - 'engines', - 'funding', - 'os', - 'cpu', - 'libc', - 'resolved', - 'integrity', - 'link', - 'dev', - 'optional', - 'devOptional', - 'inBundle', - 'hasInstallScript', - 'hasShrinkwrap', - 'extraneous', - 'deprecated', - 'workspaces', - 'requires', - 'bundled', - 'from', - 'peer', - 'overrides', -]); - -function npmEntryHasVersion(entry: unknown): boolean { - const rec = asRecord(entry); - return typeof rec?.version === 'string' && rec.version.length > 0; -} - -function npmTreeHas(deps: unknown): boolean { - const rec = asRecord(deps); - if (!rec) return false; - for (const entry of Object.values(rec)) { - if (npmEntryHasVersion(entry)) return true; - if (npmTreeHas(asRecord(entry)?.dependencies)) return true; - } - return false; -} - -function walkNpmEntry(entry: unknown, into: Set): void { - addUnknown(ownKeys(entry), NPM_ENTRY, into); - const nested = asRecord(entry)?.dependencies; - const rec = asRecord(nested); - if (!rec) return; - for (const child of Object.values(rec)) { - if (asRecord(child)) walkNpmEntry(child, into); - } -} - -function npmKeys(text: string): string[] { - const root = asRecord(parseJson(text)); - if (!root) return []; - const packages = asRecord(root.packages); - let usable = false; - if (packages) { - for (const [key, entry] of Object.entries(packages)) { - if (key && npmEntryHasVersion(entry)) usable = true; - } - } - if (!usable && npmTreeHas(root.dependencies)) usable = true; - if (!usable) return []; - const into = new Set(); - addUnknown(ownKeys(root), NPM_ROOT, into); - if (packages) { - for (const entry of Object.values(packages)) walkNpmEntry(entry, into); - } - if (asRecord(root.dependencies)) walkNpmEntry({ dependencies: root.dependencies }, into); - return sorted(into); -} - -// ── pnpm-lock.yaml ─────────────────────────────────────────────────────────── - -const PNPM_ROOT = setOf([ - 'lockfileVersion', - 'settings', - 'importers', - 'packages', - 'snapshots', - 'time', - 'neverBuiltDependencies', - 'onlyBuiltDependencies', - 'overrides', - 'packageExtensionsChecksum', - 'patchedDependencies', - 'pnpmfileChecksum', - 'ignoredOptionalDependencies', - 'dependencies', - 'devDependencies', - 'optionalDependencies', - 'peerDependencies', - 'specifiers', -]); -const PNPM_SETTINGS = setOf([ - 'autoInstallPeers', - 'excludeLinksFromLockfile', - 'peersSuffixMaxLength', - 'injectWorkspacePackages', -]); -const PNPM_PACKAGE = setOf([ - 'resolution', - 'engines', - 'cpu', - 'os', - 'libc', - 'deprecated', - 'hasBin', - 'optional', - 'dev', - 'requiresBuild', - 'prepare', - 'patched', - 'id', - 'name', - 'version', - 'dependencies', - 'optionalDependencies', - 'peerDependencies', - 'peerDependenciesMeta', - 'transitivePeerDependencies', - 'bundledDependencies', -]); -const PNPM_SNAPSHOT = setOf([ - 'dependencies', - 'optionalDependencies', - 'transitivePeerDependencies', - 'optional', - 'dev', - 'id', - 'patched', -]); -const PNPM_IMPORTER = setOf([ - 'dependencies', - 'devDependencies', - 'optionalDependencies', - 'peerDependencies', - 'dependenciesMeta', - 'optionalDependenciesMeta', - 'specifiers', -]); -const PNPM_DEP = setOf(['specifier', 'version']); -const PNPM_DEP_SECTIONS = ['dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies'] as const; - -function pnpmDepSectionHas(section: unknown): boolean { - const deps = asRecord(section); - if (!deps) return false; - for (const dep of Object.values(deps)) { - if (typeof dep === 'string' && dep.length > 0) return true; - const rec = asRecord(dep); - if (typeof rec?.version === 'string' && rec.version.length > 0) return true; - } - return false; -} - -function notePnpmDepSection(section: unknown, into: Set): void { - const deps = asRecord(section); - if (!deps) return; - for (const dep of Object.values(deps)) addUnknown(ownKeys(dep), PNPM_DEP, into); -} - -function pnpmKeys(text: string): string[] { - const root = asRecord(parseYamlDoc(text)); - if (!root) return []; - const packages = asRecord(root.packages); - const snapshots = asRecord(root.snapshots); - const importers = asRecord(root.importers); - let usable = Boolean(packages && Object.keys(packages).length > 0); - if (!usable && snapshots && Object.keys(snapshots).length > 0) usable = true; - if (!usable && importers) { - for (const imp of Object.values(importers)) { - const rec = asRecord(imp); - if (!rec) continue; - if (PNPM_DEP_SECTIONS.some((section) => pnpmDepSectionHas(rec[section]))) usable = true; - } - } - if (!usable && PNPM_DEP_SECTIONS.some((section) => pnpmDepSectionHas(root[section]))) usable = true; - if (!usable) return []; - const into = new Set(); - addUnknown(ownKeys(root), PNPM_ROOT, into); - addUnknown(ownKeys(root.settings), PNPM_SETTINGS, into); - if (packages) { - for (const entry of Object.values(packages)) addUnknown(ownKeys(entry), PNPM_PACKAGE, into); - } - if (snapshots) { - for (const entry of Object.values(snapshots)) addUnknown(ownKeys(entry), PNPM_SNAPSHOT, into); - } - if (importers) { - for (const imp of Object.values(importers)) { - addUnknown(ownKeys(imp), PNPM_IMPORTER, into); - const rec = asRecord(imp); - if (!rec) continue; - for (const section of PNPM_DEP_SECTIONS) notePnpmDepSection(rec[section], into); - } - } - for (const section of PNPM_DEP_SECTIONS) notePnpmDepSection(root[section], into); - return sorted(into); -} - -// ── yarn.lock ──────────────────────────────────────────────────────────────── - -const YARN_FIELDS = setOf([ - 'version', - 'resolved', - 'integrity', - 'dependencies', - 'optionalDependencies', - 'peerDependencies', - 'dependenciesMeta', - 'peerDependenciesMeta', - 'checksum', - 'languageName', - 'linkType', - 'bin', - 'conditions', - 'resolution', - 'cacheKey', -]); - -function yarnKeys(text: string): string[] { - let descriptor = false; - let version = false; - const into = new Set(); - for (const raw of text.split('\n')) { - if (!raw.trim() || raw.trimStart().startsWith('#')) continue; - if (!/^\s/.test(raw)) { - if (raw.includes('@')) descriptor = true; - continue; - } - const field = /^ {2}([A-Za-z_][A-Za-z0-9_-]*)\b/.exec(raw); - if (!field) continue; - if (field[1] === 'version' && /^ {2}version:?\s+\S/.test(raw)) version = true; - addUnknown([field[1]!], YARN_FIELDS, into); - } - return descriptor && version ? sorted(into) : []; -} - -// ── TOML locks: Cargo, poetry, uv, pdm ─────────────────────────────────────── - -const TOML_ROOT = setOf([ - 'version', - 'package', - 'metadata', - 'patch', - 'replace', - 'requires-python', - 'revision', - 'resolution-markers', - 'manifest', - 'conflicts', - 'supported-markers', - 'options', - 'dependency-groups', - 'extras', -]); -const TOML_PACKAGE = setOf([ - 'name', - 'version', - 'source', - 'checksum', - 'dependencies', - 'replace', - 'optional', - 'description', - 'python-versions', - 'files', - 'extras', - 'category', - 'groups', - 'markers', - 'develop', - 'dev-dependencies', - 'optional-dependencies', - 'metadata', - 'wheels', - 'sdist', - 'dev', - 'resolution-markers', - 'environments', - 'summary', - 'marker', - 'index', -]); - -function tomlPackages(doc: Record): unknown[] { - const list = doc.package; - if (Array.isArray(list)) return list; - return list ? [list] : []; -} - -function tomlKeys(text: string): string[] { - const doc = parseTomlDoc(text); - if (!doc) return []; - const packages = tomlPackages(doc); - const usable = packages.some((entry) => { - const rec = asRecord(entry); - return typeof rec?.name === 'string' && typeof rec.version === 'string' && rec.version.length > 0; - }); - if (!usable) return []; - const into = new Set(); - addUnknown(ownKeys(doc), TOML_ROOT, into); - for (const entry of packages) addUnknown(ownKeys(entry), TOML_PACKAGE, into); - return sorted(into); -} - -// ── Gemfile.lock ───────────────────────────────────────────────────────────── - -const GEM_SECTIONS = setOf([ - 'GEM', - 'GIT', - 'PATH', - 'PLATFORMS', - 'DEPENDENCIES', - 'BUNDLED WITH', - 'CHECKSUMS', - 'PLUGIN SOURCE', -]); - -function gemfileKeys(text: string): string[] { - if (!/^ {4}\S+ \(\d/m.test(text)) return []; - const into = new Set(); - for (const raw of text.split('\n')) { - if (!raw || /^[ \t]/.test(raw)) continue; - const header = raw.trim(); - if (!/^[A-Z][A-Z ]+$/.test(header)) continue; - addUnknown([header], GEM_SECTIONS, into); - } - return sorted(into); -} - -// ── composer.lock ──────────────────────────────────────────────────────────── - -const COMPOSER_ROOT = setOf([ - '_readme', - 'content-hash', - 'packages', - 'packages-dev', - 'aliases', - 'minimum-stability', - 'stability-flags', - 'prefer-stable', - 'prefer-lowest', - 'platform', - 'platform-dev', - 'plugin-api-version', -]); -const COMPOSER_PKG = setOf([ - 'name', - 'version', - 'version_normalized', - 'source', - 'dist', - 'require', - 'require-dev', - 'conflict', - 'replace', - 'provide', - 'suggest', - 'bin', - 'type', - 'extra', - 'autoload', - 'autoload-dev', - 'notification-url', - 'license', - 'authors', - 'description', - 'homepage', - 'keywords', - 'support', - 'funding', - 'time', - 'abandoned', - 'default-branch', - 'transport-options', - 'scripts', - 'target-dir', -]); - -function composerListHas(list: unknown): boolean { - if (!Array.isArray(list)) return false; - return list.some((entry) => { - const rec = asRecord(entry); - return typeof rec?.name === 'string' && typeof rec.version === 'string' && rec.version.length > 0; - }); -} - -function composerKeys(text: string): string[] { - const root = asRecord(parseJson(text)); - if (!root) return []; - if (!composerListHas(root.packages) && !composerListHas(root['packages-dev'])) return []; - const into = new Set(); - addUnknown(ownKeys(root), COMPOSER_ROOT, into); - for (const section of ['packages', 'packages-dev']) { - const list = root[section]; - if (!Array.isArray(list)) continue; - for (const entry of list) addUnknown(ownKeys(entry), COMPOSER_PKG, into); - } - return sorted(into); -} - -// ── Pipfile.lock ───────────────────────────────────────────────────────────── - -const PIPFILE_ROOT = setOf(['_meta', 'default', 'develop']); -const PIPFILE_ENTRY = setOf(['version', 'markers', 'hashes', 'index', 'extras']); - -function pipfileSectionHas(section: unknown): boolean { - const deps = asRecord(section); - if (!deps) return false; - return Object.values(deps).some((entry) => typeof asRecord(entry)?.version === 'string'); -} - -function pipfileKeys(text: string): string[] { - const root = asRecord(parseJson(text)); - if (!root) return []; - if (!pipfileSectionHas(root.default) && !pipfileSectionHas(root.develop)) return []; - const into = new Set(); - addUnknown(ownKeys(root), PIPFILE_ROOT, into); - for (const section of ['default', 'develop']) { - const deps = asRecord(root[section]); - if (!deps) continue; - for (const entry of Object.values(deps)) addUnknown(ownKeys(entry), PIPFILE_ENTRY, into); - } - return sorted(into); -} - -// ── packages.lock.json ─────────────────────────────────────────────────────── - -const NUGET_ROOT = setOf(['version', 'dependencies']); -const NUGET_ENTRY = setOf(['type', 'requested', 'resolved', 'contentHash', 'dependencies', 'framework']); - -function nugetKeys(text: string): string[] { - const root = asRecord(parseJson(text)); - const frameworks = asRecord(root?.dependencies); - if (!root || !frameworks) return []; - let usable = false; - const into = new Set(); - for (const framework of Object.values(frameworks)) { - const packages = asRecord(framework); - if (!packages) continue; - for (const entry of Object.values(packages)) { - const rec = asRecord(entry); - if (typeof rec?.resolved === 'string' && rec.resolved.length > 0) usable = true; - addUnknown(ownKeys(entry), NUGET_ENTRY, into); - } - } - if (!usable) return []; - addUnknown(ownKeys(root), NUGET_ROOT, into); - return sorted(into); -} - -// ── Package.resolved ───────────────────────────────────────────────────────── - -const SWIFT_ROOT = setOf(['version', 'originHash', 'pins', 'object']); -const SWIFT_OBJECT = setOf(['pins']); -const SWIFT_PIN = setOf(['identity', 'kind', 'location', 'state', 'package', 'repositoryURL']); -const SWIFT_STATE = setOf(['version', 'revision', 'branch']); - -function swiftPins(root: Record): unknown[] { - if (Array.isArray(root.pins)) return root.pins; - const object = asRecord(root.object); - return Array.isArray(object?.pins) ? object.pins : []; -} - -function swiftKeys(text: string): string[] { - const root = asRecord(parseJson(text)); - if (!root) return []; - const pins = swiftPins(root); - const usable = pins.some((pin) => typeof asRecord(asRecord(pin)?.state)?.version === 'string'); - if (!usable) return []; - const into = new Set(); - addUnknown(ownKeys(root), SWIFT_ROOT, into); - addUnknown(ownKeys(root.object), SWIFT_OBJECT, into); - for (const pin of pins) { - addUnknown(ownKeys(pin), SWIFT_PIN, into); - addUnknown(ownKeys(asRecord(pin)?.state), SWIFT_STATE, into); - } - return sorted(into); -} - -// ── pubspec.lock ───────────────────────────────────────────────────────────── - -const PUB_ROOT = setOf(['packages', 'sdks']); -const PUB_ENTRY = setOf(['dependency', 'description', 'source', 'version']); - -function pubspecKeys(text: string): string[] { - const root = asRecord(parseYamlDoc(text)); - const packages = asRecord(root?.packages); - if (!root || !packages) return []; - let usable = false; - const into = new Set(); - for (const entry of Object.values(packages)) { - const rec = asRecord(entry); - if (typeof rec?.version === 'string' && rec.version.length > 0) usable = true; - addUnknown(ownKeys(entry), PUB_ENTRY, into); - } - if (!usable) return []; - addUnknown(ownKeys(root), PUB_ROOT, into); - return sorted(into); -} - -/** - * Sorted unique unknown optional keys on a lockfile that already resolves at - * least one package. Empty when the file has no resolved package, is not a - * known lockfile, or does not parse. Never throws. - */ -export function unknownOptionalLockfileKeys(basename: string, text: string): string[] { - switch (basename.toLowerCase()) { - case 'package-lock.json': - case 'npm-shrinkwrap.json': - return npmKeys(text); - case 'pnpm-lock.yaml': - return pnpmKeys(text); - case 'yarn.lock': - return yarnKeys(text); - case 'cargo.lock': - case 'poetry.lock': - case 'uv.lock': - case 'pdm.lock': - return tomlKeys(text); - case 'gemfile.lock': - return gemfileKeys(text); - case 'composer.lock': - return composerKeys(text); - case 'pipfile.lock': - return pipfileKeys(text); - case 'packages.lock.json': - return nugetKeys(text); - case 'package.resolved': - return swiftKeys(text); - case 'pubspec.lock': - return pubspecKeys(text); - default: - return []; - } -} - -/** Stable warning line. `relPath` is the lockfile path relative to the project root. */ -export function unknownOptionalFieldWarning(relPath: string, key: string): string { - return `warning: ${relPath}: unknown optional lockfile field "${key}"; kept the dependencies this file already resolved`; -} - -/** One line per key, sorted by key name. Unsafe key names are dropped. */ -export function formatUnknownOptionalFieldWarnings(relPath: string, keys: readonly string[]): string[] { - const unique = new Set(); - for (const key of keys) { - if (SAFE_KEY.test(key)) unique.add(key); - } - return [...unique] - .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) - .map((key) => unknownOptionalFieldWarning(relPath, key)); -} - -function relativeLockfilePath(root: string, absPath: string): string { - const rel = path.relative(root, absPath); - if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) return path.basename(absPath); - return rel.split(path.sep).join('/'); -} - -function contentKey(absPath: string, text: string): string { - let hash = 0x811c9dc5; - for (let i = 0; i < text.length; i++) { - hash ^= text.charCodeAt(i); - hash = Math.imul(hash, 0x01000193); - } - return `${absPath}\0${(hash >>> 0).toString(16)}:${text.length}`; -} - -/** - * Write the unknown-optional-field warnings for one lockfile. A repeated read - * of the same bytes, or the same key on a later read, does not write again. - */ -export function warnUnknownOptionalLockfile(root: string, absPath: string, text: string): void { - const id = contentKey(absPath, text); - if (seenContent.has(id)) return; - seenContent.add(id); - const rel = relativeLockfilePath(root, absPath); - const fresh: string[] = []; - for (const key of unknownOptionalLockfileKeys(path.basename(absPath), text)) { - const mark = `${absPath}\0${key}`; - if (emittedKeys.has(mark)) continue; - emittedKeys.add(mark); - fresh.push(key); - } - for (const line of formatUnknownOptionalFieldWarnings(rel, fresh)) { - process.stderr.write(`${line}\n`); - } -} diff --git a/src/engine/lockfile-unknown.test.ts b/src/engine/lockfile-unknown.test.ts new file mode 100644 index 00000000..9c07e903 --- /dev/null +++ b/src/engine/lockfile-unknown.test.ts @@ -0,0 +1,302 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseCargoLock } from '../core-open/scanners/cargo-lockfile.js'; +import { parseGemfileLock } from '../core-open/scanners/gemfile-lock.js'; +import { parseGradleLockfile } from '../core-open/scanners/gradle-lockfile.js'; +import { parsePackageLock, parsePnpmLock, parseYarnLock } from '../core-open/scanners/npm-lockfile.js'; +import { parsePipfileLock, parsePyTomlLock } from '../core-open/scanners/python-lockfile.js'; +import { LockfileParseError, assertLockfileText } from '../core-open/utils/lockfile-parse.js'; +import { + LockfileUnknownFields, + notePnpmLockText, + resetUnknownOptionalLockfileWarnings, +} from '../core-open/utils/lockfile-unknown.js'; +import { fullDependencyGraph, lockfileVersion } from './lockfile.js'; + +const SECRET = 'super-secret-token'; +const WARN = (file: string, key: string) => + `${file}: unknown optional lockfile field "${key}"; continuing with the fields this version understands.`; + +let root: string; +let stderr: string[]; + +beforeEach(() => { + resetUnknownOptionalLockfileWarnings(); + root = fs.mkdtempSync(path.join(os.tmpdir(), 'vg-lock-unknown-')); + stderr = []; + vi.spyOn(process.stderr, 'write').mockImplementation((chunk) => { + stderr.push(String(chunk)); + return true; + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); + resetUnknownOptionalLockfileWarnings(); + fs.rmSync(root, { recursive: true, force: true }); +}); + +const write = (name: string, body: string): void => fs.writeFileSync(path.join(root, name), body); + +function warnings(): string[] { + return stderr.map((line) => line.replace(/\n$/, '')).filter((line) => line.includes('unknown optional lockfile field')); +} + +describe('lockfile unknown optional fields', () => { + it('keeps an npm dependency set and warns once per unknown key, sorted', () => { + const json = { + lockfileVersion: 3, + futureOptional: `Bearer ${SECRET}`, + alphaField: true, + packages: { + '': { name: 'app', dependencies: { 'left-pad': '^1.3.0' } }, + 'node_modules/left-pad': { version: '1.3.0', integrity: 'sha512-abc', futureOptional: SECRET }, + 'node_modules/left-pad/node_modules/nested': { version: '9.9.9' }, + }, + }; + expect(parsePackageLock(json).get('left-pad')).toBe('1.3.0'); + expect(warnings()).toEqual([WARN('package-lock.json', 'alphaField'), WARN('package-lock.json', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + expect(warnings().join('\n')).not.toContain('Bearer'); + expect(warnings().join('\n')).not.toContain('sha512'); + + warnings().splice(0, warnings().length); + stderr.length = 0; + parsePackageLock(json); + expect(warnings()).toEqual([]); + }); + + it('keeps npm graph edges when a package entry has an unknown optional key', () => { + write( + 'package-lock.json', + JSON.stringify({ + lockfileVersion: 3, + packages: { + '': { name: 'app', dependencies: { chalk: '^5.3.0' } }, + 'node_modules/chalk': { version: '5.3.0', dependencies: { 'ansi-styles': '^6.0.0' }, futureOptional: SECRET }, + 'node_modules/ansi-styles': { version: '6.2.1', integrity: 'sha512-abc' }, + }, + }), + ); + const graph = fullDependencyGraph(root); + expect(graph?.components).toEqual([ + { package: 'ansi-styles', version: '6.2.1' }, + { package: 'chalk', version: '5.3.0' }, + ]); + expect(graph?.edges?.get('chalk@5.3.0')).toEqual(['ansi-styles@6.2.1']); + expect(warnings()).toEqual([WARN('package-lock.json', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + }); + + it('keeps a pnpm importer pin and warns on one unknown top-level key', () => { + const doc = { + lockfileVersion: '9.0', + futureOptional: SECRET, + importers: { + '.': { dependencies: { 'left-pad': { specifier: '^1.3.0', version: '1.3.0' } } }, + }, + }; + expect(parsePnpmLock(doc).get('left-pad')).toBe('1.3.0'); + expect(warnings()).toEqual([WARN('pnpm-lock.yaml', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + + write( + 'pnpm-lock.yaml', + [ + "lockfileVersion: '9.0'", + `futureOptional: ${SECRET}`, + 'importers:', + ' .:', + ' dependencies:', + ' left-pad:', + ' specifier: ^1.3.0', + ' version: 1.3.0', + 'packages:', + ' left-pad@1.3.0:', + ' resolution: {integrity: sha512-abc}', + ].join('\n'), + ); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + const graph = fullDependencyGraph(root); + expect(graph?.components).toEqual([{ package: 'left-pad', version: '1.3.0' }]); + expect(lockfileVersion(root, 'npm', 'left-pad')).toBe('1.3.0'); + expect(warnings()).toEqual([WARN('pnpm-lock.yaml', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + }); + + it('does not warn on a real pnpm-lock.yaml', () => { + const lockPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../pnpm-lock.yaml'); + const text = fs.readFileSync(lockPath, 'utf8'); + const notes = new LockfileUnknownFields(); + notePnpmLockText(text, 'pnpm-lock.yaml', notes); + expect(notes.messages()).toEqual([]); + }); + + it('keeps a yarn pin and warns on one unknown field', () => { + const text = ['left-pad@^1.3.0:', ' version "1.3.0"', ` futureOptional "${SECRET}"`, ''].join('\n'); + expect(parseYarnLock(text).byName.get('left-pad')).toBe('1.3.0'); + expect(warnings()).toEqual([WARN('yarn.lock', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + + write('yarn.lock', text); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'left-pad', version: '1.3.0' }]); + expect(warnings()).toEqual([WARN('yarn.lock', 'futureOptional')]); + }); + + it('keeps poetry, uv, and Pipfile pins and warns on one unknown key', () => { + const poetry = ['[[package]]', 'name = "click"', 'version = "8.1.3"', `futureOptional = "${SECRET}"`, ''].join('\n'); + expect(parsePyTomlLock(poetry, 'poetry.lock', undefined, 'poetry').get('click')).toBe('8.1.3'); + expect(warnings()).toEqual([WARN('poetry.lock', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + + write('poetry.lock', poetry); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'click', version: '8.1.3' }]); + expect(warnings()).toEqual([WARN('poetry.lock', 'futureOptional')]); + + const uv = ['version = 1', '[[package]]', 'name = "httpx"', 'version = "0.27.0"', 'futureOptional = "1"', ''].join('\n'); + expect(parsePyTomlLock(uv, 'uv.lock', undefined, 'uv').get('httpx')).toBe('0.27.0'); + + const pip = { + _meta: { hash: { sha256: 'abc' } }, + default: { requests: { version: '==2.31.0', futureOptional: SECRET } }, + develop: {}, + }; + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + expect(parsePipfileLock(pip).get('requests')).toBe('2.31.0'); + expect(warnings()).toEqual([WARN('Pipfile.lock', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + }); + + it('keeps a Cargo pin and warns on one unknown package key', () => { + const text = ['version = 4', '[[package]]', 'name = "serde"', 'version = "1.0.203"', `futureOptional = "${SECRET}"`, ''].join('\n'); + expect(parseCargoLock(text).get('serde')).toEqual(['1.0.203']); + expect(warnings()).toEqual([WARN('Cargo.lock', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + + write('Cargo.lock', text); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'serde', version: '1.0.203' }]); + expect(warnings()).toEqual([WARN('Cargo.lock', 'futureOptional')]); + }); + + it('keeps composer, NuGet, Swift, and pub pins and warns on one unknown key', () => { + write( + 'composer.lock', + JSON.stringify({ + 'content-hash': 'abc', + futureOptional: SECRET, + packages: [{ name: 'symfony/console', version: 'v7.1.0', description: 'cli' }], + }), + ); + expect(lockfileVersion(root, 'php', 'symfony/console')).toBe('7.1.0'); + expect(warnings()).toEqual([WARN('composer.lock', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + + fs.rmSync(path.join(root, 'composer.lock')); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + write( + 'packages.lock.json', + JSON.stringify({ + version: 1, + futureOptional: true, + dependencies: { 'net8.0': { 'Newtonsoft.Json': { type: 'Direct', resolved: '13.0.3' } } }, + }), + ); + expect(lockfileVersion(root, 'dotnet', 'Newtonsoft.Json')).toBe('13.0.3'); + expect(warnings()).toEqual([WARN('packages.lock.json', 'futureOptional')]); + + fs.rmSync(path.join(root, 'packages.lock.json')); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + write( + 'Package.resolved', + JSON.stringify({ + version: 3, + futureOptional: true, + pins: [{ identity: 'vapor', state: { version: '4.89.0' } }], + }), + ); + expect(lockfileVersion(root, 'swift', 'vapor')).toBe('4.89.0'); + expect(warnings()).toEqual([WARN('Package.resolved', 'futureOptional')]); + + fs.rmSync(path.join(root, 'Package.resolved')); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + write( + 'pubspec.lock', + [ + 'packages:', + ' provider:', + ' dependency: "direct main"', + ' version: "6.1.2"', + ' futureOptional: true', + 'sdks:', + ' dart: ">=3.0.0 <4.0.0"', + ].join('\n'), + ); + expect(lockfileVersion(root, 'dart', 'provider')).toBe('6.1.2'); + expect(warnings()).toEqual([WARN('pubspec.lock', 'futureOptional')]); + }); + + it('omits a field name that is not a plain identifier and does not print its text', () => { + const notes = new LockfileUnknownFields(); + notes.note('package-lock.json', `Authorization: Bearer ${SECRET}`); + expect(notes.messages()).toEqual([ + 'package-lock.json: unknown optional lockfile field (name omitted); continuing with the fields this version understands.', + ]); + expect(notes.messages().join('\n')).not.toContain(SECRET); + expect(notes.messages().join('\n')).not.toContain('Bearer'); + expect(notes.messages().join('\n')).not.toContain('Authorization'); + }); + + it('keeps Gradle and Gemfile rows when optional commentary is present, without a field warning', () => { + const gradle = '# generated\ncom.google.guava:guava:31.1-jre=compileClasspath\nempty=annotationProcessor\n'; + expect(parseGradleLockfile(gradle).get('com.google.guava:guava')).toBe('31.1-jre'); + write('gradle.lockfile', gradle); + expect(lockfileVersion(root, 'java', 'com.google.guava:guava')).toBe('31.1-jre'); + + const gem = ['GEM', ' remote: https://rubygems.org/', ' specs:', ' rails (7.1.2)', 'FUTURE_OPTIONAL', ' note: true', ''].join('\n'); + expect(parseGemfileLock(gem).get('rails')).toBe('7.1.2'); + write('Gemfile.lock', gem); + fs.rmSync(path.join(root, 'gradle.lockfile')); + expect(lockfileVersion(root, 'ruby', 'rails')).toBe('7.1.2'); + expect(warnings()).toEqual([]); + }); + + it('still fails closed when required lockfile structure is broken, without echoing contents', () => { + const cases: Array<[string, string, string]> = [ + ['package-lock.json', `{"packages":{"node_modules/left-pad":{"version":"1.3.0","integrity":"${SECRET}"`, 'JSON'], + ['pnpm-lock.yaml', `lockfileVersion: '9.0'\npackages:\n left-pad@1.3.0:\n resolution: {integrity: ${SECRET}`, 'YAML'], + ['Cargo.lock', `[[package]]\nname = "serde"\nversion = "${SECRET}`, 'TOML'], + ['yarn.lock', `"left-pad@^1.3.0":\n futureOptional "${SECRET}"\n`, 'yarn.lock'], + ['gradle.lockfile', `not a lock line ${SECRET}\n`, 'gradle.lockfile'], + ['Gemfile.lock', ` rails (${SECRET}\n`, 'Gemfile.lock'], + ['go.sum', `github.com/gin-contrib/sse v1.1.0 ${SECRET}\n`, 'go.sum'], + ]; + for (const [file, body, kind] of cases) { + write(file, body); + expect(() => assertLockfileText(path.join(root, file), body, kind as 'JSON'), file).toThrow(LockfileParseError); + try { + assertLockfileText(path.join(root, file), body, kind as 'JSON'); + } catch (err) { + const message = (err as Error).message; + expect(message, file).toContain(file); + expect(message, file).toContain('package manager'); + expect(message, file).not.toContain(SECRET); + } + fs.rmSync(path.join(root, file)); + } + expect(warnings()).toEqual([]); + }); +}); diff --git a/src/engine/lockfile.ts b/src/engine/lockfile.ts index 6774929c..bcc88606 100644 --- a/src/engine/lockfile.ts +++ b/src/engine/lockfile.ts @@ -6,8 +6,22 @@ import { LockfileParseError, parseLockfileJson, } from '../core-open/utils/lockfile-parse.js'; +import { + emitUnknownOptionalLockfileWarnings, + lockfileWarningPath, + LockfileUnknownFields, + noteComposerLock, + noteNugetLock, + notePackageLock, + notePackageResolved, + notePipfileLock, + notePnpmLockText, + notePubspecLockText, + noteTomlLockText, + noteYarnLockText, + type TomlLockKind, +} from '../core-open/utils/lockfile-unknown.js'; import type { DepRecord } from './drift.js'; -import { warnUnknownOptionalLockfile } from './lockfile-optional.js'; /** * Read a lockfile that may be absent. A missing file is `undefined` (the @@ -30,9 +44,6 @@ function readCheckedText(root: string, file: string): string | undefined { if (text === undefined) return undefined; const kind = lockfileKind(file); if (kind) assertLockfileText(abs, text, kind); - // Syntax already passed. An unknown optional field warns and the text is - // still returned, so the dependency set is kept. - warnUnknownOptionalLockfile(root, abs, text); return text; } @@ -40,9 +51,22 @@ function readCheckedJson(root: string, file: string): unknown | undefined { const abs = path.join(root, file); const text = readOptionalText(abs); if (text === undefined) return undefined; - const parsed = parseLockfileJson(abs, text); - warnUnknownOptionalLockfile(root, abs, text); - return parsed; + return parseLockfileJson(abs, text); +} + +function tomlKind(file: string): TomlLockKind | undefined { + if (file === 'Cargo.lock') return 'cargo'; + if (file === 'poetry.lock') return 'poetry'; + if (file === 'uv.lock') return 'uv'; + if (file === 'pdm.lock') return 'pdm'; + return undefined; +} + +/** Record unknown optional keys on a lockfile that already parsed. */ +function reportLockfileFields(root: string, file: string, fill: (rel: string, notes: LockfileUnknownFields) => void): void { + const notes = new LockfileUnknownFields(); + fill(lockfileWarningPath(file, root), notes); + emitUnknownOptionalLockfileWarnings(notes); } /** @@ -85,6 +109,7 @@ function gradleLock(root: string, name: string): string | undefined { function packageResolved(root: string, name: string): string | undefined { const parsed = readCheckedJson(root, 'Package.resolved'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'Package.resolved', (rel, notes) => notePackageResolved(parsed, rel, notes)); const data = parsed as { pins?: Array<{ identity?: string; package?: string; state?: { version?: string } }>; object?: { pins?: Array<{ identity?: string; package?: string; state?: { version?: string } }> }; @@ -103,6 +128,7 @@ function packageResolved(root: string, name: string): string | undefined { function pubspecLock(root: string, name: string): string | undefined { const text = readCheckedText(root, 'pubspec.lock'); if (text === undefined) return undefined; + reportLockfileFields(root, 'pubspec.lock', (rel, notes) => notePubspecLockText(text, rel, notes)); let inPkg = false; for (const line of text.split('\n')) { const key = /^ ([A-Za-z0-9_.]+):\s*$/.exec(line); @@ -136,6 +162,7 @@ function gemfileLock(root: string, name: string): string | undefined { function composerLock(root: string, name: string): string | undefined { const parsed = readCheckedJson(root, 'composer.lock'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'composer.lock', (rel, notes) => noteComposerLock(parsed, rel, notes)); const data = parsed as Record>; for (const section of ['packages', 'packages-dev']) { const arr = data[section]; @@ -150,6 +177,7 @@ function composerLock(root: string, name: string): string | undefined { function packagesLock(root: string, name: string): string | undefined { const parsed = readCheckedJson(root, 'packages.lock.json'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'packages.lock.json', (rel, notes) => noteNugetLock(parsed, rel, notes)); const data = parsed as { dependencies?: Record> }; const target = name.toLowerCase(); for (const fw of Object.values(data.dependencies ?? {})) { @@ -188,6 +216,8 @@ function pypiLockVersion(root: string, name: string): string | undefined { function tomlPackageLock(root: string, file: string, name: string, normalize: (s: string) => string): string | undefined { const text = readCheckedText(root, file); if (text === undefined) return undefined; + const kind = tomlKind(file); + if (kind) reportLockfileFields(root, file, (rel, notes) => noteTomlLockText(text, rel, notes, kind)); const target = normalize(name); for (const block of text.split(/\[\[package\]\]/)) { const nm = /(?:^|\n)\s*name\s*=\s*"([^"]+)"/.exec(block); @@ -203,6 +233,7 @@ function tomlPackageLock(root: string, file: string, name: string, normalize: (s function pipfileLock(root: string, name: string): string | undefined { const parsed = readCheckedJson(root, 'Pipfile.lock'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'Pipfile.lock', (rel, notes) => notePipfileLock(parsed, rel, notes)); const data = parsed as Record>; const target = pep503(name); for (const section of ['default', 'develop']) { @@ -308,6 +339,7 @@ interface NpmV2Package { function npmLockGraph(root: string): LockfileGraph | undefined { const parsed = readCheckedJson(root, 'package-lock.json'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'package-lock.json', (rel, notes) => notePackageLock(parsed, rel, notes)); const data = parsed as { packages?: Record; dependencies?: Record }>; @@ -392,6 +424,7 @@ function walkNpmV1Tree( function pnpmLockTree(root: string): LockfileComponent[] | undefined { const text = readCheckedText(root, 'pnpm-lock.yaml'); if (text === undefined) return undefined; + reportLockfileFields(root, 'pnpm-lock.yaml', (rel, notes) => notePnpmLockText(text, rel, notes)); const section = sectionOf(text, 'packages'); if (!section) return undefined; const out = new Map(); @@ -412,6 +445,7 @@ function pnpmLockTree(root: string): LockfileComponent[] | undefined { function yarnLockTree(root: string): LockfileComponent[] | undefined { const text = readCheckedText(root, 'yarn.lock'); if (text === undefined) return undefined; + reportLockfileFields(root, 'yarn.lock', (rel, notes) => noteYarnLockText(text, rel, notes)); const out = new Map(); let pendingNames: string[] = []; for (const line of text.split('\n')) { @@ -479,6 +513,7 @@ function yarnHeaderRealName(spec: string): string | undefined { function cargoLockTree(root: string): LockfileComponent[] | undefined { const text = readCheckedText(root, 'Cargo.lock'); if (text === undefined) return undefined; + reportLockfileFields(root, 'Cargo.lock', (rel, notes) => noteTomlLockText(text, rel, notes, 'cargo')); const out = new Map(); for (const block of text.split(/\[\[package\]\]/)) { const nm = /(?:^|\n)\s*name\s*=\s*"([^"]+)"/.exec(block); @@ -518,6 +553,8 @@ function goSumTree(root: string): LockfileComponent[] | undefined { function poetryLikeLockTree(root: string, file: string): LockfileComponent[] | undefined { const text = readCheckedText(root, file); if (text === undefined) return undefined; + const kind = tomlKind(file); + if (kind) reportLockfileFields(root, file, (rel, notes) => noteTomlLockText(text, rel, notes, kind)); const out = new Map(); for (const block of text.split(/\[\[package\]\]/)) { const nm = /(?:^|\n)\s*name\s*=\s*"([^"]+)"/.exec(block); @@ -531,6 +568,7 @@ function poetryLikeLockTree(root: string, file: string): LockfileComponent[] | u function packageLockVersion(root: string, name: string): string | undefined { const parsed = readCheckedJson(root, 'package-lock.json'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'package-lock.json', (rel, notes) => notePackageLock(parsed, rel, notes)); const data = parsed as { packages?: Record; dependencies?: Record }; // v2/v3: packages keyed by install path; the top-level dep is "node_modules/". const top = data.packages?.[`node_modules/${name}`]?.version; @@ -556,6 +594,7 @@ function packageLockVersion(root: string, name: string): string | undefined { function pnpmLockVersion(root: string, name: string): string | undefined { const text = readCheckedText(root, 'pnpm-lock.yaml'); if (text === undefined) return undefined; + reportLockfileFields(root, 'pnpm-lock.yaml', (rel, notes) => notePnpmLockText(text, rel, notes)); const importers = sectionOf(text, 'importers'); if (!importers) return undefined; // Entries look like: @@ -590,6 +629,7 @@ function sectionOf(text: string, key: string): string | undefined { function yarnLockVersion(root: string, name: string): string | undefined { const text = readCheckedText(root, 'yarn.lock'); if (text === undefined) return undefined; + reportLockfileFields(root, 'yarn.lock', (rel, notes) => noteYarnLockText(text, rel, notes)); let inBlock = false; for (const line of text.split('\n')) { if (line && !/^\s/.test(line) && !line.startsWith('#')) { diff --git a/src/reporting/scanners/dependency-graph.ts b/src/reporting/scanners/dependency-graph.ts index c7c23109..66bb4be8 100644 --- a/src/reporting/scanners/dependency-graph.ts +++ b/src/reporting/scanners/dependency-graph.ts @@ -1,7 +1,13 @@ import * as path from 'node:path'; import { readTextFile, pathExists, findPackageJsonFiles, readJsonFile, FileCache } from '../../core-open/index.js'; import { assertLockfileText, parseLockfileJson } from '../../core-open/utils/lockfile-parse.js'; -import { warnUnknownOptionalLockfile } from '../../engine/lockfile-optional.js'; +import { + lockfileWarningPath, + notePackageLock, + notePnpmLockText, + noteYarnLockText, + withLockfileNotes, +} from '../../core-open/utils/lockfile-unknown.js'; import type { PackageJson, DependencyGraphResult, DuplicatedPackage, PhantomDependency } from '../../core-open/index.js'; interface LockEntry { @@ -37,6 +43,10 @@ function parseNpmLock(content: string, filePath: string): LockEntry[] { dependencies?: Record }>; } | null; if (lock && typeof lock === 'object') { + withLockfileNotes(undefined, (bucket) => { + notePackageLock(lock, lockfileWarningPath(filePath), bucket); + return undefined; + }); // v2/v3 format: `packages` keyed by path like "node_modules/lodash" if (lock.packages && typeof lock.packages === 'object') { @@ -102,19 +112,24 @@ export async function scanDependencyGraph(rootDir: string, cache?: FileCache): P result.lockfileType = 'pnpm'; const content = await _readTextFile(pnpmLock); assertLockfileText(pnpmLock, content, 'YAML'); - warnUnknownOptionalLockfile(rootDir, pnpmLock, content); entries = parsePnpmLock(content); + withLockfileNotes(undefined, (bucket) => { + notePnpmLockText(content, lockfileWarningPath(pnpmLock), bucket); + return undefined; + }); } else if (await _pathExists(npmLock)) { result.lockfileType = 'npm'; const content = await _readTextFile(npmLock); entries = parseNpmLock(content, npmLock); - warnUnknownOptionalLockfile(rootDir, npmLock, content); } else if (await _pathExists(yarnLock)) { result.lockfileType = 'yarn'; const content = await _readTextFile(yarnLock); assertLockfileText(yarnLock, content, 'yarn.lock'); - warnUnknownOptionalLockfile(rootDir, yarnLock, content); entries = parseYarnLock(content); + withLockfileNotes(undefined, (bucket) => { + noteYarnLockText(content, lockfileWarningPath(yarnLock), bucket); + return undefined; + }); } if (entries.length === 0) return result;