diff --git a/CHANGELOG.md b/CHANGELOG.md index ec0672cb..ab198d26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ backward compatible. ### Added +- **Unknown optional lockfile fields warn and continue.** When a lockfile still lists dependencies vg can read, an extra field does not drop that graph or the SBOM rows built from it. The command prints one warning per lockfile path and field name, sorted by path then field: `pnpm-lock.yaml: unknown optional lockfile field ""; continuing with the fields this version understands.` A field name that is not a plain identifier is reported as `(name omitted)`. A truncated lockfile, or one missing the structure its format requires, still exits non-zero. + - **`vg build --attest` and `vg build --verify` are documented.** The docs cover the DSSE in-toto statement written to `.vibgrate/attestation.intoto.jsonl`, the Ed25519 key diff --git a/DOCS.md b/DOCS.md index 9a314a0e..6c86163c 100644 --- a/DOCS.md +++ b/DOCS.md @@ -27,6 +27,7 @@ For a quick overview, see the [README](./README.md). This document covers everyt - [CycloneDX type and SPDX primaryPackagePurpose](#cyclonedx-type-and-spdx-primarypackagepurpose) - [Production, development, and optional scope](./docs/sbom-dependency-scope.md) - [vg scan](#vg-scan) + - [Unknown lockfile fields](#unknown-lockfile-fields) - [Offline scan with a package-version manifest](#offline-scan-with-a-package-version-manifest) - [Vulnerabilities and exposure attribution](#vulnerabilities-and-exposure-attribution) - [Go modules: pseudo-versions and +incompatible](#go-modules-pseudo-versions-and-incompatible) @@ -2128,6 +2129,22 @@ By default, the scan writes `.vibgrate/scan_result.json`. Use `--no-local-artifa For offline drift scoring, pass `--package-manifest ` with a downloaded manifest bundle such as `https://github.com/vibgrate/manifests/latest-packages.zip`. The manifest shape, the fail-closed errors, and what offline mode skips are in [Offline scan with a package-version manifest](#offline-scan-with-a-package-version-manifest). +### Unknown lockfile fields + +Package managers add fields to lockfiles as they release. When the file still lists dependencies vg can read, an extra field does not erase that graph or the SBOM rows built from it. vg keeps the package names, versions, and dependency edges it understands. + +vg prints one warning on stderr for each unknown field name in that file. Lines are sorted by lockfile path, then by field name. The same path and field name is printed once. A warning looks like this: + +```text +pnpm-lock.yaml: unknown optional lockfile field "futureOptional"; continuing with the fields this version understands. +``` + +The path is relative to the directory you ran the command from, when the lockfile is under that directory. Otherwise the warning uses the file name only. The warning names the field. It does not print the field's value, an environment value, or an authorization header. A field name that is not a plain identifier is reported as `(name omitted)`. + +A lockfile that is truncated, empty, or missing the structure its format requires still fails the command. The error names the file and the format, tells you to regenerate the file with your package manager, and the process exits non-zero. The error does not include the file's contents. + +`vg scan` and `vg sbom export` follow this for npm, pnpm, yarn, poetry, uv, pdm, Pipfile, Cargo, Composer, NuGet, Swift package pins, and pub. A Gradle lock line, a Gemfile spec, and a `go.sum` line do not have optional field names. A line that does not match that format is invalid, and the command fails. + ### Offline scan with a package-version manifest `--offline` scores drift, and with `--vulns` it matches known vulnerabilities, from files on the machine. `--package-manifest` is the local file those lookups read. diff --git a/src/core-open/scanners/cargo-lockfile.ts b/src/core-open/scanners/cargo-lockfile.ts index 1d04ed90..dcb59894 100644 --- a/src/core-open/scanners/cargo-lockfile.ts +++ b/src/core-open/scanners/cargo-lockfile.ts @@ -15,6 +15,7 @@ import * as path from 'node:path'; import * as semver from 'semver'; import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { type LockfileUnknownFields, noteTomlLockDoc, withLockfileNotes } from '../utils/lockfile-unknown.js'; import { parseToml } from '../utils/toml.js'; import type { LockfileIo } from './npm-lockfile.js'; export type { LockfileIo } from './npm-lockfile.js'; @@ -30,9 +31,20 @@ export interface CargoLockIndex { * Parse a Cargo.lock into name → locked versions. A crate may appear more than once (multiple * incompatible majors in the tree), so versions accumulate into a list per name. */ -export function parseCargoLock(text: string): Map { +export function parseCargoLock( + text: string, + relPath = 'Cargo.lock', + notes?: LockfileUnknownFields, +): Map { + return withLockfileNotes(notes, (bucket) => { + const doc = parseToml(text); + noteTomlLockDoc(doc, relPath, bucket, 'cargo'); + return parseCargoLockBody(doc); + }); +} + +function parseCargoLockBody(doc: Record | null): Map { const out = new Map(); - const doc = parseToml(text); const packages = doc?.package; if (!Array.isArray(packages)) return out; for (const entry of packages) { @@ -70,7 +82,7 @@ export async function loadCargoLockIndex(dir: string, io: LockfileIo): Promise pickLockedVersion(map.get(name), spec) }; } catch (err) { diff --git a/src/core-open/scanners/dart-scanner.ts b/src/core-open/scanners/dart-scanner.ts index f8a4d67b..ff6c4291 100644 --- a/src/core-open/scanners/dart-scanner.ts +++ b/src/core-open/scanners/dart-scanner.ts @@ -5,6 +5,7 @@ import * as path from 'node:path'; import * as semver from 'semver'; import { readTextFile, FileCache } from '../utils/fs.js'; import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { lockfileWarningPath, notePubspecLockText, withLockfileNotes } from '../utils/lockfile-unknown.js'; import { withTimeout } from '../utils/timeout.js'; import { PubCache } from './pub-cache.js'; import type { ProjectScan, DependencyRow, DetectedFramework } from '../types.js'; @@ -188,6 +189,10 @@ async function parsePubspecLock(filePath: string, cache?: FileCache): Promise { + notePubspecLockText(content, lockfileWarningPath(filePath), bucket); + return undefined; + }); let currentPackage: string | null = null; for (const line of content.split(/\r?\n/)) { diff --git a/src/core-open/scanners/npm-lockfile.ts b/src/core-open/scanners/npm-lockfile.ts index 03e98764..fcfd2efd 100644 --- a/src/core-open/scanners/npm-lockfile.ts +++ b/src/core-open/scanners/npm-lockfile.ts @@ -17,6 +17,13 @@ import * as path from 'node:path'; import { parse as parseYaml } from 'yaml'; import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { + type LockfileUnknownFields, + notePackageLock, + notePnpmLockDoc, + noteYarnLockText, + withLockfileNotes, +} from '../utils/lockfile-unknown.js'; export type NpmLockSource = 'package-lock' | 'yarn' | 'pnpm'; @@ -37,7 +44,18 @@ interface PackageLockShape { const TOP_LEVEL_NODE_MODULES = /^node_modules\/((?:@[^/]+\/)?[^/]+)$/; /** name → top-level installed version from a package-lock.json (v2/v3 `packages`, falling back to v1 `dependencies`). */ -export function parsePackageLock(json: unknown): Map { +export function parsePackageLock( + json: unknown, + relPath = 'package-lock.json', + notes?: LockfileUnknownFields, +): Map { + return withLockfileNotes(notes, (bucket) => { + notePackageLock(json, relPath, bucket); + return parsePackageLockBody(json); + }); +} + +function parsePackageLockBody(json: unknown): Map { const out = new Map(); if (!json || typeof json !== 'object') return out; const lock = json as PackageLockShape; @@ -79,7 +97,18 @@ function specName(spec: string): string | null { * specs ending in `:`, followed by an indented `version: x.y.z`. Returns both a precise `name@range` * map and a name fallback. */ -export function parseYarnLock(text: string): YarnLockIndex { +export function parseYarnLock( + text: string, + relPath = 'yarn.lock', + notes?: LockfileUnknownFields, +): YarnLockIndex { + return withLockfileNotes(notes, (bucket) => { + noteYarnLockText(text, relPath, bucket); + return parseYarnLockBody(text); + }); +} + +function parseYarnLockBody(text: string): YarnLockIndex { const bySpec = new Map(); const byName = new Map(); let currentSpecs: string[] = []; @@ -123,7 +152,18 @@ function cleanPnpmVersion(v: string): string | null { * and the top-level `dependencies`/`devDependencies`/`optionalDependencies` (v5/v6), whose values are * either a version string or `{ specifier, version }`. First occurrence wins (the root importer). */ -export function parsePnpmLock(doc: unknown): Map { +export function parsePnpmLock( + doc: unknown, + relPath = 'pnpm-lock.yaml', + notes?: LockfileUnknownFields, +): Map { + return withLockfileNotes(notes, (bucket) => { + notePnpmLockDoc(doc, relPath, bucket); + return parsePnpmLockBody(doc); + }); +} + +function parsePnpmLockBody(doc: unknown): Map { const out = new Map(); if (!doc || typeof doc !== 'object') return out; const d = doc as Record; @@ -180,7 +220,7 @@ export async function loadNpmLockIndex(dir: string, io: LockfileIo): Promise map.get(name) ?? null }; } // package-lock next: structured JSON and unambiguous. @@ -192,7 +232,7 @@ export async function loadNpmLockIndex(dir: string, io: LockfileIo): Promise map.get(name) ?? null }; } if (await io.exists(yarnPath).catch(() => false)) { @@ -200,7 +240,7 @@ export async function loadNpmLockIndex(dir: string, io: LockfileIo): Promise; try { assertLockfileText(yarnPath, text, 'yarn.lock'); - parsed = parseYarnLock(text); + parsed = parseYarnLock(text, yarnPath); } catch (err) { rethrowLockfileParseError(err); throw new LockfileParseError(yarnPath, 'yarn.lock'); diff --git a/src/core-open/scanners/php-scanner.ts b/src/core-open/scanners/php-scanner.ts index e5993ca3..618e95c7 100644 --- a/src/core-open/scanners/php-scanner.ts +++ b/src/core-open/scanners/php-scanner.ts @@ -5,6 +5,7 @@ import * as path from 'node:path'; import * as semver from 'semver'; import { FileCache, readJsonFile } from '../utils/fs.js'; import { LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { lockfileWarningPath, noteComposerLock, withLockfileNotes } from '../utils/lockfile-unknown.js'; import { withTimeout } from '../utils/timeout.js'; import { ComposerCache } from './composer-cache.js'; import type { ProjectScan, DependencyRow, DetectedFramework } from '../types.js'; @@ -146,19 +147,23 @@ async function parseComposerLock(filePath: string, cache?: FileCache): Promise) { - if (pkg.name && pkg.version) { - resolved.set(pkg.name, pkg.version); + withLockfileNotes(undefined, (bucket) => { + noteComposerLock(data, lockfileWarningPath(filePath), bucket); + // Parse packages + for (const pkg of (data.packages ?? []) as Array<{ name?: string; version?: string }>) { + if (pkg.name && pkg.version) { + resolved.set(pkg.name, pkg.version); + } } - } - - // Parse dev packages - for (const pkg of (data['packages-dev'] ?? []) as Array<{ name?: string; version?: string }>) { - if (pkg.name && pkg.version) { - resolved.set(pkg.name, pkg.version); + + // Parse dev packages + for (const pkg of (data['packages-dev'] ?? []) as Array<{ name?: string; version?: string }>) { + if (pkg.name && pkg.version) { + resolved.set(pkg.name, pkg.version); + } } - } + return resolved; + }); } catch (err) { rethrowLockfileParseError(err); throw new LockfileParseError(filePath, 'JSON'); diff --git a/src/core-open/scanners/python-lockfile.ts b/src/core-open/scanners/python-lockfile.ts index 8a9f0204..deced737 100644 --- a/src/core-open/scanners/python-lockfile.ts +++ b/src/core-open/scanners/python-lockfile.ts @@ -21,6 +21,13 @@ */ import * as path from 'node:path'; import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { + type LockfileUnknownFields, + type TomlLockKind, + notePipfileLock, + noteTomlLockDoc, + withLockfileNotes, +} from '../utils/lockfile-unknown.js'; import { parseToml } from '../utils/toml.js'; import type { LockfileIo } from './npm-lockfile.js'; export type { LockfileIo } from './npm-lockfile.js'; @@ -48,9 +55,21 @@ function bareVersion(v: string): string | null { * name → version from a TOML lockfile of `[[package]]` tables (poetry / uv / pdm). First occurrence * wins. Keys are PEP 503 normalized. */ -export function parsePyTomlLock(text: string): Map { +export function parsePyTomlLock( + text: string, + relPath = 'poetry.lock', + notes?: LockfileUnknownFields, + kind: TomlLockKind = 'poetry', +): Map { + return withLockfileNotes(notes, (bucket) => { + const doc = parseToml(text); + noteTomlLockDoc(doc, relPath, bucket, kind); + return parsePyTomlLockBody(doc); + }); +} + +function parsePyTomlLockBody(doc: Record | null): Map { const out = new Map(); - const doc = parseToml(text); const packages = doc?.package; if (!Array.isArray(packages)) return out; for (const entry of packages) { @@ -64,7 +83,18 @@ export function parsePyTomlLock(text: string): Map { } /** name → version from a Pipfile.lock (JSON `default` + `develop` maps). Keys are PEP 503 normalized. */ -export function parsePipfileLock(json: unknown): Map { +export function parsePipfileLock( + json: unknown, + relPath = 'Pipfile.lock', + notes?: LockfileUnknownFields, +): Map { + return withLockfileNotes(notes, (bucket) => { + notePipfileLock(json, relPath, bucket); + return parsePipfileLockBody(json); + }); +} + +function parsePipfileLockBody(json: unknown): Map { const out = new Map(); if (!json || typeof json !== 'object') return out; const doc = json as Record; @@ -96,7 +126,7 @@ export async function loadPythonLockIndex(dir: string, io: LockfileIo): Promise< try { const text = await io.readText(p); assertLockfileText(p, text, 'TOML'); - const map = parsePyTomlLock(text); + const map = parsePyTomlLock(text, p, undefined, source); if (map.size) return { source, size: map.size, resolve: (n) => map.get(normalizePyName(n)) ?? null }; } catch (err) { rethrowLockfileParseError(err); @@ -112,7 +142,7 @@ export async function loadPythonLockIndex(dir: string, io: LockfileIo): Promise< rethrowLockfileParseError(err); throw new LockfileParseError(pipfile, 'JSON'); } - const map = parsePipfileLock(json); + const map = parsePipfileLock(json, pipfile); if (map.size) return { source: 'pipfile', size: map.size, resolve: (n) => map.get(normalizePyName(n)) ?? null }; } return null; diff --git a/src/core-open/scanners/swift-scanner.ts b/src/core-open/scanners/swift-scanner.ts index 9c4aec6f..54a1d3be 100644 --- a/src/core-open/scanners/swift-scanner.ts +++ b/src/core-open/scanners/swift-scanner.ts @@ -5,6 +5,7 @@ import * as path from 'node:path'; import * as semver from 'semver'; import { readTextFile, FileCache } from '../utils/fs.js'; import { parseLockfileJson, rethrowLockfileParseError } from '../utils/lockfile-parse.js'; +import { lockfileWarningPath, notePackageResolved, withLockfileNotes } from '../utils/lockfile-unknown.js'; import { withTimeout } from '../utils/timeout.js'; import { SwiftCache } from './swift-cache.js'; import type { ProjectScan, DependencyRow, DetectedFramework } from '../types.js'; @@ -135,6 +136,11 @@ function parsePackageResolved(filePath: string, content: string): Map; } | null; + withLockfileNotes(undefined, (bucket) => { + notePackageResolved(data, lockfileWarningPath(filePath), bucket); + return undefined; + }); + if (!data || typeof data !== 'object' || !data.pins) return resolved; for (const pin of data.pins) { if (!pin.identity || !pin.state?.version) continue; diff --git a/src/core-open/utils/lockfile-unknown.ts b/src/core-open/utils/lockfile-unknown.ts new file mode 100644 index 00000000..024c2be7 --- /dev/null +++ b/src/core-open/utils/lockfile-unknown.ts @@ -0,0 +1,448 @@ +// VENDORED from @vibgrate/core-open (packages/vibgrate-core-open) by +// scripts/vendor-core-open.mjs. Do not edit here — change the source package +// and re-run the vendor script. Apache-2.0. +/** + * Unknown optional lockfile fields. + * + * A key this version does not model is not a reason to drop a dependency + * set that is otherwise readable. Callers record the key, keep parsing, and + * emit one warning per file per key name. The warning names the key and the + * lockfile path. It never includes the key's value, an environment value, or + * an absolute path (those can carry home directories or registry tokens). + * + * Corrupt or missing required structure stays fail-closed in the syntax + * checker. This module does not accept a broken document. + */ +import * as path from 'node:path'; + +/** Identifier-like key. Anything else can be a header or a pasted secret. */ +const SAFE_KEY = /^[A-Za-z_][A-Za-z0-9_-]{0,63}$/; +const SAFE_REL = /^[A-Za-z0-9_@.+-][A-Za-z0-9_@.+/-]{0,180}$/; + +const emitted = new Set(); + +export function resetUnknownOptionalLockfileWarnings(): void { + emitted.clear(); +} + +/** + * Path safe to print: root-relative when the file sits under the process + * working directory, otherwise the file name alone. + */ +export function lockfileWarningPath(filePath: string, root?: string): string { + const abs = path.resolve(root ?? process.cwd(), filePath); + const fromCwd = safeRelative(path.relative(process.cwd(), abs)); + if (fromCwd) return fromCwd; + if (root) { + const fromRoot = safeRelative(path.relative(root, abs)); + if (fromRoot) return fromRoot; + } + const base = path.basename(abs); + return safeRelative(base) ?? 'lockfile'; +} + +function safeRelative(rel: string): string | undefined { + const posix = rel.replace(/\\/g, '/').replace(/^\.\//, ''); + if (!posix || posix.startsWith('..') || posix.split('/').includes('..')) return undefined; + if (!SAFE_REL.test(posix)) return undefined; + return posix; +} + +export function formatUnknownOptionalLockfileField(relPath: string, key: string): string { + return `${relPath}: unknown optional lockfile field "${key}"; continuing with the fields this version understands.`; +} + +export function formatOmittedUnknownOptionalLockfileField(relPath: string): string { + return `${relPath}: unknown optional lockfile field (name omitted); continuing with the fields this version understands.`; +} + +export class LockfileUnknownFields { + private readonly byFile = new Map>(); + private readonly omitted = new Set(); + + note(relPath: string, key: string): void { + const file = lockfileWarningPath(relPath); + if (!SAFE_KEY.test(key)) { + this.omitted.add(file); + return; + } + let set = this.byFile.get(file); + if (!set) { + set = new Set(); + this.byFile.set(file, set); + } + set.add(key); + } + + noteObject(relPath: string, value: unknown, known: ReadonlySet): void { + if (!value || typeof value !== 'object' || Array.isArray(value)) return; + for (const key of Object.keys(value as object)) { + if (known.has(key)) continue; + this.note(relPath, key); + } + } + + /** Sorted by path, then by key name. One line per pair. */ + messages(): string[] { + const files = new Set([...this.byFile.keys(), ...this.omitted]); + const out: string[] = []; + for (const file of [...files].sort()) { + const keys = [...(this.byFile.get(file) ?? [])].sort(); + for (const key of keys) out.push(formatUnknownOptionalLockfileField(file, key)); + if (this.omitted.has(file)) out.push(formatOmittedUnknownOptionalLockfileField(file)); + } + return out; + } +} + +export function emitUnknownOptionalLockfileWarnings( + notes: LockfileUnknownFields, + write: (line: string) => void = (line) => { + process.stderr.write(`${line}\n`); + }, +): string[] { + const fresh: string[] = []; + for (const message of notes.messages()) { + if (emitted.has(message)) continue; + emitted.add(message); + fresh.push(message); + write(message); + } + return fresh; +} + +/** Fill `notes` when the caller owns them; otherwise warn on stderr. */ +export function withLockfileNotes( + notes: LockfileUnknownFields | undefined, + fill: (bucket: LockfileUnknownFields) => T, +): T { + const bucket = notes ?? new LockfileUnknownFields(); + const result = fill(bucket); + if (!notes) emitUnknownOptionalLockfileWarnings(bucket); + return result; +} + +// ── Known keys ────────────────────────────────────────────────────────────── +// A key in these sets is part of the documented lockfile shape. It is not an +// "unknown" field, even when this reader does not copy the value (integrity, +// checksum, and similar). Missing a real key here would warn on every project. + +export const NPM_LOCK_TOP: ReadonlySet = new Set([ + 'name', 'version', 'lockfileVersion', 'requires', 'packages', 'dependencies', +]); + +export const NPM_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'resolved', 'integrity', 'link', 'dev', 'optional', 'devOptional', + 'inBundle', 'hasInstallScript', 'hasShrinkwrap', 'bin', 'license', 'engines', + 'dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies', + 'peerDependenciesMeta', 'bundledDependencies', 'bundleDependencies', 'os', 'cpu', + 'libc', 'funding', 'deprecated', 'extraneous', 'workspaces', 'requires', 'from', + 'bundled', 'peer', 'licenseFile', 'scripts', +]); + +export const PNPM_LOCK_TOP: ReadonlySet = new Set([ + 'lockfileVersion', 'settings', 'importers', 'packages', 'snapshots', 'time', + 'neverBuiltDependencies', 'onlyBuiltDependencies', 'overrides', 'packageExtensions', + 'packageExtensionsChecksum', 'patchedDependencies', 'catalogs', 'catalog', + 'ignoredOptionalDependencies', 'configDependencies', 'pnpmfileChecksum', + // pnpm v5/v6 put these maps at the top of the file. v9 moved them under importers. + 'dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies', + 'specifiers', 'dependenciesMeta', +]); + +export const PNPM_LOCK_IMPORTER: ReadonlySet = new Set([ + 'dependencies', 'devDependencies', 'optionalDependencies', 'peerDependencies', + 'specifiers', 'dependenciesMeta', 'publishDirectory', +]); + +export const PNPM_LOCK_DEP: ReadonlySet = new Set(['specifier', 'version', 'optional']); + +export const PNPM_LOCK_BLOCK: ReadonlySet = new Set([ + 'bundledDependencies', 'cpu', 'dependencies', 'deprecated', 'dev', 'devDependencies', + 'engines', 'hasBin', 'hash', 'libc', 'name', 'optional', 'optionalDependencies', 'os', + 'path', 'peerDependencies', 'peerDependenciesMeta', 'requiresBuild', 'resolution', + 'transitivePeerDependencies', 'version', +]); + +export const YARN_LOCK_FIELDS: ReadonlySet = new Set([ + 'version', 'resolved', 'integrity', 'dependencies', 'optionalDependencies', + 'dependenciesMeta', 'peerDependencies', 'peerDependenciesMeta', 'checksum', + 'languageName', 'linkType', 'resolution', 'bin', 'conditions', 'cacheKey', +]); + +export const CARGO_LOCK_TOP: ReadonlySet = new Set(['version', 'package', 'metadata', 'patch']); +export const CARGO_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'source', 'checksum', 'dependencies', 'replace', 'explicit', +]); + +export const POETRY_LOCK_TOP: ReadonlySet = new Set(['package', 'metadata']); +export const POETRY_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'description', 'optional', 'python-versions', 'files', + 'dependencies', 'extras', 'category', 'groups', 'markers', 'source', 'develop', + 'dev-dependencies', +]); +export const POETRY_LOCK_METADATA: ReadonlySet = new Set([ + 'lock-version', 'python-versions', 'content-hash', +]); + +export const UV_LOCK_TOP: ReadonlySet = new Set([ + 'version', 'requires-python', 'package', 'manifest', 'resolution-markers', +]); +export const UV_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'source', 'dependencies', 'optional-dependencies', + 'dev-dependencies', 'wheels', 'sdist', 'metadata', 'provides-extras', + 'resolution-markers', +]); + +export const PDM_LOCK_TOP: ReadonlySet = new Set(['metadata', 'package']); +export const PDM_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'requires_python', 'summary', 'dependencies', 'files', + 'marker', 'extras', 'groups', +]); +export const PDM_LOCK_METADATA: ReadonlySet = new Set([ + 'groups', 'strategy', 'lock_version', 'content_hash', +]); + +export type TomlLockKind = 'cargo' | 'poetry' | 'uv' | 'pdm'; + +export const PIPFILE_TOP: ReadonlySet = new Set(['_meta', 'default', 'develop']); +export const PIPFILE_META: ReadonlySet = new Set(['hash', 'pipfile-spec', 'requires', 'sources']); +export const PIPFILE_ENTRY: ReadonlySet = new Set([ + 'hashes', 'markers', 'version', 'index', 'extras', 'subdirectory', + 'editable', 'git', 'ref', 'path', 'file', 'branch', 'tag', 'revision', +]); + +export const COMPOSER_LOCK_TOP: ReadonlySet = new Set([ + 'content-hash', 'packages', 'packages-dev', 'aliases', 'minimum-stability', + 'stability-flags', 'prefer-stable', 'prefer-lowest', 'platform', 'platform-dev', + 'platform-overrides', 'plugin-api-version', +]); +export const COMPOSER_LOCK_PACKAGE: ReadonlySet = new Set([ + 'name', 'version', 'version_normalized', 'source', 'dist', 'require', 'require-dev', + 'conflict', 'replace', 'provide', 'suggest', 'type', 'extra', 'autoload', 'autoload-dev', + 'notification-url', 'license', 'authors', 'description', 'homepage', 'keywords', 'support', + 'funding', 'time', 'bin', 'archive', 'abandoned', 'transport-options', 'installation-source', + 'target-dir', 'include-path', 'scripts', 'repositories', 'config', 'minimum-stability', + 'prefer-stable', 'default-branch', 'aliases', +]); + +export const NUGET_LOCK_TOP: ReadonlySet = new Set(['version', 'dependencies']); +export const NUGET_LOCK_ENTRY: ReadonlySet = new Set([ + 'type', 'requested', 'resolved', 'contentHash', 'dependencies', +]); + +export const SWIFT_RESOLVED_TOP: ReadonlySet = new Set(['version', 'originHash', 'pins', 'object']); +export const SWIFT_PIN: ReadonlySet = new Set(['identity', 'kind', 'location', 'state', 'package']); +export const SWIFT_PIN_STATE: ReadonlySet = new Set(['version', 'revision', 'branch']); + +export const PUBSPEC_LOCK_TOP: ReadonlySet = new Set(['packages', 'sdks']); +export const PUBSPEC_LOCK_FIELDS: ReadonlySet = new Set([ + 'dependency', 'description', 'source', 'version', +]); + +function tomlSets(kind: TomlLockKind): { + top: ReadonlySet; + pkg: ReadonlySet; + meta?: ReadonlySet; +} { + switch (kind) { + case 'cargo': + return { top: CARGO_LOCK_TOP, pkg: CARGO_LOCK_PACKAGE }; + case 'poetry': + return { top: POETRY_LOCK_TOP, pkg: POETRY_LOCK_PACKAGE, meta: POETRY_LOCK_METADATA }; + case 'uv': + return { top: UV_LOCK_TOP, pkg: UV_LOCK_PACKAGE }; + case 'pdm': + return { top: PDM_LOCK_TOP, pkg: PDM_LOCK_PACKAGE, meta: PDM_LOCK_METADATA }; + default: { + const _exhaustive: never = kind; + return _exhaustive; + } + } +} + +export function notePackageLock(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, NPM_LOCK_TOP); + if (!doc || typeof doc !== 'object') return; + const data = doc as { packages?: unknown; dependencies?: unknown }; + if (data.packages && typeof data.packages === 'object' && !Array.isArray(data.packages)) { + for (const entry of Object.values(data.packages as object)) { + notes.noteObject(relPath, entry, NPM_LOCK_PACKAGE); + } + } + noteNpmDependencyTree(data.dependencies, relPath, notes); +} + +function noteNpmDependencyTree(deps: unknown, relPath: string, notes: LockfileUnknownFields): void { + if (!deps || typeof deps !== 'object' || Array.isArray(deps)) return; + for (const entry of Object.values(deps as object)) { + notes.noteObject(relPath, entry, NPM_LOCK_PACKAGE); + if (entry && typeof entry === 'object') { + noteNpmDependencyTree((entry as { dependencies?: unknown }).dependencies, relPath, notes); + } + } +} + +export function notePnpmLockDoc(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, PNPM_LOCK_TOP); + if (!doc || typeof doc !== 'object') return; + const data = doc as Record; + notePnpmDependencyBlocks(data, relPath, notes); + const importers = data.importers; + if (!importers || typeof importers !== 'object' || Array.isArray(importers)) return; + for (const imp of Object.values(importers as object)) { + notes.noteObject(relPath, imp, PNPM_LOCK_IMPORTER); + if (imp && typeof imp === 'object') notePnpmDependencyBlocks(imp as Record, relPath, notes); + } +} + +function notePnpmDependencyBlocks(data: Record, relPath: string, notes: LockfileUnknownFields): void { + for (const section of ['dependencies', 'devDependencies', 'optionalDependencies'] as const) { + const deps = data[section]; + if (!deps || typeof deps !== 'object' || Array.isArray(deps)) continue; + for (const entry of Object.values(deps as object)) { + notes.noteObject(relPath, entry, PNPM_LOCK_DEP); + } + } +} + +const PNPM_TOP_LINE = /^([A-Za-z_][A-Za-z0-9_-]*):/; +const PNPM_BLOCK_LINE = /^ {4}([A-Za-z_][A-Za-z0-9_-]*):/; +const PNPM_DEP_LINE = /^ {8}([A-Za-z_][A-Za-z0-9_-]*):/; + +/** Text form of pnpm-lock.yaml. Package names sit at other indents and are not keys. */ +export function notePnpmLockText(text: string, relPath: string, notes: LockfileUnknownFields): void { + for (const line of text.split('\n')) { + const top = PNPM_TOP_LINE.exec(line); + if (top) { + if (!PNPM_LOCK_TOP.has(top[1]!)) notes.note(relPath, top[1]!); + continue; + } + const block = PNPM_BLOCK_LINE.exec(line); + if (block) { + if (!PNPM_LOCK_BLOCK.has(block[1]!)) notes.note(relPath, block[1]!); + continue; + } + const dep = PNPM_DEP_LINE.exec(line); + if (dep && !PNPM_LOCK_DEP.has(dep[1]!)) notes.note(relPath, dep[1]!); + } +} + +const YARN_FIELD_LINE = /^ {2}([A-Za-z_][A-Za-z0-9_-]*)\b/; + +export function noteYarnLockText(text: string, relPath: string, notes: LockfileUnknownFields): void { + for (const line of text.split('\n')) { + const field = YARN_FIELD_LINE.exec(line); + if (field && !YARN_LOCK_FIELDS.has(field[1]!)) notes.note(relPath, field[1]!); + } +} + +const TOML_ASSIGN = /^([A-Za-z_][A-Za-z0-9_-]*)\s*=/; + +function tomlAssignmentKeys(block: string): string[] { + const keys: string[] = []; + for (const line of block.split('\n')) { + const match = TOML_ASSIGN.exec(line); + if (match) keys.push(match[1]!); + } + return keys; +} + +export function noteTomlLockDoc(doc: unknown, relPath: string, notes: LockfileUnknownFields, kind: TomlLockKind): void { + const sets = tomlSets(kind); + notes.noteObject(relPath, doc, sets.top); + if (!doc || typeof doc !== 'object') return; + const data = doc as { package?: unknown; metadata?: unknown }; + if (Array.isArray(data.package)) { + for (const entry of data.package) notes.noteObject(relPath, entry, sets.pkg); + } + if (sets.meta) notes.noteObject(relPath, data.metadata, sets.meta); +} + +export function noteTomlLockText(text: string, relPath: string, notes: LockfileUnknownFields, kind: TomlLockKind): void { + const sets = tomlSets(kind); + // Preamble assignments only. A file that opens with `[[package]]` has no + // top-level keys; package names inside later tables are not fields. + const tableAt = text.search(/^\s*\[/m); + const head = tableAt === -1 ? text : text.slice(0, tableAt); + for (const key of tomlAssignmentKeys(head)) { + if (!sets.top.has(key)) notes.note(relPath, key); + } + for (const block of text.split(/\[\[package\]\]/).slice(1)) { + const body = block.split(/\n\[/)[0] ?? block; + for (const key of tomlAssignmentKeys(body)) { + if (!sets.pkg.has(key)) notes.note(relPath, key); + } + } + if (!sets.meta) return; + const meta = /\[metadata\][^\n]*\n([\s\S]*?)(?=\n\[|$)/.exec(text); + if (!meta) return; + for (const key of tomlAssignmentKeys(meta[1]!)) { + if (!sets.meta.has(key)) notes.note(relPath, key); + } +} + +export function notePipfileLock(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, PIPFILE_TOP); + if (!doc || typeof doc !== 'object') return; + const data = doc as Record; + notes.noteObject(relPath, data._meta, PIPFILE_META); + for (const section of ['default', 'develop']) { + const deps = data[section]; + if (!deps || typeof deps !== 'object' || Array.isArray(deps)) continue; + for (const entry of Object.values(deps as object)) notes.noteObject(relPath, entry, PIPFILE_ENTRY); + } +} + +export function noteComposerLock(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, COMPOSER_LOCK_TOP); + if (!doc || typeof doc !== 'object') return; + const data = doc as Record; + for (const section of ['packages', 'packages-dev']) { + const arr = data[section]; + if (!Array.isArray(arr)) continue; + for (const entry of arr) notes.noteObject(relPath, entry, COMPOSER_LOCK_PACKAGE); + } +} + +export function noteNugetLock(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, NUGET_LOCK_TOP); + if (!doc || typeof doc !== 'object') return; + const deps = (doc as { dependencies?: unknown }).dependencies; + if (!deps || typeof deps !== 'object' || Array.isArray(deps)) return; + for (const framework of Object.values(deps as object)) { + if (!framework || typeof framework !== 'object' || Array.isArray(framework)) continue; + for (const entry of Object.values(framework as object)) { + notes.noteObject(relPath, entry, NUGET_LOCK_ENTRY); + } + } +} + +export function notePackageResolved(doc: unknown, relPath: string, notes: LockfileUnknownFields): void { + notes.noteObject(relPath, doc, SWIFT_RESOLVED_TOP); + if (!doc || typeof doc !== 'object') return; + const data = doc as { pins?: unknown; object?: { pins?: unknown } }; + const pins = Array.isArray(data.pins) ? data.pins : data.object?.pins; + if (!Array.isArray(pins)) return; + for (const pin of pins) { + notes.noteObject(relPath, pin, SWIFT_PIN); + if (pin && typeof pin === 'object') { + notes.noteObject(relPath, (pin as { state?: unknown }).state, SWIFT_PIN_STATE); + } + } +} + +const PUBSPEC_TOP_LINE = /^([A-Za-z_][A-Za-z0-9_-]*):/; +const PUBSPEC_FIELD_LINE = /^ {4}([A-Za-z_][A-Za-z0-9_-]*):/; + +export function notePubspecLockText(text: string, relPath: string, notes: LockfileUnknownFields): void { + for (const line of text.split('\n')) { + const top = PUBSPEC_TOP_LINE.exec(line); + if (top) { + if (!PUBSPEC_LOCK_TOP.has(top[1]!)) notes.note(relPath, top[1]!); + continue; + } + const field = PUBSPEC_FIELD_LINE.exec(line); + if (field && !PUBSPEC_LOCK_FIELDS.has(field[1]!)) notes.note(relPath, field[1]!); + } +} diff --git a/src/engine/lockfile-unknown.test.ts b/src/engine/lockfile-unknown.test.ts new file mode 100644 index 00000000..9c07e903 --- /dev/null +++ b/src/engine/lockfile-unknown.test.ts @@ -0,0 +1,302 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseCargoLock } from '../core-open/scanners/cargo-lockfile.js'; +import { parseGemfileLock } from '../core-open/scanners/gemfile-lock.js'; +import { parseGradleLockfile } from '../core-open/scanners/gradle-lockfile.js'; +import { parsePackageLock, parsePnpmLock, parseYarnLock } from '../core-open/scanners/npm-lockfile.js'; +import { parsePipfileLock, parsePyTomlLock } from '../core-open/scanners/python-lockfile.js'; +import { LockfileParseError, assertLockfileText } from '../core-open/utils/lockfile-parse.js'; +import { + LockfileUnknownFields, + notePnpmLockText, + resetUnknownOptionalLockfileWarnings, +} from '../core-open/utils/lockfile-unknown.js'; +import { fullDependencyGraph, lockfileVersion } from './lockfile.js'; + +const SECRET = 'super-secret-token'; +const WARN = (file: string, key: string) => + `${file}: unknown optional lockfile field "${key}"; continuing with the fields this version understands.`; + +let root: string; +let stderr: string[]; + +beforeEach(() => { + resetUnknownOptionalLockfileWarnings(); + root = fs.mkdtempSync(path.join(os.tmpdir(), 'vg-lock-unknown-')); + stderr = []; + vi.spyOn(process.stderr, 'write').mockImplementation((chunk) => { + stderr.push(String(chunk)); + return true; + }); +}); + +afterEach(() => { + vi.restoreAllMocks(); + resetUnknownOptionalLockfileWarnings(); + fs.rmSync(root, { recursive: true, force: true }); +}); + +const write = (name: string, body: string): void => fs.writeFileSync(path.join(root, name), body); + +function warnings(): string[] { + return stderr.map((line) => line.replace(/\n$/, '')).filter((line) => line.includes('unknown optional lockfile field')); +} + +describe('lockfile unknown optional fields', () => { + it('keeps an npm dependency set and warns once per unknown key, sorted', () => { + const json = { + lockfileVersion: 3, + futureOptional: `Bearer ${SECRET}`, + alphaField: true, + packages: { + '': { name: 'app', dependencies: { 'left-pad': '^1.3.0' } }, + 'node_modules/left-pad': { version: '1.3.0', integrity: 'sha512-abc', futureOptional: SECRET }, + 'node_modules/left-pad/node_modules/nested': { version: '9.9.9' }, + }, + }; + expect(parsePackageLock(json).get('left-pad')).toBe('1.3.0'); + expect(warnings()).toEqual([WARN('package-lock.json', 'alphaField'), WARN('package-lock.json', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + expect(warnings().join('\n')).not.toContain('Bearer'); + expect(warnings().join('\n')).not.toContain('sha512'); + + warnings().splice(0, warnings().length); + stderr.length = 0; + parsePackageLock(json); + expect(warnings()).toEqual([]); + }); + + it('keeps npm graph edges when a package entry has an unknown optional key', () => { + write( + 'package-lock.json', + JSON.stringify({ + lockfileVersion: 3, + packages: { + '': { name: 'app', dependencies: { chalk: '^5.3.0' } }, + 'node_modules/chalk': { version: '5.3.0', dependencies: { 'ansi-styles': '^6.0.0' }, futureOptional: SECRET }, + 'node_modules/ansi-styles': { version: '6.2.1', integrity: 'sha512-abc' }, + }, + }), + ); + const graph = fullDependencyGraph(root); + expect(graph?.components).toEqual([ + { package: 'ansi-styles', version: '6.2.1' }, + { package: 'chalk', version: '5.3.0' }, + ]); + expect(graph?.edges?.get('chalk@5.3.0')).toEqual(['ansi-styles@6.2.1']); + expect(warnings()).toEqual([WARN('package-lock.json', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + }); + + it('keeps a pnpm importer pin and warns on one unknown top-level key', () => { + const doc = { + lockfileVersion: '9.0', + futureOptional: SECRET, + importers: { + '.': { dependencies: { 'left-pad': { specifier: '^1.3.0', version: '1.3.0' } } }, + }, + }; + expect(parsePnpmLock(doc).get('left-pad')).toBe('1.3.0'); + expect(warnings()).toEqual([WARN('pnpm-lock.yaml', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + + write( + 'pnpm-lock.yaml', + [ + "lockfileVersion: '9.0'", + `futureOptional: ${SECRET}`, + 'importers:', + ' .:', + ' dependencies:', + ' left-pad:', + ' specifier: ^1.3.0', + ' version: 1.3.0', + 'packages:', + ' left-pad@1.3.0:', + ' resolution: {integrity: sha512-abc}', + ].join('\n'), + ); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + const graph = fullDependencyGraph(root); + expect(graph?.components).toEqual([{ package: 'left-pad', version: '1.3.0' }]); + expect(lockfileVersion(root, 'npm', 'left-pad')).toBe('1.3.0'); + expect(warnings()).toEqual([WARN('pnpm-lock.yaml', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + }); + + it('does not warn on a real pnpm-lock.yaml', () => { + const lockPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../pnpm-lock.yaml'); + const text = fs.readFileSync(lockPath, 'utf8'); + const notes = new LockfileUnknownFields(); + notePnpmLockText(text, 'pnpm-lock.yaml', notes); + expect(notes.messages()).toEqual([]); + }); + + it('keeps a yarn pin and warns on one unknown field', () => { + const text = ['left-pad@^1.3.0:', ' version "1.3.0"', ` futureOptional "${SECRET}"`, ''].join('\n'); + expect(parseYarnLock(text).byName.get('left-pad')).toBe('1.3.0'); + expect(warnings()).toEqual([WARN('yarn.lock', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + + write('yarn.lock', text); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'left-pad', version: '1.3.0' }]); + expect(warnings()).toEqual([WARN('yarn.lock', 'futureOptional')]); + }); + + it('keeps poetry, uv, and Pipfile pins and warns on one unknown key', () => { + const poetry = ['[[package]]', 'name = "click"', 'version = "8.1.3"', `futureOptional = "${SECRET}"`, ''].join('\n'); + expect(parsePyTomlLock(poetry, 'poetry.lock', undefined, 'poetry').get('click')).toBe('8.1.3'); + expect(warnings()).toEqual([WARN('poetry.lock', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + + write('poetry.lock', poetry); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'click', version: '8.1.3' }]); + expect(warnings()).toEqual([WARN('poetry.lock', 'futureOptional')]); + + const uv = ['version = 1', '[[package]]', 'name = "httpx"', 'version = "0.27.0"', 'futureOptional = "1"', ''].join('\n'); + expect(parsePyTomlLock(uv, 'uv.lock', undefined, 'uv').get('httpx')).toBe('0.27.0'); + + const pip = { + _meta: { hash: { sha256: 'abc' } }, + default: { requests: { version: '==2.31.0', futureOptional: SECRET } }, + develop: {}, + }; + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + expect(parsePipfileLock(pip).get('requests')).toBe('2.31.0'); + expect(warnings()).toEqual([WARN('Pipfile.lock', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + }); + + it('keeps a Cargo pin and warns on one unknown package key', () => { + const text = ['version = 4', '[[package]]', 'name = "serde"', 'version = "1.0.203"', `futureOptional = "${SECRET}"`, ''].join('\n'); + expect(parseCargoLock(text).get('serde')).toEqual(['1.0.203']); + expect(warnings()).toEqual([WARN('Cargo.lock', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + + write('Cargo.lock', text); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + expect(fullDependencyGraph(root)?.components).toEqual([{ package: 'serde', version: '1.0.203' }]); + expect(warnings()).toEqual([WARN('Cargo.lock', 'futureOptional')]); + }); + + it('keeps composer, NuGet, Swift, and pub pins and warns on one unknown key', () => { + write( + 'composer.lock', + JSON.stringify({ + 'content-hash': 'abc', + futureOptional: SECRET, + packages: [{ name: 'symfony/console', version: 'v7.1.0', description: 'cli' }], + }), + ); + expect(lockfileVersion(root, 'php', 'symfony/console')).toBe('7.1.0'); + expect(warnings()).toEqual([WARN('composer.lock', 'futureOptional')]); + expect(warnings().join('\n')).not.toContain(SECRET); + + fs.rmSync(path.join(root, 'composer.lock')); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + write( + 'packages.lock.json', + JSON.stringify({ + version: 1, + futureOptional: true, + dependencies: { 'net8.0': { 'Newtonsoft.Json': { type: 'Direct', resolved: '13.0.3' } } }, + }), + ); + expect(lockfileVersion(root, 'dotnet', 'Newtonsoft.Json')).toBe('13.0.3'); + expect(warnings()).toEqual([WARN('packages.lock.json', 'futureOptional')]); + + fs.rmSync(path.join(root, 'packages.lock.json')); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + write( + 'Package.resolved', + JSON.stringify({ + version: 3, + futureOptional: true, + pins: [{ identity: 'vapor', state: { version: '4.89.0' } }], + }), + ); + expect(lockfileVersion(root, 'swift', 'vapor')).toBe('4.89.0'); + expect(warnings()).toEqual([WARN('Package.resolved', 'futureOptional')]); + + fs.rmSync(path.join(root, 'Package.resolved')); + resetUnknownOptionalLockfileWarnings(); + stderr.length = 0; + write( + 'pubspec.lock', + [ + 'packages:', + ' provider:', + ' dependency: "direct main"', + ' version: "6.1.2"', + ' futureOptional: true', + 'sdks:', + ' dart: ">=3.0.0 <4.0.0"', + ].join('\n'), + ); + expect(lockfileVersion(root, 'dart', 'provider')).toBe('6.1.2'); + expect(warnings()).toEqual([WARN('pubspec.lock', 'futureOptional')]); + }); + + it('omits a field name that is not a plain identifier and does not print its text', () => { + const notes = new LockfileUnknownFields(); + notes.note('package-lock.json', `Authorization: Bearer ${SECRET}`); + expect(notes.messages()).toEqual([ + 'package-lock.json: unknown optional lockfile field (name omitted); continuing with the fields this version understands.', + ]); + expect(notes.messages().join('\n')).not.toContain(SECRET); + expect(notes.messages().join('\n')).not.toContain('Bearer'); + expect(notes.messages().join('\n')).not.toContain('Authorization'); + }); + + it('keeps Gradle and Gemfile rows when optional commentary is present, without a field warning', () => { + const gradle = '# generated\ncom.google.guava:guava:31.1-jre=compileClasspath\nempty=annotationProcessor\n'; + expect(parseGradleLockfile(gradle).get('com.google.guava:guava')).toBe('31.1-jre'); + write('gradle.lockfile', gradle); + expect(lockfileVersion(root, 'java', 'com.google.guava:guava')).toBe('31.1-jre'); + + const gem = ['GEM', ' remote: https://rubygems.org/', ' specs:', ' rails (7.1.2)', 'FUTURE_OPTIONAL', ' note: true', ''].join('\n'); + expect(parseGemfileLock(gem).get('rails')).toBe('7.1.2'); + write('Gemfile.lock', gem); + fs.rmSync(path.join(root, 'gradle.lockfile')); + expect(lockfileVersion(root, 'ruby', 'rails')).toBe('7.1.2'); + expect(warnings()).toEqual([]); + }); + + it('still fails closed when required lockfile structure is broken, without echoing contents', () => { + const cases: Array<[string, string, string]> = [ + ['package-lock.json', `{"packages":{"node_modules/left-pad":{"version":"1.3.0","integrity":"${SECRET}"`, 'JSON'], + ['pnpm-lock.yaml', `lockfileVersion: '9.0'\npackages:\n left-pad@1.3.0:\n resolution: {integrity: ${SECRET}`, 'YAML'], + ['Cargo.lock', `[[package]]\nname = "serde"\nversion = "${SECRET}`, 'TOML'], + ['yarn.lock', `"left-pad@^1.3.0":\n futureOptional "${SECRET}"\n`, 'yarn.lock'], + ['gradle.lockfile', `not a lock line ${SECRET}\n`, 'gradle.lockfile'], + ['Gemfile.lock', ` rails (${SECRET}\n`, 'Gemfile.lock'], + ['go.sum', `github.com/gin-contrib/sse v1.1.0 ${SECRET}\n`, 'go.sum'], + ]; + for (const [file, body, kind] of cases) { + write(file, body); + expect(() => assertLockfileText(path.join(root, file), body, kind as 'JSON'), file).toThrow(LockfileParseError); + try { + assertLockfileText(path.join(root, file), body, kind as 'JSON'); + } catch (err) { + const message = (err as Error).message; + expect(message, file).toContain(file); + expect(message, file).toContain('package manager'); + expect(message, file).not.toContain(SECRET); + } + fs.rmSync(path.join(root, file)); + } + expect(warnings()).toEqual([]); + }); +}); diff --git a/src/engine/lockfile.ts b/src/engine/lockfile.ts index 94ec0a67..bcc88606 100644 --- a/src/engine/lockfile.ts +++ b/src/engine/lockfile.ts @@ -6,6 +6,21 @@ import { LockfileParseError, parseLockfileJson, } from '../core-open/utils/lockfile-parse.js'; +import { + emitUnknownOptionalLockfileWarnings, + lockfileWarningPath, + LockfileUnknownFields, + noteComposerLock, + noteNugetLock, + notePackageLock, + notePackageResolved, + notePipfileLock, + notePnpmLockText, + notePubspecLockText, + noteTomlLockText, + noteYarnLockText, + type TomlLockKind, +} from '../core-open/utils/lockfile-unknown.js'; import type { DepRecord } from './drift.js'; /** @@ -39,6 +54,21 @@ function readCheckedJson(root: string, file: string): unknown | undefined { return parseLockfileJson(abs, text); } +function tomlKind(file: string): TomlLockKind | undefined { + if (file === 'Cargo.lock') return 'cargo'; + if (file === 'poetry.lock') return 'poetry'; + if (file === 'uv.lock') return 'uv'; + if (file === 'pdm.lock') return 'pdm'; + return undefined; +} + +/** Record unknown optional keys on a lockfile that already parsed. */ +function reportLockfileFields(root: string, file: string, fill: (rel: string, notes: LockfileUnknownFields) => void): void { + const notes = new LockfileUnknownFields(); + fill(lockfileWarningPath(file, root), notes); + emitUnknownOptionalLockfileWarnings(notes); +} + /** * Lockfile-pinned version resolution (VG-LIB-SUPERSET-PLAN A.2 / D13). * @@ -79,6 +109,7 @@ function gradleLock(root: string, name: string): string | undefined { function packageResolved(root: string, name: string): string | undefined { const parsed = readCheckedJson(root, 'Package.resolved'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'Package.resolved', (rel, notes) => notePackageResolved(parsed, rel, notes)); const data = parsed as { pins?: Array<{ identity?: string; package?: string; state?: { version?: string } }>; object?: { pins?: Array<{ identity?: string; package?: string; state?: { version?: string } }> }; @@ -97,6 +128,7 @@ function packageResolved(root: string, name: string): string | undefined { function pubspecLock(root: string, name: string): string | undefined { const text = readCheckedText(root, 'pubspec.lock'); if (text === undefined) return undefined; + reportLockfileFields(root, 'pubspec.lock', (rel, notes) => notePubspecLockText(text, rel, notes)); let inPkg = false; for (const line of text.split('\n')) { const key = /^ ([A-Za-z0-9_.]+):\s*$/.exec(line); @@ -130,6 +162,7 @@ function gemfileLock(root: string, name: string): string | undefined { function composerLock(root: string, name: string): string | undefined { const parsed = readCheckedJson(root, 'composer.lock'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'composer.lock', (rel, notes) => noteComposerLock(parsed, rel, notes)); const data = parsed as Record>; for (const section of ['packages', 'packages-dev']) { const arr = data[section]; @@ -144,6 +177,7 @@ function composerLock(root: string, name: string): string | undefined { function packagesLock(root: string, name: string): string | undefined { const parsed = readCheckedJson(root, 'packages.lock.json'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'packages.lock.json', (rel, notes) => noteNugetLock(parsed, rel, notes)); const data = parsed as { dependencies?: Record> }; const target = name.toLowerCase(); for (const fw of Object.values(data.dependencies ?? {})) { @@ -182,6 +216,8 @@ function pypiLockVersion(root: string, name: string): string | undefined { function tomlPackageLock(root: string, file: string, name: string, normalize: (s: string) => string): string | undefined { const text = readCheckedText(root, file); if (text === undefined) return undefined; + const kind = tomlKind(file); + if (kind) reportLockfileFields(root, file, (rel, notes) => noteTomlLockText(text, rel, notes, kind)); const target = normalize(name); for (const block of text.split(/\[\[package\]\]/)) { const nm = /(?:^|\n)\s*name\s*=\s*"([^"]+)"/.exec(block); @@ -197,6 +233,7 @@ function tomlPackageLock(root: string, file: string, name: string, normalize: (s function pipfileLock(root: string, name: string): string | undefined { const parsed = readCheckedJson(root, 'Pipfile.lock'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'Pipfile.lock', (rel, notes) => notePipfileLock(parsed, rel, notes)); const data = parsed as Record>; const target = pep503(name); for (const section of ['default', 'develop']) { @@ -302,6 +339,7 @@ interface NpmV2Package { function npmLockGraph(root: string): LockfileGraph | undefined { const parsed = readCheckedJson(root, 'package-lock.json'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'package-lock.json', (rel, notes) => notePackageLock(parsed, rel, notes)); const data = parsed as { packages?: Record; dependencies?: Record }>; @@ -386,6 +424,7 @@ function walkNpmV1Tree( function pnpmLockTree(root: string): LockfileComponent[] | undefined { const text = readCheckedText(root, 'pnpm-lock.yaml'); if (text === undefined) return undefined; + reportLockfileFields(root, 'pnpm-lock.yaml', (rel, notes) => notePnpmLockText(text, rel, notes)); const section = sectionOf(text, 'packages'); if (!section) return undefined; const out = new Map(); @@ -406,6 +445,7 @@ function pnpmLockTree(root: string): LockfileComponent[] | undefined { function yarnLockTree(root: string): LockfileComponent[] | undefined { const text = readCheckedText(root, 'yarn.lock'); if (text === undefined) return undefined; + reportLockfileFields(root, 'yarn.lock', (rel, notes) => noteYarnLockText(text, rel, notes)); const out = new Map(); let pendingNames: string[] = []; for (const line of text.split('\n')) { @@ -473,6 +513,7 @@ function yarnHeaderRealName(spec: string): string | undefined { function cargoLockTree(root: string): LockfileComponent[] | undefined { const text = readCheckedText(root, 'Cargo.lock'); if (text === undefined) return undefined; + reportLockfileFields(root, 'Cargo.lock', (rel, notes) => noteTomlLockText(text, rel, notes, 'cargo')); const out = new Map(); for (const block of text.split(/\[\[package\]\]/)) { const nm = /(?:^|\n)\s*name\s*=\s*"([^"]+)"/.exec(block); @@ -512,6 +553,8 @@ function goSumTree(root: string): LockfileComponent[] | undefined { function poetryLikeLockTree(root: string, file: string): LockfileComponent[] | undefined { const text = readCheckedText(root, file); if (text === undefined) return undefined; + const kind = tomlKind(file); + if (kind) reportLockfileFields(root, file, (rel, notes) => noteTomlLockText(text, rel, notes, kind)); const out = new Map(); for (const block of text.split(/\[\[package\]\]/)) { const nm = /(?:^|\n)\s*name\s*=\s*"([^"]+)"/.exec(block); @@ -525,6 +568,7 @@ function poetryLikeLockTree(root: string, file: string): LockfileComponent[] | u function packageLockVersion(root: string, name: string): string | undefined { const parsed = readCheckedJson(root, 'package-lock.json'); if (!parsed || typeof parsed !== 'object') return undefined; + reportLockfileFields(root, 'package-lock.json', (rel, notes) => notePackageLock(parsed, rel, notes)); const data = parsed as { packages?: Record; dependencies?: Record }; // v2/v3: packages keyed by install path; the top-level dep is "node_modules/". const top = data.packages?.[`node_modules/${name}`]?.version; @@ -550,6 +594,7 @@ function packageLockVersion(root: string, name: string): string | undefined { function pnpmLockVersion(root: string, name: string): string | undefined { const text = readCheckedText(root, 'pnpm-lock.yaml'); if (text === undefined) return undefined; + reportLockfileFields(root, 'pnpm-lock.yaml', (rel, notes) => notePnpmLockText(text, rel, notes)); const importers = sectionOf(text, 'importers'); if (!importers) return undefined; // Entries look like: @@ -584,6 +629,7 @@ function sectionOf(text: string, key: string): string | undefined { function yarnLockVersion(root: string, name: string): string | undefined { const text = readCheckedText(root, 'yarn.lock'); if (text === undefined) return undefined; + reportLockfileFields(root, 'yarn.lock', (rel, notes) => noteYarnLockText(text, rel, notes)); let inBlock = false; for (const line of text.split('\n')) { if (line && !/^\s/.test(line) && !line.startsWith('#')) { diff --git a/src/reporting/scanners/dependency-graph.ts b/src/reporting/scanners/dependency-graph.ts index a446026a..66bb4be8 100644 --- a/src/reporting/scanners/dependency-graph.ts +++ b/src/reporting/scanners/dependency-graph.ts @@ -1,6 +1,13 @@ import * as path from 'node:path'; import { readTextFile, pathExists, findPackageJsonFiles, readJsonFile, FileCache } from '../../core-open/index.js'; import { assertLockfileText, parseLockfileJson } from '../../core-open/utils/lockfile-parse.js'; +import { + lockfileWarningPath, + notePackageLock, + notePnpmLockText, + noteYarnLockText, + withLockfileNotes, +} from '../../core-open/utils/lockfile-unknown.js'; import type { PackageJson, DependencyGraphResult, DuplicatedPackage, PhantomDependency } from '../../core-open/index.js'; interface LockEntry { @@ -36,6 +43,10 @@ function parseNpmLock(content: string, filePath: string): LockEntry[] { dependencies?: Record }>; } | null; if (lock && typeof lock === 'object') { + withLockfileNotes(undefined, (bucket) => { + notePackageLock(lock, lockfileWarningPath(filePath), bucket); + return undefined; + }); // v2/v3 format: `packages` keyed by path like "node_modules/lodash" if (lock.packages && typeof lock.packages === 'object') { @@ -102,6 +113,10 @@ export async function scanDependencyGraph(rootDir: string, cache?: FileCache): P const content = await _readTextFile(pnpmLock); assertLockfileText(pnpmLock, content, 'YAML'); entries = parsePnpmLock(content); + withLockfileNotes(undefined, (bucket) => { + notePnpmLockText(content, lockfileWarningPath(pnpmLock), bucket); + return undefined; + }); } else if (await _pathExists(npmLock)) { result.lockfileType = 'npm'; const content = await _readTextFile(npmLock); @@ -111,6 +126,10 @@ export async function scanDependencyGraph(rootDir: string, cache?: FileCache): P const content = await _readTextFile(yarnLock); assertLockfileText(yarnLock, content, 'yarn.lock'); entries = parseYarnLock(content); + withLockfileNotes(undefined, (bucket) => { + noteYarnLockText(content, lockfileWarningPath(yarnLock), bucket); + return undefined; + }); } if (entries.length === 0) return result;