feat(lockfile): warn and keep packages when an optional field is unknown

CHANGELOG.md

@@ -14,6 +14,8 @@ backward compatible.
 
 ### Added
 
+- **Unknown optional lockfile fields warn and continue.** When a lockfile still lists dependencies vg can read, an extra field does not drop that graph or the SBOM rows built from it. The command prints one warning per lockfile path and field name, sorted by path then field: `pnpm-lock.yaml: unknown optional lockfile field "<name>"; continuing with the fields this version understands.` A field name that is not a plain identifier is reported as `(name omitted)`. A truncated lockfile, or one missing the structure its format requires, still exits non-zero.
+
 - **`vg build --attest` and `vg build --verify` are documented.** The docs
   cover the DSSE in-toto statement written to
   `.vibgrate/attestation.intoto.jsonl`, the Ed25519 key

DOCS.md

@@ -27,6 +27,7 @@ For a quick overview, see the [README](./README.md). This document covers everyt
     - [CycloneDX type and SPDX primaryPackagePurpose](#cyclonedx-type-and-spdx-primarypackagepurpose)
     - [Production, development, and optional scope](./docs/sbom-dependency-scope.md)
   - [vg scan](#vg-scan)
+    - [Unknown lockfile fields](#unknown-lockfile-fields)
     - [Offline scan with a package-version manifest](#offline-scan-with-a-package-version-manifest)
     - [Vulnerabilities and exposure attribution](#vulnerabilities-and-exposure-attribution)
       - [Go modules: pseudo-versions and +incompatible](#go-modules-pseudo-versions-and-incompatible)
@@ -2128,6 +2129,22 @@ By default, the scan writes `.vibgrate/scan_result.json`. Use `--no-local-artifa
 
 For offline drift scoring, pass `--package-manifest <file>` with a downloaded manifest bundle such as `https://github.com/vibgrate/manifests/latest-packages.zip`. The manifest shape, the fail-closed errors, and what offline mode skips are in [Offline scan with a package-version manifest](#offline-scan-with-a-package-version-manifest).
 
+### Unknown lockfile fields
+
+Package managers add fields to lockfiles as they release. When the file still lists dependencies vg can read, an extra field does not erase that graph or the SBOM rows built from it. vg keeps the package names, versions, and dependency edges it understands.
+
+vg prints one warning on stderr for each unknown field name in that file. Lines are sorted by lockfile path, then by field name. The same path and field name is printed once. A warning looks like this:
+
+```text
+pnpm-lock.yaml: unknown optional lockfile field "futureOptional"; continuing with the fields this version understands.
+```
+
+The path is relative to the directory you ran the command from, when the lockfile is under that directory. Otherwise the warning uses the file name only. The warning names the field. It does not print the field's value, an environment value, or an authorization header. A field name that is not a plain identifier is reported as `(name omitted)`.
+
+A lockfile that is truncated, empty, or missing the structure its format requires still fails the command. The error names the file and the format, tells you to regenerate the file with your package manager, and the process exits non-zero. The error does not include the file's contents.
+
+`vg scan` and `vg sbom export` follow this for npm, pnpm, yarn, poetry, uv, pdm, Pipfile, Cargo, Composer, NuGet, Swift package pins, and pub. A Gradle lock line, a Gemfile spec, and a `go.sum` line do not have optional field names. A line that does not match that format is invalid, and the command fails.
+
 ### Offline scan with a package-version manifest
 
 `--offline` scores drift, and with `--vulns` it matches known vulnerabilities, from files on the machine. `--package-manifest` is the local file those lookups read.

src/core-open/scanners/cargo-lockfile.ts

@@ -15,6 +15,7 @@
 import * as path from 'node:path';
 import * as semver from 'semver';
 import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js';
+import { type LockfileUnknownFields, noteTomlLockDoc, withLockfileNotes } from '../utils/lockfile-unknown.js';
 import { parseToml } from '../utils/toml.js';
 import type { LockfileIo } from './npm-lockfile.js';
 export type { LockfileIo } from './npm-lockfile.js';
@@ -30,9 +31,20 @@ export interface CargoLockIndex {
  * Parse a Cargo.lock into name → locked versions. A crate may appear more than once (multiple
  * incompatible majors in the tree), so versions accumulate into a list per name.
  */
-export function parseCargoLock(text: string): Map<string, string[]> {
+export function parseCargoLock(
+  text: string,
+  relPath = 'Cargo.lock',
+  notes?: LockfileUnknownFields,
+): Map<string, string[]> {
+  return withLockfileNotes(notes, (bucket) => {
+    const doc = parseToml(text);
+    noteTomlLockDoc(doc, relPath, bucket, 'cargo');
+    return parseCargoLockBody(doc);
+  });
+}
+
+function parseCargoLockBody(doc: Record<string, unknown> | null): Map<string, string[]> {
   const out = new Map<string, string[]>();
-  const doc = parseToml(text);
   const packages = doc?.package;
   if (!Array.isArray(packages)) return out;
   for (const entry of packages) {
@@ -70,7 +82,7 @@ export async function loadCargoLockIndex(dir: string, io: LockfileIo): Promise<C
   try {
     const text = await io.readText(lockPath);
     assertLockfileText(lockPath, text, 'TOML');
-    const map = parseCargoLock(text);
+    const map = parseCargoLock(text, lockPath);
     if (!map.size) return null;
     return { size: map.size, resolve: (name, spec) => pickLockedVersion(map.get(name), spec) };
   } catch (err) {

src/core-open/scanners/dart-scanner.ts

@@ -5,6 +5,7 @@ import * as path from 'node:path';
 import * as semver from 'semver';
 import { readTextFile, FileCache } from '../utils/fs.js';
 import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js';
+import { lockfileWarningPath, notePubspecLockText, withLockfileNotes } from '../utils/lockfile-unknown.js';
 import { withTimeout } from '../utils/timeout.js';
 import { PubCache } from './pub-cache.js';
 import type { ProjectScan, DependencyRow, DetectedFramework } from '../types.js';
@@ -188,6 +189,10 @@ async function parsePubspecLock(filePath: string, cache?: FileCache): Promise<Ma
   assertLockfileText(filePath, content, 'YAML');
 
   try {
+    withLockfileNotes(undefined, (bucket) => {
+      notePubspecLockText(content, lockfileWarningPath(filePath), bucket);
+      return undefined;
+    });
     let currentPackage: string | null = null;
 
     for (const line of content.split(/\r?\n/)) {

src/core-open/scanners/npm-lockfile.ts

@@ -17,6 +17,13 @@
 import * as path from 'node:path';
 import { parse as parseYaml } from 'yaml';
 import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js';
+import {
+  type LockfileUnknownFields,
+  notePackageLock,
+  notePnpmLockDoc,
+  noteYarnLockText,
+  withLockfileNotes,
+} from '../utils/lockfile-unknown.js';
 
 export type NpmLockSource = 'package-lock' | 'yarn' | 'pnpm';
 
@@ -37,7 +44,18 @@ interface PackageLockShape {
 const TOP_LEVEL_NODE_MODULES = /^node_modules\/((?:@[^/]+\/)?[^/]+)$/;
 
 /** name → top-level installed version from a package-lock.json (v2/v3 `packages`, falling back to v1 `dependencies`). */
-export function parsePackageLock(json: unknown): Map<string, string> {
+export function parsePackageLock(
+  json: unknown,
+  relPath = 'package-lock.json',
+  notes?: LockfileUnknownFields,
+): Map<string, string> {
+  return withLockfileNotes(notes, (bucket) => {
+    notePackageLock(json, relPath, bucket);
+    return parsePackageLockBody(json);
+  });
+}
+
+function parsePackageLockBody(json: unknown): Map<string, string> {
   const out = new Map<string, string>();
   if (!json || typeof json !== 'object') return out;
   const lock = json as PackageLockShape;
@@ -79,7 +97,18 @@ function specName(spec: string): string | null {
  * specs ending in `:`, followed by an indented `version: x.y.z`. Returns both a precise `name@range`
  * map and a name fallback.
  */
-export function parseYarnLock(text: string): YarnLockIndex {
+export function parseYarnLock(
+  text: string,
+  relPath = 'yarn.lock',
+  notes?: LockfileUnknownFields,
+): YarnLockIndex {
+  return withLockfileNotes(notes, (bucket) => {
+    noteYarnLockText(text, relPath, bucket);
+    return parseYarnLockBody(text);
+  });
+}
+
+function parseYarnLockBody(text: string): YarnLockIndex {
   const bySpec = new Map<string, string>();
   const byName = new Map<string, string>();
   let currentSpecs: string[] = [];
@@ -123,7 +152,18 @@ function cleanPnpmVersion(v: string): string | null {
  * and the top-level `dependencies`/`devDependencies`/`optionalDependencies` (v5/v6), whose values are
  * either a version string or `{ specifier, version }`. First occurrence wins (the root importer).
  */
-export function parsePnpmLock(doc: unknown): Map<string, string> {
+export function parsePnpmLock(
+  doc: unknown,
+  relPath = 'pnpm-lock.yaml',
+  notes?: LockfileUnknownFields,
+): Map<string, string> {
+  return withLockfileNotes(notes, (bucket) => {
+    notePnpmLockDoc(doc, relPath, bucket);
+    return parsePnpmLockBody(doc);
+  });
+}
+
+function parsePnpmLockBody(doc: unknown): Map<string, string> {
   const out = new Map<string, string>();
   if (!doc || typeof doc !== 'object') return out;
   const d = doc as Record<string, unknown>;
@@ -180,7 +220,7 @@ export async function loadNpmLockIndex(dir: string, io: LockfileIo): Promise<Npm
       rethrowLockfileParseError(err);
       throw new LockfileParseError(pnpmPath, 'YAML');
     }
-    const map = parsePnpmLock(doc);
+    const map = parsePnpmLock(doc, pnpmPath);
     if (map.size) return { source: 'pnpm', size: map.size, resolve: (name) => map.get(name) ?? null };
   }
   // package-lock next: structured JSON and unambiguous.
@@ -192,15 +232,15 @@ export async function loadNpmLockIndex(dir: string, io: LockfileIo): Promise<Npm
       rethrowLockfileParseError(err);
       throw new LockfileParseError(lockPath, 'JSON');
     }
-    const map = parsePackageLock(json);
+    const map = parsePackageLock(json, lockPath);
     if (map.size) return { source: 'package-lock', size: map.size, resolve: (name) => map.get(name) ?? null };
   }
   if (await io.exists(yarnPath).catch(() => false)) {
     const text = await io.readText(yarnPath);
     let parsed: ReturnType<typeof parseYarnLock>;
     try {
       assertLockfileText(yarnPath, text, 'yarn.lock');
-      parsed = parseYarnLock(text);
+      parsed = parseYarnLock(text, yarnPath);
     } catch (err) {
       rethrowLockfileParseError(err);
       throw new LockfileParseError(yarnPath, 'yarn.lock');

src/core-open/scanners/php-scanner.ts

@@ -5,6 +5,7 @@ import * as path from 'node:path';
 import * as semver from 'semver';
 import { FileCache, readJsonFile } from '../utils/fs.js';
 import { LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js';
+import { lockfileWarningPath, noteComposerLock, withLockfileNotes } from '../utils/lockfile-unknown.js';
 import { withTimeout } from '../utils/timeout.js';
 import { ComposerCache } from './composer-cache.js';
 import type { ProjectScan, DependencyRow, DetectedFramework } from '../types.js';
@@ -146,19 +147,23 @@ async function parseComposerLock(filePath: string, cache?: FileCache): Promise<M
   }
 
   try {
-    // Parse packages
-    for (const pkg of (data.packages ?? []) as Array<{ name?: string; version?: string }>) {
-      if (pkg.name && pkg.version) {
-        resolved.set(pkg.name, pkg.version);
+    withLockfileNotes(undefined, (bucket) => {
+      noteComposerLock(data, lockfileWarningPath(filePath), bucket);
+      // Parse packages
+      for (const pkg of (data.packages ?? []) as Array<{ name?: string; version?: string }>) {
+        if (pkg.name && pkg.version) {
+          resolved.set(pkg.name, pkg.version);
+        }
       }
-    }
-
-    // Parse dev packages
-    for (const pkg of (data['packages-dev'] ?? []) as Array<{ name?: string; version?: string }>) {
-      if (pkg.name && pkg.version) {
-        resolved.set(pkg.name, pkg.version);
+
+      // Parse dev packages
+      for (const pkg of (data['packages-dev'] ?? []) as Array<{ name?: string; version?: string }>) {
+        if (pkg.name && pkg.version) {
+          resolved.set(pkg.name, pkg.version);
+        }
       }
-    }
+      return resolved;
+    });
   } catch (err) {
     rethrowLockfileParseError(err);
     throw new LockfileParseError(filePath, 'JSON');

src/core-open/scanners/python-lockfile.ts

@@ -21,6 +21,13 @@
  */
 import * as path from 'node:path';
 import { assertLockfileText, LockfileParseError, rethrowLockfileParseError } from '../utils/lockfile-parse.js';
+import {
+  type LockfileUnknownFields,
+  type TomlLockKind,
+  notePipfileLock,
+  noteTomlLockDoc,
+  withLockfileNotes,
+} from '../utils/lockfile-unknown.js';
 import { parseToml } from '../utils/toml.js';
 import type { LockfileIo } from './npm-lockfile.js';
 export type { LockfileIo } from './npm-lockfile.js';
@@ -48,9 +55,21 @@ function bareVersion(v: string): string | null {
  * name → version from a TOML lockfile of `[[package]]` tables (poetry / uv / pdm). First occurrence
  * wins. Keys are PEP 503 normalized.
  */
-export function parsePyTomlLock(text: string): Map<string, string> {
+export function parsePyTomlLock(
+  text: string,
+  relPath = 'poetry.lock',
+  notes?: LockfileUnknownFields,
+  kind: TomlLockKind = 'poetry',
+): Map<string, string> {
+  return withLockfileNotes(notes, (bucket) => {
+    const doc = parseToml(text);
+    noteTomlLockDoc(doc, relPath, bucket, kind);
+    return parsePyTomlLockBody(doc);
+  });
+}
+
+function parsePyTomlLockBody(doc: Record<string, unknown> | null): Map<string, string> {
   const out = new Map<string, string>();
-  const doc = parseToml(text);
   const packages = doc?.package;
   if (!Array.isArray(packages)) return out;
   for (const entry of packages) {
@@ -64,7 +83,18 @@ export function parsePyTomlLock(text: string): Map<string, string> {
 }
 
 /** name → version from a Pipfile.lock (JSON `default` + `develop` maps). Keys are PEP 503 normalized. */
-export function parsePipfileLock(json: unknown): Map<string, string> {
+export function parsePipfileLock(
+  json: unknown,
+  relPath = 'Pipfile.lock',
+  notes?: LockfileUnknownFields,
+): Map<string, string> {
+  return withLockfileNotes(notes, (bucket) => {
+    notePipfileLock(json, relPath, bucket);
+    return parsePipfileLockBody(json);
+  });
+}
+
+function parsePipfileLockBody(json: unknown): Map<string, string> {
   const out = new Map<string, string>();
   if (!json || typeof json !== 'object') return out;
   const doc = json as Record<string, unknown>;
@@ -96,7 +126,7 @@ export async function loadPythonLockIndex(dir: string, io: LockfileIo): Promise<
     try {
       const text = await io.readText(p);
       assertLockfileText(p, text, 'TOML');
-      const map = parsePyTomlLock(text);
+      const map = parsePyTomlLock(text, p, undefined, source);
       if (map.size) return { source, size: map.size, resolve: (n) => map.get(normalizePyName(n)) ?? null };
     } catch (err) {
       rethrowLockfileParseError(err);
@@ -112,7 +142,7 @@ export async function loadPythonLockIndex(dir: string, io: LockfileIo): Promise<
       rethrowLockfileParseError(err);
       throw new LockfileParseError(pipfile, 'JSON');
     }
-    const map = parsePipfileLock(json);
+    const map = parsePipfileLock(json, pipfile);
     if (map.size) return { source: 'pipfile', size: map.size, resolve: (n) => map.get(normalizePyName(n)) ?? null };
   }
   return null;

src/core-open/scanners/swift-scanner.ts

@@ -5,6 +5,7 @@ import * as path from 'node:path';
 import * as semver from 'semver';
 import { readTextFile, FileCache } from '../utils/fs.js';
 import { parseLockfileJson, rethrowLockfileParseError } from '../utils/lockfile-parse.js';
+import { lockfileWarningPath, notePackageResolved, withLockfileNotes } from '../utils/lockfile-unknown.js';
 import { withTimeout } from '../utils/timeout.js';
 import { SwiftCache } from './swift-cache.js';
 import type { ProjectScan, DependencyRow, DetectedFramework } from '../types.js';
@@ -135,6 +136,11 @@ function parsePackageResolved(filePath: string, content: string): Map<string, st
     }>;
   } | null;
 
+  withLockfileNotes(undefined, (bucket) => {
+    notePackageResolved(data, lockfileWarningPath(filePath), bucket);
+    return undefined;
+  });
+
   if (!data || typeof data !== 'object' || !data.pins) return resolved;
   for (const pin of data.pins) {
     if (!pin.identity || !pin.state?.version) continue;