diff --git a/CHANGELOG.md b/CHANGELOG.md index 428b130b..034675d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,15 @@ backward compatible. ## [Unreleased] +### Changed + +- **`vg build` and `vg scan` do not follow symlinks, and they say so.** A + directory symlink is not entered, so a link to its parent cannot hang the + walk, and a file symlink is not read. When any are skipped, one stderr + notice lists the count and the first few root-relative paths. Nothing is + printed when there are none. Output files, the exit code, and JSON on + stdout stay the same. + ### Added - **Unknown optional lockfile fields warn and continue.** When a lockfile still lists dependencies vg can read, an extra field does not drop that graph or the SBOM rows built from it. The command prints one warning per lockfile path and field name, sorted by path then field: `pnpm-lock.yaml: unknown optional lockfile field ""; continuing with the fields this version understands.` A field name that is not a plain identifier is reported as `(name omitted)`. A truncated lockfile, or one missing the structure its format requires, still exits non-zero. diff --git a/DOCS.md b/DOCS.md index 3b62bc80..7f9fb200 100644 --- a/DOCS.md +++ b/DOCS.md @@ -110,6 +110,7 @@ For a quick overview, see the [README](./README.md). This document covers everyt - [vibgrate.config.ts](#vibgrateconfigts) - [Thresholds](#thresholds) - [Scanner Toggles](#scanner-toggles) + - [Symlinks](#symlinks) - [Extended Scanners](#extended-scanners) - [Platform Matrix](#platform-matrix) - [Dependency Risk](#dependency-risk) @@ -2128,6 +2129,8 @@ Switches (flags that take no value, such as `--vulns`, `--offline` or `--no-grap By default, the scan writes `.vibgrate/scan_result.json`. Use `--no-local-artifacts` or `--max-privacy` to suppress local JSON artifact files. +`vg scan` does not follow symlinks while it indexes the tree. A skipped link is named once on stderr. See [Symlinks](#symlinks). + For offline drift scoring, pass `--package-manifest ` with a downloaded manifest bundle such as `https://github.com/vibgrate/manifests/latest-packages.zip`. The manifest shape, the fail-closed errors, and what offline mode skips are in [Offline scan with a package-version manifest](#offline-scan-with-a-package-version-manifest). ### Unknown lockfile fields @@ -2718,6 +2721,8 @@ Maps source code into a graph artifact, enabling all downstream queries (`vg sho | `--attestation ` | `.vibgrate/attestation.intoto.jsonl` | Where `--attest` writes, and where `--verify` reads | | `--pub ` | — | Public key PEM that pins the signer for `--verify` | +`vg build` does not follow symlinks while it discovers files. A skipped link is named once on stderr. See [Symlinks](#symlinks). `.gitignore` and `--exclude` still apply. + **Local by default — no git churn.** The first time vg writes into `.vibgrate/` it also creates `.vibgrate/.gitignore`, keeping the graph artifacts (`graph.json`, `graph.html`, `GRAPH_REPORT.md`, `facts.jsonl`, `mcp-navigation.json`) and the cache out of git — so builds, auto-refreshes, and MCP use never leave your branch dirty. Run `vg share` when you want the map committed for your team (it rewrites that ignore file). vg never touches an existing `.vibgrate/.gitignore`, so edit it (or leave it empty) to manage the ignores yourself. Maven `import` edges, and the Gradle files that stay document nodes, are in [Maven and Gradle manifests](#maven-and-gradle-manifests). How a GitHub Actions `uses:` pin is copied into the map: [Action pins in the code map](./docs/ci/github-actions.md#action-pins-in-the-code-map). @@ -5031,6 +5036,32 @@ of observed memory — so identical input still produces an identical `graph.json`. To give the build more room instead of limiting it, raise the Node heap: `NODE_OPTIONS=--max-old-space-size=8192`. +### Symlinks + +`vg build` and `vg scan` do not follow symlinks. While walking, a directory +symlink is not entered and a file symlink is not read. That includes a link +that points at its own parent, so a cycle cannot make the walk hang. The link +is left out of the map and out of the scan. Output files do not change, and +the exit code stays the same. + +The directory you pass as the root is opened even when that path itself is a +symlink. Links found underneath it are not followed. To include the files a +link points at, point the root at that target, or pass a narrower directory +that already contains them. To leave a link out without a notice, list it in +`.gitignore` or pass `--exclude`. + +When a walk skips one or more symlinks, vg prints one notice on stderr and +nothing when there are none. The notice is the count, then the first few +root-relative paths in sorted order (at most five; further links are a +`+N more` count). Paths are relative to the root you passed. Absolute paths +are not printed. The line goes to stderr, so `--format json` and +`vg build --json` keep a JSON document on stdout. `--quiet` hides promotional +text only; it does not hide this notice. + +```text +notice: skipped 3 symlinks (alias.ts, nested/cycle, via). vg does not follow symlinks. Point the root at the link target, or pass --exclude or a narrower root. +``` + --- ## Extended Scanners diff --git a/src/core-open/utils/fs.ts b/src/core-open/utils/fs.ts index 915e88e5..5e19c4e3 100644 --- a/src/core-open/utils/fs.ts +++ b/src/core-open/utils/fs.ts @@ -17,6 +17,7 @@ import { UnsafeRootError, type WalkBudgetState, } from './root-safety.js'; +import { emitSkippedSymlinkNotice, rememberSkippedSymlink } from './skipped-symlinks.js'; const execFileAsync = promisify(execFile); @@ -490,6 +491,10 @@ export class FileCache { if (budgetError) return; budgetError = noteWalkEntry(budgetState); }; + // Symlinks are not followed. Dirent.isDirectory() is false for a link, so a + // link to a parent cannot re-enter this walk. One notice is printed after + // the walk finishes. The prelude count stays quiet so a scan says it once. + const skippedSymlinks: string[] = []; async function walk(dir: string, gitignoreLevels: GitignoreLevel[]) { if (budgetError) return; @@ -542,6 +547,15 @@ export class FileCache { if (isExcluded && isExcluded(relPath)) continue; if (budgetError) break; + if (e.isSymbolicLink()) { + // Ignored as a file or as a directory: the operator already asked to + // skip it, so it does not join the notice. + if (!(isGitignored(levels, absPath, false) || isGitignored(levels, absPath, true))) { + rememberSkippedSymlink(skippedSymlinks, relPath); + } + continue; + } + if (e.isDirectory()) { if (isSkippedDirName(e.name) || extraSkip.has(e.name)) continue; // Respect .gitignore (root + any nested ones seen so far) so @@ -577,6 +591,7 @@ export class FileCache { await walk(rootDir, []); if (budgetError) throw budgetError; + emitSkippedSymlinkNotice(skippedSymlinks); let totalDirs = 0; const rootNameIndex = new Map(); @@ -885,6 +900,10 @@ async function countTreeWithBudget( const relPath = path.relative(rootDir, absPath); if (isExcluded && isExcluded(relPath)) continue; + // Not followed. The indexing walk prints the notice; this count does not, + // so vg scan does not say the same thing twice. + if (e.isSymbolicLink()) continue; + if (e.isDirectory()) { if (isSkippedDirName(e.name) || extraSkip.has(e.name)) continue; if (isGitignored(levels, absPath, true)) continue; @@ -975,6 +994,8 @@ export async function countFilesInDir(dir: string, recursive = true): Promise[] = []; for (const e of entries) { + // Not followed. A link to a parent cannot re-enter this count. + if (e.isSymbolicLink()) continue; if (e.isDirectory()) { if (!recursive) continue; if (isSkippedDirName(e.name) || extraSkip.has(e.name)) continue; @@ -1009,6 +1030,8 @@ export async function bytesInDir(dir: string, recursive = true): Promise } const subs: Promise[] = []; for (const e of entries) { + // Not followed. A link to a parent cannot re-enter this size walk. + if (e.isSymbolicLink()) continue; if (e.isDirectory()) { if (!recursive) continue; if (isSkippedDirName(e.name) || extraSkip.has(e.name)) continue; @@ -1044,6 +1067,7 @@ export async function findFiles( budgetError = noteWalkEntry(budgetState); }; const results: string[] = []; + const skippedSymlinks: string[] = []; const cores = typeof os.availableParallelism === 'function' ? os.availableParallelism() : os.cpus().length || 4; const maxConcurrentReads = Math.max(8, Math.min(64, cores * 4)); const readDirSemaphore = new Semaphore(maxConcurrentReads); @@ -1061,6 +1085,10 @@ export async function findFiles( for (const e of entries) { if (budgetError) break; + if (e.isSymbolicLink()) { + rememberSkippedSymlink(skippedSymlinks, path.relative(rootDir, path.join(dir, e.name))); + continue; + } if (e.isDirectory()) { if (isSkippedDirName(e.name)) continue; note(); @@ -1081,6 +1109,7 @@ export async function findFiles( await walk(rootDir); if (budgetError) throw budgetError; + emitSkippedSymlinkNotice(skippedSymlinks); return results; } diff --git a/src/core-open/utils/skipped-symlinks.ts b/src/core-open/utils/skipped-symlinks.ts new file mode 100644 index 00000000..17e71d26 --- /dev/null +++ b/src/core-open/utils/skipped-symlinks.ts @@ -0,0 +1,65 @@ +// VENDORED from @vibgrate/core-open (packages/vibgrate-core-open) by +// scripts/vendor-core-open.mjs. Do not edit here — change the source package +// and re-run the vendor script. Apache-2.0. +import * as path from 'node:path'; + +/** + * How many root-relative paths one notice lists. Further links are a count. + * The walk sorts before capping, so the same tree always names the same paths. + */ +export const SKIPPED_SYMLINK_NOTICE_CAP = 5; + +function comparePath(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +/** + * Root-relative POSIX path, or '' when `absPath` is the root or escapes it. + * Absolute paths are never returned. + */ +export function walkRelativePath(root: string, absPath: string): string { + const rel = path.relative(root, absPath); + if (!rel || rel === '.' || rel.startsWith('..')) return ''; + return rel.split(path.sep).join('/'); +} + +/** Record one skipped symlink. Empty and out-of-root paths are dropped. */ +export function rememberSkippedSymlink(bucket: string[], relPath: string): void { + const rel = relPath.split(path.sep).join('/'); + if (!rel || rel === '.' || rel.startsWith('..') || rel.startsWith('/')) return; + bucket.push(rel); +} + +/** + * One stderr line when a walk skips symlinks. `null` when `relPaths` is empty. + * Paths are de-duplicated, sorted, and capped. Absolute paths are omitted. + */ +export function formatSkippedSymlinkNotice(relPaths: readonly string[]): string | null { + const seen = new Set(); + const unique: string[] = []; + for (const raw of relPaths) { + const rel = raw.split('\\').join('/'); + if (!rel || rel === '.' || rel.startsWith('../') || rel === '..' || rel.startsWith('/')) continue; + if (/^[A-Za-z]:/.test(rel)) continue; + if (seen.has(rel)) continue; + seen.add(rel); + unique.push(rel); + } + unique.sort(comparePath); + if (unique.length === 0) return null; + const shown = unique.slice(0, SKIPPED_SYMLINK_NOTICE_CAP); + const extra = unique.length - shown.length; + const list = extra > 0 ? `${shown.join(', ')}, +${extra} more` : shown.join(', '); + const noun = unique.length === 1 ? 'symlink' : 'symlinks'; + return ( + `notice: skipped ${unique.length} ${noun} (${list}). ` + + 'vg does not follow symlinks. Point the root at the link target, or pass --exclude or a narrower root.' + ); +} + +/** Write {@link formatSkippedSymlinkNotice} to stderr. No-op when there is nothing to say. */ +export function emitSkippedSymlinkNotice(relPaths: readonly string[]): void { + const line = formatSkippedSymlinkNotice(relPaths); + if (!line) return; + process.stderr.write(`${line}\n`); +} diff --git a/src/engine/discover.ts b/src/engine/discover.ts index 2f523416..c5113b9a 100644 --- a/src/engine/discover.ts +++ b/src/engine/discover.ts @@ -6,6 +6,7 @@ import { requireDataConfig } from '../core-open/config.js'; import { dropBlankPatterns, gitignoreWithoutBlankLines } from '../core-open/utils/glob.js'; import { assertLockfileFile, lockfileKind } from '../core-open/utils/lockfile-parse.js'; import { assertSafeWalkRoot, createWalkBudget, noteWalkEntry } from '../core-open/utils/root-safety.js'; +import { emitSkippedSymlinkNotice } from '../core-open/utils/skipped-symlinks.js'; /** * Deterministic file discovery. @@ -248,6 +249,10 @@ export function discover(options: DiscoverOptions): DiscoveredFile[] { const found = new Map(); const budget = createWalkBudget(root, options.maxEntries); + // Symlinks are not followed. Dirent.isDirectory() / isFile() are false for a + // link, so a directory link to its parent cannot re-enter this walk. The + // notice lists the ones this walk skipped; ignored links stay quiet. + const skippedSymlinks: string[] = []; const considerFile = (abs: string): void => { const rel = toPosix(path.relative(root, abs)); @@ -274,6 +279,11 @@ export function discover(options: DiscoverOptions): DiscoveredFile[] { for (const entry of entries) { const abs = path.join(dir, entry.name); const rel = toPosix(path.relative(root, abs)); + if (entry.isSymbolicLink()) { + if (rel && (rootIg.ignores(rel) || rootIg.ignores(`${rel}/`))) continue; + if (rel) skippedSymlinks.push(rel); + continue; + } if (entry.isDirectory()) { if (isSkippedDirName(entry.name)) continue; if (rel && rootIg.ignores(`${rel}/`)) continue; @@ -297,6 +307,7 @@ export function discover(options: DiscoverOptions): DiscoveredFile[] { } else if (stat.isFile()) considerFile(scope); } + emitSkippedSymlinkNotice(skippedSymlinks); return [...found.values()].sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0)); } diff --git a/src/engine/docs-ingest.ts b/src/engine/docs-ingest.ts index 21c7ff05..64da6fbb 100644 --- a/src/engine/docs-ingest.ts +++ b/src/engine/docs-ingest.ts @@ -408,6 +408,8 @@ export function discoverDocs(options: DiscoverDocsOptions): DiscoveredDoc[] { if (found.size >= maxFiles) return; const abs = path.join(dir, entry.name); const rel = toPosix(path.relative(root, abs)); + // Not followed. discover() prints the one notice for this tree. + if (entry.isSymbolicLink()) continue; if (entry.isDirectory()) { if (isSkippedDirName(entry.name)) continue; // Workflows / .github must be walked even if other tools ignore them diff --git a/test/symlink-walk.test.ts b/test/symlink-walk.test.ts new file mode 100644 index 00000000..e064feea --- /dev/null +++ b/test/symlink-walk.test.ts @@ -0,0 +1,189 @@ +import { describe, it, expect, afterEach, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { discover } from '../src/engine/discover.js'; +import { FileCache } from '../src/core-open/utils/fs.js'; +import { runCoreScan } from '../src/core-open/index.js'; + +const NOTICE = + 'notice: skipped 3 symlinks (alias.ts, nested/cycle, via). vg does not follow symlinks. Point the root at the link target, or pass --exclude or a narrower root.'; + +function symlinksSupported(): boolean { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'vg-symlink-probe-')); + try { + fs.writeFileSync(path.join(dir, 't'), 'x'); + try { + fs.symlinkSync('t', path.join(dir, 'l')); + return true; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'EPERM' || code === 'ENOTSUP' || code === 'EACCES' || code === 'EINVAL') return false; + throw err; + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +const SYMLINKS = symlinksSupported(); + +/** + * Real files, a directory symlink to its parent, a directory symlink to a + * sibling, and a file symlink. Following `nested/cycle` would walk the parent + * again and not terminate. + */ +function symlinkFixture(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vg-symlink-')); + fs.writeFileSync(path.join(root, 'keep.ts'), 'export const keep = 1;\n'); + fs.mkdirSync(path.join(root, 'hidden')); + fs.writeFileSync(path.join(root, 'hidden', 'secret.ts'), 'export const secret = 1;\n'); + fs.mkdirSync(path.join(root, 'nested')); + fs.symlinkSync('..', path.join(root, 'nested', 'cycle')); + fs.symlinkSync('hidden', path.join(root, 'via')); + fs.symlinkSync('keep.ts', path.join(root, 'alias.ts')); + return root; +} + +function captureStderr(run: () => Promise | void): Promise { + const chunks: string[] = []; + const spy = vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array) => { + chunks.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8')); + return true; + }); + return Promise.resolve(run()).finally(() => spy.mockRestore()).then(() => chunks.join('')); +} + +describe.skipIf(!SYMLINKS)('vg build discovery and vg scan walk skip symlinks', () => { + const dirs: string[] = []; + afterEach(() => { + while (dirs.length) fs.rmSync(dirs.pop()!, { recursive: true, force: true }); + }); + + it('discover finishes, does not descend, and prints one stable stderr notice', async () => { + const root = symlinkFixture(); + dirs.push(root); + const stdout: string[] = []; + const outSpy = vi.spyOn(process.stdout, 'write').mockImplementation((chunk: string | Uint8Array) => { + stdout.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8')); + return true; + }); + + let first: string[] = []; + let second: string[] = []; + const stderr1 = await captureStderr(() => { + first = discover({ root }).map((f) => f.rel); + }); + const stderr2 = await captureStderr(() => { + second = discover({ root }).map((f) => f.rel); + }); + outSpy.mockRestore(); + + expect(first).toEqual(['hidden/secret.ts', 'keep.ts']); + expect(second).toEqual(first); + expect(first.some((rel) => rel.startsWith('via/') || rel.includes('cycle/'))).toBe(false); + expect(first).not.toContain('alias.ts'); + expect(stderr1).toBe(`${NOTICE}\n`); + expect(stderr2).toBe(stderr1); + expect(stderr1).not.toContain(root); + expect(stdout.join('')).not.toContain('symlink'); + }, 5_000); + + it('does not mention a symlink the operator already excluded', async () => { + const root = symlinkFixture(); + dirs.push(root); + const stderr = await captureStderr(() => { + const rels = discover({ root, exclude: ['alias.ts'] }).map((f) => f.rel); + expect(rels).toEqual(['hidden/secret.ts', 'keep.ts']); + }); + expect(stderr).toBe( + 'notice: skipped 2 symlinks (nested/cycle, via). vg does not follow symlinks. Point the root at the link target, or pass --exclude or a narrower root.\n', + ); + expect(stderr).not.toContain('alias.ts'); + }); + + it('the scan walk finishes with the same files and the same notice', async () => { + const root = symlinkFixture(); + dirs.push(root); + + const read = async (): Promise<{ files: string[]; dirs: string[] }> => { + const entries = await new FileCache().walkDir(root); + const files = entries.filter((e) => e.isFile).map((e) => e.relPath.split(path.sep).join('/')).sort(); + const directories = entries.filter((e) => e.isDirectory).map((e) => e.relPath.split(path.sep).join('/')).sort(); + return { files, dirs: directories }; + }; + + let first: { files: string[]; dirs: string[] } | undefined; + let second: { files: string[]; dirs: string[] } | undefined; + const stderr1 = await captureStderr(async () => { + first = await read(); + }); + const stderr2 = await captureStderr(async () => { + second = await read(); + }); + + expect(first).toEqual({ files: ['hidden/secret.ts', 'keep.ts'], dirs: ['hidden', 'nested'] }); + expect(second).toEqual(first); + expect(stderr1).toBe(`${NOTICE}\n`); + expect(stderr2).toBe(stderr1); + expect(stderr1).not.toContain(root); + }, 5_000); + + it('prints nothing when the tree has no symlinks', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'vg-nosym-')); + dirs.push(root); + fs.writeFileSync(path.join(root, 'keep.ts'), 'export const keep = 1;\n'); + const stderr = await captureStderr(() => { + expect(discover({ root }).map((f) => f.rel)).toEqual(['keep.ts']); + }); + expect(stderr).toBe(''); + }); + + it('vg scan keeps JSON on stdout and writes no extra files', async () => { + const root = symlinkFixture(); + dirs.push(root); + fs.writeFileSync( + path.join(root, 'package.json'), + JSON.stringify({ name: 'symlink-fixture', version: '1.0.0' }), + ); + const before = fs.readdirSync(root).sort(); + const logs: string[] = []; + const logSpy = vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => { + logs.push(args.map((a) => String(a)).join(' ')); + }); + const stderrChunks: string[] = []; + const errSpy = vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array) => { + stderrChunks.push(typeof chunk === 'string' ? chunk : Buffer.from(chunk).toString('utf8')); + return true; + }); + + const artifact = await runCoreScan(root, { + format: 'json', + concurrency: 1, + offline: true, + noLocalArtifacts: true, + quiet: true, + }); + logSpy.mockRestore(); + errSpy.mockRestore(); + + const stdout = logs.join('\n'); + const stderr = stderrChunks.join(''); + expect(artifact).toBeTruthy(); + expect(stdout).not.toContain('notice: skipped'); + expect(stdout).not.toContain(root); + const jsonText = logs.find((line) => line.trim().startsWith('{')); + expect(jsonText).toBeTruthy(); + const parsed = JSON.parse(jsonText!) as { schemaVersion?: string }; + expect(parsed.schemaVersion).toBe('1.0'); + expect(JSON.stringify(parsed)).not.toContain('via/secret.ts'); + expect(JSON.stringify(parsed)).not.toContain('nested/cycle'); + expect(JSON.stringify(parsed)).not.toContain('alias.ts'); + const notices = stderr.match(/notice: skipped \d+ symlinks/g) ?? []; + expect(notices).toEqual(['notice: skipped 3 symlinks']); + expect(stderr).toContain('alias.ts, nested/cycle, via'); + expect(stderr).not.toContain(root); + expect(fs.readdirSync(root).sort()).toEqual(before); + expect(fs.existsSync(path.join(root, '.vibgrate'))).toBe(false); + }, 60_000); +});