diff --git a/CHANGELOG.md b/CHANGELOG.md index dd9b356fb..489d6808e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ Versioning](https://semver.org/spec/v2.0.0.html). version, shrinks by one entry per write, and Wireit reminds you once a day while it holds more than twice the limit. To free it at once, run `rm -rf .wireit/*/cache`. See [#71](https://github.com/google/wireit/issues/71). +- Set `WIREIT_CACHE_WORKTREES=true` to store a linked Git worktree's local + cache in the main worktree's corresponding `.wireit` folders. Any other value + keeps each worktree's own cache. Fingerprints and GitHub Actions caching are + unchanged. ### Fixed diff --git a/README.md b/README.md index 005b49f45..aaaef7b43 100644 --- a/README.md +++ b/README.md @@ -372,6 +372,17 @@ Note the limit is applied per script, so a package with many cached scripts will still use a multiple of this space. To free all of it at once, use `rm -rf .wireit/*/cache .wireit/*/temp .wireit/trash`. +Linked Git worktrees (`git worktree add`) keep their own `.wireit` cache +folders. Set `WIREIT_CACHE_WORKTREES=true` to store a linked worktree's local +cache in the main worktree's corresponding `.wireit` folders. Shared entries +are named from package-relative paths, so both worktrees hit the same entry. +Any other value keeps each worktree's own cache and the same entry names as +before. Lock, fingerprint, and output-manifest files stay in the current +worktree. Fingerprints and GitHub Actions caching are unchanged. + +Restored files are copied verbatim. Wireit does not rewrite output contents +that embed another checkout's absolute path. + ### GitHub Actions caching In _[GitHub Actions](https://github.com/features/actions)_ mode, Wireit caches @@ -895,6 +906,7 @@ The following environment variables affect the behavior of Wireit: | `WIREIT_FAILURES` | [How to handle script failures](#failures-and-errors).

Options:
| | `WIREIT_LOGGER` | How to present progress and results on the command line.

Options:
| | `WIREIT_CACHE_MAX_ENTRIES` | Maximum number of [local cache](#local-caching) entries to retain per script.

Defaults to `10`.

Must be a positive integer or `infinity`. | +| `WIREIT_CACHE_WORKTREES` | Set to `true` to store a linked Git worktree's [local cache](#local-caching) in the main worktree's corresponding `.wireit` folders.

Must be exactly `true`. If unset or any other value, interpreted as `false`. Does not affect fingerprints or GitHub Actions caching. | | `WIREIT_DEBUG_LOG_FILE` | Path to a file which will receive detailed event logging. | | `WIREIT_MAX_OPEN_FILES` | Limits the number of file descriptors Wireit will have open concurrently. Prevents resource exhaustion when checking large numbers of cached files. Set to a lower number if you hit file descriptor limits.

Defaults to `200`.

Must be a positive integer. | | `WIREIT_PARALLEL` | [Maximum number of scripts to run at one time](#parallelism).

Defaults to 2×logical CPU cores.

Must be a positive integer or `infinity`. | diff --git a/package.json b/package.json index 9f7935f16..9f5113d97 100644 --- a/package.json +++ b/package.json @@ -183,7 +183,7 @@ "output": [] }, "test:local-cache": { - "command": "node --test --test-reporter=dot lib/test/local-cache.test.js", + "command": "node --test --test-reporter=dot lib/test/local-cache.test.js lib/test/fingerprint.test.js lib/test/cache-root.test.js", "env": { "NODE_OPTIONS": "--enable-source-maps" }, diff --git a/src/caching/local-cache.ts b/src/caching/local-cache.ts index ef543be84..993359e2f 100644 --- a/src/caching/local-cache.ts +++ b/src/caching/local-cache.ts @@ -6,10 +6,11 @@ import * as fs from '../util/fs.js'; import * as pathlib from 'path'; -import {createHash, randomBytes} from 'crypto'; +import {randomBytes} from 'crypto'; import {getPackageDataDir, getScriptDataDir} from '../util/script-data-dir.js'; import {copyEntries} from '../util/copy.js'; import {glob} from '../util/glob.js'; +import {resolveCachePackageDir} from '../util/cache-root.js'; import type {Cache, CacheHit} from './cache.js'; import type {ScriptReference} from '../config.js'; @@ -66,8 +67,9 @@ const REMIND_OVER_LIMIT_EVERY_MS = 24 * 60 * 60 * 1000; */ export class LocalCache implements Cache { readonly #maxEntries: number; + readonly #shareWorktrees: boolean; - /** Packages used this run, whose trash {@link sweepTrash} empties. */ + /** Cache package dirs used this run, whose trash {@link sweepTrash} empties. */ readonly #packageDirs = new Set(); /** Messages for the user, which the next {@link sweepTrash} returns. */ @@ -79,16 +81,25 @@ export class LocalCache implements Cache { */ readonly #remindedPackages = new Set(); - /** @param maxEntries Entries to retain per script, or Infinity for all. */ - constructor(maxEntries: number) { + /** + * @param maxEntries Entries to retain per script, or Infinity for all. + * @param options When `shareWorktrees` is true, linked git worktrees write + * cache into the main worktree's matching `.wireit`. + */ + constructor(maxEntries: number, options?: {shareWorktrees?: boolean}) { this.#maxEntries = maxEntries; + this.#shareWorktrees = options?.shareWorktrees === true; } async get( script: ScriptReference, fingerprint: Fingerprint, ): Promise { - const cacheDir = this.#getCacheDir(script, fingerprint); + const cacheDir = this.#getCacheDir( + await this.#cachePackageDir(script), + script, + fingerprint, + ); try { await fs.access(cacheDir); } catch (error) { @@ -105,12 +116,17 @@ export class LocalCache implements Cache { script: ScriptReference, fingerprint: Fingerprint, ): Promise { - this.#packageDirs.add(script.packageDir); + const cachePackageDir = await this.#cachePackageDir(script); + this.#packageDirs.add(cachePackageDir); // Recency lives in the mtime, so there is no index file to maintain. atime // won't do, because filesystems are commonly mounted noatime or relatime. const now = new Date(); try { - await fs.utimes(this.#getCacheDir(script, fingerprint), now, now); + await fs.utimes( + this.#getCacheDir(cachePackageDir, script, fingerprint), + now, + now, + ); } catch { // No entry, or one we can't stamp (read-only mount, foreign owner). A hit // is still a hit; the entry just ages as though only ever written. @@ -122,8 +138,32 @@ export class LocalCache implements Cache { fingerprint: Fingerprint, absoluteFiles: AbsoluteEntry[], ): Promise { - this.#packageDirs.add(script.packageDir); - const absCacheDir = this.#getCacheDir(script, fingerprint); + const cachePackageDir = await this.#cachePackageDir(script); + this.#packageDirs.add(cachePackageDir); + const absCacheDir = this.#getCacheDir(cachePackageDir, script, fingerprint); + await (this.#shareWorktrees + ? this.#writeShared(script, cachePackageDir, absCacheDir, absoluteFiles) + : this.#writeExclusive( + cachePackageDir, + script, + absCacheDir, + absoluteFiles, + )); + await this.#evictLeastRecentlyUsed(script, pathlib.basename(absCacheDir)); + return true; + } + + /** + * Copy into a temp folder and rename into place, so a killed Wireit can't + * leave a partial entry. An entry that already exists is an error: the + * Executor checks for a hit before it runs the script. + */ + async #writeExclusive( + cachePackageDir: string, + script: ScriptReference, + absCacheDir: string, + absoluteFiles: AbsoluteEntry[], + ): Promise { if (absoluteFiles.length === 0) { // No temp folder, because an empty "output" runs without the lock. // @@ -136,28 +176,80 @@ export class LocalCache implements Cache { // checked for an existing cache hit. throw new Error(`Did not expect ${absCacheDir} to already exist.`); } - await this.#evictLeastRecentlyUsed(script, pathlib.basename(absCacheDir)); - return true; + return; + } + await this.#writeThroughTemp( + cachePackageDir, + script, + absoluteFiles, + absCacheDir, + ); + // The script lock covers this temp folder. Anything still in it was left + // by a killed or failed write. + await this.#trashLeftoverTemp(cachePackageDir, script); + } + + /** + * Temp dir plus rename, so two worktrees racing on one shared entry leave a + * single complete directory. If the destination already exists, that write + * won and this one is a hit. Does not sweep other temps: the script lock + * stays in this worktree, so another worktree may be writing into the same + * shared temp folder. + */ + async #writeShared( + script: ScriptReference, + cachePackageDir: string, + absCacheDir: string, + absoluteFiles: AbsoluteEntry[], + ): Promise { + if (absoluteFiles.length === 0) { + // No temp folder, because an empty "output" runs without the lock. + // mkdir is a hit when the directory already exists, so two worktrees + // caching the same empty output don't throw. + await fs.mkdir(absCacheDir, {recursive: true}); + return; + } + const tempDir = pathlib.join( + this.#getScriptTempDir(cachePackageDir, script), + randomBytes(8).toString('hex'), + ); + await fs.mkdir(this.#getScriptCacheDir(cachePackageDir, script), { + recursive: true, + }); + try { + await copyEntries(absoluteFiles, script.packageDir, tempDir); + await fs.rename(tempDir, absCacheDir); + } catch (error) { + await fs.rmTree(tempDir).catch(() => {}); + const {code} = error as {code?: string}; + if (code === 'EEXIST' || code === 'ENOTEMPTY' || code === 'EPERM') { + try { + await fs.access(absCacheDir); + return; + } catch { + throw error; + } + } + throw error; } - await this.#writeThroughTemp(script, absoluteFiles, absCacheDir); - await Promise.all([ - this.#evictLeastRecentlyUsed(script, pathlib.basename(absCacheDir)), - this.#trashLeftoverTemp(script), - ]); - return true; } async #writeThroughTemp( + cachePackageDir: string, script: ScriptReference, absoluteFiles: AbsoluteEntry[], absCacheDir: string, ): Promise { // Short, so a path that fits the Windows limit in the cache fits here. + // Next to the cache entry, so the rename stays on one filesystem when a + // linked worktree's cache lives in the main worktree. const tempDir = pathlib.join( - this.#getScriptTempDir(script), + this.#getScriptTempDir(cachePackageDir, script), randomBytes(8).toString('hex'), ); - await fs.mkdir(this.#getScriptCacheDir(script), {recursive: true}); + await fs.mkdir(this.#getScriptCacheDir(cachePackageDir, script), { + recursive: true, + }); try { await copyEntries(absoluteFiles, script.packageDir, tempDir); await fs.rename(tempDir, absCacheDir); @@ -170,8 +262,11 @@ export class LocalCache implements Cache { } /** Needs the script's lock, and must run after this write's rename. */ - async #trashLeftoverTemp(script: ScriptReference): Promise { - const tempDir = this.#getScriptTempDir(script); + async #trashLeftoverTemp( + cachePackageDir: string, + script: ScriptReference, + ): Promise { + const tempDir = this.#getScriptTempDir(cachePackageDir, script); let leftovers; try { leftovers = await fs.readdir(tempDir, {withFileTypes: true}); @@ -180,7 +275,7 @@ export class LocalCache implements Cache { } await Promise.allSettled( leftovers.map((entry) => - this.#moveToTrash(script.packageDir, pathlib.join(tempDir, entry.name)), + this.#moveToTrash(cachePackageDir, pathlib.join(tempDir, entry.name)), ), ); } @@ -219,7 +314,8 @@ export class LocalCache implements Cache { return; } try { - const cacheDir = this.#getScriptCacheDir(script); + const cachePackageDir = await this.#cachePackageDir(script); + const cacheDir = this.#getScriptCacheDir(cachePackageDir, script); const entries = await fs.readdir(cacheDir, {withFileTypes: true}); if (entries.length <= this.#maxEntries) { return; @@ -243,7 +339,7 @@ export class LocalCache implements Cache { // allSettled, so one entry we can't move (EPERM on Windows, while // something holds it open) doesn't block evicting the rest. await Promise.allSettled( - doomed.map(({path}) => this.#moveToTrash(script.packageDir, path)), + doomed.map(({path}) => this.#moveToTrash(cachePackageDir, path)), ); const numLeft = entries.length - doomed.length; if (numLeft > REMIND_OVER_LIMIT_FACTOR * this.#maxEntries) { @@ -350,18 +446,34 @@ export class LocalCache implements Cache { return pathlib.join(getPackageDataDir(packageDir), 'trash'); } - #getScriptCacheDir(script: ScriptReference): string { - return pathlib.join(getScriptDataDir(script), 'cache'); + async #cachePackageDir(script: ScriptReference): Promise { + return this.#shareWorktrees + ? resolveCachePackageDir(script.packageDir, {shareWorktrees: true}) + : script.packageDir; } - #getScriptTempDir(script: ScriptReference): string { - return pathlib.join(getScriptDataDir(script), 'temp'); + #getScriptCacheDir(cachePackageDir: string, script: ScriptReference): string { + return pathlib.join( + getScriptDataDir({packageDir: cachePackageDir, name: script.name}), + 'cache', + ); } - #getCacheDir(script: ScriptReference, fingerprint: Fingerprint): string { + #getScriptTempDir(cachePackageDir: string, script: ScriptReference): string { + return pathlib.join( + getScriptDataDir({packageDir: cachePackageDir, name: script.name}), + 'temp', + ); + } + + #getCacheDir( + cachePackageDir: string, + script: ScriptReference, + fingerprint: Fingerprint, + ): string { return pathlib.join( - this.#getScriptCacheDir(script), - createHash('sha256').update(fingerprint.string).digest('hex'), + this.#getScriptCacheDir(cachePackageDir, script), + fingerprint.localCacheEntryName(this.#shareWorktrees), ); } } diff --git a/src/cli-options.ts b/src/cli-options.ts index 282ee3efb..6f1ebe76d 100644 --- a/src/cli-options.ts +++ b/src/cli-options.ts @@ -64,6 +64,7 @@ export interface Options { numWorkers: number; cache: 'local' | 'github' | 'none'; cacheMaxEntries: number; + cacheWorktrees: boolean; failureMode: FailureMode; agent: Agent; logger: Logger; @@ -220,6 +221,9 @@ export const getOptions = async (): Promise> => { } } + // Same rule as CI: only the exact string "true" enables it. + const cacheWorktrees = process.env['WIREIT_CACHE_WORKTREES'] === 'true'; + const failureModeResult = ((): Result => { const str = process.env['WIREIT_FAILURES']; if (!str) { @@ -307,6 +311,7 @@ export const getOptions = async (): Promise> => { numWorkers: numWorkersResult.value, cache: cacheResult.value, cacheMaxEntries: cacheMaxEntriesResult.value, + cacheWorktrees, failureMode: failureModeResult.value, agent, logger, diff --git a/src/cli.ts b/src/cli.ts index 58c133ef0..dbb2860ab 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -62,7 +62,9 @@ const run = async (options: Options): Promise> => { case 'local': { // Import dynamically so that we import fewer unnecessary modules. const {LocalCache} = await import('./caching/local-cache.js'); - cache = new LocalCache(options.cacheMaxEntries); + cache = new LocalCache(options.cacheMaxEntries, { + shareWorktrees: options.cacheWorktrees, + }); break; } case 'github': { diff --git a/src/fingerprint.ts b/src/fingerprint.ts index b18ca0e2e..0660d86ed 100644 --- a/src/fingerprint.ts +++ b/src/fingerprint.ts @@ -5,6 +5,7 @@ */ import {createHash} from 'crypto'; +import * as pathlib from 'path'; import {createReadStream} from './util/fs.js'; import {glob} from './util/glob.js'; import {scriptReferenceToString} from './config.js'; @@ -138,6 +139,9 @@ export class Fingerprint { const filteredDependencyFingerprints: Array< [ScriptReferenceString, FingerprintSha256HexDigest] > = []; + const portableDependencyFingerprints: Array< + [ScriptReferenceString, string] + > = []; for (const [dep, depFingerprint] of dependencyFingerprints) { if (!dep.cascade) { // cascade: false means the fingerprint of the dependency isn't @@ -151,6 +155,19 @@ export class Fingerprint { scriptReferenceToString(dep.config), depFingerprint.hash, ]); + // Package-relative, and hashed from the dependency's portable form, so + // two worktrees of one repo share a local-cache entry. This is not part + // of {@link Fingerprint.string}. + portableDependencyFingerprints.push([ + scriptReferenceToString({ + packageDir: pathRelativeToPackage( + script.packageDir, + dep.config.packageDir, + ), + name: dep.config.name, + }), + depFingerprint.localCacheEntryName(true), + ]); } let fileHashes: Array<[string, FileSha256HexDigest]>; @@ -193,7 +210,7 @@ export class Fingerprint { } erroredFilePaths.push(absolutePath); } - return [file.path, hash.digest('hex') as FileSha256HexDigest]; + return [absolutePath, hash.digest('hex') as FileSha256HexDigest]; }), ); @@ -258,12 +275,36 @@ export class Fingerprint { env: script.env, }; fingerprint.#data = data as FingerprintData; + fingerprint.#portableString = JSON.stringify({ + ...data, + files: Object.fromEntries( + fileHashes + .map(([filePath, hash]): [string, FileSha256HexDigest] => [ + pathRelativeToPackage(script.packageDir, filePath), + hash, + ]) + .sort(([aFile], [bFile]) => aFile.localeCompare(bFile)), + ), + dependencies: Object.fromEntries( + portableDependencyFingerprints.sort(([aRef], [bRef]) => + aRef.localeCompare(bRef), + ), + ), + }); return {ok: true, value: fingerprint}; } #str?: FingerprintString; #data?: FingerprintData; #hash?: FingerprintSha256HexDigest; + /** + * Same fields as {@link string}, but file paths and dependency package dirs + * are package-relative and dependency hashes come from each dependency's + * portable form. Set only by {@link Fingerprint.compute}. Local-cache entry + * names use this when worktree sharing is on. Freshness and GitHub Actions + * caching keep using {@link string}. + */ + #portableString?: string; get string(): FingerprintString { if (this.#str === undefined) { @@ -288,7 +329,31 @@ export class Fingerprint { return this.#hash; } + /** + * Directory name of a local-cache entry. + * + * With worktree sharing off, this is {@link hash}, so existing cache folders + * stay valid. With sharing on, it is the hash of {@link #portableString}. + * A fingerprint built with {@link Fingerprint.fromString} has no portable + * form and falls back to {@link hash}. + */ + localCacheEntryName(shareWorktrees: boolean): string { + const source = + shareWorktrees && this.#portableString !== undefined + ? this.#portableString + : this.string; + return source === this.string + ? this.hash + : createHash('sha256').update(source).digest('hex'); + } + equal(other: Fingerprint): boolean { return this.string === other.string; } } + +/** Package-relative so a shared local-cache entry matches across worktrees. */ +const pathRelativeToPackage = (packageDir: string, path: string): string => { + const relative = pathlib.relative(packageDir, path); + return relative === '' ? '.' : relative; +}; diff --git a/src/test/cache-root.test.ts b/src/test/cache-root.test.ts new file mode 100644 index 000000000..a8d1e99b8 --- /dev/null +++ b/src/test/cache-root.test.ts @@ -0,0 +1,69 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import {test} from 'node:test'; +import * as assert from 'node:assert'; +import * as pathlib from 'path'; +import {realpath} from '../util/fs.js'; +import {detectWorktree, resolveCachePackageDir} from '../util/cache-root.js'; +import {FilesystemTestRig} from './util/filesystem-test-rig.js'; +import {git, initRepo} from './util/git.js'; + +void test('main worktree uses its own package dir', async () => { + await using rig = await FilesystemTestRig.setup(); + initRepo(rig.temp); + await rig.write('README.md', 'x'); + git(rig.temp, ['add', '.']); + git(rig.temp, ['commit', '-m', 'init']); + const pkg = rig.resolve('packages/foo'); + await rig.mkdir('packages/foo'); + const info = await detectWorktree(pkg); + assert.ok(info); + assert.equal(await realpath(info.worktreeRoot), await realpath(rig.temp)); + assert.equal(await realpath(info.mainWorktreeRoot), await realpath(rig.temp)); + assert.equal(await resolveCachePackageDir(pkg, {shareWorktrees: true}), pkg); +}); + +void test('linked worktree keeps its own cache package dir unless sharing', async () => { + await using rig = await FilesystemTestRig.setup(); + const main = rig.resolve('main'); + const linked = rig.resolve('linked'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'README.md'), 'x'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + git(main, ['worktree', 'add', linked, '-b', 'other']); + const linkedPkg = pathlib.join(linked, 'packages', 'foo'); + await rig.mkdir(pathlib.join('linked', 'packages', 'foo')); + assert.equal(await resolveCachePackageDir(linkedPkg), linkedPkg); +}); + +void test('linked worktree cache package dir maps onto the main tree when sharing', async () => { + await using rig = await FilesystemTestRig.setup(); + const main = rig.resolve('main'); + const linked = rig.resolve('linked'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'README.md'), 'x'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + git(main, ['worktree', 'add', linked, '-b', 'other']); + const mainPkg = pathlib.join(main, 'packages', 'foo'); + const linkedPkg = pathlib.join(linked, 'packages', 'foo'); + await rig.mkdir(pathlib.join('main', 'packages', 'foo')); + await rig.mkdir(pathlib.join('linked', 'packages', 'foo')); + const info = await detectWorktree(linkedPkg); + assert.ok(info); + assert.equal(await realpath(info.worktreeRoot), await realpath(linked)); + assert.equal(await realpath(info.mainWorktreeRoot), await realpath(main)); + assert.equal( + await realpath( + await resolveCachePackageDir(linkedPkg, {shareWorktrees: true}), + ), + await realpath(mainPkg), + ); +}); diff --git a/src/test/cli-options.test.ts b/src/test/cli-options.test.ts index ea81c9b94..6c5e6ad2b 100644 --- a/src/test/cli-options.test.ts +++ b/src/test/cli-options.test.ts @@ -60,6 +60,7 @@ async function assertOptions( watch: false, cache: 'local', cacheMaxEntries: 10, + cacheWorktrees: false, numWorkers: 10, failureMode: 'no-new', logger: 'QuietLogger', @@ -607,4 +608,46 @@ for (const {agent, runCmd, testCmd, startCmd, needsExtraDashes} of commands) { }), ); } + + void test( + `${agent} WIREIT_CACHE_WORKTREES=true`, + rigTest(async ({rig}) => { + await assertOptions( + rig, + `${runCmd} main ${extraDashes}`, + { + agent, + script: { + packageDir: rig.temp, + name: 'main', + }, + cacheWorktrees: true, + }, + { + WIREIT_CACHE_WORKTREES: 'true', + }, + ); + }), + ); + + void test( + `${agent} WIREIT_CACHE_WORKTREES=yes is false`, + rigTest(async ({rig}) => { + await assertOptions( + rig, + `${runCmd} main ${extraDashes}`, + { + agent, + script: { + packageDir: rig.temp, + name: 'main', + }, + cacheWorktrees: false, + }, + { + WIREIT_CACHE_WORKTREES: 'yes', + }, + ); + }), + ); } diff --git a/src/test/fingerprint.test.ts b/src/test/fingerprint.test.ts new file mode 100644 index 000000000..d7ff0d04c --- /dev/null +++ b/src/test/fingerprint.test.ts @@ -0,0 +1,105 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import {test} from 'node:test'; +import * as assert from 'node:assert'; +import * as pathlib from 'path'; +import {Fingerprint} from '../fingerprint.js'; +import {scriptReferenceToString} from '../config.js'; +import {FilesystemTestRig} from './util/filesystem-test-rig.js'; + +import type {ArrayNode, JsonAstNode} from '../util/ast.js'; +import type {Dependency, StandardScriptConfig} from '../config.js'; +import type {FingerprintString} from '../fingerprint.js'; + +const stringNode = (value: string): JsonAstNode => ({ + type: 'string', + offset: 0, + length: value.length, + value, +}); + +const arrayNode = (values: string[]): ArrayNode => ({ + node: {type: 'array', offset: 0, length: 0, value: undefined}, + values, +}); + +const scriptAt = (packageDir: string): StandardScriptConfig => ({ + packageDir, + name: 'compile', + state: 'valid', + command: stringNode('tsc'), + extraArgs: [], + clean: true, + files: arrayNode(['src/a.ts', 'input.txt']), + output: arrayNode(['lib/**']), + service: undefined, + env: {}, + dependencies: [], + services: [], + scriptAstNode: undefined, + configAstNode: undefined, + declaringFile: { + path: pathlib.join(packageDir, 'package.json'), + contents: '{}', + }, + failures: [], +}); + +const depAt = ( + packageDir: string, + fingerprint: Fingerprint, +): [Dependency, Fingerprint] => [ + { + cascade: true, + config: scriptAt(packageDir), + specifier: stringNode('compile'), + }, + fingerprint, +]; + +void test('fingerprint stays absolute; shared cache entry name matches', async () => { + await using rig = await FilesystemTestRig.setup(); + await rig.write({ + 'a/packages/foo/src/a.ts': 'export const a = 1;', + 'a/packages/foo/input.txt': 'v0', + 'b/packages/foo/src/a.ts': 'export const a = 1;', + 'b/packages/foo/input.txt': 'v0', + }); + const depFingerprint = Fingerprint.fromString( + '{"fullyTracked":true}' as FingerprintString, + ); + const fooA = rig.resolve('a/packages/foo'); + const fooB = rig.resolve('b/packages/foo'); + const resultA = await Fingerprint.compute(scriptAt(fooA), [ + depAt(rig.resolve('a/packages/dep'), depFingerprint), + ]); + const resultB = await Fingerprint.compute(scriptAt(fooB), [ + depAt(rig.resolve('b/packages/dep'), depFingerprint), + ]); + assert.ok(resultA.ok); + assert.ok(resultB.ok); + assert.notEqual(resultA.value.string, resultB.value.string); + assert.deepEqual( + Object.keys(resultA.value.data.files).sort(), + [pathlib.join(fooA, 'src', 'a.ts'), pathlib.join(fooA, 'input.txt')].sort(), + ); + assert.deepEqual(Object.keys(resultA.value.data.dependencies), [ + scriptReferenceToString({ + packageDir: rig.resolve('a/packages/dep'), + name: 'compile', + }), + ]); + assert.equal(resultA.value.localCacheEntryName(false), resultA.value.hash); + assert.notEqual( + resultA.value.localCacheEntryName(false), + resultB.value.localCacheEntryName(false), + ); + assert.equal( + resultA.value.localCacheEntryName(true), + resultB.value.localCacheEntryName(true), + ); +}); diff --git a/src/test/local-cache.test.ts b/src/test/local-cache.test.ts index 1a7f0279b..7ab3065d0 100644 --- a/src/test/local-cache.test.ts +++ b/src/test/local-cache.test.ts @@ -14,6 +14,7 @@ import {Fingerprint} from '../fingerprint.js'; import {getScriptDataDir} from '../util/script-data-dir.js'; import {FilesystemTestRig} from './util/filesystem-test-rig.js'; import {FsGate} from './util/fs-gate.js'; +import {git, initRepo} from './util/git.js'; import type {AbsoluteEntry} from '../util/glob.js'; import type {FingerprintString} from '../fingerprint.js'; @@ -24,7 +25,10 @@ import type {ScriptReference} from '../config.js'; const SCRIPT_NAME = 'a'; -async function setup(maxEntries: number): Promise< +async function setup( + maxEntries: number, + shareWorktrees = false, +): Promise< { rig: FilesystemTestRig; cache: LocalCache; @@ -61,7 +65,7 @@ async function setup(maxEntries: number): Promise< packageDir: rig.resolve('.'), name: SCRIPT_NAME, }; - const cache = new LocalCache(maxEntries); + const cache = new LocalCache(maxEntries, {shareWorktrees}); const cacheDir = pathlib.join(getScriptDataDir(script), 'cache'); const trashDir = rig.resolve(pathlib.join('.wireit', 'trash')); @@ -485,3 +489,66 @@ void test('a write that fails leaves no entry, and deletes its temp copy', async await ctx.cacheOutput('v0'); assert.deepEqual(await ctx.entryHashes(), [hashOf('v0')]); }); + +void test('a second set of the same entry throws', async () => { + await using ctx = await setup(10); + await ctx.cacheOutput('v0'); + await assert.rejects(() => ctx.cacheOutput('v0')); +}); + +void test('a second shared set of an empty output is a hit', async () => { + await using ctx = await setup(10, true); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + assert.equal(await ctx.cache.set(ctx.script, fingerprint('v0'), []), true); + assert.equal(await ctx.cache.set(ctx.script, fingerprint('v0'), []), true); + assert.deepEqual(await ctx.entryHashes(), [hashOf('v0')]); +}); + +void test('a second set of a shared entry is a hit, not an error', async () => { + await using ctx = await setup(10, true); + // Stop the worktree walk at this temp dir, including when the checkout + // running the test is itself a linked worktree. + await ctx.rig.mkdir('.git'); + await ctx.cacheOutput('v0'); + await ctx.cacheOutput('v0'); + assert.deepEqual(await ctx.entryHashes(), [hashOf('v0')]); +}); + +void test('linked worktree restores from the main worktree local cache', async () => { + await using rig = await FilesystemTestRig.setup(); + const main = rig.resolve('main'); + const linked = rig.resolve('linked'); + await rig.mkdir('main'); + initRepo(main); + await rig.write(pathlib.join('main', 'README.md'), 'x'); + git(main, ['add', '.']); + git(main, ['commit', '-m', 'init']); + git(main, ['worktree', 'add', linked, '-b', 'other']); + + const mainScript = {packageDir: main, name: SCRIPT_NAME}; + const linkedScript = {packageDir: linked, name: SCRIPT_NAME}; + const cache = new LocalCache(10, {shareWorktrees: true}); + const outputEntry = { + path: pathlib.join(main, 'output'), + dirent: { + isFile: () => true, + isDirectory: () => false, + isSymbolicLink: () => false, + }, + } as AbsoluteEntry; + await rig.write(pathlib.join('main', 'output'), 'from-main'); + assert.equal( + await cache.set(mainScript, fingerprint('v0'), [outputEntry]), + true, + ); + + const linkedCacheDir = pathlib.join(getScriptDataDir(linkedScript), 'cache'); + await assert.rejects(fs.readdir(linkedCacheDir), {code: 'ENOENT'}); + + const hit = await cache.get(linkedScript, fingerprint('v0')); + assert.ok(hit); + await hit.apply(); + assert.equal(await rig.read(pathlib.join('linked', 'output')), 'from-main'); +}); diff --git a/src/test/util/filesystem-test-rig.ts b/src/test/util/filesystem-test-rig.ts index 48059bcae..072c44409 100644 --- a/src/test/util/filesystem-test-rig.ts +++ b/src/test/util/filesystem-test-rig.ts @@ -31,6 +31,12 @@ export class FilesystemTestRig { } } + static async setup(): Promise { + const rig = new FilesystemTestRig(); + await rig.setup(); + return rig; + } + /** * Initialize the temporary filesystem. */ @@ -40,6 +46,10 @@ export class FilesystemTestRig { await this.mkdir('.'); } + async [Symbol.asyncDispose](): Promise { + await this.cleanup(); + } + /** * Delete the temporary filesystem. */ diff --git a/src/test/util/git.ts b/src/test/util/git.ts new file mode 100644 index 000000000..b345b49bb --- /dev/null +++ b/src/test/util/git.ts @@ -0,0 +1,22 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import {execFileSync} from 'child_process'; + +export const git = (cwd: string, args: string[]) => { + execFileSync('git', args, { + cwd, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); +}; + +export const initRepo = (dir: string) => { + git(dir, ['init']); + git(dir, ['config', 'user.email', 'wireit@example.com']); + git(dir, ['config', 'user.name', 'Wireit Test']); + git(dir, ['config', 'commit.gpgsign', 'false']); +}; diff --git a/src/test/util/test-rig.ts b/src/test/util/test-rig.ts index 2372766cc..bec5402fc 100644 --- a/src/test/util/test-rig.ts +++ b/src/test/util/test-rig.ts @@ -44,7 +44,7 @@ export class WireitTestRig extends FilesystemTestRig implements AsyncDisposable { - static async setup(): Promise { + static override async setup(): Promise { const rig = new WireitTestRig(); await rig.setup(); return rig; @@ -273,7 +273,7 @@ export class WireitTestRig return command; } - async [Symbol.asyncDispose]() { + override async [Symbol.asyncDispose]() { await this.cleanup(); } diff --git a/src/util/cache-root.ts b/src/util/cache-root.ts new file mode 100644 index 000000000..9a3dd391c --- /dev/null +++ b/src/util/cache-root.ts @@ -0,0 +1,162 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as fs from './fs.js'; +import * as pathlib from 'path'; + +export interface WorktreeInfo { + worktreeRoot: string; + mainWorktreeRoot: string; +} + +/** + * Directory whose `.wireit/` folder should hold this package's local cache + * (not lock/fingerprint/manifest). + * + * Linked git worktrees share the main worktree only when `shareWorktrees` is + * true. Otherwise `packageDir` (Wireit's historical layout). + */ +export const resolveCachePackageDir = async ( + packageDir: string, + options?: {shareWorktrees?: boolean}, +): Promise => { + if (options?.shareWorktrees !== true) { + return packageDir; + } + const absPackageDir = pathlib.resolve(packageDir); + const worktree = await detectWorktree(absPackageDir); + if ( + worktree === undefined || + worktree.worktreeRoot === worktree.mainWorktreeRoot + ) { + return absPackageDir; + } + return pathlib.join( + worktree.mainWorktreeRoot, + pathlib.relative(worktree.worktreeRoot, absPackageDir), + ); +}; + +export const detectWorktree = async ( + startDir: string, +): Promise => { + const worktreeRoot = await findGitAncestor(pathlib.resolve(startDir)); + if (worktreeRoot === undefined) { + return undefined; + } + const dotGit = pathlib.join(worktreeRoot, '.git'); + const stat = await lstatOrUndefined(dotGit); + if (stat === undefined) { + return undefined; + } + if (stat.isDirectory()) { + return {worktreeRoot, mainWorktreeRoot: worktreeRoot}; + } + if (!stat.isFile()) { + return undefined; + } + const gitDir = await readGitdirPointer(dotGit, worktreeRoot); + if (gitDir === undefined) { + return undefined; + } + const mainWorktreeRoot = await readMainWorktreeRoot(gitDir); + if (mainWorktreeRoot === undefined) { + return undefined; + } + return {worktreeRoot, mainWorktreeRoot}; +}; + +const findGitAncestor = async (dir: string): Promise => { + // Nearest `.git`. A submodule working tree has its own `.git` file, so a + // package inside one does not walk up into the superproject. + // https://git-scm.com/docs/gitsubmodules + const stat = await lstatOrUndefined(pathlib.join(dir, '.git')); + if (stat !== undefined) { + return dir; + } + const parent = pathlib.dirname(dir); + return parent === dir ? undefined : findGitAncestor(parent); +}; + +const lstatOrUndefined = async ( + path: string, +): Promise> | undefined> => { + try { + return await fs.lstat(path); + } catch (error) { + const {code} = error as {code: string}; + if (code === /* does not exist */ 'ENOENT') { + return undefined; + } + throw error; + } +}; + +const readFileOrUndefined = async ( + path: string, +): Promise => { + try { + return await fs.readFile(path, 'utf8'); + } catch (error) { + const {code} = error as {code: string}; + if (code === /* does not exist */ 'ENOENT') { + return undefined; + } + throw error; + } +}; + +const readGitdirPointer = async ( + dotGitFile: string, + worktreeRoot: string, +): Promise => { + const content = await readFileOrUndefined(dotGitFile); + if (content === undefined) { + return undefined; + } + const prefix = 'gitdir: '; + const line = content.split('\n')[0] ?? ''; + if (!line.startsWith(prefix)) { + return undefined; + } + const pointed = line.slice(prefix.length).trim(); + const resolved = pathlib.isAbsolute(pointed) + ? pointed + : pathlib.resolve(worktreeRoot, pointed); + return realpathOrSelf(resolved); +}; + +const readMainWorktreeRoot = async ( + gitDir: string, +): Promise => { + // Linked worktrees point `commondir` at the main git dir. A submodule git + // dir has none, so this returns undefined and the cache stays put. + // https://git-scm.com/docs/gitrepository-layout + const commondir = ( + await readFileOrUndefined(pathlib.join(gitDir, 'commondir')) + )?.trim(); + if (commondir === undefined || commondir === '') { + return undefined; + } + const gitCommon = pathlib.isAbsolute(commondir) + ? commondir + : pathlib.resolve(gitDir, commondir); + const resolved = await realpathOrSelf(gitCommon); + const main = pathlib.dirname(resolved); + return main === resolved ? undefined : main; +}; + +const realpathOrSelf = async (path: string): Promise => { + try { + return await fs.realpath(path); + } catch (error) { + const {code} = error as {code: string}; + if (code === /* does not exist */ 'ENOENT') { + return path; + } + throw error; + } +}; diff --git a/src/util/fs.ts b/src/util/fs.ts index e1dd2f8f8..299a2a4ae 100644 --- a/src/util/fs.ts +++ b/src/util/fs.ts @@ -141,6 +141,15 @@ export async function lstat(path: string): Promise { } } +export async function realpath(path: string): Promise { + const reservation = await fileBudget.reserve(); + try { + return await fs.realpath(path); + } finally { + reservation[Symbol.dispose](); + } +} + export async function stat(path: string): Promise { const reservation = await fileBudget.reserve(); try {