diff --git a/packages/typescript/src/api/async/api.ts b/packages/typescript/src/api/async/api.ts index 208313dac3502..89b61d2c8ddd1 100644 --- a/packages/typescript/src/api/async/api.ts +++ b/packages/typescript/src/api/async/api.ts @@ -354,9 +354,29 @@ export class API implements FormatDiagnosticsHo } async updateSnapshot(params?: FromLSP extends true ? LSPUpdateSnapshotParams : UpdateSnapshotParams): Promise { + return this.updateSnapshotWorker(params); + } + + /** @internal */ + async updateSnapshotFrom(baseSnapshot: Snapshot, params?: UpdateSnapshotParams): Promise { + if (!this.activeSnapshots.has(baseSnapshot) || baseSnapshot.isDisposed()) { + throw new Error("Cannot update an inactive snapshot"); + } + if (baseSnapshot !== this.latestSnapshot) { + // TODO: Support forking active memory/cache snapshots once the server-side + // ownership, project state, and cache semantics have been worked out. + throw new Error("Snapshot.update can only update the latest snapshot"); + } + return this.updateSnapshotWorker(params, baseSnapshot); + } + + private async updateSnapshotWorker( + params?: LSPUpdateSnapshotParams | UpdateSnapshotParams, + baseSnapshot?: Snapshot, + ): Promise { await this.ensureInitialized(); - const requestParams = toUpdateSnapshotRequest(params); + const requestParams = toUpdateSnapshotRequest(params, baseSnapshot?.id); const data = await this.client.apiRequest("updateSnapshot", requestParams); // Retain cached source files from previous snapshot for unchanged files @@ -485,6 +505,15 @@ export class API implements FormatDiagnosticsHo createProgramOptions: CreateProgramOptions, oldProgram?: Program, fileChanges?: APIFileChanges, + ): Promise { + return this.createProgramWorker(rootFiles, createProgramOptions, oldProgram, fileChanges); + } + + private async createProgramWorker( + rootFiles: readonly DocumentIdentifier[], + createProgramOptions: CreateProgramOptions, + oldProgram?: Program, + fileChanges?: APIFileChanges, ): Promise { await this.ensureInitialized(); @@ -524,6 +553,10 @@ export class API implements FormatDiagnosticsHo type EnsureInitialized = () => Promise; // @sync: type EnsureInitialized = (() => void) & { gen(): Generator; }; +interface SnapshotOwner extends FormatDiagnosticsHost { + updateSnapshotFrom(baseSnapshot: Snapshot, params?: UpdateSnapshotParams): Promise; +} + export class InternalAPI { private client: Client; private ensureInitialized: EnsureInitialized; @@ -560,6 +593,7 @@ export class Snapshot { private disposed: boolean = false; private disposePromise: Promise | undefined; private onDispose: () => void; + private api: SnapshotOwner; private snapshotRegistry: SnapshotObjectRegistry; readonly internal: SnapshotInternalAPI; @@ -568,18 +602,19 @@ export class Snapshot { client: Client, sourceFileCache: SourceFileCache, toPath: (fileName: string) => Path, - formatDiagnosticsHost: FormatDiagnosticsHost, + api: SnapshotOwner, onDispose: () => void, ) { this.id = data.snapshot; this.client = client; this.toPath = toPath; + this.api = api; this.onDispose = onDispose; this.projectMap = new Map(); this.snapshotRegistry = new SnapshotObjectRegistry(client, this.id, projectId => this.projectMap.get(projectId)); for (const projData of data.projects) { - const project = new Project(projData, this.id, client, sourceFileCache, toPath, formatDiagnosticsHost, this.snapshotRegistry); + const project = new Project(projData, this.id, client, sourceFileCache, toPath, api, this.snapshotRegistry); this.projectMap.set(toPath(projData.configFileName), project); } @@ -606,10 +641,18 @@ export class Snapshot { return this.projectMap.get(this.toPath(data.configFileName)); } + /** + * Creates the next snapshot, layering its filesystem over this snapshot's + * filesystem. This snapshot must still be active and be the latest snapshot. + */ + async update(params?: UpdateSnapshotParams): Promise { + this.ensureNotDisposed(); + return this.api.updateSnapshotFrom(this, params); + } + [globalThis.Symbol.dispose](): void { void this.dispose(); } - dispose(): Promise { return this.disposePromise ??= this.disposeWorker(); } @@ -1396,10 +1439,9 @@ export class Program implements FormatDiagnosticsHost { } /** - * Emits files to the configured filesystem. - * - * When the API has a virtual filesystem with a `writeFile` callback, output - * is written there. Otherwise, the server writes directly to the host filesystem. + * Emits files to the configured filesystem. Layer and host filesystems are + * written through; full filesystems remain immutable and return emitted + * files in {@link EmitResult.fileSystem}. */ async emit(emitOnly?: EmitOnly): Promise { const response = await this.client.apiRequest("emit", { @@ -1407,10 +1449,17 @@ export class Program implements FormatDiagnosticsHost { project: this.project.id, ...(emitOnly !== undefined ? { emitOnly } : {}), }); + const fileSystem = response.emittedFilesContents.length + ? { + kind: "layer" as const, + files: Object.fromEntries(response.emittedFiles.map((fileName, index) => [fileName, response.emittedFilesContents[index]])), + } + : undefined; return { emitSkipped: response.emitSkipped, diagnostics: response.diagnostics, emittedFiles: response.emittedFiles, + ...(fileSystem ? { fileSystem } : {}), }; } diff --git a/packages/typescript/src/api/async/types.ts b/packages/typescript/src/api/async/types.ts index 8df18f408c5b7..8c59e3b65a43b 100644 --- a/packages/typescript/src/api/async/types.ts +++ b/packages/typescript/src/api/async/types.ts @@ -8,7 +8,10 @@ import type { NamedTupleMember, ParameterDeclaration, } from "../../ast/ast.ts"; -import type { Diagnostic } from "../proto.ts"; +import type { + Diagnostic, + RequestFileSystem, +} from "../proto.ts"; import type { NodeHandle, Signature, @@ -401,6 +404,8 @@ export interface EmitResult { readonly emitSkipped: boolean; readonly diagnostics: readonly Diagnostic[]; readonly emittedFiles: readonly string[]; + /** Emitted files captured as a filesystem layer suitable for {@link Snapshot.update}. */ + readonly fileSystem?: RequestFileSystem | undefined; } export interface EmitOutput { diff --git a/packages/typescript/src/api/fs.ts b/packages/typescript/src/api/fs.ts index 0deb7b6a668e2..0e5b3f0c3ad2b 100644 --- a/packages/typescript/src/api/fs.ts +++ b/packages/typescript/src/api/fs.ts @@ -1,4 +1,18 @@ -import { getPathComponents } from "./path.ts"; +import getExePath from "#getExePath"; +import { dirname } from "node:path"; +import { + getPathComponents, + normalizePath, +} from "./path.ts"; +import type { + RequestDirectoryEntries, + RequestFileSystem, + RequestSymlink, +} from "./proto.generated.ts"; +import { + type DocumentIdentifier, + resolveFileName, +} from "./proto.ts"; export interface FileSystemEntries { files: string[]; @@ -24,6 +38,140 @@ export interface FileSystem { /** The callback names supported by the Go server for virtual FS delegation. */ export const fsCallbackNames = ["readFile", "fileExists", "directoryExists", "getAccessibleEntries", "realpath", "writeFile"] as const; +export interface CreateFileSystemOptions { + /** Complete directory listings. Full filesystems derive these from `files` when omitted. */ + directories?: Record; + symlinks?: Record; + /** Files or directory trees hidden from an underlying snapshot or host filesystem. */ + removedPaths?: readonly string[]; +} + +export interface CreateFileSystemWithLibOptions extends CreateFileSystemOptions { + /** Default library directory used by a custom or non-embedded compiler executable. */ + defaultLibraryPath?: string; +} + +/** + * Files supplied to a request filesystem. String identifiers are file names; + * use `{ uri }` when supplying a document URI so it can be decoded correctly. + */ +export type RequestFileEntries = readonly (readonly [id: DocumentIdentifier, content: string])[]; + +/** Creates a full request filesystem, deriving directory listings when omitted. */ +export function createFileSystem( + files: RequestFileEntries, + options: CreateFileSystemOptions = {}, +): RequestFileSystem { + return createRequestFileSystem("full", files, options); +} + +/** + * Creates a full request filesystem with the compiler's default library + * directory mounted read-only through the host filesystem. + */ +export function createFileSystemWithLib( + files: RequestFileEntries, + options: CreateFileSystemWithLibOptions = {}, +): RequestFileSystem { + const defaultLibraryPaths = options.defaultLibraryPath + ? [normalizePath(options.defaultLibraryPath)] + : [normalizePath("bundled:///libs")]; + if (!options.defaultLibraryPath) { + try { + defaultLibraryPaths.push(normalizePath(dirname(getExePath()))); + } + catch { + // A socket-connected embedded server can provide bundled libs without + // a locally installed compiler executable. + } + } + const symlinks = { ...options.symlinks }; + for (const defaultLibraryPath of defaultLibraryPaths) { + symlinks[defaultLibraryPath] ??= { target: defaultLibraryPath, host: true }; + } + return createRequestFileSystem("full", files, { + symlinks, + ...(options.directories ? { directories: options.directories } : {}), + ...(options.removedPaths?.length ? { removedPaths: options.removedPaths } : {}), + }); +} + +/** Creates a request filesystem layer, merging base directory listings when omitted. */ +export function createFileSystemLayer( + files: RequestFileEntries, + options: CreateFileSystemOptions = {}, +): RequestFileSystem { + return createRequestFileSystem("layer", files, options); +} + +function createRequestFileSystem( + kind: RequestFileSystem["kind"], + files: RequestFileEntries, + options: CreateFileSystemOptions, +): RequestFileSystem { + const normalizedFiles = new Map(); + for (const [id, content] of files) { + const fileName = normalizePath(resolveFileName(id)); + if (normalizedFiles.has(fileName)) { + throw new Error(`Duplicate request filesystem path: ${fileName}`); + } + normalizedFiles.set(fileName, content); + } + const fileRecord = Object.fromEntries(normalizedFiles); + const directories = options.directories ?? (kind === "full" ? deriveDirectoryListings(fileRecord) : undefined); + return { + kind, + files: fileRecord, + ...(directories ? { directories } : {}), + ...(options.symlinks ? { symlinks: options.symlinks } : {}), + ...(options.removedPaths?.length ? { removedPaths: [...options.removedPaths] } : {}), + }; +} + +function deriveDirectoryListings(files: Record): Record { + const listings = new Map; directories: Set; }>(); + const getListing = (directory: string) => { + let listing = listings.get(directory); + if (!listing) { + listing = { files: new Set(), directories: new Set() }; + listings.set(directory, listing); + } + return listing; + }; + + for (const inputPath of Object.keys(files)) { + const filePath = normalizePath(inputPath); + const fileName = getBaseName(filePath); + let directory = getDirectory(filePath); + getListing(directory).files.add(fileName); + + let parent = getDirectory(directory); + while (parent !== directory) { + getListing(parent).directories.add(getBaseName(directory)); + directory = parent; + parent = getDirectory(directory); + } + } + + return Object.fromEntries([...listings].map(([directory, listing]) => [directory, { + files: [...listing.files], + directories: [...listing.directories], + }])); +} + +function getDirectory(path: string): string { + const components = getPathComponents(path); + if (components.length <= 1) return components[0] ?? ""; + components.pop(); + const root = components.shift()!; + return root + components.join("/"); +} + +function getBaseName(path: string): string { + const components = getPathComponents(path); + return components.at(-1) ?? ""; +} + interface VDirectory { type: "directory"; children: Record; diff --git a/packages/typescript/src/api/path.ts b/packages/typescript/src/api/path.ts index 2d30b2a4a49e5..ef140b96c0dee 100644 --- a/packages/typescript/src/api/path.ts +++ b/packages/typescript/src/api/path.ts @@ -548,13 +548,14 @@ export function documentURIToFileName(uri: string): string { throw new Error("invalid file URI: " + uri); } + const path = decodeURIComponent(parsed.pathname); + // UNC path: file://server/share/... if (parsed.host !== "") { - return "//" + parsed.host + parsed.pathname; + return "//" + parsed.host + path; } // Local file - fix Windows path by removing leading slash before volume - const path = decodeURIComponent(parsed.pathname); if (path.length >= 3 && path.charCodeAt(0) === CharacterCodesSlash) { const [volume, rest, ok] = splitVolumePath(path.substring(1)); if (ok) { diff --git a/packages/typescript/src/api/proto.generated.ts b/packages/typescript/src/api/proto.generated.ts index b920cda9a7ef3..426cbf2adfbe1 100644 --- a/packages/typescript/src/api/proto.generated.ts +++ b/packages/typescript/src/api/proto.generated.ts @@ -189,6 +189,11 @@ export interface InitializeResponse { * All fields are optional. With no fields set, the server adopts the latest LSP state. */ export interface UpdateSnapshotParams { + /** + * Snapshot, when set, requires this to be the latest active snapshot and layers + * FileSystem over that snapshot's filesystem. Used by Snapshot.update. + */ + snapshot?: number; /** * OpenProjects lists tsconfig.json files to open/load in the new snapshot. * Opens are ref-counted and persist across snapshots until closed. @@ -201,6 +206,12 @@ export interface UpdateSnapshotParams { closeProjects?: readonly DocumentIdentifier[]; /** FileChanges describes file system changes since the last snapshot. */ fileChanges?: APIFileChanges; + /** + * FileSystem supplies file contents and directory listings for the new snapshot. + * A full filesystem is canonical and total. A filesystem layer is checked + * before falling back to the host filesystem. + */ + fileSystem?: RequestFileSystem; /** * OpenFiles lists files to keep open for the API client, mirroring LSP's * textDocument/didOpen. For each file, ancestor directories are searched for a @@ -852,6 +863,11 @@ export interface EmitResponse { emitSkipped: boolean; diagnostics: DiagnosticResponse[]; emittedFiles: string[]; + /** + * EmittedFilesContents contains contents parallel to EmittedFiles when the + * source snapshot uses a full filesystem. It is empty for write-through emits. + */ + emittedFilesContents: string[]; } export interface EmitOutputResponse { @@ -1207,6 +1223,25 @@ export interface APIFileChanges { deleted?: DocumentIdentifier[]; } +/** + * RequestFileSystem supplies file contents and, optionally, directory listings + * for a request that creates a snapshot. + */ +export interface RequestFileSystem { + kind: "full" | "layer"; + /** Files maps file names to their complete contents. */ + files: Record; + /** Directories maps directory names to complete listing results. */ + directories?: Record; + /** Symlinks maps link paths to targets in this filesystem or the host filesystem. */ + symlinks?: Record; + /** + * RemovedPaths lists files or directory trees that must be treated as missing + * even when present in an underlying snapshot or host filesystem. + */ + removedPaths?: string[]; +} + /** * SnapshotChanges describes what changed between the previous latest snapshot * and the newly created snapshot. Changes are reported per-project so clients @@ -1398,6 +1433,29 @@ export interface EmitOutputFile { sourceFileName?: string; } +/** + * RequestDirectoryEntries is a cached directory listing. Entry names are + * relative to the directory, matching vfs.GetAccessibleEntries. + */ +export interface RequestDirectoryEntries { + files: string[]; + directories: string[]; +} + +/** RequestSymlink describes a symbolic link in a request filesystem. */ +export interface RequestSymlink { + /** + * Target is resolved relative to the directory containing the link, matching + * native symbolic-link semantics. + */ + target: string; + /** + * Host routes the target through the host filesystem. This is the only way a + * full filesystem can access paths not supplied in the request filesystem. + */ + host?: boolean; +} + /** ProjectFileChanges describes what source files changed within a single project. */ export interface ProjectFileChanges { /** ChangedFiles lists source file paths whose content differs. */ diff --git a/packages/typescript/src/api/proto.ts b/packages/typescript/src/api/proto.ts index 0be9ac250a756..3f090ebf548de 100644 --- a/packages/typescript/src/api/proto.ts +++ b/packages/typescript/src/api/proto.ts @@ -80,7 +80,7 @@ export function resolveDocumentURI(identifier: DocumentIdentifier): string { return identifier.uri; } -export interface LSPUpdateSnapshotParams extends CoreUpdateSnapshotParams { +export interface LSPUpdateSnapshotParams extends Omit { /** * @deprecated Use {@link openProjects} instead. * Path to a tsconfig.json file to open in the new snapshot. @@ -94,7 +94,7 @@ export interface LSPUpdateSnapshotParams extends CoreUpdateSnapshotParams { /** * Parameters for updateSnapshot, including deprecated members handled by `toUpdateSnapshotRequest` */ -export interface UpdateSnapshotParams extends CoreUpdateSnapshotParams { +export interface UpdateSnapshotParams extends Omit { /** * @deprecated Use {@link openProjects} instead. * Path to a tsconfig.json file to open in the new snapshot. @@ -107,13 +107,14 @@ export interface UpdateSnapshotParams extends CoreUpdateSnapshotParams { * compatibility shim: a single `openProject` is folded into `openProjects` and is * never sent on the wire. */ -export function toUpdateSnapshotRequest(params?: UpdateSnapshotParams): UpdateSnapshotParams { +export function toUpdateSnapshotRequest(params?: UpdateSnapshotParams, snapshot?: number): CoreUpdateSnapshotParams { const { openProject, openProjects, ...rest } = params ?? {}; const mergedOpenProjects = openProject !== undefined ? [resolveFileName(openProject), ...(openProjects ?? [])] : openProjects; return { ...rest, + ...(snapshot !== undefined ? { snapshot } : {}), ...(mergedOpenProjects !== undefined ? { openProjects: mergedOpenProjects } : {}), }; } diff --git a/packages/typescript/src/api/sync/api.ts b/packages/typescript/src/api/sync/api.ts index 4ec27f4805f98..a8f9cba4e9fe6 100644 --- a/packages/typescript/src/api/sync/api.ts +++ b/packages/typescript/src/api/sync/api.ts @@ -555,9 +555,60 @@ export class API implements FormatDiagnosticsHo owner, "updateSnapshot", function (params?: FromLSP extends true ? LSPUpdateSnapshotParams : UpdateSnapshotParams): Snapshot { + return owner.updateSnapshotWorker(params); + }, + function* (params?: FromLSP extends true ? LSPUpdateSnapshotParams : UpdateSnapshotParams): Generator { + return yield* owner.updateSnapshotWorker.gen(params); + }, + ); + } + + /** @internal */ + get updateSnapshotFrom(): { + (baseSnapshot: Snapshot, params?: UpdateSnapshotParams): Snapshot; + gen(baseSnapshot: Snapshot, params?: UpdateSnapshotParams): Generator; + } { + const owner = this; + return cacheGeneratorMethod( + owner, + "updateSnapshotFrom", + function (baseSnapshot: Snapshot, params?: UpdateSnapshotParams): Snapshot { + if (!owner.activeSnapshots.has(baseSnapshot) || baseSnapshot.isDisposed()) { + throw new Error("Cannot update an inactive snapshot"); + } + if (baseSnapshot !== owner.latestSnapshot) { + // TODO: Support forking active memory/cache snapshots once the server-side + // ownership, project state, and cache semantics have been worked out. + throw new Error("Snapshot.update can only update the latest snapshot"); + } + return owner.updateSnapshotWorker(params, baseSnapshot); + }, + function* (baseSnapshot: Snapshot, params?: UpdateSnapshotParams): Generator { + if (!owner.activeSnapshots.has(baseSnapshot) || baseSnapshot.isDisposed()) { + throw new Error("Cannot update an inactive snapshot"); + } + if (baseSnapshot !== owner.latestSnapshot) { + // TODO: Support forking active memory/cache snapshots once the server-side + // ownership, project state, and cache semantics have been worked out. + throw new Error("Snapshot.update can only update the latest snapshot"); + } + return yield* owner.updateSnapshotWorker.gen(params, baseSnapshot); + }, + ); + } + + private get updateSnapshotWorker(): { + (params?: LSPUpdateSnapshotParams | UpdateSnapshotParams, baseSnapshot?: Snapshot): Snapshot; + gen(params?: LSPUpdateSnapshotParams | UpdateSnapshotParams, baseSnapshot?: Snapshot): Generator; + } { + const owner = this; + return cacheGeneratorMethod( + owner, + "updateSnapshotWorker", + function (params?: LSPUpdateSnapshotParams | UpdateSnapshotParams, baseSnapshot?: Snapshot): Snapshot { owner.ensureInitialized(); - const requestParams = toUpdateSnapshotRequest(params); + const requestParams = toUpdateSnapshotRequest(params, baseSnapshot?.id); const data = owner.client.apiRequest("updateSnapshot", requestParams); // Retain cached source files from previous snapshot for unchanged files @@ -586,10 +637,10 @@ export class API implements FormatDiagnosticsHo return snapshot; }, - function* (params?: FromLSP extends true ? LSPUpdateSnapshotParams : UpdateSnapshotParams): Generator { + function* (params?: LSPUpdateSnapshotParams | UpdateSnapshotParams, baseSnapshot?: Snapshot): Generator { yield* owner.ensureInitialized.gen(); - const requestParams = toUpdateSnapshotRequest(params); + const requestParams = toUpdateSnapshotRequest(params, baseSnapshot?.id); const data = yield* apiRequest("updateSnapshot", requestParams); // Retain cached source files from previous snapshot for unchanged files @@ -818,6 +869,23 @@ export class API implements FormatDiagnosticsHo return cacheGeneratorMethod( owner, "createProgram", + function (rootFiles: readonly DocumentIdentifier[], createProgramOptions: CreateProgramOptions, oldProgram?: Program, fileChanges?: APIFileChanges): Program { + return owner.createProgramWorker(rootFiles, createProgramOptions, oldProgram, fileChanges); + }, + function* (rootFiles: readonly DocumentIdentifier[], createProgramOptions: CreateProgramOptions, oldProgram?: Program, fileChanges?: APIFileChanges): Generator { + return yield* owner.createProgramWorker.gen(rootFiles, createProgramOptions, oldProgram, fileChanges); + }, + ); + } + + private get createProgramWorker(): { + (rootFiles: readonly DocumentIdentifier[], createProgramOptions: CreateProgramOptions, oldProgram?: Program, fileChanges?: APIFileChanges): Program; + gen(rootFiles: readonly DocumentIdentifier[], createProgramOptions: CreateProgramOptions, oldProgram?: Program, fileChanges?: APIFileChanges): Generator; + } { + const owner = this; + return cacheGeneratorMethod( + owner, + "createProgramWorker", function (rootFiles: readonly DocumentIdentifier[], createProgramOptions: CreateProgramOptions, oldProgram?: Program, fileChanges?: APIFileChanges): Program { owner.ensureInitialized(); @@ -894,6 +962,13 @@ export class API implements FormatDiagnosticsHo type EnsureInitialized = (() => void) & { gen(): Generator; }; +interface SnapshotOwner extends FormatDiagnosticsHost { + updateSnapshotFrom: { + (baseSnapshot: Snapshot, params?: UpdateSnapshotParams): Snapshot; + gen(baseSnapshot: Snapshot, params?: UpdateSnapshotParams): Generator; + }; +} + export class InternalAPI { private client: Client; private ensureInitialized: EnsureInitialized; @@ -974,6 +1049,7 @@ export class Snapshot { private disposed: boolean = false; private disposePromise: void | undefined; private onDispose: () => void; + private api: SnapshotOwner; private snapshotRegistry: SnapshotObjectRegistry; readonly internal: SnapshotInternalAPI; @@ -982,18 +1058,19 @@ export class Snapshot { client: Client, sourceFileCache: SourceFileCache, toPath: (fileName: string) => Path, - formatDiagnosticsHost: FormatDiagnosticsHost, + api: SnapshotOwner, onDispose: () => void, ) { this.id = data.snapshot; this.client = client; this.toPath = toPath; + this.api = api; this.onDispose = onDispose; this.projectMap = new Map(); this.snapshotRegistry = new SnapshotObjectRegistry(client, this.id, projectId => this.projectMap.get(projectId)); for (const projData of data.projects) { - const project = new Project(projData, this.id, client, sourceFileCache, toPath, formatDiagnosticsHost, this.snapshotRegistry); + const project = new Project(projData, this.id, client, sourceFileCache, toPath, api, this.snapshotRegistry); this.projectMap.set(toPath(projData.configFileName), project); } @@ -1039,10 +1116,32 @@ export class Snapshot { ); } + /** + * Creates the next snapshot, layering its filesystem over this snapshot's + * filesystem. This snapshot must still be active and be the latest snapshot. + */ + get update(): { + (params?: UpdateSnapshotParams): Snapshot; + gen(params?: UpdateSnapshotParams): Generator; + } { + const owner = this; + return cacheGeneratorMethod( + owner, + "update", + function (params?: UpdateSnapshotParams): Snapshot { + owner.ensureNotDisposed(); + return owner.api.updateSnapshotFrom(owner, params); + }, + function* (params?: UpdateSnapshotParams): Generator { + owner.ensureNotDisposed(); + return yield* owner.api.updateSnapshotFrom.gen(owner, params); + }, + ); + } + [globalThis.Symbol.dispose](): void { void this.dispose(); } - get dispose(): { (): void; gen(): Generator; @@ -2695,10 +2794,9 @@ export class Program implements FormatDiagnosticsHost { } /** - * Emits files to the configured filesystem. - * - * When the API has a virtual filesystem with a `writeFile` callback, output - * is written there. Otherwise, the server writes directly to the host filesystem. + * Emits files to the configured filesystem. Layer and host filesystems are + * written through; full filesystems remain immutable and return emitted + * files in {@link EmitResult.fileSystem}. */ get emit(): { (emitOnly?: EmitOnly): EmitResult; @@ -2714,10 +2812,17 @@ export class Program implements FormatDiagnosticsHost { project: owner.project.id, ...(emitOnly !== undefined ? { emitOnly } : {}), }); + const fileSystem = response.emittedFilesContents.length + ? { + kind: "layer" as const, + files: Object.fromEntries(response.emittedFiles.map((fileName, index) => [fileName, response.emittedFilesContents[index]])), + } + : undefined; return { emitSkipped: response.emitSkipped, diagnostics: response.diagnostics, emittedFiles: response.emittedFiles, + ...(fileSystem ? { fileSystem } : {}), }; }, function* (emitOnly?: EmitOnly): Generator { @@ -2726,10 +2831,17 @@ export class Program implements FormatDiagnosticsHost { project: owner.project.id, ...(emitOnly !== undefined ? { emitOnly } : {}), }); + const fileSystem = response.emittedFilesContents.length + ? { + kind: "layer" as const, + files: Object.fromEntries(response.emittedFiles.map((fileName, index) => [fileName, response.emittedFilesContents[index]])), + } + : undefined; return { emitSkipped: response.emitSkipped, diagnostics: response.diagnostics, emittedFiles: response.emittedFiles, + ...(fileSystem ? { fileSystem } : {}), }; }, ); diff --git a/packages/typescript/src/api/sync/types.ts b/packages/typescript/src/api/sync/types.ts index 8a6987adf024b..ffa40b3ea09bc 100644 --- a/packages/typescript/src/api/sync/types.ts +++ b/packages/typescript/src/api/sync/types.ts @@ -21,7 +21,10 @@ import type { NamedTupleMember, ParameterDeclaration, } from "../../ast/ast.ts"; -import type { Diagnostic } from "../proto.ts"; +import type { + Diagnostic, + RequestFileSystem, +} from "../proto.ts"; import type { NodeHandle, Signature, @@ -525,6 +528,8 @@ export interface EmitResult { readonly emitSkipped: boolean; readonly diagnostics: readonly Diagnostic[]; readonly emittedFiles: readonly string[]; + /** Emitted files captured as a filesystem layer suitable for {@link Snapshot.update}. */ + readonly fileSystem?: RequestFileSystem | undefined; } export interface EmitOutput { diff --git a/packages/typescript/test/async/api.test.ts b/packages/typescript/test/async/api.test.ts index e7cb081ed5375..97230f898f26c 100644 --- a/packages/typescript/test/async/api.test.ts +++ b/packages/typescript/test/async/api.test.ts @@ -69,6 +69,7 @@ import { ObjectFlags, type Signature, SignatureKind, + type Snapshot, type StringMappingType, SymbolFlags, type TemplateLiteralType, @@ -80,7 +81,12 @@ import { type TypeReference, type UnionOrIntersectionType, } from "@typescript/typescript/unstable/async"; // @sync: } from "@typescript/typescript/unstable/sync"; -import { createVirtualFileSystem } from "@typescript/typescript/unstable/fs"; +import { + createFileSystem, + createFileSystemLayer, + createFileSystemWithLib, + createVirtualFileSystem, +} from "@typescript/typescript/unstable/fs"; import type { FileSystem } from "@typescript/typescript/unstable/fs"; import assert from "node:assert"; import { globSync } from "node:fs"; @@ -582,6 +588,25 @@ describe("API", () => { // @sync-skip-block-start describe("API - automatic batching", () => { + test("initializes only once for concurrent first requests", async () => { + await using api = spawnAPI(); + const client = (api as unknown as { + client: { apiRequest(method: string, params: unknown): Promise; }; + }).client; + const apiRequest = client.apiRequest.bind(client); + let initializeCalls = 0; + client.apiRequest = (method, params) => { + if (method === "initialize") initializeCalls++; + return apiRequest(method, params); + }; + + await Promise.all([ + api.parseCommandLine(["--strict"]), + api.readConfigFile("/tsconfig.json"), + ]); + assert.equal(initializeCalls, 1); + }); + test("batches multiple concurrent requests into one automatically", async () => { await using api = new API({ cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), @@ -1743,13 +1768,28 @@ describe("Snapshot disposal", () => { await using api = spawnAPI(); const snapshot = await api.updateSnapshot({ openProject: "/tsconfig.json" }); - await Promise.all([snapshot.dispose(), snapshot.dispose()]); // @sync: snapshot.dispose(); + const firstDispose = snapshot.dispose(); + const secondDispose = snapshot.dispose(); + assert.strictEqual(firstDispose, secondDispose); + await firstDispose; // @sync: snapshot.dispose(); assert.ok(snapshot.isDisposed()); // Second dispose should not throw await snapshot.dispose(); assert.ok(snapshot.isDisposed()); }); + test("api.close waits for disposal started by using", async () => { + const api = spawnAPI(); + let snapshot: Snapshot; + { + using disposableSnapshot = await api.updateSnapshot({ openProject: "/tsconfig.json" }); + snapshot = disposableSnapshot; + } + assert.ok(snapshot.isDisposed()); + await api.close(); + await snapshot.dispose(); + }); + test("api.close disposes all active snapshots", async () => { const api = spawnAPI(); const snap1 = await api.updateSnapshot({ openProject: "/tsconfig.json" }); @@ -3147,6 +3187,616 @@ describe("readFile callback semantics", () => { }); }); +describe("updateSnapshot file systems", () => { + test("request filesystem factories derive directory listings", () => { + const memory = createFileSystem([ + ["/src/index.ts", "posix"], + ["C:\\repo\\src\\index.ts", "windows"], + ["file:///literal%20path.ts", "literal file-name string"], + [{ uri: "file:///encoded/path%20with%20spaces.ts" }, "file URI"], + [{ uri: "file:///C%3A/repo/encoded%23name.ts" }, "Windows file URI"], + [{ uri: "file://server/share/encoded%20name.ts" }, "UNC file URI"], + [{ uri: "file:///encoded/unicode%E2%80%93name.ts" }, "Unicode file URI"], + [{ uri: "file:///encoded/literal+plus.ts" }, "plus file URI"], + [{ uri: "file:///encoded/once%2520encoded.ts" }, "double-encoded file URI"], + ["vscode-remote://ssh-remote+host/workspace/src/index.ts", "remote"], + ["vscode-notebook-cell://authority/workspace/notebook.ipynb/cell.ts", "notebook"], + ]); + assert.deepEqual(memory, { + kind: "full", + files: { + "/src/index.ts": "posix", + "C:/repo/src/index.ts": "windows", + "file:///literal%20path.ts": "literal file-name string", + "/encoded/path with spaces.ts": "file URI", + "c:/repo/encoded#name.ts": "Windows file URI", + "//server/share/encoded name.ts": "UNC file URI", + "/encoded/unicode–name.ts": "Unicode file URI", + "/encoded/literal+plus.ts": "plus file URI", + "/encoded/once%20encoded.ts": "double-encoded file URI", + "vscode-remote://ssh-remote+host/workspace/src/index.ts": "remote", + "vscode-notebook-cell://authority/workspace/notebook.ipynb/cell.ts": "notebook", + }, + directories: { + "/src": { files: ["index.ts"], directories: [] }, + "/": { files: [], directories: ["src", "encoded"] }, + "C:/repo/src": { files: ["index.ts"], directories: [] }, + "C:/repo": { files: [], directories: ["src"] }, + "C:/": { files: [], directories: ["repo"] }, + "c:/repo": { files: ["encoded#name.ts"], directories: [] }, + "c:/": { files: [], directories: ["repo"] }, + "/encoded": { + files: ["path with spaces.ts", "unicode–name.ts", "literal+plus.ts", "once%20encoded.ts"], + directories: [], + }, + "//server/share": { files: ["encoded name.ts"], directories: [] }, + "//server/": { files: [], directories: ["share"] }, + "file:///": { files: ["literal%20path.ts"], directories: [] }, + "vscode-remote://ssh-remote+host/workspace/src": { files: ["index.ts"], directories: [] }, + "vscode-remote://ssh-remote+host/workspace": { files: [], directories: ["src"] }, + "vscode-remote://ssh-remote+host/": { files: [], directories: ["workspace"] }, + "vscode-notebook-cell://authority/workspace/notebook.ipynb": { files: ["cell.ts"], directories: [] }, + "vscode-notebook-cell://authority/workspace": { files: [], directories: ["notebook.ipynb"] }, + "vscode-notebook-cell://authority/": { files: [], directories: ["workspace"] }, + }, + }); + + const directories = { "/explicit": { files: ["provided.ts"], directories: [] } }; + const cache = createFileSystemLayer([["/ignored/derived.ts", "cache"]], { + directories, + removedPaths: ["/removed.ts", "/removed"], + }); + assert.equal(cache.kind, "layer"); + assert.deepEqual(cache.directories, directories); + assert.deepEqual(cache.removedPaths, ["/removed.ts", "/removed"]); + + assert.throws( + () => + createFileSystem([ + ["/duplicate.ts", "path"], + [{ uri: "file:///duplicate.ts" }, "URI"], + ]), + /Duplicate request filesystem path: \/duplicate\.ts/, + ); + + const prototypeFileSystem = createFileSystem([["__proto__", "prototype"]]); + assert.equal(prototypeFileSystem.files["__proto__"], "prototype"); + assert.ok(Object.hasOwn(prototypeFileSystem.files, "__proto__")); + + assert.throws( + () => + createFileSystem([ + ["/normalized/duplicate.ts", "forward slash"], + ["\\normalized\\duplicate.ts", "backslash"], + ]), + /Duplicate request filesystem path: \/normalized\/duplicate\.ts/, + ); + }); + + test("full file system is total and does not invoke host callbacks", async () => { + const callbackCalls: string[] = []; + const host = createVirtualFileSystem({ + "/host.ts": `export const source = "host";`, + }); + const fs: FileSystem = { + readFile: path => { + callbackCalls.push(`readFile:${path}`); + return host.readFile!(path); + }, + fileExists: path => { + callbackCalls.push(`fileExists:${path}`); + return host.fileExists!(path); + }, + directoryExists: path => { + callbackCalls.push(`directoryExists:${path}`); + return host.directoryExists!(path); + }, + getAccessibleEntries: path => { + callbackCalls.push(`getAccessibleEntries:${path}`); + return host.getAccessibleEntries!(path); + }, + realpath: path => { + callbackCalls.push(`realpath:${path}`); + return path; + }, + writeFile: (path, content) => { + callbackCalls.push(`writeFile:${path}`); + host.writeFile!(path, content); + }, + }; + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs, + }); + + using snapshot = await api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: { + kind: "full", + files: { + "/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true }, include: ["src/**/*.ts"] }), + "/src/index.ts": `export const source = "memory";`, + }, + directories: { + "/": { files: ["tsconfig.json"], directories: ["src"] }, + "/src": { files: ["index.ts"], directories: [] }, + }, + }, + }); + const project = snapshot.getProject("/tsconfig.json")!; + const sourceFile = await project.program.getSourceFile("/src/index.ts"); + assert.equal(sourceFile?.text, `export const source = "memory";`); + assert.equal(await project.program.getSourceFile("/host.ts"), undefined); + assert.deepEqual(callbackCalls, []); + }); + + test("full file system with lib resolves the default library", async () => { + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + }); + using snapshot = await api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystemWithLib(Object.entries({ + "/tsconfig.json": JSON.stringify({ compilerOptions: { strict: true }, files: ["src/main.ts"] }), + "/src/main.ts": `export const values: Array = [];`, + })), + }); + const program = snapshot.getProject("/tsconfig.json")!.program; + assert.deepEqual(await program.getGlobalDiagnostics(), []); + const sourceFileNames = await program.getSourceFileNames(); + const defaultLibraryName = sourceFileNames.find(fileName => fileName.includes("/lib.") && fileName.endsWith(".d.ts")); + assert.ok(defaultLibraryName, JSON.stringify(sourceFileNames)); + const defaultLibrary = await program.getSourceFile(defaultLibraryName); + assert.ok(defaultLibrary); + assert.equal(await program.isSourceFileDefaultLibrary(defaultLibrary), true); + }); + + test("full file system accepts paths decoded from VS Code document URIs", async () => { + const fileDocument = { uri: "file:///workspace/file%20name.ts" }; + const remoteDocument = { uri: "vscode-remote://ssh-remote+host/workspace/src/remote%20name.ts" }; + const notebookDocument = { uri: "vscode-notebook-cell:/workspace/notebook.ipynb/cell%20name.ts" }; + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + }); + using snapshot = await api.updateSnapshot({ + openFiles: [fileDocument, remoteDocument, notebookDocument], + fileSystem: createFileSystem([ + [fileDocument, `export const file = true;`], + [remoteDocument, `export const remote = true;`], + [notebookDocument, `export const cell = true;`], + ]), + }); + const fileProject = await snapshot.getDefaultProjectForFile(fileDocument); + const remoteProject = await snapshot.getDefaultProjectForFile(remoteDocument); + const notebookProject = await snapshot.getDefaultProjectForFile(notebookDocument); + assert.equal((await fileProject?.program.getSourceFile(fileDocument))?.text, `export const file = true;`); + assert.equal((await remoteProject?.program.getSourceFile(remoteDocument))?.text, `export const remote = true;`); + assert.equal((await notebookProject?.program.getSourceFile(notebookDocument))?.text, `export const cell = true;`); + }); + + test("file system layer bypasses callbacks on hits and falls back on misses", async () => { + const readFileCalls: string[] = []; + const directoryCalls: string[] = []; + const host = createVirtualFileSystem({ + "/src/fallback.ts": `export const fallback = true;`, + }); + const fs: FileSystem = { + ...host, + readFile: path => { + readFileCalls.push(path); + return host.readFile!(path); + }, + getAccessibleEntries: path => { + directoryCalls.push(path); + return host.getAccessibleEntries!(path); + }, + }; + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs, + }); + + using snapshot = await api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: { + kind: "layer", + files: { + "/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true }, include: ["src/**/*.ts"] }), + "/src/index.ts": `export const cached = true;`, + }, + directories: { + "/": { files: ["tsconfig.json"], directories: ["src"] }, + "/src": { files: ["fallback.ts", "index.ts"], directories: [] }, + }, + }, + }); + const project = snapshot.getProject("/tsconfig.json")!; + assert.equal((await project.program.getSourceFile("/src/index.ts"))?.text, `export const cached = true;`); + assert.equal((await project.program.getSourceFile("/src/fallback.ts"))?.text, `export const fallback = true;`); + + assert.ok(!readFileCalls.includes("/tsconfig.json")); + assert.ok(!readFileCalls.includes("/src/index.ts")); + assert.ok(readFileCalls.includes("/src/fallback.ts")); + assert.ok(!directoryCalls.includes("/")); + assert.ok(!directoryCalls.includes("/src")); + }); + + test("file system layer factory preserves host directory entries", async () => { + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: createVirtualFileSystem({ + "/src/from-host.ts": `export const host = true;`, + }), + }); + + using snapshot = await api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystemLayer([ + ["/tsconfig.json", JSON.stringify({ compilerOptions: { noLib: true }, include: ["src/**/*.ts"] })], + ["/src/from-cache.ts", `export const cache = true;`], + ]), + }); + const program = snapshot.getProject("/tsconfig.json")!.program; + assert.deepEqual( + [...await program.getSourceFileNames()].sort(), + ["/src/from-cache.ts", "/src/from-host.ts"], + ); + }); + + test("full file system resolves packages through internal monorepo symlinks", async () => { + const callbackCalls: string[] = []; + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: { + readFile: path => { + callbackCalls.push(path); + return undefined; + }, + }, + }); + + using snapshot = await api.updateSnapshot({ + openProject: "/project/tsconfig.json", + fileSystem: { + kind: "full", + files: { + "/project/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true, moduleResolution: "node" }, files: ["index.ts"] }), + "/project/index.ts": `import { value } from "pkg"; export { value };`, + "/packages/pkg/index.d.ts": `export declare const value: number;`, + }, + symlinks: { + "/project/node_modules/pkg": { target: "/packages/pkg" }, + }, + }, + }); + const project = snapshot.getProject("/project/tsconfig.json")!; + assert.equal( + (await project.program.getSourceFile("/packages/pkg/index.d.ts"))?.text, + `export declare const value: number;`, + ); + assert.deepEqual(callbackCalls, []); + }); + + test("full file system resolves relative symlink targets", async () => { + const callbackCalls: string[] = []; + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: { + readFile: path => { + callbackCalls.push(path); + return undefined; + }, + }, + }); + + using snapshot = await api.updateSnapshot({ + openProject: "/project/tsconfig.json", + fileSystem: { + kind: "full", + files: { + "/project/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true }, files: ["index.ts"] }), + "/project/index.ts": `export { value } from "./pkg";`, + "/packages/pkg/index.d.ts": `export declare const value: number;`, + }, + symlinks: { + "/project/pkg": { target: "../packages/pkg" }, + }, + }, + }); + const project = snapshot.getProject("/project/tsconfig.json")!; + assert.equal( + (await project.program.getSourceFile("/project/pkg/index.d.ts"))?.text, + `export declare const value: number;`, + ); + assert.deepEqual(callbackCalls, []); + }); + + test("Snapshot.update layers filesystem edits and removals", async () => { + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + }); + using snapshot = await api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystem(Object.entries({ + "/tsconfig.json": JSON.stringify({ + compilerOptions: { noLib: true }, + include: ["src/**/*.ts"], + }), + "/src/keep.ts": `export const keep = true;`, + "/src/change.ts": `export const version = "old";`, + "/src/remove.ts": `export const remove = true;`, + "/src/removed/gone.ts": `export const gone = true;`, + })), + }); + + using updated = await snapshot.update({ + fileSystem: createFileSystemLayer( + Object.entries({ + "/src/change.ts": `export const version = "new";`, + "/src/added.ts": `export const added = true;`, + }), + { + removedPaths: ["/src/remove.ts", "/src/removed"], + }, + ), + }); + const project = updated.getProject("/tsconfig.json")!; + assert.equal((await project.program.getSourceFile("/src/keep.ts"))?.text, `export const keep = true;`); + assert.equal((await project.program.getSourceFile("/src/change.ts"))?.text, `export const version = "new";`); + assert.equal((await project.program.getSourceFile("/src/added.ts"))?.text, `export const added = true;`); + assert.equal(await project.program.getSourceFile("/src/remove.ts"), undefined); + assert.equal(await project.program.getSourceFile("/src/removed/gone.ts"), undefined); + await assert.rejects(() => snapshot.update(), /can only update the latest snapshot/); // @sync: assert.throws(() => snapshot.update(), /can only update the latest snapshot/); + + using updatedAgain = await updated.update({ + fileSystem: createFileSystemLayer( + Object.entries({ + "/src/added.ts": `export const added = "updated again";`, + }), + { + removedPaths: ["/src/change.ts"], + }, + ), + }); + const updatedAgainProject = updatedAgain.getProject("/tsconfig.json")!; + assert.equal((await updatedAgainProject.program.getSourceFile("/src/keep.ts"))?.text, `export const keep = true;`); + assert.equal((await updatedAgainProject.program.getSourceFile("/src/added.ts"))?.text, `export const added = "updated again";`); + assert.equal(await updatedAgainProject.program.getSourceFile("/src/change.ts"), undefined); + }); + + test("eager snapshot disposal does not retain filesystem history", async () => { + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + }); + let snapshot: Snapshot = await api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystem([ + ["/tsconfig.json", JSON.stringify({ compilerOptions: { noLib: true }, files: ["pkg/index.ts"] })], + ["/pkg/index.ts", ""], + ]), + }); + try { + let content = ""; + for (const character of "export const x = 1") { + const oldSnapshot: Snapshot = snapshot; + content += character; + snapshot = await oldSnapshot.update({ + fileSystem: createFileSystemLayer([["/pkg/index.ts", content]]), + }); + await oldSnapshot.dispose(); + assert.equal(oldSnapshot.isDisposed(), true); + } + + const program = snapshot.getProject("/tsconfig.json")!.program; + assert.equal((await program.getSourceFile("/pkg/index.ts"))?.text, "export const x = 1"); + } + finally { + await snapshot.dispose(); + } + }); + + test("Snapshot.update treats a full filesystem as a total replacement", async () => { + const host = createVirtualFileSystem({ + "/host.ts": `export const source = "host";`, + }); + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: host, + }); + using snapshot = await api.updateSnapshot(); + using replaced = await snapshot.update({ + openProject: "/tsconfig.json", + fileSystem: createFileSystem([ + ["/tsconfig.json", JSON.stringify({ compilerOptions: { noLib: true }, files: ["memory.ts", "host.ts"] })], + ["/memory.ts", `export const source = "memory";`], + ]), + }); + const program = replaced.getProject("/tsconfig.json")!.program; + assert.equal((await program.getSourceFile("/memory.ts"))?.text, `export const source = "memory";`); + assert.equal(await program.getSourceFile("/host.ts"), undefined); + }); + + test("Snapshot.update applies target changes through inherited symlinks", async () => { + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + }); + using snapshot = await api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystem( + Object.entries({ + "/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true }, files: ["src/main.ts"] }), + "/src/main.ts": `import "./link/change"; import "./link/added"; import "./link/remove";`, + "/target/change.ts": `export const version = "old";`, + "/target/remove.ts": `export const removed = true;`, + }), + { + symlinks: { + "/src/link": { target: "/target" }, + }, + }, + ), + }); + + using updated = await snapshot.update({ + fileSystem: createFileSystemLayer( + Object.entries({ + "/target/change.ts": `export const version = "new";`, + "/target/added.ts": `export const added = true;`, + }), + { + removedPaths: ["/target/remove.ts"], + }, + ), + }); + const program = updated.getProject("/tsconfig.json")!.program; + assert.equal((await program.getSourceFile("/src/link/change.ts"))?.text, `export const version = "new";`); + assert.equal((await program.getSourceFile("/src/link/added.ts"))?.text, `export const added = true;`); + assert.equal(await program.getSourceFile("/src/link/remove.ts"), undefined); + }); + + test("full filesystem emit returns outputs without mutating the host", async () => { + const hostWrites: string[] = []; + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: { + writeFile: path => { + hostWrites.push(path); + }, + }, + }); + using snapshot = await api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystem(Object.entries({ + "/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true, outDir: "/out", rootDir: "/src" }, files: ["src/main.ts"] }), + "/src/main.ts": `export const value: number = 1;`, + })), + }); + const program = snapshot.getProject("/tsconfig.json")!.program; + const result = await program.emit(); + assert.deepEqual(result.emittedFiles, ["/out/main.js"]); + assert.deepEqual(result.fileSystem, { + kind: "layer", + files: { + "/out/main.js": `export const value = 1;\n`, + }, + }); + assert.deepEqual(hostWrites, []); + + using updated = await snapshot.update({ fileSystem: result.fileSystem!, openFiles: ["/out/main.js"] }); + const outputProject = await updated.getDefaultProjectForFile("/out/main.js"); + assert.equal((await updated.getProject("/tsconfig.json")!.program.getSourceFile("/src/main.ts"))?.text, `export const value: number = 1;`); + assert.equal((await outputProject?.program.getSourceFile("/out/main.js"))?.text, `export const value = 1;\n`); + }); + + test("filesystem layer emit writes through to the host", async () => { + const host = createVirtualFileSystem({}); + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: host, + }); + using snapshot = await api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystemLayer(Object.entries({ + "/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true, outDir: "/out", rootDir: "/src" }, files: ["src/main.ts"] }), + "/src/main.ts": `export const value: number = 1;`, + })), + }); + const program = snapshot.getProject("/tsconfig.json")!.program; + const result = await program.emit(); + assert.equal(result.fileSystem, undefined); + assert.equal(host.readFile!("/out/main.js"), `export const value = 1;\n`); + }); + + test("full file system can link node_modules from the host", async () => { + const readFileCalls: string[] = []; + const directoryExistsCalls: string[] = []; + const fileExistsCalls: string[] = []; + const host = createVirtualFileSystem({ + "/host/node_modules/pkg/index.d.ts": `export declare const value: string;`, + }); + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: { + ...host, + directoryExists: path => { + directoryExistsCalls.push(path); + return host.directoryExists!(path); + }, + fileExists: path => { + const exists = host.fileExists!(path); + fileExistsCalls.push(`${path}:${exists}`); + return exists; + }, + readFile: path => { + readFileCalls.push(path); + return host.readFile!(path); + }, + }, + }); + + using snapshot = await api.updateSnapshot({ + openProject: "/project/tsconfig.json", + fileSystem: { + kind: "full", + files: { + "/project/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true, moduleResolution: "node" }, files: ["index.ts"] }), + "/project/index.ts": `import { value } from "pkg"; export { value };`, + }, + symlinks: { + "/project/node_modules": { target: "/host/node_modules", host: true }, + }, + }, + }); + const project = snapshot.getProject("/project/tsconfig.json")!; + const sourceFileNames = await project.program.getSourceFileNames(); + assert.ok( + sourceFileNames.includes("/host/node_modules/pkg/index.d.ts"), + JSON.stringify({ sourceFileNames, readFileCalls, directoryExistsCalls, fileExistsCalls }), + ); + assert.equal( + (await project.program.getSourceFile("/host/node_modules/pkg/index.d.ts"))?.text, + `export declare const value: string;`, + ); + assert.ok(readFileCalls.includes("/host/node_modules/pkg/index.d.ts")); + assert.ok(!readFileCalls.some(path => path.startsWith("/project/node_modules"))); + }); + + test("Snapshot.update host symlinks bypass an inherited full filesystem", async () => { + const host = createVirtualFileSystem({ + "/host/node_modules/pkg/index.d.ts": `export declare const value: string;`, + }); + await using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: host, + }); + using snapshot = await api.updateSnapshot({ + openProject: "/project/tsconfig.json", + fileSystem: createFileSystem([ + ["/project/tsconfig.json", JSON.stringify({ compilerOptions: { noLib: true, moduleResolution: "node" }, files: ["index.ts"] })], + ["/project/index.ts", `import { value } from "pkg"; export { value };`], + ]), + }); + using updated = await snapshot.update({ + fileSystem: createFileSystemLayer([], { + symlinks: { + "/project/node_modules": { target: "/host/node_modules", host: true }, + }, + }), + }); + const project = updated.getProject("/project/tsconfig.json")!; + assert.equal( + (await project.program.getSourceFile("/host/node_modules/pkg/index.d.ts"))?.text, + `export declare const value: string;`, + ); + }); + + // TODO: Add request filesystem coverage for `tsc -b` and `tsc -b --clean` + // once build and clean are exposed through the client API. In particular, + // verify that clean removes synthetic outputs and that build-mode re-timestamping + // of emitted-but-unchanged files works for full filesystems, which currently + // do not model modification times. +}); + describe("Checker - isArrayType / isTupleType", () => { test("number[] is array, not tuple", async () => { await using api = spawnAPI({ diff --git a/packages/typescript/test/sync/api-generators.test.ts b/packages/typescript/test/sync/api-generators.test.ts index 546bc5a0d38fc..154ff74281ba8 100644 --- a/packages/typescript/test/sync/api-generators.test.ts +++ b/packages/typescript/test/sync/api-generators.test.ts @@ -140,6 +140,9 @@ const publicGeneratorExemptions = new Map([ const privateGeneratorGetters = new Set([ "API.ensureInitialized", "API.initializeWorker", + "API.updateSnapshotFrom", + "API.updateSnapshotWorker", + "API.createProgramWorker", "Checker.getIntrinsicType", "Checker.getWellKnownSignatures", "Checker.getWellKnownSymbols", @@ -910,6 +913,21 @@ describe("API - generator batching", () => { runParityBatch(api, cases); assert.deepEqual(temporaryProjects, ["/tsconfig.json", "/tsconfig.json"]); + const snapshotGeneratorAPI = spawnAPI(parityFiles); + const snapshotSyncAPI = spawnAPI(parityFiles); + try { + const generatorBase = snapshotGeneratorAPI.batch(snapshotGeneratorAPI.updateSnapshot.gen({ openProject: "/tsconfig.json" }))[0]; + const syncBase = snapshotSyncAPI.updateSnapshot({ openProject: "/tsconfig.json" }); + const generatorUpdated = snapshotGeneratorAPI.batch(generatorBase.update.gen())[0]; + const syncUpdated = syncBase.update(); + assertSnapshotsEquivalent(generatorUpdated, syncUpdated, "Snapshot.update"); + exercisedMethods.add("Snapshot.update"); + } + finally { + snapshotGeneratorAPI.close(); + snapshotSyncAPI.close(); + } + const destructiveAPI = spawnAPI(parityFiles); const disposableSnapshot = destructiveAPI.batch(destructiveAPI.updateSnapshot.gen({ openProject: "/tsconfig.json" }))[0]; destructiveAPI.batch(disposableSnapshot.dispose.gen()); diff --git a/packages/typescript/test/sync/api.test.ts b/packages/typescript/test/sync/api.test.ts index cb2322eaa2e16..68be8d38e0262 100644 --- a/packages/typescript/test/sync/api.test.ts +++ b/packages/typescript/test/sync/api.test.ts @@ -56,7 +56,12 @@ import { createVariableStatement, } from "@typescript/typescript/unstable/ast/factory"; import { visitEachChild } from "@typescript/typescript/unstable/ast/visitor"; -import { createVirtualFileSystem } from "@typescript/typescript/unstable/fs"; +import { + createFileSystem, + createFileSystemLayer, + createFileSystemWithLib, + createVirtualFileSystem, +} from "@typescript/typescript/unstable/fs"; import type { FileSystem } from "@typescript/typescript/unstable/fs"; import { API, @@ -79,6 +84,7 @@ import { ObjectFlags, type Signature, SignatureKind, + type Snapshot, type StringMappingType, SymbolFlags, type TemplateLiteralType, @@ -1652,6 +1658,9 @@ describe("Snapshot disposal", () => { using api = spawnAPI(); const snapshot = api.updateSnapshot({ openProject: "/tsconfig.json" }); + const firstDispose = snapshot.dispose(); + const secondDispose = snapshot.dispose(); + assert.strictEqual(firstDispose, secondDispose); snapshot.dispose(); assert.ok(snapshot.isDisposed()); // Second dispose should not throw @@ -1659,6 +1668,18 @@ describe("Snapshot disposal", () => { assert.ok(snapshot.isDisposed()); }); + test("api.close waits for disposal started by using", () => { + const api = spawnAPI(); + let snapshot: Snapshot; + { + using disposableSnapshot = api.updateSnapshot({ openProject: "/tsconfig.json" }); + snapshot = disposableSnapshot; + } + assert.ok(snapshot.isDisposed()); + api.close(); + snapshot.dispose(); + }); + test("api.close disposes all active snapshots", () => { const api = spawnAPI(); const snap1 = api.updateSnapshot({ openProject: "/tsconfig.json" }); @@ -3056,6 +3077,616 @@ describe("readFile callback semantics", () => { }); }); +describe("updateSnapshot file systems", () => { + test("request filesystem factories derive directory listings", () => { + const memory = createFileSystem([ + ["/src/index.ts", "posix"], + ["C:\\repo\\src\\index.ts", "windows"], + ["file:///literal%20path.ts", "literal file-name string"], + [{ uri: "file:///encoded/path%20with%20spaces.ts" }, "file URI"], + [{ uri: "file:///C%3A/repo/encoded%23name.ts" }, "Windows file URI"], + [{ uri: "file://server/share/encoded%20name.ts" }, "UNC file URI"], + [{ uri: "file:///encoded/unicode%E2%80%93name.ts" }, "Unicode file URI"], + [{ uri: "file:///encoded/literal+plus.ts" }, "plus file URI"], + [{ uri: "file:///encoded/once%2520encoded.ts" }, "double-encoded file URI"], + ["vscode-remote://ssh-remote+host/workspace/src/index.ts", "remote"], + ["vscode-notebook-cell://authority/workspace/notebook.ipynb/cell.ts", "notebook"], + ]); + assert.deepEqual(memory, { + kind: "full", + files: { + "/src/index.ts": "posix", + "C:/repo/src/index.ts": "windows", + "file:///literal%20path.ts": "literal file-name string", + "/encoded/path with spaces.ts": "file URI", + "c:/repo/encoded#name.ts": "Windows file URI", + "//server/share/encoded name.ts": "UNC file URI", + "/encoded/unicode–name.ts": "Unicode file URI", + "/encoded/literal+plus.ts": "plus file URI", + "/encoded/once%20encoded.ts": "double-encoded file URI", + "vscode-remote://ssh-remote+host/workspace/src/index.ts": "remote", + "vscode-notebook-cell://authority/workspace/notebook.ipynb/cell.ts": "notebook", + }, + directories: { + "/src": { files: ["index.ts"], directories: [] }, + "/": { files: [], directories: ["src", "encoded"] }, + "C:/repo/src": { files: ["index.ts"], directories: [] }, + "C:/repo": { files: [], directories: ["src"] }, + "C:/": { files: [], directories: ["repo"] }, + "c:/repo": { files: ["encoded#name.ts"], directories: [] }, + "c:/": { files: [], directories: ["repo"] }, + "/encoded": { + files: ["path with spaces.ts", "unicode–name.ts", "literal+plus.ts", "once%20encoded.ts"], + directories: [], + }, + "//server/share": { files: ["encoded name.ts"], directories: [] }, + "//server/": { files: [], directories: ["share"] }, + "file:///": { files: ["literal%20path.ts"], directories: [] }, + "vscode-remote://ssh-remote+host/workspace/src": { files: ["index.ts"], directories: [] }, + "vscode-remote://ssh-remote+host/workspace": { files: [], directories: ["src"] }, + "vscode-remote://ssh-remote+host/": { files: [], directories: ["workspace"] }, + "vscode-notebook-cell://authority/workspace/notebook.ipynb": { files: ["cell.ts"], directories: [] }, + "vscode-notebook-cell://authority/workspace": { files: [], directories: ["notebook.ipynb"] }, + "vscode-notebook-cell://authority/": { files: [], directories: ["workspace"] }, + }, + }); + + const directories = { "/explicit": { files: ["provided.ts"], directories: [] } }; + const cache = createFileSystemLayer([["/ignored/derived.ts", "cache"]], { + directories, + removedPaths: ["/removed.ts", "/removed"], + }); + assert.equal(cache.kind, "layer"); + assert.deepEqual(cache.directories, directories); + assert.deepEqual(cache.removedPaths, ["/removed.ts", "/removed"]); + + assert.throws( + () => + createFileSystem([ + ["/duplicate.ts", "path"], + [{ uri: "file:///duplicate.ts" }, "URI"], + ]), + /Duplicate request filesystem path: \/duplicate\.ts/, + ); + + const prototypeFileSystem = createFileSystem([["__proto__", "prototype"]]); + assert.equal(prototypeFileSystem.files["__proto__"], "prototype"); + assert.ok(Object.hasOwn(prototypeFileSystem.files, "__proto__")); + + assert.throws( + () => + createFileSystem([ + ["/normalized/duplicate.ts", "forward slash"], + ["\\normalized\\duplicate.ts", "backslash"], + ]), + /Duplicate request filesystem path: \/normalized\/duplicate\.ts/, + ); + }); + + test("full file system is total and does not invoke host callbacks", () => { + const callbackCalls: string[] = []; + const host = createVirtualFileSystem({ + "/host.ts": `export const source = "host";`, + }); + const fs: FileSystem = { + readFile: path => { + callbackCalls.push(`readFile:${path}`); + return host.readFile!(path); + }, + fileExists: path => { + callbackCalls.push(`fileExists:${path}`); + return host.fileExists!(path); + }, + directoryExists: path => { + callbackCalls.push(`directoryExists:${path}`); + return host.directoryExists!(path); + }, + getAccessibleEntries: path => { + callbackCalls.push(`getAccessibleEntries:${path}`); + return host.getAccessibleEntries!(path); + }, + realpath: path => { + callbackCalls.push(`realpath:${path}`); + return path; + }, + writeFile: (path, content) => { + callbackCalls.push(`writeFile:${path}`); + host.writeFile!(path, content); + }, + }; + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs, + }); + + using snapshot = api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: { + kind: "full", + files: { + "/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true }, include: ["src/**/*.ts"] }), + "/src/index.ts": `export const source = "memory";`, + }, + directories: { + "/": { files: ["tsconfig.json"], directories: ["src"] }, + "/src": { files: ["index.ts"], directories: [] }, + }, + }, + }); + const project = snapshot.getProject("/tsconfig.json")!; + const sourceFile = project.program.getSourceFile("/src/index.ts"); + assert.equal(sourceFile?.text, `export const source = "memory";`); + assert.equal(project.program.getSourceFile("/host.ts"), undefined); + assert.deepEqual(callbackCalls, []); + }); + + test("full file system with lib resolves the default library", () => { + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + }); + using snapshot = api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystemWithLib(Object.entries({ + "/tsconfig.json": JSON.stringify({ compilerOptions: { strict: true }, files: ["src/main.ts"] }), + "/src/main.ts": `export const values: Array = [];`, + })), + }); + const program = snapshot.getProject("/tsconfig.json")!.program; + assert.deepEqual(program.getGlobalDiagnostics(), []); + const sourceFileNames = program.getSourceFileNames(); + const defaultLibraryName = sourceFileNames.find(fileName => fileName.includes("/lib.") && fileName.endsWith(".d.ts")); + assert.ok(defaultLibraryName, JSON.stringify(sourceFileNames)); + const defaultLibrary = program.getSourceFile(defaultLibraryName); + assert.ok(defaultLibrary); + assert.equal(program.isSourceFileDefaultLibrary(defaultLibrary), true); + }); + + test("full file system accepts paths decoded from VS Code document URIs", () => { + const fileDocument = { uri: "file:///workspace/file%20name.ts" }; + const remoteDocument = { uri: "vscode-remote://ssh-remote+host/workspace/src/remote%20name.ts" }; + const notebookDocument = { uri: "vscode-notebook-cell:/workspace/notebook.ipynb/cell%20name.ts" }; + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + }); + using snapshot = api.updateSnapshot({ + openFiles: [fileDocument, remoteDocument, notebookDocument], + fileSystem: createFileSystem([ + [fileDocument, `export const file = true;`], + [remoteDocument, `export const remote = true;`], + [notebookDocument, `export const cell = true;`], + ]), + }); + const fileProject = snapshot.getDefaultProjectForFile(fileDocument); + const remoteProject = snapshot.getDefaultProjectForFile(remoteDocument); + const notebookProject = snapshot.getDefaultProjectForFile(notebookDocument); + assert.equal((fileProject?.program.getSourceFile(fileDocument))?.text, `export const file = true;`); + assert.equal((remoteProject?.program.getSourceFile(remoteDocument))?.text, `export const remote = true;`); + assert.equal((notebookProject?.program.getSourceFile(notebookDocument))?.text, `export const cell = true;`); + }); + + test("file system layer bypasses callbacks on hits and falls back on misses", () => { + const readFileCalls: string[] = []; + const directoryCalls: string[] = []; + const host = createVirtualFileSystem({ + "/src/fallback.ts": `export const fallback = true;`, + }); + const fs: FileSystem = { + ...host, + readFile: path => { + readFileCalls.push(path); + return host.readFile!(path); + }, + getAccessibleEntries: path => { + directoryCalls.push(path); + return host.getAccessibleEntries!(path); + }, + }; + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs, + }); + + using snapshot = api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: { + kind: "layer", + files: { + "/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true }, include: ["src/**/*.ts"] }), + "/src/index.ts": `export const cached = true;`, + }, + directories: { + "/": { files: ["tsconfig.json"], directories: ["src"] }, + "/src": { files: ["fallback.ts", "index.ts"], directories: [] }, + }, + }, + }); + const project = snapshot.getProject("/tsconfig.json")!; + assert.equal((project.program.getSourceFile("/src/index.ts"))?.text, `export const cached = true;`); + assert.equal((project.program.getSourceFile("/src/fallback.ts"))?.text, `export const fallback = true;`); + + assert.ok(!readFileCalls.includes("/tsconfig.json")); + assert.ok(!readFileCalls.includes("/src/index.ts")); + assert.ok(readFileCalls.includes("/src/fallback.ts")); + assert.ok(!directoryCalls.includes("/")); + assert.ok(!directoryCalls.includes("/src")); + }); + + test("file system layer factory preserves host directory entries", () => { + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: createVirtualFileSystem({ + "/src/from-host.ts": `export const host = true;`, + }), + }); + + using snapshot = api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystemLayer([ + ["/tsconfig.json", JSON.stringify({ compilerOptions: { noLib: true }, include: ["src/**/*.ts"] })], + ["/src/from-cache.ts", `export const cache = true;`], + ]), + }); + const program = snapshot.getProject("/tsconfig.json")!.program; + assert.deepEqual( + [...program.getSourceFileNames()].sort(), + ["/src/from-cache.ts", "/src/from-host.ts"], + ); + }); + + test("full file system resolves packages through internal monorepo symlinks", () => { + const callbackCalls: string[] = []; + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: { + readFile: path => { + callbackCalls.push(path); + return undefined; + }, + }, + }); + + using snapshot = api.updateSnapshot({ + openProject: "/project/tsconfig.json", + fileSystem: { + kind: "full", + files: { + "/project/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true, moduleResolution: "node" }, files: ["index.ts"] }), + "/project/index.ts": `import { value } from "pkg"; export { value };`, + "/packages/pkg/index.d.ts": `export declare const value: number;`, + }, + symlinks: { + "/project/node_modules/pkg": { target: "/packages/pkg" }, + }, + }, + }); + const project = snapshot.getProject("/project/tsconfig.json")!; + assert.equal( + (project.program.getSourceFile("/packages/pkg/index.d.ts"))?.text, + `export declare const value: number;`, + ); + assert.deepEqual(callbackCalls, []); + }); + + test("full file system resolves relative symlink targets", () => { + const callbackCalls: string[] = []; + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: { + readFile: path => { + callbackCalls.push(path); + return undefined; + }, + }, + }); + + using snapshot = api.updateSnapshot({ + openProject: "/project/tsconfig.json", + fileSystem: { + kind: "full", + files: { + "/project/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true }, files: ["index.ts"] }), + "/project/index.ts": `export { value } from "./pkg";`, + "/packages/pkg/index.d.ts": `export declare const value: number;`, + }, + symlinks: { + "/project/pkg": { target: "../packages/pkg" }, + }, + }, + }); + const project = snapshot.getProject("/project/tsconfig.json")!; + assert.equal( + (project.program.getSourceFile("/project/pkg/index.d.ts"))?.text, + `export declare const value: number;`, + ); + assert.deepEqual(callbackCalls, []); + }); + + test("Snapshot.update layers filesystem edits and removals", () => { + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + }); + using snapshot = api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystem(Object.entries({ + "/tsconfig.json": JSON.stringify({ + compilerOptions: { noLib: true }, + include: ["src/**/*.ts"], + }), + "/src/keep.ts": `export const keep = true;`, + "/src/change.ts": `export const version = "old";`, + "/src/remove.ts": `export const remove = true;`, + "/src/removed/gone.ts": `export const gone = true;`, + })), + }); + + using updated = snapshot.update({ + fileSystem: createFileSystemLayer( + Object.entries({ + "/src/change.ts": `export const version = "new";`, + "/src/added.ts": `export const added = true;`, + }), + { + removedPaths: ["/src/remove.ts", "/src/removed"], + }, + ), + }); + const project = updated.getProject("/tsconfig.json")!; + assert.equal((project.program.getSourceFile("/src/keep.ts"))?.text, `export const keep = true;`); + assert.equal((project.program.getSourceFile("/src/change.ts"))?.text, `export const version = "new";`); + assert.equal((project.program.getSourceFile("/src/added.ts"))?.text, `export const added = true;`); + assert.equal(project.program.getSourceFile("/src/remove.ts"), undefined); + assert.equal(project.program.getSourceFile("/src/removed/gone.ts"), undefined); + assert.throws(() => snapshot.update(), /can only update the latest snapshot/); + + using updatedAgain = updated.update({ + fileSystem: createFileSystemLayer( + Object.entries({ + "/src/added.ts": `export const added = "updated again";`, + }), + { + removedPaths: ["/src/change.ts"], + }, + ), + }); + const updatedAgainProject = updatedAgain.getProject("/tsconfig.json")!; + assert.equal((updatedAgainProject.program.getSourceFile("/src/keep.ts"))?.text, `export const keep = true;`); + assert.equal((updatedAgainProject.program.getSourceFile("/src/added.ts"))?.text, `export const added = "updated again";`); + assert.equal(updatedAgainProject.program.getSourceFile("/src/change.ts"), undefined); + }); + + test("eager snapshot disposal does not retain filesystem history", () => { + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + }); + let snapshot: Snapshot = api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystem([ + ["/tsconfig.json", JSON.stringify({ compilerOptions: { noLib: true }, files: ["pkg/index.ts"] })], + ["/pkg/index.ts", ""], + ]), + }); + try { + let content = ""; + for (const character of "export const x = 1") { + const oldSnapshot: Snapshot = snapshot; + content += character; + snapshot = oldSnapshot.update({ + fileSystem: createFileSystemLayer([["/pkg/index.ts", content]]), + }); + oldSnapshot.dispose(); + assert.equal(oldSnapshot.isDisposed(), true); + } + + const program = snapshot.getProject("/tsconfig.json")!.program; + assert.equal((program.getSourceFile("/pkg/index.ts"))?.text, "export const x = 1"); + } + finally { + snapshot.dispose(); + } + }); + + test("Snapshot.update treats a full filesystem as a total replacement", () => { + const host = createVirtualFileSystem({ + "/host.ts": `export const source = "host";`, + }); + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: host, + }); + using snapshot = api.updateSnapshot(); + using replaced = snapshot.update({ + openProject: "/tsconfig.json", + fileSystem: createFileSystem([ + ["/tsconfig.json", JSON.stringify({ compilerOptions: { noLib: true }, files: ["memory.ts", "host.ts"] })], + ["/memory.ts", `export const source = "memory";`], + ]), + }); + const program = replaced.getProject("/tsconfig.json")!.program; + assert.equal((program.getSourceFile("/memory.ts"))?.text, `export const source = "memory";`); + assert.equal(program.getSourceFile("/host.ts"), undefined); + }); + + test("Snapshot.update applies target changes through inherited symlinks", () => { + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + }); + using snapshot = api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystem( + Object.entries({ + "/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true }, files: ["src/main.ts"] }), + "/src/main.ts": `import "./link/change"; import "./link/added"; import "./link/remove";`, + "/target/change.ts": `export const version = "old";`, + "/target/remove.ts": `export const removed = true;`, + }), + { + symlinks: { + "/src/link": { target: "/target" }, + }, + }, + ), + }); + + using updated = snapshot.update({ + fileSystem: createFileSystemLayer( + Object.entries({ + "/target/change.ts": `export const version = "new";`, + "/target/added.ts": `export const added = true;`, + }), + { + removedPaths: ["/target/remove.ts"], + }, + ), + }); + const program = updated.getProject("/tsconfig.json")!.program; + assert.equal((program.getSourceFile("/src/link/change.ts"))?.text, `export const version = "new";`); + assert.equal((program.getSourceFile("/src/link/added.ts"))?.text, `export const added = true;`); + assert.equal(program.getSourceFile("/src/link/remove.ts"), undefined); + }); + + test("full filesystem emit returns outputs without mutating the host", () => { + const hostWrites: string[] = []; + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: { + writeFile: path => { + hostWrites.push(path); + }, + }, + }); + using snapshot = api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystem(Object.entries({ + "/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true, outDir: "/out", rootDir: "/src" }, files: ["src/main.ts"] }), + "/src/main.ts": `export const value: number = 1;`, + })), + }); + const program = snapshot.getProject("/tsconfig.json")!.program; + const result = program.emit(); + assert.deepEqual(result.emittedFiles, ["/out/main.js"]); + assert.deepEqual(result.fileSystem, { + kind: "layer", + files: { + "/out/main.js": `export const value = 1;\n`, + }, + }); + assert.deepEqual(hostWrites, []); + + using updated = snapshot.update({ fileSystem: result.fileSystem!, openFiles: ["/out/main.js"] }); + const outputProject = updated.getDefaultProjectForFile("/out/main.js"); + assert.equal((updated.getProject("/tsconfig.json")!.program.getSourceFile("/src/main.ts"))?.text, `export const value: number = 1;`); + assert.equal((outputProject?.program.getSourceFile("/out/main.js"))?.text, `export const value = 1;\n`); + }); + + test("filesystem layer emit writes through to the host", () => { + const host = createVirtualFileSystem({}); + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: host, + }); + using snapshot = api.updateSnapshot({ + openProject: "/tsconfig.json", + fileSystem: createFileSystemLayer(Object.entries({ + "/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true, outDir: "/out", rootDir: "/src" }, files: ["src/main.ts"] }), + "/src/main.ts": `export const value: number = 1;`, + })), + }); + const program = snapshot.getProject("/tsconfig.json")!.program; + const result = program.emit(); + assert.equal(result.fileSystem, undefined); + assert.equal(host.readFile!("/out/main.js"), `export const value = 1;\n`); + }); + + test("full file system can link node_modules from the host", () => { + const readFileCalls: string[] = []; + const directoryExistsCalls: string[] = []; + const fileExistsCalls: string[] = []; + const host = createVirtualFileSystem({ + "/host/node_modules/pkg/index.d.ts": `export declare const value: string;`, + }); + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: { + ...host, + directoryExists: path => { + directoryExistsCalls.push(path); + return host.directoryExists!(path); + }, + fileExists: path => { + const exists = host.fileExists!(path); + fileExistsCalls.push(`${path}:${exists}`); + return exists; + }, + readFile: path => { + readFileCalls.push(path); + return host.readFile!(path); + }, + }, + }); + + using snapshot = api.updateSnapshot({ + openProject: "/project/tsconfig.json", + fileSystem: { + kind: "full", + files: { + "/project/tsconfig.json": JSON.stringify({ compilerOptions: { noLib: true, moduleResolution: "node" }, files: ["index.ts"] }), + "/project/index.ts": `import { value } from "pkg"; export { value };`, + }, + symlinks: { + "/project/node_modules": { target: "/host/node_modules", host: true }, + }, + }, + }); + const project = snapshot.getProject("/project/tsconfig.json")!; + const sourceFileNames = project.program.getSourceFileNames(); + assert.ok( + sourceFileNames.includes("/host/node_modules/pkg/index.d.ts"), + JSON.stringify({ sourceFileNames, readFileCalls, directoryExistsCalls, fileExistsCalls }), + ); + assert.equal( + (project.program.getSourceFile("/host/node_modules/pkg/index.d.ts"))?.text, + `export declare const value: string;`, + ); + assert.ok(readFileCalls.includes("/host/node_modules/pkg/index.d.ts")); + assert.ok(!readFileCalls.some(path => path.startsWith("/project/node_modules"))); + }); + + test("Snapshot.update host symlinks bypass an inherited full filesystem", () => { + const host = createVirtualFileSystem({ + "/host/node_modules/pkg/index.d.ts": `export declare const value: string;`, + }); + using api = new API({ + cwd: fileURLToPath(new URL("../../../../", import.meta.url).toString()), + fs: host, + }); + using snapshot = api.updateSnapshot({ + openProject: "/project/tsconfig.json", + fileSystem: createFileSystem([ + ["/project/tsconfig.json", JSON.stringify({ compilerOptions: { noLib: true, moduleResolution: "node" }, files: ["index.ts"] })], + ["/project/index.ts", `import { value } from "pkg"; export { value };`], + ]), + }); + using updated = snapshot.update({ + fileSystem: createFileSystemLayer([], { + symlinks: { + "/project/node_modules": { target: "/host/node_modules", host: true }, + }, + }), + }); + const project = updated.getProject("/project/tsconfig.json")!; + assert.equal( + (project.program.getSourceFile("/host/node_modules/pkg/index.d.ts"))?.text, + `export declare const value: string;`, + ); + }); + + // TODO: Add request filesystem coverage for `tsc -b` and `tsc -b --clean` + // once build and clean are exposed through the client API. In particular, + // verify that clean removes synthetic outputs and that build-mode re-timestamping + // of emitted-but-unchanged files works for full filesystems, which currently + // do not model modification times. +}); + describe("Checker - isArrayType / isTupleType", () => { test("number[] is array, not tuple", () => { using api = spawnAPI({ diff --git a/tsc/internal/api/proto.go b/tsc/internal/api/proto.go index c93a0a5b1ac9f..110293da83331 100644 --- a/tsc/internal/api/proto.go +++ b/tsc/internal/api/proto.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "github.com/microsoft/TypeScript/tsc/internal/api/requestfilesystem" "github.com/microsoft/TypeScript/tsc/internal/ast" "github.com/microsoft/TypeScript/tsc/internal/checker" "github.com/microsoft/TypeScript/tsc/internal/collections" @@ -344,6 +345,9 @@ type APIFileChanges struct { // UpdateSnapshotParams are the parameters for creating a new snapshot. // All fields are optional. With no fields set, the server adopts the latest LSP state. type UpdateSnapshotParams struct { + // Snapshot, when set, requires this to be the latest active snapshot and layers + // FileSystem over that snapshot's filesystem. Used by Snapshot.update. + Snapshot SnapshotID `json:"snapshot,omitempty"` // OpenProjects lists tsconfig.json files to open/load in the new snapshot. // Opens are ref-counted and persist across snapshots until closed. OpenProjects []DocumentIdentifier `json:"openProjects,omitempty"` @@ -352,6 +356,10 @@ type UpdateSnapshotParams struct { CloseProjects []DocumentIdentifier `json:"closeProjects,omitempty"` // FileChanges describes file system changes since the last snapshot. FileChanges *APIFileChanges `json:"fileChanges,omitempty"` + // FileSystem supplies file contents and directory listings for the new snapshot. + // A full filesystem is canonical and total. A filesystem layer is checked + // before falling back to the host filesystem. + FileSystem *requestfilesystem.RequestFileSystem `json:"fileSystem,omitempty"` // OpenFiles lists files to keep open for the API client, mirroring LSP's // textDocument/didOpen. For each file, ancestor directories are searched for a // tsconfig that contains it; if found, that configured project is loaded and @@ -1355,6 +1363,9 @@ type EmitResponse struct { EmitSkipped bool `json:"emitSkipped"` Diagnostics []*DiagnosticResponse `json:"diagnostics" nonnil:"true"` EmittedFiles []string `json:"emittedFiles" nonnil:"true"` + // EmittedFilesContents contains contents parallel to EmittedFiles when the + // source snapshot uses a full filesystem. It is empty for write-through emits. + EmittedFilesContents []string `json:"emittedFilesContents" nonnil:"true"` } type EmitOutputFile struct { diff --git a/tsc/internal/api/requestfilesystem/filechanges.go b/tsc/internal/api/requestfilesystem/filechanges.go new file mode 100644 index 0000000000000..dcba8e8a183ad --- /dev/null +++ b/tsc/internal/api/requestfilesystem/filechanges.go @@ -0,0 +1,53 @@ +package requestfilesystem + +import ( + "github.com/microsoft/TypeScript/tsc/internal/ls/lsconv" + "github.com/microsoft/TypeScript/tsc/internal/project" + "github.com/microsoft/TypeScript/tsc/internal/tspath" + "github.com/microsoft/TypeScript/tsc/internal/vfs" +) + +func addFileChanges(summary *project.FileChangeSummary, request *RequestFileSystem, baseFS vfs.FS, currentDirectory string) { + toPath := func(fileName string) tspath.Path { + return tspath.ToPath(fileName, currentDirectory, baseFS.UseCaseSensitiveFileNames()) + } + baseRequestFS := getRequestFileSystem(baseFS) + addChange := func(fileName string, deleted bool) { + uri := lsconv.FileNameToDocumentURI(fileName) + if deleted { + if baseFS.FileExists(fileName) { + summary.Deleted.Add(uri) + } + return + } + if baseFS.FileExists(fileName) { + summary.Changed.Add(uri) + } else { + summary.Created.Add(uri) + } + } + addChangeAndAliases := func(fileName string, deleted bool) { + addChange(fileName, deleted) + if baseRequestFS != nil { + for _, alias := range baseRequestFS.load().aliasesForPath(fileName) { + addChange(alias, deleted) + } + } + } + overlayFiles := make(map[tspath.Path]struct{}, len(request.Files)) + for fileName := range request.Files { + absoluteFileName := tspath.GetNormalizedAbsolutePath(fileName, currentDirectory) + overlayFiles[toPath(absoluteFileName)] = struct{}{} + addChangeAndAliases(absoluteFileName, false) + } + for _, removedPath := range request.RemovedPaths { + absoluteFileName := tspath.GetNormalizedAbsolutePath(removedPath, currentDirectory) + if _, replaced := overlayFiles[toPath(absoluteFileName)]; replaced { + continue + } + addChangeAndAliases(absoluteFileName, true) + } + if summary.Changed.Len()+summary.Created.Len()+summary.Deleted.Len() > 0 { + summary.IncludesWatchChangeOutsideNodeModules = true + } +} diff --git a/tsc/internal/api/requestfilesystem/requestfilesystem.go b/tsc/internal/api/requestfilesystem/requestfilesystem.go new file mode 100644 index 0000000000000..1a53f23670139 --- /dev/null +++ b/tsc/internal/api/requestfilesystem/requestfilesystem.go @@ -0,0 +1,1080 @@ +package requestfilesystem + +import ( + "errors" + "fmt" + "io/fs" + "maps" + "slices" + "strings" + "time" + + "github.com/microsoft/TypeScript/tsc/internal/tspath" + "github.com/microsoft/TypeScript/tsc/internal/vfs" +) + +// Kind controls how a request filesystem is used. +type Kind string + +const ( + // KindFull makes the supplied filesystem canonical and total. + KindFull Kind = "full" + // KindLayer checks the supplied filesystem before falling back to the host. + KindLayer Kind = "layer" +) + +// RequestDirectoryEntries is a cached directory listing. Entry names are +// relative to the directory, matching vfs.GetAccessibleEntries. +type RequestDirectoryEntries struct { + Files []string `json:"files" nonnil:"true"` + Directories []string `json:"directories" nonnil:"true"` +} + +// RequestSymlink describes a symbolic link in a request filesystem. +type RequestSymlink struct { + // Target is resolved relative to the directory containing the link, matching + // native symbolic-link semantics. + Target string `json:"target"` + // Host routes the target through the host filesystem. This is the only way a + // full filesystem can access paths not supplied in the request filesystem. + Host bool `json:"host,omitempty"` +} + +// RequestFileSystem supplies file contents and, optionally, directory listings +// for a request that creates a snapshot. +type RequestFileSystem struct { + Kind Kind `json:"kind"` + // Files maps file names to their complete contents. + Files map[string]string `json:"files" nonnil:"true"` + // Directories maps directory names to complete listing results. + Directories map[string]RequestDirectoryEntries `json:"directories,omitempty"` + // Symlinks maps link paths to targets in this filesystem or the host filesystem. + Symlinks map[string]RequestSymlink `json:"symlinks,omitempty"` + // RemovedPaths lists files or directory trees that must be treated as missing + // even when present in an underlying snapshot or host filesystem. + RemovedPaths []string `json:"removedPaths,omitempty"` +} + +// requestFileSystem is either a full filesystem or a layer over the session +// host filesystem. Layer misses deliberately go +// through base, which may itself be a callback filesystem. +type requestFileSystem struct { + kind Kind + base vfs.FS + layered bool + currentDirectory string + useCaseSensitiveNames bool + files map[tspath.Path]requestFile + directoryListings map[tspath.Path]vfs.Entries + symlinks map[tspath.Path]requestSymlink + removedPaths map[tspath.Path]struct{} + preSymlinkRemovedPaths map[tspath.Path]struct{} + sealedListings map[tspath.Path]struct{} + directories map[tspath.Path]string + derivedListings map[tspath.Path]*requestDirectoryBuilder +} + +type requestFile struct { + fileName string + content string +} + +type requestSymlink struct { + linkName string + target string + host bool +} + +type resolvedRequestPath struct { + path string + followedSymlink bool + host bool + ok bool +} + +type requestPathKind uint8 + +const ( + requestPathKindMissing requestPathKind = iota + requestPathKindFile + requestPathKindDirectory +) + +type requestPathLookup struct { + path string + kind requestPathKind + fileSystem vfs.FS + followedSymlink bool + ok bool +} + +type requestDirectoryBuilder struct { + files map[tspath.Path]string + directories map[tspath.Path]string +} + +func getRequestFileSystem(fileSystem vfs.FS) *Handle { + requestFileSystem, _ := fileSystem.(*Handle) + return requestFileSystem +} + +func getHostFileSystem(fileSystem vfs.FS) vfs.FS { + for { + requestFileSystem := getRequestFileSystem(fileSystem) + if requestFileSystem == nil { + return fileSystem + } + fileSystem = requestFileSystem.baseFileSystem() + } +} + +func newRequestFileSystemWorker(params *RequestFileSystem, base vfs.FS, currentDirectory string, layered bool) (*requestFileSystem, error) { + if params.Kind != KindFull && params.Kind != KindLayer { + return nil, fmt.Errorf("unknown request filesystem kind %q", params.Kind) + } + + result := requestFileSystem{ + kind: params.Kind, + base: base, + layered: layered, + currentDirectory: currentDirectory, + useCaseSensitiveNames: base.UseCaseSensitiveFileNames(), + files: make(map[tspath.Path]requestFile, len(params.Files)), + directoryListings: make(map[tspath.Path]vfs.Entries, len(params.Directories)), + symlinks: make(map[tspath.Path]requestSymlink, len(params.Symlinks)), + removedPaths: make(map[tspath.Path]struct{}, len(params.RemovedPaths)), + preSymlinkRemovedPaths: make(map[tspath.Path]struct{}), + sealedListings: make(map[tspath.Path]struct{}, len(params.Directories)), + } + for fileName, content := range params.Files { + absoluteFileName := result.toAbsolutePath(fileName) + path := result.toPath(absoluteFileName) + if existing, ok := result.files[path]; ok { + return nil, fmt.Errorf("duplicate request filesystem file path %q and %q", existing.fileName, absoluteFileName) + } + result.files[path] = requestFile{fileName: absoluteFileName, content: content} + } + for directoryName, entries := range params.Directories { + absoluteDirectoryName := result.toAbsolutePath(directoryName) + path := result.toPath(absoluteDirectoryName) + if _, ok := result.directoryListings[path]; ok { + return nil, fmt.Errorf("duplicate request filesystem directory path %q", absoluteDirectoryName) + } + result.directoryListings[path] = vfs.Entries{ + Files: slices.Clone(entries.Files), + Directories: slices.Clone(entries.Directories), + } + if !layered { + result.sealedListings[path] = struct{}{} + } + } + for linkName, symlink := range params.Symlinks { + absoluteLinkName := result.toAbsolutePath(linkName) + path := result.toPath(absoluteLinkName) + if existing, ok := result.symlinks[path]; ok { + return nil, fmt.Errorf("duplicate request filesystem symlink path %q and %q", existing.linkName, absoluteLinkName) + } + targetDirectory := tspath.GetDirectoryPath(absoluteLinkName) + absoluteTarget := result.toAbsolutePathFrom(symlink.Target, targetDirectory) + result.symlinks[path] = requestSymlink{ + linkName: absoluteLinkName, + target: absoluteTarget, + host: symlink.Host, + } + } + for _, path := range params.RemovedPaths { + result.removedPaths[result.toPath(result.toAbsolutePath(path))] = struct{}{} + } + result = result.rebuildDirectories() + return &result, nil +} + +func (s requestFileSystem) fallsBack() bool { + return s.layered || s.kind == KindLayer +} + +func (s requestFileSystem) baseFileSystem() vfs.FS { + return s.base +} + +func (s requestFileSystem) applyTo(base requestFileSystem) requestFileSystem { + files := maps.Clone(base.files) + directoryListings := make(map[tspath.Path]vfs.Entries, len(base.directoryListings)+len(s.directoryListings)) + for path, entries := range base.directoryListings { + directoryListings[path] = cloneEntries(entries) + } + symlinks := maps.Clone(base.symlinks) + removedPaths := maps.Clone(base.removedPaths) + preSymlinkRemovedPaths := maps.Clone(base.preSymlinkRemovedPaths) + sealedListings := maps.Clone(base.sealedListings) + + removeListingEntry := func(path tspath.Path) { + parentPath := s.toPath(tspath.GetDirectoryPath(string(path))) + entries, ok := directoryListings[parentPath] + if !ok { + return + } + name := tspath.GetBaseFileName(string(path)) + entries.Files = s.deleteEntryName(entries.Files, name) + entries.Directories = s.deleteEntryName(entries.Directories, name) + for existingName := range entries.Symlinks { + if s.equalEntryNames(existingName, name) { + delete(entries.Symlinks, existingName) + } + } + directoryListings[parentPath] = entries + } + removePath := func(path tspath.Path) { + removeListingEntry(path) + prefix := tspath.EnsureTrailingDirectorySeparator(string(path)) + for candidate := range files { + if candidate == path || strings.HasPrefix(string(candidate), prefix) { + delete(files, candidate) + } + } + for candidate := range symlinks { + if candidate == path || strings.HasPrefix(string(candidate), prefix) { + delete(symlinks, candidate) + } + } + for candidate := range directoryListings { + if candidate == path || strings.HasPrefix(string(candidate), prefix) { + delete(directoryListings, candidate) + delete(sealedListings, candidate) + } + } + } + clearPreSymlinkRemovedPath := func(path tspath.Path) { + for removedPath := range preSymlinkRemovedPaths { + if path == removedPath || strings.HasPrefix(string(removedPath), tspath.EnsureTrailingDirectorySeparator(string(path))) { + delete(preSymlinkRemovedPaths, removedPath) + } + } + } + for path := range s.removedPaths { + if !s.pathUsesSymlink(path) && base.pathUsesSymlink(path) { + preSymlinkRemovedPaths[path] = struct{}{} + } + removePath(path) + removedPaths[path] = struct{}{} + } + for path := range s.directories { + delete(files, path) + delete(symlinks, path) + } + for path, file := range s.files { + clearPreSymlinkRemovedPath(path) + removePath(path) + files[path] = file + } + for path, symlink := range s.symlinks { + clearPreSymlinkRemovedPath(path) + removePath(path) + symlinks[path] = symlink + } + for path, entries := range s.directoryListings { + if baseEntries, ok := directoryListings[path]; ok { + directoryListings[path] = mergeEntries(baseEntries, entries, s.equalEntryNames) + } else { + directoryListings[path] = cloneEntries(entries) + } + } + for path, builder := range s.derivedListings { + entries, ok := directoryListings[path] + if !ok { + continue + } + if _, explicit := s.directoryListings[path]; explicit { + continue + } + var overlay vfs.Entries + for _, name := range builder.files { + overlay.Files = append(overlay.Files, name) + } + for _, name := range builder.directories { + overlay.Directories = append(overlay.Directories, name) + } + directoryListings[path] = mergeEntries(entries, overlay, s.equalEntryNames) + } + + compacted := requestFileSystem{ + kind: base.kind, + base: base.base, + layered: base.layered, + currentDirectory: s.currentDirectory, + useCaseSensitiveNames: s.useCaseSensitiveNames, + files: files, + directoryListings: directoryListings, + symlinks: symlinks, + removedPaths: removedPaths, + preSymlinkRemovedPaths: preSymlinkRemovedPaths, + sealedListings: sealedListings, + } + return compacted.rebuildDirectories() +} + +func (s requestFileSystem) pathUsesSymlink(path tspath.Path) bool { + canonicalPath := string(path) + for linkPath := range s.symlinks { + canonicalLink := string(linkPath) + if canonicalPath == canonicalLink || strings.HasPrefix(canonicalPath, tspath.EnsureTrailingDirectorySeparator(canonicalLink)) { + return true + } + } + return false +} + +func (s requestFileSystem) isPreSymlinkRemoved(path string) bool { + canonicalPath := s.toPath(path) + for removedPath := range s.preSymlinkRemovedPaths { + if canonicalPath == removedPath || strings.HasPrefix(string(canonicalPath), tspath.EnsureTrailingDirectorySeparator(string(removedPath))) { + return true + } + } + return false +} + +func (s requestFileSystem) isRemoved(path string) bool { + canonicalPath := s.toPath(path) + for removedPath := range s.removedPaths { + if canonicalPath == removedPath || strings.HasPrefix(string(canonicalPath), tspath.EnsureTrailingDirectorySeparator(string(removedPath))) { + return true + } + } + return false +} + +func (s requestFileSystem) toAbsolutePath(path string) string { + return s.toAbsolutePathFrom(path, s.currentDirectory) +} + +func (s requestFileSystem) toAbsolutePathFrom(path string, currentDirectory string) string { + absolutePath := tspath.GetNormalizedAbsolutePath(path, currentDirectory) + if tspath.IsDiskPathRoot(absolutePath) { + return absolutePath + } + return tspath.RemoveTrailingDirectorySeparator(absolutePath) +} + +func (s requestFileSystem) toPath(path string) tspath.Path { + return tspath.ToPath(path, s.currentDirectory, s.useCaseSensitiveNames) +} + +func (s requestFileSystem) rebuildDirectories() requestFileSystem { + s.directories = make(map[tspath.Path]string) + s.derivedListings = make(map[tspath.Path]*requestDirectoryBuilder) + var registerDirectory func(string) + registerDirectory = func(directoryName string) { + directoryName = s.toAbsolutePath(directoryName) + directoryPath := s.toPath(directoryName) + if _, ok := s.directories[directoryPath]; ok { + return + } + s.directories[directoryPath] = directoryName + if s.derivedListings[directoryPath] == nil { + s.derivedListings[directoryPath] = &requestDirectoryBuilder{} + } + + parentName := tspath.GetDirectoryPath(directoryName) + parentPath := s.toPath(parentName) + if parentPath == directoryPath { + return + } + registerDirectory(parentName) + parent := s.derivedListings[parentPath] + if parent.directories == nil { + parent.directories = make(map[tspath.Path]string) + } + parent.directories[directoryPath] = tspath.GetBaseFileName(directoryName) + } + registerDirectory(s.currentDirectory) + for path, file := range s.files { + parentName := tspath.GetDirectoryPath(file.fileName) + parentPath := s.toPath(parentName) + registerDirectory(parentName) + listing := s.derivedListings[parentPath] + if listing.files == nil { + listing.files = make(map[tspath.Path]string) + } + listing.files[path] = tspath.GetBaseFileName(file.fileName) + } + for path, entries := range s.directoryListings { + directoryName := string(path) + registerDirectory(directoryName) + for _, child := range entries.Directories { + registerDirectory(tspath.CombinePaths(directoryName, child)) + } + } + for _, symlink := range s.symlinks { + registerDirectory(tspath.GetDirectoryPath(symlink.linkName)) + } + return s +} + +func (s requestFileSystem) resolvePath(path string) resolvedRequestPath { + path = s.toAbsolutePath(path) + result := resolvedRequestPath{path: path, ok: true} + seen := make(map[tspath.Path]struct{}, len(s.symlinks)) + for { + canonicalPath := string(s.toPath(result.path)) + var matchPath tspath.Path + var match requestSymlink + for linkPath, symlink := range s.symlinks { + canonicalLink := string(linkPath) + if canonicalPath != canonicalLink && !strings.HasPrefix(canonicalPath, tspath.EnsureTrailingDirectorySeparator(canonicalLink)) { + continue + } + // Resolve the first link encountered while walking from the root. This + // matches native path traversal when links happen to overlap. + if matchPath == "" || len(linkPath) < len(matchPath) { + matchPath = linkPath + match = symlink + } + } + if matchPath == "" { + result.host = s.isHostPath(result.path) + return result + } + if _, ok := seen[matchPath]; ok { + result.ok = false + return result + } + seen[matchPath] = struct{}{} + result.followedSymlink = true + suffix, ok := tspath.TrimFilePathPrefix(result.path, match.linkName, s.useCaseSensitiveNames) + if !ok { + result.ok = false + return result + } + result.path = s.toAbsolutePath(match.target + suffix) + if match.host { + result.host = true + return result + } + } +} + +// resolvePathForOverlay resolves the effective path through this snapshot layer +// and any underlying snapshot layers, stopping when this layer supplies or removes +// the resolved path. Callers in a newer layer use this to apply their own entries +// to targets of inherited symlinks before delegating the operation to the base. +func (s requestFileSystem) resolvePathForOverlay(path string) resolvedRequestPath { + resolved := s.resolvePath(path) + if !resolved.ok || resolved.host { + return resolved + } + if _, ok := s.fileAt(resolved.path); ok { + return resolved + } + if _, ok := s.directoryAt(resolved.path); ok { + return resolved + } + if !resolved.followedSymlink && s.isRemoved(path) || s.isRemoved(resolved.path) || !s.fallsBack() { + return resolved + } + baseResolved := s.resolveBasePath(resolved.path) + baseResolved.followedSymlink = baseResolved.followedSymlink || resolved.followedSymlink + return baseResolved +} + +func (s requestFileSystem) resolveBasePath(path string) resolvedRequestPath { + if base := getRequestFileSystem(s.baseFileSystem()); base != nil { + return base.load().resolvePathForOverlay(path) + } + return resolvedRequestPath{path: path, ok: true} +} + +func (s requestFileSystem) isHostPath(path string) bool { + canonicalPath := string(s.toPath(path)) + for _, symlink := range s.symlinks { + if !symlink.host { + continue + } + canonicalTarget := string(s.toPath(symlink.target)) + if canonicalPath == canonicalTarget || strings.HasPrefix(canonicalPath, tspath.EnsureTrailingDirectorySeparator(canonicalTarget)) { + return true + } + } + return false +} + +func (s requestFileSystem) aliasesForPath(path string) []string { + symlinks := make([]requestSymlink, 0, len(s.symlinks)) + for current := s; ; { + for _, symlink := range current.symlinks { + symlinks = append(symlinks, symlink) + } + base := getRequestFileSystem(current.baseFileSystem()) + if base == nil { + break + } + current = *base.load() + } + + seen := map[tspath.Path]struct{}{s.toPath(path): {}} + queue := []string{s.toAbsolutePath(path)} + var aliases []string + for len(queue) > 0 { + candidate := queue[0] + queue = queue[1:] + for _, symlink := range symlinks { + suffix, ok := tspath.TrimFilePathPrefix(candidate, symlink.target, s.useCaseSensitiveNames) + if !ok || suffix != "" && !tspath.HasTrailingDirectorySeparator(symlink.target) && !strings.HasPrefix(suffix, "/") { + continue + } + alias := s.toAbsolutePath(symlink.linkName + suffix) + aliasPath := s.toPath(alias) + if _, ok := seen[aliasPath]; ok { + continue + } + seen[aliasPath] = struct{}{} + aliases = append(aliases, alias) + queue = append(queue, alias) + } + } + return aliases +} + +func (s requestFileSystem) fileAt(path string) (requestFile, bool) { + file, ok := s.files[s.toPath(path)] + return file, ok +} + +func (s requestFileSystem) directoryAt(path string) (string, bool) { + directory, ok := s.directories[s.toPath(path)] + return directory, ok +} + +func (s requestFileSystem) pathKind(path string) requestPathKind { + if _, ok := s.fileAt(path); ok { + return requestPathKindFile + } + if _, ok := s.directoryAt(path); ok { + return requestPathKindDirectory + } + return requestPathKindMissing +} + +func (s requestFileSystem) lookupPath(path string) requestPathLookup { + absolutePath := s.toAbsolutePath(path) + if kind := s.pathKind(absolutePath); kind != requestPathKindMissing { + return requestPathLookup{path: absolutePath, kind: kind, ok: true} + } + if s.isPreSymlinkRemoved(path) { + return requestPathLookup{} + } + resolved := s.resolvePath(path) + if !resolved.ok { + return requestPathLookup{} + } + result := requestPathLookup{ + path: resolved.path, + followedSymlink: resolved.followedSymlink, + ok: true, + } + if resolved.host { + if s.isRemoved(resolved.path) { + return requestPathLookup{} + } + result.fileSystem = getHostFileSystem(s.baseFileSystem()) + result.ok = result.fileSystem != nil + return result + } + if kind := s.pathKind(resolved.path); kind != requestPathKindMissing { + result.kind = kind + return result + } + if !resolved.followedSymlink && s.isRemoved(path) { + return requestPathLookup{} + } + if s.fallsBack() { + fallback := s.resolveBasePath(resolved.path) + if !fallback.ok { + return requestPathLookup{} + } + if fallback.host { + if s.isRemoved(resolved.path) || s.isRemoved(fallback.path) { + return requestPathLookup{} + } + result.path = fallback.path + result.fileSystem = getHostFileSystem(s.baseFileSystem()) + result.ok = result.fileSystem != nil + return result + } + if kind := s.pathKind(fallback.path); kind != requestPathKindMissing { + result.path = fallback.path + result.kind = kind + return result + } + if s.isRemoved(resolved.path) || s.isRemoved(fallback.path) { + return requestPathLookup{} + } + result.fileSystem = s.baseFileSystem() + } + return result +} + +func (s requestFileSystem) mutationPath(path string) (vfs.FS, string, bool) { + if s.kind != KindLayer { + return nil, "", false + } + resolved := s.resolvePathForOverlay(path) + if !resolved.ok { + return nil, "", false + } + host := getHostFileSystem(s.baseFileSystem()) + return host, resolved.path, host != nil +} + +func cloneEntries(entries vfs.Entries) vfs.Entries { + result := vfs.Entries{ + Files: slices.Clone(entries.Files), + Directories: slices.Clone(entries.Directories), + } + if entries.Symlinks != nil { + result.Symlinks = make(map[string]struct{}, len(entries.Symlinks)) + for name := range entries.Symlinks { + result.Symlinks[name] = struct{}{} + } + } + return result +} + +func (s requestFileSystem) UseCaseSensitiveFileNames() bool { + return s.useCaseSensitiveNames +} + +func (s requestFileSystem) ReadFile(fileName string) (string, bool) { + lookup := s.lookupPath(fileName) + if !lookup.ok || lookup.kind == requestPathKindDirectory { + return "", false + } + if lookup.fileSystem != nil { + return lookup.fileSystem.ReadFile(lookup.path) + } + if file, ok := s.fileAt(lookup.path); ok { + return file.content, true + } + return "", false +} + +func (s requestFileSystem) FileExists(fileName string) bool { + lookup := s.lookupPath(fileName) + if !lookup.ok || lookup.kind == requestPathKindDirectory { + return false + } + return lookup.kind == requestPathKindFile || lookup.fileSystem != nil && lookup.fileSystem.FileExists(lookup.path) +} + +func (s requestFileSystem) DirectoryExists(directoryName string) bool { + lookup := s.lookupPath(directoryName) + if !lookup.ok || lookup.kind == requestPathKindFile { + return false + } + return lookup.kind == requestPathKindDirectory || lookup.fileSystem != nil && lookup.fileSystem.DirectoryExists(lookup.path) +} + +func (s requestFileSystem) GetAccessibleEntries(directoryName string) vfs.Entries { + if s.isPreSymlinkRemoved(directoryName) { + if entries, _, ok := s.getLocalEntries(directoryName); ok { + return s.addSymlinkEntries(directoryName, entries) + } + return vfs.Entries{Symlinks: map[string]struct{}{}} + } + resolved := s.resolvePath(directoryName) + if !resolved.ok { + return vfs.Entries{Symlinks: map[string]struct{}{}} + } + if _, ok := s.fileAt(resolved.path); ok { + return vfs.Entries{Symlinks: map[string]struct{}{}} + } + + localEntries, hasExplicitListing, hasLocalEntries := s.getLocalEntries(resolved.path) + sealedListing := s.hasSealedListing(resolved.path) + if !resolved.followedSymlink && s.isRemoved(directoryName) && !hasLocalEntries { + return vfs.Entries{Symlinks: map[string]struct{}{}} + } + fallbackPath := resolved.path + fallbackHost := false + if !resolved.host && s.fallsBack() { + fallback := s.resolveBasePath(resolved.path) + if !fallback.ok { + return vfs.Entries{Symlinks: map[string]struct{}{}} + } + fallbackPath = fallback.path + fallbackHost = fallback.host + if !fallbackHost { + if _, ok := s.fileAt(fallbackPath); ok { + return vfs.Entries{Symlinks: map[string]struct{}{}} + } + } + if !fallbackHost && s.toPath(fallbackPath) != s.toPath(resolved.path) { + targetEntries, targetExplicit, targetLocal := s.getLocalEntries(fallbackPath) + if targetLocal { + localEntries = mergeEntries(localEntries, targetEntries, s.equalEntryNames) + hasLocalEntries = true + } + hasExplicitListing = hasExplicitListing || targetExplicit + sealedListing = sealedListing || s.hasSealedListing(fallbackPath) + } + } + var result vfs.Entries + if resolved.host || fallbackHost { + hostPath := resolved.path + if fallbackHost { + hostPath = fallbackPath + } + if !s.isRemoved(directoryName) && !s.isRemoved(resolved.path) && !s.isRemoved(hostPath) { + if host := getHostFileSystem(s.baseFileSystem()); host != nil { + result = s.removeEntries(directoryName, host.GetAccessibleEntries(hostPath)) + if s.toPath(directoryName) != s.toPath(resolved.path) { + result = s.removeEntries(resolved.path, result) + } + if s.toPath(hostPath) != s.toPath(resolved.path) { + result = s.removeEntries(hostPath, result) + } + } + } + if hasLocalEntries { + result = mergeEntries(result, localEntries, s.equalEntryNames) + } + } else if !s.fallsBack() || hasExplicitListing && sealedListing { + result = localEntries + } else { + if !s.isRemoved(directoryName) && !s.isRemoved(resolved.path) && !s.isRemoved(fallbackPath) { + result = s.removeEntries(directoryName, s.baseFileSystem().GetAccessibleEntries(resolved.path)) + if s.toPath(directoryName) != s.toPath(resolved.path) { + result = s.removeEntries(resolved.path, result) + } + if s.toPath(fallbackPath) != s.toPath(resolved.path) { + result = s.removeEntries(fallbackPath, result) + } + } + if hasLocalEntries { + result = mergeEntries(result, localEntries, s.equalEntryNames) + } + } + result = s.addSymlinkEntries(resolved.path, result) + if !fallbackHost && s.toPath(fallbackPath) != s.toPath(resolved.path) { + result = s.addSymlinkEntries(fallbackPath, result) + } + result = s.removePreSymlinkEntries(directoryName, result) + return result +} + +func (s requestFileSystem) removePreSymlinkEntries(directoryName string, entries vfs.Entries) vfs.Entries { + result := cloneEntries(entries) + filter := func(values []string) []string { + return slices.DeleteFunc(values, func(name string) bool { + fileName := tspath.CombinePaths(directoryName, name) + if _, ok := s.fileAt(fileName); ok { + return false + } + if _, ok := s.directoryAt(fileName); ok { + return false + } + path := s.toPath(fileName) + for removedPath := range s.preSymlinkRemovedPaths { + if path == removedPath || strings.HasPrefix(string(path), tspath.EnsureTrailingDirectorySeparator(string(removedPath))) { + return true + } + } + return false + }) + } + result.Files = filter(result.Files) + result.Directories = filter(result.Directories) + for name := range result.Symlinks { + if len(filter([]string{name})) == 0 { + delete(result.Symlinks, name) + } + } + return result +} + +func (s requestFileSystem) hasSealedListing(directoryName string) bool { + _, ok := s.sealedListings[s.toPath(directoryName)] + return ok +} + +func (s requestFileSystem) getLocalEntries(directoryName string) (entries vfs.Entries, explicit bool, ok bool) { + path := s.toPath(directoryName) + if listing, ok := s.directoryListings[path]; ok { + return cloneEntries(listing), true, true + } + builder := s.derivedListings[path] + if builder == nil { + return vfs.Entries{}, false, false + } + for _, name := range builder.files { + entries.Files = append(entries.Files, name) + } + for _, name := range builder.directories { + entries.Directories = append(entries.Directories, name) + } + slices.Sort(entries.Files) + slices.Sort(entries.Directories) + return entries, false, true +} + +func mergeEntries(base vfs.Entries, overlay vfs.Entries, equal func(string, string) bool) vfs.Entries { + result := cloneEntries(base) + if result.Symlinks == nil { + result.Symlinks = map[string]struct{}{} + } + deleteSymlink := func(name string) { + for existingName := range result.Symlinks { + if equal(existingName, name) { + delete(result.Symlinks, existingName) + } + } + } + addFile := func(name string) { + result.Directories = slices.DeleteFunc(result.Directories, func(value string) bool { return equal(value, name) }) + if !slices.ContainsFunc(result.Files, func(value string) bool { return equal(value, name) }) { + result.Files = append(result.Files, name) + } + deleteSymlink(name) + } + addDirectory := func(name string) { + result.Files = slices.DeleteFunc(result.Files, func(value string) bool { return equal(value, name) }) + if !slices.ContainsFunc(result.Directories, func(value string) bool { return equal(value, name) }) { + result.Directories = append(result.Directories, name) + } + deleteSymlink(name) + } + for _, name := range overlay.Files { + addFile(name) + } + for _, name := range overlay.Directories { + addDirectory(name) + } + for name := range overlay.Symlinks { + result.Symlinks[name] = struct{}{} + } + slices.Sort(result.Files) + slices.Sort(result.Directories) + return result +} + +func (s requestFileSystem) removeEntries(directoryName string, entries vfs.Entries) vfs.Entries { + result := cloneEntries(entries) + filter := func(values []string) []string { + return slices.DeleteFunc(values, func(name string) bool { + return s.isRemoved(tspath.CombinePaths(directoryName, name)) + }) + } + result.Files = filter(result.Files) + result.Directories = filter(result.Directories) + for name := range result.Symlinks { + if s.isRemoved(tspath.CombinePaths(directoryName, name)) { + delete(result.Symlinks, name) + } + } + return result +} + +func (s requestFileSystem) addSymlinkEntries(directoryName string, entries vfs.Entries) vfs.Entries { + result := cloneEntries(entries) + if result.Symlinks == nil { + result.Symlinks = map[string]struct{}{} + } + + directoryPath := s.toPath(directoryName) + var links []requestSymlink + for _, symlink := range s.symlinks { + if s.toPath(tspath.GetDirectoryPath(symlink.linkName)) == directoryPath { + links = append(links, symlink) + } + } + for _, symlink := range links { + name := tspath.GetBaseFileName(symlink.linkName) + result.Files = s.deleteEntryName(result.Files, name) + result.Directories = s.deleteEntryName(result.Directories, name) + for existingName := range result.Symlinks { + if s.equalEntryNames(existingName, name) { + delete(result.Symlinks, existingName) + } + } + if s.DirectoryExists(symlink.linkName) { + result.Directories = append(result.Directories, name) + result.Symlinks[name] = struct{}{} + } else if s.FileExists(symlink.linkName) { + result.Files = append(result.Files, name) + result.Symlinks[name] = struct{}{} + } + } + slices.Sort(result.Files) + slices.Sort(result.Directories) + return result +} + +func (s requestFileSystem) deleteEntryName(values []string, value string) []string { + return slices.DeleteFunc(values, func(candidate string) bool { return s.equalEntryNames(candidate, value) }) +} + +func (s requestFileSystem) equalEntryNames(left string, right string) bool { + return tspath.GetCanonicalFileName(left, s.useCaseSensitiveNames) == tspath.GetCanonicalFileName(right, s.useCaseSensitiveNames) +} + +func (s requestFileSystem) Realpath(path string) string { + lookup := s.lookupPath(path) + if !lookup.ok { + return path + } + if lookup.fileSystem != nil { + return lookup.fileSystem.Realpath(lookup.path) + } + if lookup.kind != requestPathKindMissing || !lookup.followedSymlink { + return lookup.path + } + return path +} + +func (s requestFileSystem) WriteFile(fileName string, data string) error { + host, path, ok := s.mutationPath(fileName) + if !ok { + return vfs.ErrInvalid + } + return host.WriteFile(path, data) +} + +func (s requestFileSystem) AppendFile(fileName string, data string) error { + host, path, ok := s.mutationPath(fileName) + if !ok { + return vfs.ErrInvalid + } + return host.AppendFile(path, data) +} + +func (s requestFileSystem) Remove(path string) error { + host, path, ok := s.mutationPath(path) + if !ok { + return vfs.ErrInvalid + } + return host.Remove(path) +} + +func (s requestFileSystem) Chtimes(path string, aTime time.Time, mTime time.Time) error { + host, path, ok := s.mutationPath(path) + if !ok { + return vfs.ErrInvalid + } + return host.Chtimes(path, aTime, mTime) +} + +func (s requestFileSystem) Stat(path string) vfs.FileInfo { + lookup := s.lookupPath(path) + if !lookup.ok { + return nil + } + if lookup.fileSystem != nil { + return statFileSystem(lookup.fileSystem, lookup.path) + } + if lookup.kind == requestPathKindFile { + file, _ := s.fileAt(lookup.path) + return requestFileInfo{name: tspath.GetBaseFileName(file.fileName), size: int64(len(file.content))} + } + if lookup.kind == requestPathKindDirectory { + directoryName, _ := s.directoryAt(lookup.path) + return requestFileInfo{name: tspath.GetBaseFileName(directoryName), directory: true} + } + return nil +} + +func statFileSystem(fileSystem vfs.FS, path string) vfs.FileInfo { + if fileSystem == nil { + return nil + } + if info := fileSystem.Stat(path); info != nil { + return info + } + name := tspath.GetBaseFileName(path) + if fileSystem.DirectoryExists(path) { + return requestFileInfo{name: name, directory: true} + } + if fileSystem.FileExists(path) { + return requestFileInfo{name: name} + } + return nil +} + +func (s requestFileSystem) WalkDir(root string, walkFn vfs.WalkDirFunc) error { + originalRoot := s.toAbsolutePath(root) + resolved := s.resolvePath(originalRoot) + if !resolved.ok { + return walkFn(originalRoot, nil, vfs.ErrNotExist) + } + info := s.Stat(originalRoot) + if info == nil { + return walkFn(originalRoot, nil, vfs.ErrNotExist) + } + visited := map[string]struct{}{} + if err := s.walkDir(originalRoot, requestDirEntry{info: info}, walkFn, visited); errors.Is(err, fs.SkipAll) { + return nil + } else { + return err + } +} + +func (s requestFileSystem) walkDir(path string, entry requestDirEntry, walkFn vfs.WalkDirFunc, visited map[string]struct{}) error { + realpath := s.Realpath(path) + if _, ok := visited[realpath]; ok { + return nil + } + visited[realpath] = struct{}{} + err := walkFn(path, entry, nil) + if err != nil { + if errors.Is(err, fs.SkipDir) && entry.IsDir() { + return nil + } + return err + } + if !entry.IsDir() { + return nil + } + entries := s.GetAccessibleEntries(path) + names := append(slices.Clone(entries.Directories), entries.Files...) + slices.Sort(names) + for _, name := range names { + childPath := tspath.CombinePaths(path, name) + childInfo := s.Stat(childPath) + if childInfo == nil { + continue + } + if err := s.walkDir(childPath, requestDirEntry{info: childInfo}, walkFn, visited); err != nil { + if errors.Is(err, fs.SkipDir) { + return nil + } + return err + } + } + return nil +} + +type requestFileInfo struct { + name string + size int64 + directory bool +} + +func (i requestFileInfo) Name() string { return i.name } +func (i requestFileInfo) Size() int64 { return i.size } +func (i requestFileInfo) ModTime() time.Time { return time.Time{} } +func (i requestFileInfo) IsDir() bool { return i.directory } +func (i requestFileInfo) Sys() any { return nil } +func (i requestFileInfo) Mode() fs.FileMode { + if i.directory { + return fs.ModeDir | 0o555 + } + return 0o444 +} + +type requestDirEntry struct { + info vfs.FileInfo +} + +func (e requestDirEntry) Name() string { return e.info.Name() } +func (e requestDirEntry) IsDir() bool { return e.info.IsDir() } +func (e requestDirEntry) Type() fs.FileMode { return e.info.Mode().Type() } +func (e requestDirEntry) Info() (fs.FileInfo, error) { return e.info, nil } diff --git a/tsc/internal/api/requestfilesystem/requestfilesystem_test.go b/tsc/internal/api/requestfilesystem/requestfilesystem_test.go new file mode 100644 index 0000000000000..82643c68462fe --- /dev/null +++ b/tsc/internal/api/requestfilesystem/requestfilesystem_test.go @@ -0,0 +1,1150 @@ +package requestfilesystem + +import ( + "sync" + "testing" + "time" + + "github.com/microsoft/TypeScript/tsc/internal/project" + "github.com/microsoft/TypeScript/tsc/internal/vfs" + "github.com/microsoft/TypeScript/tsc/internal/vfs/trackingvfs" + "github.com/microsoft/TypeScript/tsc/internal/vfs/vfstest" + "gotest.tools/v3/assert" +) + +func newRequestFileSystem(params *RequestFileSystem, base vfs.FS, currentDirectory string) (*Handle, error) { + handle := &Handle{} + if err := handle.initializeFromRequest(params, base, currentDirectory); err != nil { + return nil, err + } + return handle, nil +} + +func newLayeredRequestFileSystem(params *RequestFileSystem, base vfs.FS, currentDirectory string) (*Handle, error) { + handle := &Handle{} + if err := handle.initializeLayered(params, base, currentDirectory); err != nil { + return nil, err + } + return handle, nil +} + +func (h *Handle) applyTo(base *Handle) { + h.mu.Lock() + defer h.mu.Unlock() + base.mu.Lock() + delete(base.dependents, h) + value := h.load().applyTo(*base.load()) + base.mu.Unlock() + h.value.Store(&value) + h.registerWithBaseLocked() +} + +func TestInitializeForUpdate(t *testing.T) { + t.Parallel() + + t.Run("filesystem layers over a host-backed snapshot", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{ + "/dir/host.ts": "host", + }, true) + var handle Handle + var fileChanges project.FileChangeSummary + err := handle.InitializeForUpdate(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{"/dir/cached.ts": "cached"}, + Directories: map[string]RequestDirectoryEntries{ + "/dir": {Files: []string{"cached.ts"}, Directories: []string{}}, + }, + }, nil, host, "/", &fileChanges, true) + assert.NilError(t, err) + assert.DeepEqual(t, handle.GetAccessibleEntries("/dir").Files, []string{"cached.ts", "host.ts"}) + }) + + t.Run("memory starts a new chain", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{ + "/host.ts": "host", + }, true) + base, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{"/base.ts": "base"}, + }, host, "/") + assert.NilError(t, err) + + var handle Handle + var fileChanges project.FileChangeSummary + err = handle.InitializeForUpdate(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{"/replacement.ts": "replacement"}, + }, base, host, "/", &fileChanges, true) + assert.NilError(t, err) + assert.Assert(t, handle.baseFileSystem() == host) + assert.Assert(t, getRequestFileSystem(handle.baseFileSystem()) == nil) + }) +} + +func TestConcurrentCloneAndRelease(t *testing.T) { + t.Parallel() + + host := vfstest.FromMap(map[string]string{}, true) + for range 100 { + base, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{"/base.ts": "base"}, + }, host, "/") + assert.NilError(t, err) + layered, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{"/layered.ts": "layered"}, + }, base, "/") + assert.NilError(t, err) + + var clone Handle + start := make(chan struct{}) + var waitGroup sync.WaitGroup + waitGroup.Go(func() { + <-start + clone.CloneFrom(layered) + }) + waitGroup.Go(func() { + <-start + base.Release() + }) + close(start) + waitGroup.Wait() + + layered.Release() + contents, ok := clone.ReadFile("/base.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "base") + contents, ok = clone.ReadFile("/layered.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "layered") + clone.Release() + } +} + +func TestRequestFileSystem(t *testing.T) { + t.Parallel() + + t.Run("compaction preserves host fallback", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{ + "/host.ts": "host", + }, true) + baseFS, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + }, host, "/") + assert.NilError(t, err) + base := getRequestFileSystem(baseFS) + assert.Assert(t, base != nil) + assert.Assert(t, base.baseFileSystem() == host) + assert.Assert(t, !base.FileExists("/created-after-base.ts")) + assert.NilError(t, host.WriteFile("/created-after-base.ts", "created")) + + layeredFS, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{"/layered.ts": "layered"}, + }, baseFS, "/") + assert.NilError(t, err) + layered := getRequestFileSystem(layeredFS) + assert.Assert(t, layered != nil) + assert.Assert(t, layered.baseFileSystem() == baseFS) + assert.Assert(t, layered.FileExists("/created-after-base.ts")) + assert.NilError(t, host.Remove("/created-after-base.ts")) + + layered.applyTo(base) + + assert.Assert(t, layered.baseFileSystem() == host) + assert.Assert(t, !layered.FileExists("/created-after-base.ts")) + contents, ok := layered.ReadFile("/host.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "host") + }) + + t.Run("memory is total and never falls back", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{ + "/host.ts": "host", + }, true)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/src/index.ts": "memory", + }, + }, base, "/") + assert.NilError(t, err) + + contents, ok := fileSystem.ReadFile("/src/index.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "memory") + assert.Assert(t, fileSystem.FileExists("/src/index.ts")) + assert.Assert(t, fileSystem.DirectoryExists("/src")) + assert.DeepEqual(t, fileSystem.GetAccessibleEntries("/src").Files, []string{"index.ts"}) + + _, ok = fileSystem.ReadFile("/host.ts") + assert.Assert(t, !ok) + assert.Assert(t, !fileSystem.FileExists("/host.ts")) + assert.Assert(t, !base.SeenFiles.Has("/host.ts")) + }) + + t.Run("cache hits bypass the host and misses fall back", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{ + "/fallback.ts": "fallback", + }, true)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/cached/index.ts": "cached", + }, + Directories: map[string]RequestDirectoryEntries{ + "/cached": {Files: []string{"index.ts"}, Directories: []string{}}, + }, + }, base, "/") + assert.NilError(t, err) + + contents, ok := fileSystem.ReadFile("/cached/index.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "cached") + assert.Assert(t, fileSystem.FileExists("/cached/index.ts")) + assert.Assert(t, fileSystem.DirectoryExists("/cached")) + assert.DeepEqual(t, fileSystem.GetAccessibleEntries("/cached").Files, []string{"index.ts"}) + assert.Assert(t, !base.SeenFiles.Has("/cached/index.ts")) + assert.Assert(t, !base.SeenFiles.Has("/cached")) + + contents, ok = fileSystem.ReadFile("/fallback.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "fallback") + assert.Assert(t, base.SeenFiles.Has("/fallback.ts")) + }) + + t.Run("layered memory is a total replacement", func(t *testing.T) { + t.Parallel() + fileSystem, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/memory.ts": "memory", + }, + }, vfstest.FromMap(map[string]string{"/host.ts": "host"}, true), "/") + assert.NilError(t, err) + contents, ok := fileSystem.ReadFile("/memory.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "memory") + _, ok = fileSystem.ReadFile("/host.ts") + assert.Assert(t, !ok) + }) + + t.Run("memory resolves internal file and directory symlinks", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{ + "/host.ts": "host", + }, true)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/packages/pkg/index.d.ts": "export declare const value: number;", + }, + Symlinks: map[string]RequestSymlink{ + "/project/node_modules/pkg": {Target: "../../../packages/pkg"}, + "/project/pkg.d.ts": {Target: "../packages/pkg/index.d.ts"}, + }, + }, base, "/") + assert.NilError(t, err) + + contents, ok := fileSystem.ReadFile("/project/node_modules/pkg/index.d.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "export declare const value: number;") + contents, ok = fileSystem.ReadFile("/project/pkg.d.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "export declare const value: number;") + assert.Equal(t, fileSystem.Realpath("/project/node_modules/pkg/index.d.ts"), "/packages/pkg/index.d.ts") + + entries := fileSystem.GetAccessibleEntries("/project/node_modules") + assert.DeepEqual(t, entries.Directories, []string{"pkg"}) + _, isSymlink := entries.Symlinks["pkg"] + assert.Assert(t, isSymlink) + entries = fileSystem.GetAccessibleEntries("/project") + assert.DeepEqual(t, entries.Files, []string{"pkg.d.ts"}) + _, isSymlink = entries.Symlinks["pkg.d.ts"] + assert.Assert(t, isSymlink) + assert.Assert(t, base.SeenFiles.IsEmpty()) + }) + + t.Run("cache resolves internal symlinks before the host", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{ + "/packages/pkg/index.d.ts": "host content", + }, true)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/packages/pkg/index.d.ts": "cached content", + }, + Directories: map[string]RequestDirectoryEntries{ + "/project/node_modules": {Files: []string{}, Directories: []string{}}, + }, + Symlinks: map[string]RequestSymlink{ + "/project/node_modules/pkg": {Target: "/packages/pkg"}, + }, + }, base, "/") + assert.NilError(t, err) + + contents, ok := fileSystem.ReadFile("/project/node_modules/pkg/index.d.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "cached content") + assert.Equal(t, fileSystem.Realpath("/project/node_modules/pkg/index.d.ts"), "/packages/pkg/index.d.ts") + entries := fileSystem.GetAccessibleEntries("/project/node_modules") + assert.DeepEqual(t, entries.Directories, []string{"pkg"}) + _, isSymlink := entries.Symlinks["pkg"] + assert.Assert(t, isSymlink) + assert.Assert(t, base.SeenFiles.IsEmpty()) + }) + + t.Run("cache file shadows underlying symlink realpath", func(t *testing.T) { + t.Parallel() + base := vfstest.FromMap(map[string]any{ + "/project/node_modules/pkg": vfstest.Symlink("/host/pkg"), + "/host/pkg/index.d.ts": "host content", + }, true) + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/project/node_modules/pkg/index.d.ts": "cached content", + }, + }, base, "/") + assert.NilError(t, err) + + contents, ok := fileSystem.ReadFile("/project/node_modules/pkg/index.d.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "cached content") + assert.Equal( + t, + fileSystem.Realpath("/project/node_modules/pkg/index.d.ts"), + "/project/node_modules/pkg/index.d.ts", + ) + }) + + t.Run("layered cache adds changes and blocks removed entries", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{}, true) + base, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/keep.ts": "keep", + "/change.ts": "old", + "/remove.ts": "remove", + "/removed-dir/gone.ts": "gone", + "/becomes-file/child.ts": "child", + "/becomes-directory.ts": "file", + }, + }, host, "/") + assert.NilError(t, err) + + layered, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/change.ts": "new", + "/added.ts": "added", + "/remove.ts": "replacement", + "/removed-dir/replacement.ts": "replacement", + "/becomes-file": "file", + "/becomes-directory.ts/child.ts": "child", + }, + Directories: map[string]RequestDirectoryEntries{ + "/": {Files: []string{"added.ts", "becomes-file", "change.ts", "remove.ts"}, Directories: []string{"becomes-directory.ts", "removed-dir"}}, + }, + RemovedPaths: []string{"/remove.ts", "/removed-dir"}, + }, base, "/") + assert.NilError(t, err) + + for path, expected := range map[string]string{ + "/keep.ts": "keep", + "/change.ts": "new", + "/added.ts": "added", + "/remove.ts": "replacement", + "/removed-dir/replacement.ts": "replacement", + "/becomes-file": "file", + "/becomes-directory.ts/child.ts": "child", + } { + contents, ok := layered.ReadFile(path) + assert.Assert(t, ok, path) + assert.Equal(t, contents, expected) + } + assert.Assert(t, layered.FileExists("/remove.ts")) + assert.Assert(t, layered.DirectoryExists("/removed-dir")) + assert.Assert(t, !layered.FileExists("/removed-dir/gone.ts")) + assert.Assert(t, layered.Stat("/remove.ts") != nil) + assert.Assert(t, layered.Stat("/removed-dir/replacement.ts") != nil) + assert.Equal(t, layered.Realpath("/removed-dir/replacement.ts"), "/removed-dir/replacement.ts") + assert.Assert(t, layered.FileExists("/becomes-file")) + assert.Assert(t, !layered.DirectoryExists("/becomes-file")) + assert.Assert(t, !layered.FileExists("/becomes-directory.ts")) + assert.Assert(t, layered.DirectoryExists("/becomes-directory.ts")) + assert.DeepEqual(t, layered.GetAccessibleEntries("/").Files, []string{"added.ts", "becomes-file", "change.ts", "keep.ts", "remove.ts"}) + assert.DeepEqual(t, layered.GetAccessibleEntries("/").Directories, []string{"becomes-directory.ts", "removed-dir"}) + }) + + t.Run("new layers override targets of inherited symlinks", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{}, true) + base, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/target/change.ts": "old", + "/target/keep.ts": "keep", + "/target/remove.ts": "remove", + }, + Symlinks: map[string]RequestSymlink{ + "/link": {Target: "/target"}, + }, + }, host, "/") + assert.NilError(t, err) + + layered, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/target/change.ts": "new", + "/target/added.ts": "added", + }, + RemovedPaths: []string{"/target/remove.ts"}, + }, base, "/") + assert.NilError(t, err) + + contents, ok := layered.ReadFile("/link/change.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "new") + contents, ok = layered.ReadFile("/link/added.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "added") + _, ok = layered.ReadFile("/link/remove.ts") + assert.Assert(t, !ok) + assert.DeepEqual(t, layered.GetAccessibleEntries("/link").Files, []string{"added.ts", "change.ts", "keep.ts"}) + }) + + t.Run("alias tombstones take precedence over inherited symlink targets", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{}, true) + base, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/target/file.ts": "old", + }, + Symlinks: map[string]RequestSymlink{ + "/link": {Target: "/target"}, + }, + }, host, "/") + assert.NilError(t, err) + + layered, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/target/file.ts": "new", + }, + RemovedPaths: []string{"/link"}, + }, base, "/") + assert.NilError(t, err) + + _, ok := layered.ReadFile("/link/file.ts") + assert.Assert(t, !ok) + assert.Assert(t, !layered.FileExists("/link/file.ts")) + assert.Assert(t, !layered.DirectoryExists("/link")) + assert.Assert(t, layered.Stat("/link/file.ts") == nil) + assert.Equal(t, len(layered.GetAccessibleEntries("/link").Files), 0) + }) + + t.Run("compaction preserves overlays addressed through inherited symlinks", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{}, true) + baseFS, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/target/remove.ts": "remove", + }, + Symlinks: map[string]RequestSymlink{ + "/link": {Target: "/target"}, + }, + }, host, "/") + assert.NilError(t, err) + base := getRequestFileSystem(baseFS) + + layeredFS, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{}, + RemovedPaths: []string{"/link/remove.ts"}, + }, baseFS, "/") + assert.NilError(t, err) + layered := getRequestFileSystem(layeredFS) + + _, ok := layered.ReadFile("/link/remove.ts") + assert.Assert(t, !ok) + + layered.applyTo(base) + + _, ok = layered.ReadFile("/link/remove.ts") + assert.Assert(t, !ok) + }) + + t.Run("compaction removes tombstones from explicit listings", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{}, true) + baseFS, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/dir/remove.ts": "remove", + }, + Directories: map[string]RequestDirectoryEntries{ + "/dir": {Files: []string{"remove.ts"}, Directories: []string{}}, + }, + }, host, "/") + assert.NilError(t, err) + base := getRequestFileSystem(baseFS) + + layeredFS, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{}, + RemovedPaths: []string{"/dir/remove.ts"}, + }, baseFS, "/") + assert.NilError(t, err) + layered := getRequestFileSystem(layeredFS) + assert.Equal(t, len(layered.GetAccessibleEntries("/dir").Files), 0) + + layered.applyTo(base) + + assert.Equal(t, len(layered.GetAccessibleEntries("/dir").Files), 0) + }) + + t.Run("compaction allows recreating a path removed through an inherited symlink", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{}, true) + baseFS, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/target/recreated.ts": "base", + }, + Symlinks: map[string]RequestSymlink{ + "/link": {Target: "/target"}, + }, + }, host, "/") + assert.NilError(t, err) + base := getRequestFileSystem(baseFS) + + removedFS, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + RemovedPaths: []string{"/link/recreated.ts"}, + }, baseFS, "/") + assert.NilError(t, err) + removed := getRequestFileSystem(removedFS) + removed.applyTo(base) + + recreatedFS, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/link/recreated.ts": "recreated", + }, + }, removedFS, "/") + assert.NilError(t, err) + recreated := getRequestFileSystem(recreatedFS) + contents, ok := recreated.ReadFile("/link/recreated.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "recreated") + + recreated.applyTo(removed) + + contents, ok = recreated.ReadFile("/link/recreated.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "recreated") + }) + + t.Run("compaction allows recreating a descendant of a path removed through an inherited symlink", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{}, true) + base, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/target/dir/existing.ts": "existing", + }, + Symlinks: map[string]RequestSymlink{ + "/link": {Target: "/target"}, + }, + }, host, "/") + assert.NilError(t, err) + + removed, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + RemovedPaths: []string{"/link/dir"}, + }, base, "/") + assert.NilError(t, err) + removed.applyTo(base) + + recreated, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/link/dir/recreated.ts": "recreated", + }, + }, removed, "/") + assert.NilError(t, err) + contents, ok := recreated.ReadFile("/link/dir/recreated.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "recreated") + + recreated.applyTo(removed) + + contents, ok = recreated.ReadFile("/link/dir/recreated.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "recreated") + assert.Assert(t, !recreated.FileExists("/link/dir/existing.ts")) + assert.DeepEqual(t, recreated.GetAccessibleEntries("/link/dir").Files, []string{"recreated.ts"}) + assert.DeepEqual(t, recreated.GetAccessibleEntries("/link").Directories, []string{"dir"}) + }) + + t.Run("files replacing inherited symlink target directories have empty listings", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{}, true) + base, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/target/item/child.ts": "child", + }, + Symlinks: map[string]RequestSymlink{ + "/link": {Target: "/target"}, + }, + }, host, "/") + assert.NilError(t, err) + + layered, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/target/item": "file", + }, + }, base, "/") + assert.NilError(t, err) + + assert.Assert(t, layered.FileExists("/link/item")) + assert.Assert(t, !layered.DirectoryExists("/link/item")) + assert.Equal(t, len(layered.GetAccessibleEntries("/link/item").Files), 0) + assert.Equal(t, len(layered.GetAccessibleEntries("/link/item").Directories), 0) + }) + + t.Run("cache tombstones block host hits", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{ + "/remove.ts": "host", + "/removed-dir/gone.ts": "host", + }, true)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{}, + RemovedPaths: []string{"/remove.ts", "/removed-dir"}, + }, base, "/") + assert.NilError(t, err) + + assert.Assert(t, !fileSystem.FileExists("/remove.ts")) + assert.Assert(t, !fileSystem.DirectoryExists("/removed-dir")) + assert.Assert(t, !fileSystem.FileExists("/removed-dir/gone.ts")) + assert.Assert(t, base.SeenFiles.IsEmpty()) + }) + + t.Run("compacted filesystem layers retain host fallback", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{ + "/host.ts": "host", + "/removed.ts": "host removed", + "/sealed/host.ts": "hidden from listing", + "/open/host.ts": "host listing", + "/open/layer-listed.ts": "host listed", + }, true) + baseFS, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/inherited.ts": "inherited", + "/sealed/inherited.ts": "sealed inherited", + }, + Directories: map[string]RequestDirectoryEntries{ + "/sealed": {Files: []string{"inherited.ts"}, Directories: []string{}}, + }, + RemovedPaths: []string{"/removed.ts"}, + }, host, "/") + assert.NilError(t, err) + base := getRequestFileSystem(baseFS) + + layeredFS, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/added.ts": "added", + "/sealed/added.ts": "sealed added", + }, + Directories: map[string]RequestDirectoryEntries{ + "/open": {Files: []string{"layer-listed.ts"}, Directories: []string{}}, + }, + }, baseFS, "/") + assert.NilError(t, err) + layered := getRequestFileSystem(layeredFS) + layered.applyTo(base) + assert.Assert(t, getRequestFileSystem(layered.baseFileSystem()) != base) + assert.Equal(t, layered.load().kind, KindLayer) + + for path, expected := range map[string]string{ + "/host.ts": "host", + "/inherited.ts": "inherited", + "/added.ts": "added", + } { + contents, ok := layered.ReadFile(path) + assert.Assert(t, ok, path) + assert.Equal(t, contents, expected) + } + _, ok := layered.ReadFile("/removed.ts") + assert.Assert(t, !ok) + assert.DeepEqual(t, layered.GetAccessibleEntries("/sealed").Files, []string{"added.ts", "inherited.ts"}) + assert.DeepEqual(t, layered.GetAccessibleEntries("/open").Files, []string{"host.ts", "layer-listed.ts"}) + }) + + t.Run("compacting a filesystem layer over a full filesystem produces a full filesystem", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{ + "/host.ts": "host", + }, true) + baseFS, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/target/inherited.ts": "inherited", + }, + Directories: map[string]RequestDirectoryEntries{ + "/target": {Files: []string{"inherited.ts"}, Directories: []string{}}, + }, + Symlinks: map[string]RequestSymlink{ + "/link": {Target: "/target"}, + }, + }, host, "/") + assert.NilError(t, err) + base := getRequestFileSystem(baseFS) + + layeredFS, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{ + "/target/added.ts": "added", + }, + }, baseFS, "/") + assert.NilError(t, err) + layered := getRequestFileSystem(layeredFS) + layered.applyTo(base) + assert.Equal(t, layered.load().kind, KindFull) + assert.Assert(t, getRequestFileSystem(layered.baseFileSystem()) != base) + + for path, expected := range map[string]string{ + "/link/inherited.ts": "inherited", + "/link/added.ts": "added", + } { + contents, ok := layered.ReadFile(path) + assert.Assert(t, ok, path) + assert.Equal(t, contents, expected) + } + _, ok := layered.ReadFile("/host.ts") + assert.Assert(t, !ok) + }) + + t.Run("memory routes explicit host symlinks to the host only through the link", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{ + "/host/node_modules/pkg/index.d.ts": "export declare const hostValue: string;", + "/host/outside.ts": "outside", + }, true)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/project/index.ts": `import { hostValue } from "pkg";`, + }, + Symlinks: map[string]RequestSymlink{ + "/project/node_modules": {Target: "/host/node_modules", Host: true}, + }, + }, base, "/") + assert.NilError(t, err) + + _, ok := fileSystem.ReadFile("/host/outside.ts") + assert.Assert(t, !ok) + assert.Assert(t, !base.SeenFiles.Has("/host/outside.ts")) + + contents, ok := fileSystem.ReadFile("/project/node_modules/pkg/index.d.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "export declare const hostValue: string;") + assert.Assert(t, base.SeenFiles.Has("/host/node_modules/pkg/index.d.ts")) + assert.Equal(t, fileSystem.Realpath("/project/node_modules/pkg/index.d.ts"), "/host/node_modules/pkg/index.d.ts") + + entries := fileSystem.GetAccessibleEntries("/project") + assert.DeepEqual(t, entries.Directories, []string{"node_modules"}) + _, isSymlink := entries.Symlinks["node_modules"] + assert.Assert(t, isSymlink) + }) + + t.Run("layered host symlinks bypass snapshot bases", func(t *testing.T) { + t.Parallel() + host := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{ + "/host/pkg/index.d.ts": "host", + }, true)} + base, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/memory.ts": "memory", + }, + }, host, "/") + assert.NilError(t, err) + + layered, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{}, + Symlinks: map[string]RequestSymlink{ + "/project/pkg": {Target: "/host/pkg", Host: true}, + }, + }, base, "/") + assert.NilError(t, err) + + contents, ok := layered.ReadFile("/project/pkg/index.d.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "host") + assert.Assert(t, layered.FileExists("/project/pkg/index.d.ts")) + assert.Assert(t, layered.DirectoryExists("/project/pkg")) + assert.DeepEqual(t, layered.GetAccessibleEntries("/project/pkg").Files, []string{"index.d.ts"}) + assert.Equal(t, layered.Realpath("/project/pkg/index.d.ts"), "/host/pkg/index.d.ts") + info := layered.Stat("/project/pkg/index.d.ts") + assert.Assert(t, info != nil) + assert.Equal(t, info.Name(), "index.d.ts") + assert.Assert(t, host.SeenFiles.Has("/host/pkg/index.d.ts")) + }) + + t.Run("inherited host symlinks bypass newer cache entries at the target", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{ + "/host/pkg/host.ts": "host", + "/host/pkg/removed.ts": "removed", + }, true) + base, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{}, + Symlinks: map[string]RequestSymlink{ + "/link": {Target: "/host/pkg", Host: true}, + }, + }, host, "/") + assert.NilError(t, err) + + layered, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + RemovedPaths: []string{"/link/removed.ts"}, + Files: map[string]string{ + "/host/pkg/host.ts": "cache", + "/host/pkg/cache-only.ts": "cache only", + }, + }, base, "/") + assert.NilError(t, err) + + contents, ok := layered.ReadFile("/link/host.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "host") + assert.Assert(t, !layered.FileExists("/link/cache-only.ts")) + assert.Assert(t, !layered.FileExists("/link/removed.ts")) + assert.Equal(t, layered.Stat("/link/host.ts").Size(), int64(len("host"))) + assert.DeepEqual(t, layered.GetAccessibleEntries("/link").Files, []string{"host.ts"}) + }) + + t.Run("canonical path collisions are rejected", func(t *testing.T) { + t.Parallel() + base := vfstest.FromMap(map[string]string{}, false) + + _, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + `C:\Repo\file.ts`: "first", + `c:/repo/file.ts`: "second", + }, + }, base, `C:\Workspace`) + assert.ErrorContains(t, err, "duplicate request filesystem file path") + + _, err = newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{}, + Directories: map[string]RequestDirectoryEntries{ + `C:\Repo`: {}, + `c:/repo/.`: {}, + }, + }, base, `C:\Workspace`) + assert.ErrorContains(t, err, "duplicate request filesystem directory path") + + _, err = newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{}, + Symlinks: map[string]RequestSymlink{ + `C:\Repo\link`: {Target: `C:\Target`}, + `c:/repo/link`: {Target: `C:\Other`}, + }, + }, base, `C:\Workspace`) + assert.ErrorContains(t, err, "duplicate request filesystem symlink path") + }) + + t.Run("symlink cycles are treated as missing", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{ + "/host.ts": "host", + }, true)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{}, + Symlinks: map[string]RequestSymlink{ + "/a": {Target: "/b"}, + "/b": {Target: "/a"}, + }, + }, base, "/") + assert.NilError(t, err) + + _, ok := fileSystem.ReadFile("/a/file.ts") + assert.Assert(t, !ok) + assert.Assert(t, !fileSystem.DirectoryExists("/a")) + assert.Equal(t, fileSystem.Realpath("/a"), "/a") + assert.Assert(t, base.SeenFiles.IsEmpty()) + }) + + t.Run("posix relative symlink targets resolve from the link directory", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{}, true)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/packages/pkg/index.d.ts": "export declare const value: number;", + }, + Symlinks: map[string]RequestSymlink{ + "/project/pkg": {Target: "../packages/pkg"}, + }, + }, base, `C:\Workspace`) + assert.NilError(t, err) + + contents, ok := fileSystem.ReadFile("/project/pkg/index.d.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "export declare const value: number;") + assert.Equal(t, fileSystem.Realpath("/project/pkg/index.d.ts"), "/packages/pkg/index.d.ts") + assert.Assert(t, base.SeenFiles.IsEmpty()) + }) + + t.Run("vscode document URI paths support listings symlinks and tombstones", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{}, true)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "vscode-remote://ssh-remote+host/workspace/src/index.ts": "index", + "vscode-remote://ssh-remote+host/workspace/packages/pkg/a.ts": "package", + }, + Symlinks: map[string]RequestSymlink{ + "vscode-remote://ssh-remote+host/workspace/src/pkg": {Target: "../packages/pkg"}, + }, + RemovedPaths: []string{ + "vscode-remote://ssh-remote+host/workspace/packages/pkg/removed.ts", + }, + }, base, "/") + assert.NilError(t, err) + + contents, ok := fileSystem.ReadFile("vscode-remote://ssh-remote+host/workspace/src/index.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "index") + contents, ok = fileSystem.ReadFile("vscode-remote://ssh-remote+host/workspace/src/pkg/a.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "package") + assert.Equal( + t, + fileSystem.Realpath("vscode-remote://ssh-remote+host/workspace/src/pkg/a.ts"), + "vscode-remote://ssh-remote+host/workspace/packages/pkg/a.ts", + ) + assert.DeepEqual( + t, + fileSystem.GetAccessibleEntries("vscode-remote://ssh-remote+host/workspace/src").Files, + []string{"index.ts"}, + ) + assert.DeepEqual( + t, + fileSystem.GetAccessibleEntries("vscode-remote://ssh-remote+host/workspace/src").Directories, + []string{"pkg"}, + ) + assert.Assert(t, !fileSystem.FileExists("vscode-remote://ssh-remote+host/workspace/src/pkg/removed.ts")) + assert.Assert(t, base.SeenFiles.IsEmpty()) + }) + + t.Run("windows paths resolve symlinks case insensitively", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{ + "C:/Host/outside.ts": "outside", + }, false)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + `C:\Repo\Packages\Pkg\Index.d.ts`: "export declare const windowsValue: number;", + }, + Directories: map[string]RequestDirectoryEntries{ + `C:\Repo\Project\node_modules`: {Files: []string{}, Directories: []string{"pkg"}}, + }, + Symlinks: map[string]RequestSymlink{ + `C:\Repo\Project\node_modules\PKG`: {Target: `..\..\Packages\Pkg`}, + `C:\Repo\Project\Current.d.ts`: {Target: `..\Packages\Pkg\Index.d.ts`}, + }, + }, base, `C:\Workspace`) + assert.NilError(t, err) + + contents, ok := fileSystem.ReadFile(`c:\repo\project\NODE_MODULES\pkg\INDEX.D.TS`) + assert.Assert(t, ok) + assert.Equal(t, contents, "export declare const windowsValue: number;") + contents, ok = fileSystem.ReadFile(`C:\REPO\PROJECT\current.d.ts`) + assert.Assert(t, ok) + assert.Equal(t, contents, "export declare const windowsValue: number;") + assert.Equal( + t, + fileSystem.Realpath(`c:\repo\project\node_modules\pkg\index.d.ts`), + "C:/Repo/Packages/Pkg/index.d.ts", + ) + + entries := fileSystem.GetAccessibleEntries(`c:\REPO\project\NODE_MODULES`) + assert.DeepEqual(t, entries.Directories, []string{"PKG"}) + _, isSymlink := entries.Symlinks["PKG"] + assert.Assert(t, isSymlink) + assert.Assert(t, base.SeenFiles.IsEmpty()) + }) + + t.Run("case insensitive symlink matching handles unicode byte length changes", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{}, false)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "C:/Repo/target.ts": "target", + }, + Symlinks: map[string]RequestSymlink{ + "C:/Repo/K": {Target: "C:/Repo/target.ts"}, + }, + }, base, "C:/Repo") + assert.NilError(t, err) + + contents, ok := fileSystem.ReadFile("c:/repo/k") + assert.Assert(t, ok) + assert.Equal(t, contents, "target") + }) + + t.Run("request filesystems are immutable and cache mutations write through to the host", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{ + "/host.ts": "host", + }, true) + memory, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + "/src/a.ts": "a", + }, + }, host, "/") + assert.NilError(t, err) + assert.ErrorIs(t, memory.WriteFile("/src/b.ts", "b"), vfs.ErrInvalid) + assert.ErrorIs(t, memory.AppendFile("/src/a.ts", "b"), vfs.ErrInvalid) + assert.ErrorIs(t, memory.Remove("/src"), vfs.ErrInvalid) + contents, ok := memory.ReadFile("/src/a.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "a") + + cache, err := newLayeredRequestFileSystem(&RequestFileSystem{ + Kind: KindLayer, + Files: map[string]string{}, + }, memory, "/") + assert.NilError(t, err) + assert.NilError(t, cache.WriteFile("/written.ts", "written")) + assert.NilError(t, cache.AppendFile("/written.ts", " appended")) + contents, ok = host.ReadFile("/written.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "written appended") + assert.NilError(t, cache.Remove("/written.ts")) + assert.Assert(t, !host.FileExists("/written.ts")) + }) + + t.Run("cache mutations follow inherited request symlinks", func(t *testing.T) { + t.Parallel() + host := vfstest.FromMap(map[string]string{ + "/target/write.ts": "target", + "/target/append.ts": "target", + "/target/remove.ts": "target", + "/target/times.ts": "target", + "/link/write.ts": "alias", + "/link/append.ts": "alias", + "/link/remove.ts": "alias", + "/link/times.ts": "alias", + }, true) + base, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{}, + Symlinks: map[string]RequestSymlink{ + "/link": {Target: "/target"}, + }, + }, host, "/") + assert.NilError(t, err) + cache, err := newLayeredRequestFileSystem(&RequestFileSystem{Kind: KindLayer}, base, "/") + assert.NilError(t, err) + + assert.NilError(t, cache.WriteFile("/link/write.ts", "written")) + contents, ok := host.ReadFile("/target/write.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "written") + + assert.NilError(t, cache.AppendFile("/link/append.ts", " appended")) + contents, ok = host.ReadFile("/target/append.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "target appended") + + assert.NilError(t, cache.Remove("/link/remove.ts")) + assert.Assert(t, !host.FileExists("/target/remove.ts")) + + modified := time.Unix(123, 0) + assert.NilError(t, cache.Chtimes("/link/times.ts", modified, modified)) + assert.Equal(t, host.Stat("/target/times.ts").ModTime(), modified) + }) + + t.Run("mixed windows and posix roots support cross-root and relative symlinks", func(t *testing.T) { + t.Parallel() + base := &trackingvfs.FS{Inner: vfstest.FromMap(map[string]string{ + "C:/Host/node_modules/host-pkg/index.d.ts": "export declare const hostValue: boolean;", + }, false)} + fileSystem, err := newRequestFileSystem(&RequestFileSystem{ + Kind: KindFull, + Files: map[string]string{ + `C:\Repo\Packages\windows-pkg\index.d.ts`: "export declare const windowsValue: number;", + "/repo/packages/posix-pkg/index.d.ts": "export declare const posixValue: string;", + }, + Symlinks: map[string]RequestSymlink{ + // Cross between drive-letter and POSIX roots in both directions. + `C:\Repo\Project\node_modules\posix-pkg`: {Target: "/repo/packages/posix-pkg"}, + "/repo/project/node_modules/windows-pkg": {Target: `C:\Repo\Packages\windows-pkg`}, + // Windows symlink targets read from disk may be relative to the link's directory. + `C:\Repo\Project\windows-pkg.d.ts`: {Target: `..\Packages\windows-pkg\index.d.ts`}, + `C:\Repo\Project\node_modules\host-pkg`: { + Target: `..\..\..\Host\node_modules\host-pkg`, + Host: true, + }, + }, + }, base, `C:\Workspace`) + assert.NilError(t, err) + + contents, ok := fileSystem.ReadFile(`c:\REPO\project\NODE_MODULES\POSIX-PKG\INDEX.D.TS`) + assert.Assert(t, ok) + assert.Equal(t, contents, "export declare const posixValue: string;") + contents, ok = fileSystem.ReadFile("/REPO/PROJECT/NODE_MODULES/WINDOWS-PKG/INDEX.D.TS") + assert.Assert(t, ok) + assert.Equal(t, contents, "export declare const windowsValue: number;") + contents, ok = fileSystem.ReadFile(`c:\repo\project\WINDOWS-PKG.D.TS`) + assert.Assert(t, ok) + assert.Equal(t, contents, "export declare const windowsValue: number;") + contents, ok = fileSystem.ReadFile(`C:\Repo\Project\node_modules\HOST-PKG\index.d.ts`) + assert.Assert(t, ok) + assert.Equal(t, contents, "export declare const hostValue: boolean;") + + assert.Equal( + t, + fileSystem.Realpath(`c:\repo\project\node_modules\posix-pkg\index.d.ts`), + "/repo/packages/posix-pkg/index.d.ts", + ) + assert.Equal( + t, + fileSystem.Realpath("/repo/project/node_modules/windows-pkg/index.d.ts"), + "C:/Repo/Packages/windows-pkg/index.d.ts", + ) + assert.Assert(t, base.SeenFiles.Has("C:/Host/node_modules/host-pkg/index.d.ts")) + }) +} diff --git a/tsc/internal/api/requestfilesystem/requestfilesystemhandle.go b/tsc/internal/api/requestfilesystem/requestfilesystemhandle.go new file mode 100644 index 0000000000000..0b70c4fe33bcf --- /dev/null +++ b/tsc/internal/api/requestfilesystem/requestfilesystemhandle.go @@ -0,0 +1,250 @@ +package requestfilesystem + +import ( + "sync" + "sync/atomic" + "time" + + "github.com/microsoft/TypeScript/tsc/internal/project" + "github.com/microsoft/TypeScript/tsc/internal/vfs" +) + +// Handle is a request filesystem whose backing layers can be compacted as snapshots are released. +type Handle struct { + mu sync.Mutex + value atomic.Pointer[requestFileSystem] + dependents map[*Handle]struct{} + released bool +} + +func (h *Handle) load() *requestFileSystem { + return h.value.Load() +} + +func (h *Handle) initialize(value requestFileSystem) { + h.mu.Lock() + defer h.mu.Unlock() + if h.Initialized() { + panic("request filesystem handle already initialized") + } + h.value.Store(&value) + h.registerWithBaseLocked() +} + +// Initialized reports whether the handle contains a request filesystem. +func (h *Handle) Initialized() bool { + return h.load() != nil +} + +func (h *Handle) initializeFromRequest(params *RequestFileSystem, base vfs.FS, currentDirectory string) error { + value, err := newRequestFileSystemWorker(params, base, currentDirectory, false) + if err != nil { + return err + } + h.initialize(*value) + return nil +} + +func (h *Handle) initializeLayered(params *RequestFileSystem, base vfs.FS, currentDirectory string) error { + if params.Kind != KindLayer { + return h.initializeFromRequest(params, base, currentDirectory) + } + value, err := newRequestFileSystemWorker(params, base, currentDirectory, true) + if err != nil { + return err + } + h.initialize(*value) + return nil +} + +// InitializeForUpdate initializes the handle from a snapshot update request and its optional base. +func (h *Handle) InitializeForUpdate(params *RequestFileSystem, base *Handle, host vfs.FS, currentDirectory string, fileChanges *project.FileChangeSummary, hasBaseSnapshot bool) error { + if params == nil { + if base != nil { + h.CloneFrom(base) + } + return nil + } + if params.Kind == KindLayer && hasBaseSnapshot { + baseFS := host + if base != nil { + baseFS = base + } + addFileChanges(fileChanges, params, baseFS, currentDirectory) + return h.initializeLayered(params, baseFS, currentDirectory) + } + return h.initializeFromRequest(params, host, currentDirectory) +} + +// FS returns this handle as a filesystem, or a nil interface when it is uninitialized. +func (h *Handle) FS() vfs.FS { + if !h.Initialized() { + return nil + } + return h +} + +// CloneFrom initializes a zero-value handle with an independently managed copy of source. +func (h *Handle) CloneFrom(source *Handle) { + if source == nil { + return + } + value := *source.load() + h.mu.Lock() + defer h.mu.Unlock() + if h.Initialized() { + panic("request filesystem handle already initialized") + } + h.value.Store(&value) + h.registerWithBaseLocked() +} + +// Release removes this handle from the dependency graph and compacts live dependents. +func (h *Handle) Release() { + if h == nil { + return + } + if h.load() == nil { + return + } + h.mu.Lock() + if h.released { + h.mu.Unlock() + return + } + h.released = true + h.mu.Unlock() + + h.compactDependents() + h.unregisterFromBase() +} + +func (h *Handle) registerWithBaseLocked() { + for { + base := h.layeredBase() + if base == nil { + return + } + base.mu.Lock() + if base.released { + value := h.load().applyTo(*base.load()) + h.value.Store(&value) + base.mu.Unlock() + continue + } + if base.dependents == nil { + base.dependents = make(map[*Handle]struct{}) + } + base.dependents[h] = struct{}{} + base.mu.Unlock() + return + } +} + +func (h *Handle) unregisterFromBase() { + if base := h.layeredBase(); base != nil { + base.mu.Lock() + delete(base.dependents, h) + base.mu.Unlock() + } +} + +func (h *Handle) layeredBase() *Handle { + value := h.load() + if !value.layered { + return nil + } + return getRequestFileSystem(value.baseFileSystem()) +} + +func (h *Handle) compactDependents() { + for { + h.mu.Lock() + var dependent *Handle + for candidate := range h.dependents { + dependent = candidate + delete(h.dependents, candidate) + break + } + value := h.load() + h.mu.Unlock() + if dependent == nil { + return + } + + dependent.mu.Lock() + if !dependent.released && dependent.layeredBase() == h { + compacted := dependent.load().applyTo(*value) + dependent.value.Store(&compacted) + dependent.registerWithBaseLocked() + } + dependent.mu.Unlock() + dependent.compactDependents() + } +} + +func (h *Handle) baseFileSystem() vfs.FS { + return h.load().baseFileSystem() +} + +// HasFullFileSystem reports whether any backing layer is a full filesystem. +func (h *Handle) HasFullFileSystem() bool { + for h != nil { + value := h.load() + if value.kind == KindFull { + return true + } + h = getRequestFileSystem(value.baseFileSystem()) + } + return false +} + +func (h *Handle) UseCaseSensitiveFileNames() bool { + return h.load().UseCaseSensitiveFileNames() +} + +func (h *Handle) ReadFile(fileName string) (string, bool) { + return h.load().ReadFile(fileName) +} + +func (h *Handle) FileExists(fileName string) bool { + return h.load().FileExists(fileName) +} + +func (h *Handle) DirectoryExists(directoryName string) bool { + return h.load().DirectoryExists(directoryName) +} + +func (h *Handle) GetAccessibleEntries(directoryName string) vfs.Entries { + return h.load().GetAccessibleEntries(directoryName) +} + +func (h *Handle) Realpath(path string) string { + return h.load().Realpath(path) +} + +func (h *Handle) WriteFile(fileName string, data string) error { + return h.load().WriteFile(fileName, data) +} + +func (h *Handle) AppendFile(fileName string, data string) error { + return h.load().AppendFile(fileName, data) +} + +func (h *Handle) Remove(path string) error { + return h.load().Remove(path) +} + +func (h *Handle) Chtimes(path string, aTime time.Time, mTime time.Time) error { + return h.load().Chtimes(path, aTime, mTime) +} + +func (h *Handle) Stat(path string) vfs.FileInfo { + return h.load().Stat(path) +} + +func (h *Handle) WalkDir(root string, walkFn vfs.WalkDirFunc) error { + return h.load().WalkDir(root, walkFn) +} + +var _ vfs.FS = (*Handle)(nil) diff --git a/tsc/internal/api/session.go b/tsc/internal/api/session.go index 3cc6a23a0babd..96195228faf02 100644 --- a/tsc/internal/api/session.go +++ b/tsc/internal/api/session.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "github.com/microsoft/TypeScript/tsc/internal/api/encoder" + "github.com/microsoft/TypeScript/tsc/internal/api/requestfilesystem" "github.com/microsoft/TypeScript/tsc/internal/ast" "github.com/microsoft/TypeScript/tsc/internal/astnav" "github.com/microsoft/TypeScript/tsc/internal/checker" @@ -43,8 +44,9 @@ var sessionIDCounter atomic.Uint64 // Multiple clients may hold references to the same snapshot via ref counting; // the registries are cleaned up when refCount reaches zero. type snapshotData struct { - snapshot *project.Snapshot - refCount int + snapshot *project.Snapshot + fileSystem requestfilesystem.Handle + refCount int // Symbol IDs come from ast.GetSymbolId, a global atomic counter, so the same // *ast.Symbol pointer always has the same unique ID across all projects in the @@ -484,6 +486,23 @@ func (s *Session) retainSnapshotData(handle SnapshotID) (*snapshotData, error) { return sd, nil } +// retainLatestSnapshotData atomically verifies that handle identifies the latest +// active snapshot and takes a temporary reference that pins it for an update. +// The caller must pair a successful call with releaseSnapshot, including on errors. +func (s *Session) retainLatestSnapshotData(handle SnapshotID) (*snapshotData, error) { + s.snapshotsMu.Lock() + defer s.snapshotsMu.Unlock() + if handle != s.latestSnapshot { + return nil, fmt.Errorf("%w: snapshot %d is not the latest snapshot", ErrClientError, handle) + } + sd := s.snapshots[handle] + if sd == nil { + return nil, fmt.Errorf("%w: snapshot %d not found", ErrClientError, handle) + } + sd.refCount++ + return sd, nil +} + func (s *Session) releaseSnapshot(handle SnapshotID) error { s.snapshotsMu.Lock() sd := s.snapshots[handle] @@ -492,14 +511,58 @@ func (s *Session) releaseSnapshot(handle SnapshotID) error { return fmt.Errorf("%w: snapshot %d not found", ErrClientError, handle) } sd.refCount-- - if sd.refCount <= 0 { - delete(s.snapshots, handle) - sd.snapshot.Deref(s.projectSession) + if sd.refCount > 0 { + s.snapshotsMu.Unlock() + return nil } + delete(s.snapshots, snapshotHandle(sd.snapshot)) s.snapshotsMu.Unlock() + + sd.snapshot.Deref(s.projectSession) + sd.fileSystem.Release() return nil } +func newSnapshotData() *snapshotData { + sd := &snapshotData{ + refCount: 1, + symbolRegistry: make(map[SymbolID]*ast.Symbol), + symbolCanonicalProjects: make(map[SymbolID]ProjectID), + projectRegistries: make(map[ProjectID]*projectRegistryData), + } + return sd +} + +func (s *Session) registerSnapshotData(sd *snapshotData, updateLatest bool) (SnapshotID, *snapshotData) { + handle := snapshotHandle(sd.snapshot) + s.snapshotsMu.Lock() + existingSD := s.snapshots[handle] + if existingSD != nil { + existingSD.refCount++ + } else { + s.snapshots[handle] = sd + } + var previous *snapshotData + if updateLatest { + previous = s.snapshots[s.latestSnapshot] + s.latestSnapshot = handle + } + s.snapshotsMu.Unlock() + + if existingSD != nil { + sd.snapshot.Deref(s.projectSession) + sd.fileSystem.Release() + } + return handle, previous +} + +func (sd *snapshotData) fileSystemHandle() *requestfilesystem.Handle { + if !sd.fileSystem.Initialized() { + return nil + } + return &sd.fileSystem +} + // checkerSetup holds the common context needed by handlers that require a type checker. type checkerSetup struct { sd *snapshotData @@ -980,9 +1043,31 @@ func (s *Session) handleUpdateSnapshot(ctx context.Context, params *UpdateSnapsh s.updateMu.Lock() defer s.updateMu.Unlock() + var baseSD *snapshotData + if params.Snapshot != 0 { + var err error + baseSD, err = s.retainLatestSnapshotData(params.Snapshot) + if err != nil { + return nil, err + } + // Release only the temporary pin acquired above; the client's Snapshot + // continues to own its existing reference even if this update fails. + defer func() { _ = s.releaseSnapshot(params.Snapshot) }() + } + fileChanges := s.toFileChangeSummary(params.FileChanges) apiRequest := &project.APISnapshotRequest{} + var baseRequestFileSystem *requestfilesystem.Handle + if baseSD != nil { + baseRequestFileSystem = baseSD.fileSystemHandle() + } + sd := newSnapshotData() + if err := sd.fileSystem.InitializeForUpdate(params.FileSystem, baseRequestFileSystem, s.projectSession.FS(), s.projectSession.GetCurrentDirectory(), &fileChanges, baseSD != nil); err != nil { + return nil, fmt.Errorf("%w: %w", ErrClientError, err) + } + apiRequest.FileSystem = sd.fileSystem.FS() + apiRequest.ReplaceFileSystem = params.FileSystem != nil // Open projects: only take a new ref for projects we aren't already holding open. var openedProjects []tspath.Path @@ -1051,8 +1136,10 @@ func (s *Session) handleUpdateSnapshot(ctx context.Context, params *UpdateSnapsh if err != nil { // APIUpdate returns a ref'd snapshot even on error; release it. snapshot.Deref(s.projectSession) + sd.fileSystem.Release() return nil, fmt.Errorf("%w: failed to update snapshot: %w", ErrClientError, err) } + sd.snapshot = snapshot // Commit ref tracking now that the update succeeded. for _, configPath := range openedProjects { @@ -1068,31 +1155,8 @@ func (s *Session) handleUpdateSnapshot(ctx context.Context, params *UpdateSnapsh s.openFiles.Delete(path) } - // Create or ref-count snapshot data, then atomically read the previous latest - // snapshot (the diff base) and advance latestSnapshot to the new handle. - // If the same snapshot ID is returned (no changes), we increment the ref count - // so each client-side Snapshot can be disposed independently. - handle := snapshotHandle(snapshot) - s.snapshotsMu.Lock() - sd, exists := s.snapshots[handle] - if exists { - // Same snapshot already stored — release the caller's ref since - // the stored snapshot already has one, and bump the API refcount. - snapshot.Deref(s.projectSession) - sd.refCount++ - } else { - sd = &snapshotData{ - snapshot: snapshot, - refCount: 1, - symbolRegistry: make(map[SymbolID]*ast.Symbol), - symbolCanonicalProjects: make(map[SymbolID]ProjectID), - projectRegistries: make(map[ProjectID]*projectRegistryData), - } - s.snapshots[handle] = sd - } - prevSD := s.snapshots[s.latestSnapshot] - s.latestSnapshot = handle - s.snapshotsMu.Unlock() + // Atomically advance latestSnapshot and retain duplicate handles independently. + handle, prevSD := s.registerSnapshotData(sd, true) // Build projects list projects := snapshot.ProjectCollection.Projects() @@ -1128,29 +1192,17 @@ func (s *Session) handleUpdateTemporarySnapshot(ctx context.Context, params *Upd defer func() { _ = s.releaseSnapshot(params.Snapshot) }() uri := params.File.ToURI(s.projectSession.GetCurrentDirectory()) + sd := newSnapshotData() + sd.fileSystem.CloneFrom(baseSD.fileSystemHandle()) - snapshot, err := s.projectSession.APIUpdateTemporary(ctx, baseSD.snapshot, uri, params.NewText) + snapshot, err := s.projectSession.APIUpdateTemporary(ctx, baseSD.snapshot, sd.fileSystem.FS(), uri, params.NewText) if err != nil { + sd.fileSystem.Release() return nil, fmt.Errorf("%w: failed to update temporary snapshot: %w", ErrClientError, err) } + sd.snapshot = snapshot - handle := snapshotHandle(snapshot) - s.snapshotsMu.Lock() - sd, exists := s.snapshots[handle] - if exists { - snapshot.Deref(s.projectSession) - sd.refCount++ - } else { - sd = &snapshotData{ - snapshot: snapshot, - refCount: 1, - symbolRegistry: make(map[SymbolID]*ast.Symbol), - symbolCanonicalProjects: make(map[SymbolID]ProjectID), - projectRegistries: make(map[ProjectID]*projectRegistryData), - } - s.snapshots[handle] = sd - } - s.snapshotsMu.Unlock() + handle, _ := s.registerSnapshotData(sd, false) // Build projects list projects := snapshot.ProjectCollection.Projects() @@ -1199,7 +1251,9 @@ func (s *Session) handleCreateProgram(ctx context.Context, params *CreateProgram return nil, err } } + sd := newSnapshotData() + fileChanges := s.toFileChangeSummary(params.FileChanges) snapshot := s.projectSession.APICreateProgram( ctx, rootFileNames, @@ -1208,31 +1262,17 @@ func (s *Session) handleCreateProgram(ctx context.Context, params *CreateProgram core.Map(params.CreateProgramOptions.ConfigFileParsingDiagnostics, func(d *DiagnosticResponse) *ast.Diagnostic { return d.ToDiagnostic() }), oldSnapshot, oldProject, - s.toFileChangeSummary(params.FileChanges), + fileChanges, ) project := snapshot.ProjectCollection.InferredProject() if project == nil { snapshot.Deref(s.projectSession) + sd.fileSystem.Release() return nil, fmt.Errorf("%w: failed to create synthetic project", ErrClientError) } + sd.snapshot = snapshot - handle := snapshotHandle(snapshot) - s.snapshotsMu.Lock() - if sd, exists := s.snapshots[handle]; exists { - // Same snapshot already stored: use the existing retained ref and only bump API refcount. - snapshot.Deref(s.projectSession) - sd.refCount++ - } else { - sd = &snapshotData{ - snapshot: snapshot, - refCount: 1, - symbolRegistry: make(map[SymbolID]*ast.Symbol), - symbolCanonicalProjects: make(map[SymbolID]ProjectID), - projectRegistries: make(map[ProjectID]*projectRegistryData), - } - s.snapshots[handle] = sd - } - s.snapshotsMu.Unlock() + handle, _ := s.registerSnapshotData(sd, false) return &CreateProgramResponse{ Snapshot: handle, @@ -2757,8 +2797,24 @@ func (s *Session) handleEmit(ctx context.Context, params *EmitParams) (*EmitResp if err != nil { return nil, err } - options.WriteFile = func(fileName string, text string, _ *compiler.WriteFileData) error { - return s.projectSession.FS().WriteFile(fileName, text) + var outputFiles map[string]string + sd, err := s.getSnapshotData(params.Snapshot) + if err != nil { + return nil, err + } + if fileSystem := sd.fileSystemHandle(); fileSystem != nil && fileSystem.HasFullFileSystem() { + outputFiles = make(map[string]string) + var outputMu sync.Mutex + options.WriteFile = func(fileName string, text string, _ *compiler.WriteFileData) error { + outputMu.Lock() + outputFiles[fileName] = text + outputMu.Unlock() + return nil + } + } else { + options.WriteFile = func(fileName string, text string, _ *compiler.WriteFileData) error { + return s.projectSession.FS().WriteFile(fileName, text) + } } result, err := emitProgram(ctx, program, options) if err != nil { @@ -2768,10 +2824,18 @@ func (s *Session) handleEmit(ctx context.Context, params *EmitParams) (*EmitResp if emittedFiles == nil { emittedFiles = []string{} } + emittedFilesContents := []string{} + if outputFiles != nil { + emittedFilesContents = make([]string, len(emittedFiles)) + for i, fileName := range emittedFiles { + emittedFilesContents[i] = outputFiles[fileName] + } + } return &EmitResponse{ - EmitSkipped: result.EmitSkipped, - Diagnostics: nonNilDiagnostics(result.Diagnostics), - EmittedFiles: emittedFiles, + EmitSkipped: result.EmitSkipped, + Diagnostics: nonNilDiagnostics(result.Diagnostics), + EmittedFiles: emittedFiles, + EmittedFilesContents: emittedFilesContents, }, nil } @@ -3720,6 +3784,7 @@ func (s *Session) Close() { defer s.snapshotsMu.Unlock() for handle, sd := range s.snapshots { sd.snapshot.Deref(s.projectSession) + sd.fileSystem.Release() delete(s.snapshots, handle) } } diff --git a/tsc/internal/api/session_requestfilesystem_test.go b/tsc/internal/api/session_requestfilesystem_test.go new file mode 100644 index 0000000000000..249c107c9556c --- /dev/null +++ b/tsc/internal/api/session_requestfilesystem_test.go @@ -0,0 +1,452 @@ +package api + +import ( + "context" + "errors" + "fmt" + "strconv" + "sync" + "testing" + + "github.com/microsoft/TypeScript/tsc/internal/api/requestfilesystem" + "github.com/microsoft/TypeScript/tsc/internal/testutil/projecttestutil" + "github.com/microsoft/TypeScript/tsc/internal/tspath" + "gotest.tools/v3/assert" +) + +func TestUpdateSnapshotUsesFullFileSystem(t *testing.T) { + t.Parallel() + + projectSession, _ := projecttestutil.Setup(map[string]any{ + "/host.ts": "host", + }) + defer projectSession.Close() + session := NewSession(projectSession, nil) + defer session.Close() + + response, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + OpenProjects: []DocumentIdentifier{{FileName: "/tsconfig.json"}}, + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindFull, + Files: map[string]string{ + "/tsconfig.json": `{ "compilerOptions": { "noLib": true }, "files": ["src/index.ts"] }`, + "/src/index.ts": `export const value = "memory";`, + "/src/other.ts": `export const other = true;`, + }, + }, + }) + assert.NilError(t, err) + assert.Equal(t, len(response.Projects), 1) + assert.Equal(t, response.Projects[0].ConfigFileName, "/tsconfig.json") + + snapshot := session.snapshots[response.Snapshot].snapshot + contents, ok := snapshot.ReadFile("/src/index.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, `export const value = "memory";`) + _, ok = snapshot.ReadFile("/host.ts") + assert.Assert(t, !ok) + + // Carrying the same filesystem forward without a delta must preserve + // incremental state instead of forcing a full program rebuild. + program := snapshot.ProjectCollection.GetProjectByPath(tspath.Path("/tsconfig.json")).GetProgram() + unchanged, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{Snapshot: response.Snapshot}) + assert.NilError(t, err) + unchangedSnapshot := session.snapshots[unchanged.Snapshot].snapshot + assert.Assert(t, unchangedSnapshot.ProjectCollection.GetProjectByPath(tspath.Path("/tsconfig.json")).GetProgram() == program) + response = unchanged + + // Supplying a new filesystem replaces inherited snapshot disk caches even + // when the caller does not redundantly list every file in FileChanges. + response, err = session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindFull, + Files: map[string]string{ + "/tsconfig.json": `{ "compilerOptions": { "noLib": true }, "files": ["src/index.ts", "src/other.ts"] }`, + "/src/index.ts": `export const value = "updated";`, + "/src/other.ts": `export const other = true;`, + }, + }, + }) + assert.NilError(t, err) + snapshot = session.snapshots[response.Snapshot].snapshot + contents, ok = snapshot.ReadFile("/src/index.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, `export const value = "updated";`) + + // Temporary snapshots retain the base snapshot's supplied filesystem for + // every file other than the temporary overlay. + temporary, err := session.handleUpdateTemporarySnapshot(context.Background(), &UpdateTemporarySnapshotParams{ + Snapshot: response.Snapshot, + File: DocumentIdentifier{FileName: "/src/index.ts"}, + NewText: `export const value = "temporary";`, + }) + assert.NilError(t, err) + temporarySnapshot := session.snapshots[temporary.Snapshot].snapshot + contents, ok = temporarySnapshot.ReadFile("/src/index.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, `export const value = "temporary";`) + contents, ok = temporarySnapshot.ReadFile("/src/other.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, `export const other = true;`) +} + +func TestSnapshotUpdateFullFileSystemIsTotal(t *testing.T) { + t.Parallel() + + projectSession, _ := projecttestutil.Setup(map[string]any{ + "/host.ts": "host", + }) + defer projectSession.Close() + session := NewSession(projectSession, nil) + defer session.Close() + + base, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{}) + assert.NilError(t, err) + replaced, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + Snapshot: base.Snapshot, + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindFull, + Files: map[string]string{ + "/memory.ts": "memory", + }, + }, + }) + assert.NilError(t, err) + + snapshot := session.snapshots[replaced.Snapshot].snapshot + contents, ok := snapshot.ReadFile("/memory.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, "memory") + _, ok = snapshot.ReadFile("/host.ts") + assert.Assert(t, !ok) +} + +func TestSnapshotUpdateCarriesHostFileSystemWithoutOverride(t *testing.T) { + t.Parallel() + + projectSession, _ := projecttestutil.Setup(map[string]any{ + "/tsconfig.json": `{ "compilerOptions": { "noLib": true }, "files": ["index.ts"] }`, + "/index.ts": `export const value = true;`, + }) + defer projectSession.Close() + session := NewSession(projectSession, nil) + defer session.Close() + + base, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + OpenProjects: []DocumentIdentifier{{FileName: "/tsconfig.json"}}, + }) + assert.NilError(t, err) + baseSnapshot := session.snapshots[base.Snapshot].snapshot + program := baseSnapshot.ProjectCollection.GetProjectByPath(tspath.Path("/tsconfig.json")).GetProgram() + + updated, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{Snapshot: base.Snapshot}) + assert.NilError(t, err) + updatedSnapshot := session.snapshots[updated.Snapshot].snapshot + assert.Assert(t, updatedSnapshot.ProjectCollection.GetProjectByPath(tspath.Path("/tsconfig.json")).GetProgram() == program) +} + +func TestEmitFromLayerOverFullFileSystemReturnsFileContents(t *testing.T) { + t.Parallel() + + projectSession, _ := projecttestutil.Setup(map[string]any{}) + defer projectSession.Close() + session := NewSession(projectSession, nil) + defer session.Close() + ctx := context.Background() + + base, err := session.handleUpdateSnapshot(ctx, &UpdateSnapshotParams{ + OpenProjects: []DocumentIdentifier{{FileName: "/tsconfig.json"}}, + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindFull, + Files: map[string]string{ + "/tsconfig.json": `{ "compilerOptions": { "noLib": true, "outDir": "/out" }, "files": ["src/main.ts"] }`, + "/src/main.ts": `export const value: number = 1;`, + }, + }, + }) + assert.NilError(t, err) + layered, err := session.handleUpdateSnapshot(ctx, &UpdateSnapshotParams{ + Snapshot: base.Snapshot, + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindLayer, + Files: map[string]string{}, + }, + }) + assert.NilError(t, err) + assert.Equal(t, len(layered.Projects), 1) + + emitted, err := session.handleEmit(ctx, &EmitParams{ + Snapshot: layered.Snapshot, + Project: layered.Projects[0].Id, + }) + assert.NilError(t, err) + assert.DeepEqual(t, emitted.EmittedFiles, []string{"/out/src/main.js"}) + assert.DeepEqual(t, emitted.EmittedFilesContents, []string{"export const value = 1;\n"}) + + _, err = session.handleRelease(ctx, &ReleaseParams{Snapshot: base.Snapshot}) + assert.NilError(t, err) + emittedAfterRelease, err := session.handleEmit(ctx, &EmitParams{ + Snapshot: layered.Snapshot, + Project: layered.Projects[0].Id, + }) + assert.NilError(t, err) + assert.DeepEqual(t, emittedAfterRelease.EmittedFiles, emitted.EmittedFiles) + assert.DeepEqual(t, emittedAfterRelease.EmittedFilesContents, emitted.EmittedFilesContents) +} + +func TestReleaseSnapshotCompactsSoleLayeredFileSystem(t *testing.T) { + t.Parallel() + + projectSession, _ := projecttestutil.Setup(map[string]any{ + "/host.ts": "host", + }) + defer projectSession.Close() + session := NewSession(projectSession, nil) + defer session.Close() + + base, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindFull, + Files: map[string]string{ + "/inherited.ts": "inherited", + "/changed.ts": "old", + "/removed.ts": "removed", + }, + }, + }) + assert.NilError(t, err) + baseFileSystem := session.snapshots[base.Snapshot].fileSystemHandle() + assert.Assert(t, baseFileSystem != nil) + + layered, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + Snapshot: base.Snapshot, + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindLayer, + Files: map[string]string{ + "/changed.ts": "new", + "/added.ts": "added", + }, + RemovedPaths: []string{"/removed.ts"}, + }, + }) + assert.NilError(t, err) + layeredSnapshotData := session.snapshots[layered.Snapshot] + layeredSnapshot := layeredSnapshotData.snapshot + layeredFileSystem := layeredSnapshotData.fileSystemHandle() + assert.Assert(t, layeredFileSystem != nil) + assert.Equal(t, session.snapshots[base.Snapshot].refCount, 1) + + _, err = session.handleRelease(context.Background(), &ReleaseParams{Snapshot: base.Snapshot}) + assert.NilError(t, err) + assert.Assert(t, session.snapshots[base.Snapshot] == nil) + + for path, expected := range map[string]string{ + "/inherited.ts": "inherited", + "/changed.ts": "new", + "/added.ts": "added", + } { + contents, readOK := layeredSnapshot.ReadFile(path) + assert.Assert(t, readOK, path) + assert.Equal(t, contents, expected) + } + _, ok := layeredSnapshot.ReadFile("/removed.ts") + assert.Assert(t, !ok) + _, ok = layeredSnapshot.ReadFile("/host.ts") + assert.Assert(t, !ok) + assert.Assert(t, layeredFileSystem.HasFullFileSystem()) +} + +func TestEagerSnapshotReleaseDoesNotRetainFileSystemHistory(t *testing.T) { + t.Parallel() + + projectSession, _ := projecttestutil.Setup(map[string]any{}) + defer projectSession.Close() + session := NewSession(projectSession, nil) + defer session.Close() + + response, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindFull, + Files: map[string]string{ + "/pkg/index.ts": "", + }, + }, + }) + assert.NilError(t, err) + + content := "" + for _, character := range "export const x = 1" { + oldSnapshot := response.Snapshot + content += string(character) + response, err = session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + Snapshot: oldSnapshot, + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindLayer, + Files: map[string]string{ + "/pkg/index.ts": content, + }, + }, + }) + assert.NilError(t, err) + _, err = session.handleRelease(context.Background(), &ReleaseParams{Snapshot: oldSnapshot}) + assert.NilError(t, err) + + assert.Equal(t, len(session.snapshots), 1) + current := session.snapshots[response.Snapshot] + assert.Assert(t, current != nil) + assert.Equal(t, current.refCount, 1) + fileSystem := current.fileSystemHandle() + assert.Assert(t, fileSystem != nil) + assert.Assert(t, fileSystem.HasFullFileSystem()) + actual, ok := current.snapshot.ReadFile("/pkg/index.ts") + assert.Assert(t, ok) + assert.Equal(t, actual, content) + } +} + +func TestSnapshotReleaseCompactsChainedFileSystems(t *testing.T) { + t.Parallel() + + projectSession, _ := projecttestutil.Setup(map[string]any{}) + defer projectSession.Close() + session := NewSession(projectSession, nil) + defer session.Close() + + responses := make([]*UpdateSnapshotResponse, 4) + var err error + responses[0], err = session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindFull, + Files: map[string]string{"/pkg/index.ts": "0"}, + }, + }) + assert.NilError(t, err) + for i := 1; i < len(responses); i++ { + responses[i], err = session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + Snapshot: responses[i-1].Snapshot, + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindLayer, + Files: map[string]string{"/pkg/index.ts": strconv.Itoa(i)}, + }, + }) + assert.NilError(t, err) + } + + _, err = session.handleRelease(context.Background(), &ReleaseParams{Snapshot: responses[0].Snapshot}) + assert.NilError(t, err) + assert.Assert(t, session.snapshots[responses[0].Snapshot] == nil) + + for i := 1; i < len(responses); i++ { + current := session.snapshots[responses[i].Snapshot] + assert.Assert(t, current != nil) + assert.Equal(t, current.refCount, 1) + fileSystem := current.fileSystemHandle() + assert.Assert(t, fileSystem != nil) + assert.Assert(t, fileSystem.HasFullFileSystem()) + contents, ok := current.snapshot.ReadFile("/pkg/index.ts") + assert.Assert(t, ok) + assert.Equal(t, contents, strconv.Itoa(i)) + } +} + +func TestTemporarySnapshotRetainsLayeredFileSystemHistory(t *testing.T) { + t.Parallel() + + projectSession, _ := projecttestutil.Setup(map[string]any{}) + defer projectSession.Close() + session := NewSession(projectSession, nil) + defer session.Close() + + base, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindFull, + Files: map[string]string{"/pkg/index.ts": "base"}, + }, + }) + assert.NilError(t, err) + layered, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + Snapshot: base.Snapshot, + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindLayer, + Files: map[string]string{"/pkg/index.ts": "layered"}, + }, + }) + assert.NilError(t, err) + layeredFileSystem := session.snapshots[layered.Snapshot].fileSystemHandle() + temporary, err := session.handleUpdateTemporarySnapshot(context.Background(), &UpdateTemporarySnapshotParams{ + Snapshot: layered.Snapshot, + File: DocumentIdentifier{FileName: "/pkg/index.ts"}, + NewText: "temporary", + }) + assert.NilError(t, err) + + _, err = session.handleRelease(context.Background(), &ReleaseParams{Snapshot: layered.Snapshot}) + assert.NilError(t, err) + _, err = session.handleRelease(context.Background(), &ReleaseParams{Snapshot: base.Snapshot}) + assert.NilError(t, err) + + current := session.snapshots[temporary.Snapshot] + assert.Assert(t, current != nil) + fileSystem := current.fileSystemHandle() + assert.Assert(t, fileSystem != nil) + assert.Assert(t, fileSystem != layeredFileSystem) + assert.Assert(t, fileSystem.HasFullFileSystem()) +} + +func TestSnapshotReleaseCompactionSupportsConcurrentReaders(t *testing.T) { + t.Parallel() + + projectSession, _ := projecttestutil.Setup(map[string]any{}) + defer projectSession.Close() + session := NewSession(projectSession, nil) + defer session.Close() + + files := make(map[string]string, 1024) + for index := range 1024 { + files[fmt.Sprintf("/pkg/file%d.ts", index)] = strconv.Itoa(index) + } + base, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + FileSystem: &requestfilesystem.RequestFileSystem{Kind: requestfilesystem.KindFull, Files: files}, + }) + assert.NilError(t, err) + layered, err := session.handleUpdateSnapshot(context.Background(), &UpdateSnapshotParams{ + Snapshot: base.Snapshot, + FileSystem: &requestfilesystem.RequestFileSystem{ + Kind: requestfilesystem.KindLayer, + Files: map[string]string{"/pkg/file0.ts": "updated"}, + }, + }) + assert.NilError(t, err) + fileSystem := session.snapshots[layered.Snapshot].fileSystemHandle() + + started := make(chan struct{}) + done := make(chan struct{}) + readerError := make(chan error, 1) + var waitGroup sync.WaitGroup + waitGroup.Go(func() { + close(started) + for { + select { + case <-done: + return + default: + contents, ok := fileSystem.ReadFile("/pkg/file0.ts") + if !ok || contents != "updated" { + readerError <- fmt.Errorf("unexpected overridden file: %q, %t", contents, ok) + return + } + if !fileSystem.FileExists("/pkg/file1023.ts") { + readerError <- errors.New("inherited file disappeared") + return + } + } + } + }) + <-started + _, err = session.handleRelease(context.Background(), &ReleaseParams{Snapshot: base.Snapshot}) + assert.NilError(t, err) + close(done) + waitGroup.Wait() + close(readerError) + assert.NilError(t, <-readerError) +} diff --git a/tsc/internal/project/api.go b/tsc/internal/project/api.go index 52d7eeb5fcfd3..57119e3b7663d 100644 --- a/tsc/internal/project/api.go +++ b/tsc/internal/project/api.go @@ -8,6 +8,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/ast" "github.com/microsoft/TypeScript/tsc/internal/core" "github.com/microsoft/TypeScript/tsc/internal/lsp/lsproto" + "github.com/microsoft/TypeScript/tsc/internal/vfs" ) // APIUpdate creates a new snapshot incorporating the given file changes and the @@ -24,11 +25,20 @@ func (s *Session) APIUpdate(ctx context.Context, apiFileChanges FileChangeSummar fileChanges, overlays, ataChanges, _ := s.flushChanges(ctx) mergeFileChangeSummary(&fileChanges, apiFileChanges) + var fs vfs.FS + var replaceFileSystem bool + if apiRequest != nil { + fs = apiRequest.FileSystem + replaceFileSystem = apiRequest.ReplaceFileSystem + } newSnapshot := s.updateSnapshotRef(ctx, overlays, SnapshotChange{ - apiRequest: apiRequest, - fileChanges: fileChanges, - ataChanges: ataChanges, + apiRequest: apiRequest, + fs: fs, + fileSystemOverride: fs != nil, + replaceFileSystem: replaceFileSystem, + fileChanges: fileChanges, + ataChanges: ataChanges, }) return newSnapshot, newSnapshot.apiError } @@ -39,7 +49,7 @@ func (s *Session) APIUpdate(ctx context.Context, apiFileChanges FileChangeSummar // An error is returned if the file name does not have a recognized script extension. // On success, the returned snapshot carries a single reference (the clone ref); // the caller must call snapshot.Deref(s) when done. -func (s *Session) APIUpdateTemporary(ctx context.Context, baseSnapshot *Snapshot, uri lsproto.DocumentUri, newText string) (*Snapshot, error) { +func (s *Session) APIUpdateTemporary(ctx context.Context, baseSnapshot *Snapshot, fileSystem vfs.FS, uri lsproto.DocumentUri, newText string) (*Snapshot, error) { path := uri.Path(baseSnapshot.UseCaseSensitiveFileNames()) overlays := maps.Clone(baseSnapshot.fs.overlays) @@ -59,9 +69,14 @@ func (s *Session) APIUpdateTemporary(ctx context.Context, baseSnapshot *Snapshot fileChanges.Opened = uri } overlays[path] = newOverlay(uri.FileName(), newText, version, scriptKind) + if fileSystem == nil { + fileSystem = baseSnapshot.fs.fs + } newSnapshot := baseSnapshot.Clone(ctx, SnapshotChange{ - fileChanges: fileChanges, + fs: fileSystem, + fileSystemOverride: baseSnapshot.fileSystemOverride, + fileChanges: fileChanges, ResourceRequest: ResourceRequest{ Documents: []lsproto.DocumentUri{uri}, }, diff --git a/tsc/internal/project/snapshot.go b/tsc/internal/project/snapshot.go index 97b2961d3f5c8..62f3a6d0fc097 100644 --- a/tsc/internal/project/snapshot.go +++ b/tsc/internal/project/snapshot.go @@ -23,6 +23,7 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/project/logging" "github.com/microsoft/TypeScript/tsc/internal/sourcemap" "github.com/microsoft/TypeScript/tsc/internal/tspath" + "github.com/microsoft/TypeScript/tsc/internal/vfs" "github.com/microsoft/TypeScript/tsc/internal/vfs/vfsmatch" ) @@ -53,6 +54,9 @@ type Snapshot struct { builderLogs *logging.LogTree apiError error + // fileSystemOverride indicates that this snapshot was built from a filesystem + // supplied by an API update rather than the session host filesystem. + fileSystemOverride bool } func (s *Snapshot) contentMapperWatchState() ([]string, *collections.Set[tspath.Path]) { @@ -345,6 +349,17 @@ func (s *Snapshot) UseCaseSensitiveFileNames() bool { return s.fs.fs.UseCaseSensitiveFileNames() } +// FileSystem returns the filesystem backing this snapshot. +func (s *Snapshot) FileSystem() vfs.FS { + return s.fs.fs +} + +// HasFileSystemOverride reports whether this snapshot uses an API-supplied +// filesystem instead of the session host filesystem. +func (s *Snapshot) HasFileSystemOverride() bool { + return s.fileSystemOverride +} + func (s *Snapshot) ReadFile(fileName string) (string, bool) { handle := s.GetFile(fileName) if handle == nil { @@ -374,6 +389,10 @@ type APISnapshotRequest struct { CloseProjects *collections.Set[tspath.Path] OpenFiles *collections.Set[lsproto.DocumentUri] CloseFiles *collections.Set[tspath.Path] + FileSystem vfs.FS + // ReplaceFileSystem indicates that FileSystem is a new source rather than the + // unchanged filesystem carried forward from the base snapshot. + ReplaceFileSystem bool } type ProjectTreeRequest struct { @@ -419,6 +438,11 @@ type ResourceRequest struct { type SnapshotChange struct { ResourceRequest reason UpdateReason + // fs overrides the session filesystem for this snapshot. It is used by API + // snapshots that supply their own memory or cache filesystem. + fs vfs.FS + fileSystemOverride bool + replaceFileSystem bool // fileChanges are the changes that have occurred since the last snapshot. fileChanges FileChangeSummary // compilerOptionsForInferredProjects is the compiler options to use for inferred projects. @@ -515,7 +539,17 @@ func (s *Snapshot) Clone( inferredContentMappers = change.contentMapperContributions.Mappers inferredContentMapperExtensions = change.contentMapperContributions.Extensions } - fs := newSnapshotFSBuilder(session.fs.fs, s.fs.overlays, overlays, s.fs.diskFiles, s.fs.diskDirectories, s.fs.nodeModulesRealpathAliases, session.options.PositionEncoding, s.toPath) + baseFS := session.fs.fs + if change.fs != nil { + baseFS = change.fs + } + // A supplied filesystem must take precedence over disk files inherited from + // the previous snapshot. Likewise, returning to the session host must not retain + // files from a previous total memory filesystem. + if change.replaceFileSystem || s.fileSystemOverride != change.fileSystemOverride { + change.fileChanges.InvalidateAll = true + } + fs := newSnapshotFSBuilder(baseFS, s.fs.overlays, overlays, s.fs.diskFiles, s.fs.diskDirectories, s.fs.nodeModulesRealpathAliases, session.options.PositionEncoding, s.toPath) change.fileChanges = s.processFileChanges(fs, change.fileChanges, logger, change.contentMapperContributions) compilerOptionsForInferredProjects := s.compilerOptionsForInferredProjects @@ -691,6 +725,7 @@ func (s *Snapshot) Clone( newSnapshot.inferredProjectContentMapperExtensions = inferredContentMapperExtensions newSnapshot.builderLogs = logger newSnapshot.apiError = apiError + newSnapshot.fileSystemOverride = change.fileSystemOverride for _, project := range newSnapshot.ProjectCollection.Projects() { if project.Program != nil {