diff --git a/src/vs/platform/agentHost/node/devContainerAgentHostService.ts b/src/vs/platform/agentHost/node/devContainerAgentHostService.ts index 5f07958e2a62ae..ee589a9c47600f 100644 --- a/src/vs/platform/agentHost/node/devContainerAgentHostService.ts +++ b/src/vs/platform/agentHost/node/devContainerAgentHostService.ts @@ -10,13 +10,13 @@ import { EventEmitter as NodeEventEmitter } from 'events'; import { lstat, rename, rm, stat, writeFile } from 'fs/promises'; import { Duplex } from 'stream'; import { CancellationToken, CancellationTokenSource } from '../../../base/common/cancellation.js'; -import { CancellationError, getErrorMessage } from '../../../base/common/errors.js'; +import { CancellationError, getErrorMessage, isCancellationError } from '../../../base/common/errors.js'; import { Emitter } from '../../../base/common/event.js'; import { join, posix } from '../../../base/common/path.js'; import { StopWatch } from '../../../base/common/stopwatch.js'; import { findExecutable } from '../../../base/node/processes.js'; import { Disposable, DisposableMap, DisposableStore, IDisposable, toDisposable } from '../../../base/common/lifecycle.js'; -import { vArray, vLiteral, vObj, vString } from '../../../base/common/validation.js'; +import { vArray, vLiteral, vObj, vOptionalProp, vString } from '../../../base/common/validation.js'; import { localize } from '../../../nls.js'; import { ILogService } from '../../log/common/log.js'; import { IProductService } from '../../product/common/productService.js'; @@ -44,6 +44,7 @@ import { } from './sshRemoteAgentHostHelpers.js'; import { ensureRemoteAgentHostCliInstalled } from './remoteAgentHostCliInstaller.js'; import { prepareOwnerOnlyDirectory } from './localAgentHostMetadata.js'; +import { buildCreateDevContainerCacheCommand, buildLinkDevContainerServerCacheCommand, canAddDevContainerServerCacheMount, devContainerServerCacheMount, getDevContainerCliCachePath, getDevContainerServerCachePath } from './devContainerServerCache.js'; const LOG_PREFIX = '[DevContainerAgentHost]'; const DETECT_MUSL_COMMAND = 'if [ -e /etc/alpine-release ]; then printf musl; elif command -v ldd >/dev/null 2>&1; then case "$(ldd --version 2>&1)" in *musl*) printf musl;; esac; fi'; @@ -108,6 +109,7 @@ interface IDevContainerMount { readonly Type: string; readonly Source: string; readonly Destination: string; + readonly Name?: string; } interface IGitIdentity { @@ -125,6 +127,7 @@ const devContainerMountsValidator = vArray(vObj({ Type: vString(), Source: vString(), Destination: vString(), + Name: vOptionalProp(vString()), })); /** Testable relay abstraction owned by the shared-process launcher. */ @@ -201,9 +204,10 @@ export abstract class DevContainerAgentHostService extends Disposable implements try { this._logService.info(`${LOG_PREFIX} Starting Dev Container for ${config.workspaceFolder}`); + const cacheMountArgs = await this._getServerCacheMountArgs(config.connectionId, config.workspaceFolder, tokenSource.token); const up = await this._runDevContainer( config.connectionId, - ['up', ...DEV_CONTAINER_LOG_ARGS, '--workspace-folder', config.workspaceFolder], + ['up', ...DEV_CONTAINER_LOG_ARGS, '--workspace-folder', config.workspaceFolder, ...cacheMountArgs], tokenSource.token, ); const upResult = parseDevContainerUpResult(up.stdout); @@ -242,6 +246,7 @@ export abstract class DevContainerAgentHostService extends Disposable implements const serverDataFolderName = this._productService.serverDataFolderName ?? '.vscode-server-oss'; const quality = this._productService.quality || 'insider'; + const cliCacheDir = await this._configureCaches(config.connectionId, upResult.containerId, serverDataFolderName, platform, exec, tokenSource.token); const cliInstallation = await ensureRemoteAgentHostCliInstalled(exec, platform, { serverDataFolderName, quality, @@ -249,6 +254,8 @@ export abstract class DevContainerAgentHostService extends Disposable implements reportInstalling: () => this._logService.info(`${LOG_PREFIX} Installing VS Code CLI in Dev Container...`), logService: this._logService, logPrefix: LOG_PREFIX, + cliCacheDir, + reportCacheStatus: message => this._reportOutput(config.connectionId, `${message}\n`), }); const { cliBin } = cliInstallation; const cliDataDir = getRemoteCLIDataDir(serverDataFolderName); @@ -313,6 +320,76 @@ export abstract class DevContainerAgentHostService extends Disposable implements } } + private async _getServerCacheMountArgs(connectionId: string, workspaceFolder: string, token: CancellationToken): Promise { + try { + const config = await this._runDevContainer(connectionId, ['read-configuration', ...DEV_CONTAINER_LOG_ARGS, '--workspace-folder', workspaceFolder, '--include-merged-configuration'], token); + if (config.code !== 0) { + throw new Error(`Cannot read Dev Container configuration (exit ${config.code}): ${config.stderr}`); + } + if (canAddDevContainerServerCacheMount(config.stdout)) { + return ['--mount', devContainerServerCacheMount]; + } + this._logService.info(`${LOG_PREFIX} Keeping configured container mounts; server cache sharing requires an existing vscode volume mount`); + } catch (error) { + if (isCancellationError(error) || token.isCancellationRequested) { + throw error; + } + this._logService.warn(`${LOG_PREFIX} Cannot add optional shared server cache mount`, error); + this._reportOutput(connectionId, `Cannot add optional shared server cache mount: ${getErrorMessage(error)}\n`); + } + return []; + } + + private async _configureCaches(connectionId: string, containerId: string, serverDataFolderName: string, platform: { os: string; arch: string }, exec: ISshExec, token: CancellationToken): Promise { + const reportError = (kind: string, error: Error) => { + this._logService.warn(`${LOG_PREFIX} Shared ${kind} cache unavailable; keeping the existing CLI cache`, error); + this._reportOutput(connectionId, `Shared ${kind} cache unavailable; keeping the existing CLI cache: ${getErrorMessage(error)}\n`); + }; + try { + const mounts = await this._getContainerMounts(connectionId, containerId, token); + if (!mounts.some(mount => mount.Type === 'volume' && mount.Name === 'vscode' && mount.Destination === '/vscode') + || mounts.some(mount => mount.Destination.startsWith('/vscode/'))) { + throw new Error('The shared vscode volume is not mounted at /vscode without nested mounts'); + } + const { stdout } = await exec('id -u; id -g'); + const [uid, gid] = stdout.trim().split(/\r?\n/); + let cliCacheDir: string | undefined; + for (const kind of ['server', 'CLI'] as const) { + if (kind === 'CLI' && !this._productService.commit) { + continue; + } + try { + const cachePath = kind === 'server' ? getDevContainerServerCachePath(serverDataFolderName, platform) : getDevContainerCliCachePath(serverDataFolderName, platform); + const created = await this._runLocalCommand('docker', [ + 'exec', '--user', 'root', containerId, '/bin/sh', '-c', buildCreateDevContainerCacheCommand(cachePath, uid, gid), + ], await this._resolveShellEnvironment(), token); + if (created.code !== 0) { + throw new Error(`Unable to prepare shared ${kind} cache (exit ${created.code}): ${created.stderr}`); + } + if (kind === 'server') { + await exec(buildLinkDevContainerServerCacheCommand(serverDataFolderName, cachePath)); + this._logService.info(`${LOG_PREFIX} Using shared server cache at ${cachePath}`); + this._reportOutput(connectionId, `Using shared server cache at ${cachePath}\n`); + } else { + cliCacheDir = cachePath; + } + } catch (error) { + if (isCancellationError(error) || token.isCancellationRequested) { + throw error; + } + reportError(kind, error); + } + } + return cliCacheDir; + } catch (error) { + if (isCancellationError(error) || token.isCancellationRequested) { + throw error; + } + reportError('server', error); + return undefined; + } + } + private async _configureGitSafeDirectory(connectionId: string, containerId: string, remoteWorkspaceFolder: string, exec: ISshExec, token: CancellationToken): Promise { const rootResult = await exec( `command -v git >/dev/null 2>&1 && ROOT_FOLDER="$(git -C ${shellEscape(remoteWorkspaceFolder)} rev-parse --show-toplevel)" && test "$(stat -c %u "$ROOT_FOLDER")" != "$(id -u)" && printf '%s' "$ROOT_FOLDER"`, diff --git a/src/vs/platform/agentHost/node/devContainerServerCache.ts b/src/vs/platform/agentHost/node/devContainerServerCache.ts new file mode 100644 index 00000000000000..15f008870c7151 --- /dev/null +++ b/src/vs/platform/agentHost/node/devContainerServerCache.ts @@ -0,0 +1,109 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { posix } from '../../../base/common/path.js'; +import { vArray, vObj, vOptionalProp, vString, vUnion } from '../../../base/common/validation.js'; +import { getRemoteCLIDataDir, shellEscape } from './sshRemoteAgentHostHelpers.js'; + +export const devContainerServerCacheMount = 'type=volume,source=vscode,target=/vscode,external=true'; + +const configurationValidator = vObj({ + configuration: vObj({ + dockerComposeFile: vOptionalProp(vUnion(vString(), vArray(vString()))), + workspaceMount: vOptionalProp(vString()), + runArgs: vOptionalProp(vArray(vString())), + }), + mergedConfiguration: vObj({ + mounts: vOptionalProp(vArray(vUnion(vString(), vObj({ target: vString() })))), + }), +}); + +/** Compose controls its own mounts; do not shadow a configured mount to add an optional cache. */ +export function canAddDevContainerServerCacheMount(output: string): boolean { + const { content, error } = configurationValidator.validate(JSON.parse(output)); + if (error) { + throw new Error(`Invalid Dev Container configuration: ${error.message}`); + } + const { configuration, mergedConfiguration } = content; + const usesCachePath = (value: string) => /(?:^|[=:])\/vscode(?:\/|[,:\s]|$)/.test(value); + return configuration.dockerComposeFile === undefined + && !configuration.runArgs?.some(usesCachePath) + && !usesCachePath(configuration.workspaceMount ?? '') + && !mergedConfiguration.mounts?.some(mount => usesCachePath(typeof mount === 'string' ? mount : mount.target)); +} + +export function getDevContainerServerCachePath(serverDataFolderName: string, platform: { os: string; arch: string }): string { + return getDevContainerCachePath(serverDataFolderName, platform, 'servers'); +} + +export function getDevContainerCliCachePath(serverDataFolderName: string, platform: { os: string; arch: string }): string { + return getDevContainerCachePath(serverDataFolderName, platform, 'bin'); +} + +function getDevContainerCachePath(serverDataFolderName: string, platform: { os: string; arch: string }, kind: 'servers' | 'bin'): string { + getRemoteCLIDataDir(serverDataFolderName); + const sharedFolderName = serverDataFolderName.replace(/^\.+/, ''); + if (!sharedFolderName) { + throw new Error('Invalid Dev Container server data folder'); + } + if (!['linux', 'alpine'].includes(platform.os) || !['x64', 'arm64', 'armhf'].includes(platform.arch)) { + throw new Error(`Unsupported Dev Container server cache platform: ${platform.os}-${platform.arch}`); + } + return posix.join('/vscode', sharedFolderName, 'cli', kind, `${platform.os}-${platform.arch}`); +} + +/** Creates only missing cache directories, without changing ownership of an existing shared cache. */ +export function buildCreateDevContainerCacheCommand(cachePath: string, uid: string, gid: string, cacheRoot = '/vscode'): string { + if (!/^\d+$/.test(uid) || !/^\d+$/.test(gid) + || !posix.isAbsolute(cacheRoot) || cacheRoot.endsWith('/') || posix.normalize(cacheRoot) !== cacheRoot + || !cachePath.startsWith(`${cacheRoot}/`) || cachePath.endsWith('/') || posix.normalize(cachePath) !== cachePath) { + throw new Error('Invalid Dev Container server cache path or user'); + } + const parents: string[] = []; + for (let parent = posix.dirname(cachePath); parent !== cacheRoot; parent = posix.dirname(parent)) { + parents.unshift(parent); + } + return [ + 'set -eu', + 'umask 022', + `test ! -L ${shellEscape(cacheRoot)}`, + `test -d ${shellEscape(cacheRoot)}`, + ...parents.flatMap(parent => [ + `test ! -L ${shellEscape(parent)}`, + `test -d ${shellEscape(parent)} || mkdir ${shellEscape(parent)} || test -d ${shellEscape(parent)}`, + ]), + `test ! -L ${shellEscape(cachePath)}`, + `if mkdir ${shellEscape(cachePath)} 2>/dev/null; then chown ${shellEscape(`${uid}:${gid}`)} ${shellEscape(cachePath)}; else test -d ${shellEscape(cachePath)}; fi`, + ].join('\n'); +} + +/** Linux containers support ln -T, which prevents races from placing a link inside an existing directory. */ +export function buildLinkDevContainerServerCacheCommand(serverDataFolderName: string, cachePath: string): string { + return [ + 'set -eu', + `cli_dir=${getRemoteCLIDataDir(serverDataFolderName)}`, + `cache_dir=${shellEscape(cachePath)}`, + 'servers="$cli_dir/servers"', + 'if [ -L "$servers" ]; then', + ' if [ "$(readlink "$servers")" != "$cache_dir" ]; then echo "Preserving existing servers symlink" >&2; exit 1; fi', + 'elif [ -e "$servers" ]; then', + ' echo "Preserving existing private server cache" >&2; exit 1', + 'fi', + 'cache_unavailable() {', + ' if [ -L "$servers" ] && [ "$(readlink "$servers")" = "$cache_dir" ]; then rm -- "$servers"; fi', + ' echo "$1" >&2', + ' exit 1', + '}', + 'test -d "$cache_dir" && test -w "$cache_dir" || cache_unavailable "Shared server cache is not writable"', + 'for entry in "$cache_dir/lru.json" "$cache_dir/.locks"; do', + ' if [ -L "$entry" ] || { [ -e "$entry" ] && [ ! -w "$entry" ]; }; then cache_unavailable "Shared server cache metadata is not writable"; fi', + 'done', + 'mkdir -p "$cli_dir"', + 'if [ ! -L "$servers" ]; then', + ' ln -sT "$cache_dir" "$servers" || { test -L "$servers" && test "$(readlink "$servers")" = "$cache_dir"; }', + 'fi', + 'printf "%s" "$cache_dir"', + ].join('\n'); +} diff --git a/src/vs/platform/agentHost/node/remoteAgentHostCliCache.ts b/src/vs/platform/agentHost/node/remoteAgentHostCliCache.ts new file mode 100644 index 00000000000000..0e5409bae3f657 --- /dev/null +++ b/src/vs/platform/agentHost/node/remoteAgentHostCliCache.ts @@ -0,0 +1,74 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { posix } from '../../../base/common/path.js'; +import { getRemoteCLIArchiveName, getRemoteCLIBin, getRemoteCLIInstallRoot, shellEscape } from './sshRemoteAgentHostHelpers.js'; + +/** Downloads immutable CLI entries under a process lock, then installs an independent private copy. */ +export function buildInstallRemoteCliFromCacheCommand(cacheDir: string, serverDataFolderName: string, quality: string, commit: string, url: string): string { + const cliBin = getRemoteCLIBin(serverDataFolderName, quality, commit); + if (!posix.isAbsolute(cacheDir) || posix.normalize(cacheDir) !== cacheDir || !/^[a-z]+$/.test(quality)) { + throw new Error('Invalid CLI cache directory or quality'); + } + const archive = getRemoteCLIArchiveName(quality); + const key = `${quality}-${commit}`; + const commitGlob = '[0-9a-f]'.repeat(40); + return [ + 'set -eu', + 'umask 022', + `cache_dir=${shellEscape(cacheDir)}`, + `cache_key=${shellEscape(key)}`, + `archive=${shellEscape(archive)}`, + `install_root=${getRemoteCLIInstallRoot(serverDataFolderName)}`, + 'test ! -L "$cache_dir" && test -d "$cache_dir" && test -w "$cache_dir"', + 'command -v flock >/dev/null || { echo "CLI cache requires flock" >&2; exit 1; }', + 'test ! -L "$cache_dir/.locks"', + 'mkdir -p "$cache_dir/.locks"', + 'test ! -L "$cache_dir/.locks/$cache_key"', + 'exec 9>"$cache_dir/.locks/$cache_key"', + 'flock 9', + 'entry="$cache_dir/$cache_key"', + 'staging="$cache_dir/.$cache_key.staging"', + 'private_tmp=', + 'trap \'rm -rf -- "$staging"; if [ -n "$private_tmp" ]; then rm -rf -- "$private_tmp"; fi\' 0', + 'trap \'exit 1\' 1 2 15', + 'test ! -L "$entry"', + 'rm -rf -- "$staging"', + 'if [ ! -e "$entry" ]; then', + ' mkdir -p "$staging/content"', + ` curl -fsSL ${shellEscape(url)} -o "$staging/archive.tar.gz"`, + ' tar xzf "$staging/archive.tar.gz" -C "$staging/content"', + ' test ! -L "$staging/content/$archive" && test -f "$staging/content/$archive"', + ' chmod +x "$staging/content/$archive"', + ' version=$("$staging/content/$archive" --version)', + ` case "$version" in *${shellEscape(commit)}*) ;; *) echo "Downloaded CLI does not match the requested commit" >&2; exit 1 ;; esac`, + ' mv -T "$staging/content" "$entry"', + 'fi', + 'test ! -L "$entry/$archive" && test -f "$entry/$archive" && test -x "$entry/$archive"', + 'version=$("$entry/$archive" --version)', + `case "$version" in *${shellEscape(commit)}*) ;; *) echo "Cached CLI does not match the requested commit" >&2; exit 1 ;; esac`, + 'mkdir -p "$install_root"', + 'private_tmp=$(mktemp -d "$install_root/.cli-install-XXXXXX")', + 'cp "$entry/$archive" "$private_tmp/$archive"', + `mv -fT "$private_tmp/$archive" ${cliBin}`, + 'touch "$entry"', + 'rm -rf -- "$staging" "$private_tmp"', + 'trap - 0', + 'flock -u 9', + // Pruning takes the same entry locks as copying; a private install never depends on cache lifetime. + '(', + ' cd "$cache_dir"', + ` ls -1dt -- ${quality}-${commitGlob} 2>/dev/null | tail -n +6 | while IFS= read -r old; do`, + ' if [ "$old" != "$cache_key" ]; then', + ' (', + ' test ! -L ".locks/$old"', + ' exec 8>".locks/$old"', + ' if flock -n 8; then rm -rf -- "$old"; fi', + ' )', + ' fi', + ' done', + ')', + ].join('\n'); +} diff --git a/src/vs/platform/agentHost/node/remoteAgentHostCliInstaller.ts b/src/vs/platform/agentHost/node/remoteAgentHostCliInstaller.ts index ccf678096ece6c..35ce1af1c527bc 100644 --- a/src/vs/platform/agentHost/node/remoteAgentHostCliInstaller.ts +++ b/src/vs/platform/agentHost/node/remoteAgentHostCliInstaller.ts @@ -3,7 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { isCancellationError } from '../../../base/common/errors.js'; import { ILogService } from '../../log/common/log.js'; +import { buildInstallRemoteCliFromCacheCommand } from './remoteAgentHostCliCache.js'; import { buildCLIDownloadUrl, buildCleanupOldCLIsCommand, @@ -22,6 +24,8 @@ export interface IRemoteAgentHostCliInstallOptions { readonly reportInstalling: () => void; readonly logService: ILogService; readonly logPrefix?: string; + readonly cliCacheDir?: string; + readonly reportCacheStatus?: (message: string) => void; } /** The resolved CLI path and whether this invocation installed it. */ @@ -77,7 +81,26 @@ async function ensurePinnedCliInstalled( ].join(' && '); try { - await exec(installCommand); + let installedFromCache = false; + if (options.cliCacheDir) { + try { + await exec(buildInstallRemoteCliFromCacheCommand(options.cliCacheDir, options.serverDataFolderName, options.quality, commit, url)); + installedFromCache = true; + const message = `Installed private CLI copy from shared cache at ${options.cliCacheDir}`; + options.logService.info(`${logPrefix} ${message}`); + options.reportCacheStatus?.(message); + } catch (error) { + if (isCancellationError(error)) { + throw error; + } + const message = `Shared CLI cache unavailable; downloading a private copy: ${error instanceof Error ? error.message : String(error)}`; + options.logService.warn(`${logPrefix} ${message}`); + options.reportCacheStatus?.(message); + } + } + if (!installedFromCache) { + await exec(installCommand); + } const { code: versionCode } = await exec(`${cliBin} --version`, { ignoreExitCode: true }); if (versionCode !== 0) { throw new Error(`CLI at ${cliBin} failed --version check after install (exit code ${versionCode})`); @@ -86,6 +109,9 @@ async function ensurePinnedCliInstalled( await exec(buildCleanupOldCLIsCommand(options.serverDataFolderName, options.quality), { ignoreExitCode: true }); return { cliBin, installed: true }; } catch (error) { + if (isCancellationError(error)) { + throw error; + } const message = error instanceof Error ? error.message : String(error); options.logService.warn(`${logPrefix} Could not install matching CLI for commit ${commit}: ${message}. Looking for a fallback CLI...`); const fallback = await findFallbackCli(exec, options); diff --git a/src/vs/platform/agentHost/test/node/devContainerAgentHostService.test.ts b/src/vs/platform/agentHost/test/node/devContainerAgentHostService.test.ts index 271d485be755b7..e7bf96955b8700 100644 --- a/src/vs/platform/agentHost/test/node/devContainerAgentHostService.test.ts +++ b/src/vs/platform/agentHost/test/node/devContainerAgentHostService.test.ts @@ -26,6 +26,7 @@ import { IRequestService } from '../../../request/common/request.js'; import { URI } from '../../../../base/common/uri.js'; import { DevContainerAgentHostMainService, getDevContainerCliPath, getDevContainerExecArgs, IDevContainerRelay, parseDevContainerMounts, parseDevContainerUpResult, waitForDevContainerRelayConnection } from '../../node/devContainerAgentHostService.js'; import { ISshExec } from '../../node/sshRemoteAgentHostHelpers.js'; +import { devContainerServerCacheMount } from '../../node/devContainerServerCache.js'; class TestRelay implements IDevContainerRelay { readonly sent: string[] = []; @@ -72,8 +73,13 @@ class TestDevContainerAgentHostMainService extends DevContainerAgentHostMainServ gitRootFolder: string | undefined; gitRootReportedAsDubiousOwnership = false; safeDirectories: readonly string[] = []; - containerMounts: readonly { readonly Type: string; readonly Source: string; readonly Destination: string }[] = []; + containerMounts: readonly { readonly Type: string; readonly Source: string; readonly Destination: string; readonly Name?: string }[] = [ + { Type: 'volume', Source: '/volumes/vscode', Destination: '/vscode', Name: 'vscode' }, + ]; containerMountsError: Error | undefined; + cacheSetupError: Error | undefined; + cliCacheSetupError: Error | undefined; + cacheMountConfigured = false; hostDirectoryOwnedByCurrentUser = true; readonly checkedHostDirectories: string[] = []; readonly hostGitConfig = new Map(); @@ -92,6 +98,7 @@ class TestDevContainerAgentHostMainService extends DevContainerAgentHostMainServ private readonly _existingCertificateFiles: ReadonlySet = new Set(), testTmpDir = '/tmp', logService: NullLogService = new NullLogService(), + commit?: string, ) { const configurationService = new TestConfigurationService({ 'http.systemCertificates': systemCertificates }); super( @@ -99,7 +106,7 @@ class TestDevContainerAgentHostMainService extends DevContainerAgentHostMainServ new class extends mock() { override readonly quality = 'insider'; override readonly serverDataFolderName = '.vscode-server-oss'; - override readonly commit = undefined; + override readonly commit = commit; }(), NullTelemetryService, configurationService, @@ -175,10 +182,13 @@ class TestDevContainerAgentHostMainService extends DevContainerAgentHostMainServ protected override _runDevContainer(connectionId: string, args: readonly string[]): Promise<{ stdout: string; stderr: string; code: number }> { this.devContainerArgs.push([...args]); + if (args[0] === 'read-configuration') { + return Promise.resolve({ stdout: JSON.stringify({ configuration: {}, mergedConfiguration: { mounts: this.cacheMountConfigured ? [{ target: '/vscode' }] : [] } }), stderr: '', code: 0 }); + } if (args[0] === 'exec') { return Promise.resolve({ stdout: '', stderr: '', code: 0 }); } - assert.deepStrictEqual(args, ['up', '--log-level', 'debug', '--workspace-folder', '/workspace']); + assert.deepStrictEqual(args, ['up', '--log-level', 'debug', '--workspace-folder', '/workspace', ...this.cacheMountConfigured ? [] : ['--mount', devContainerServerCacheMount]]); this._reportOutput(connectionId, 'Starting Dev Container\n'); return Promise.resolve({ stdout: `[1 ms] Starting...\n${JSON.stringify({ outcome: 'success', containerId: 'container-id', remoteWorkspaceFolder: this.remoteWorkspaceFolder })}\n`, @@ -209,6 +219,12 @@ class TestDevContainerAgentHostMainService extends DevContainerAgentHostMainServ code: value === undefined ? 1 : 0, }); } + if (command === 'docker' && args[0] === 'exec' && args[1] === '--user' && args[2] === 'root') { + if (args.at(-1)?.includes('/cli/bin/') && this.cliCacheSetupError) { + throw this.cliCacheSetupError; + } + return Promise.resolve({ stdout: '', stderr: '', code: 0 }); + } throw new Error(`Unexpected local command: ${command} ${args.join(' ')}`); } @@ -219,6 +235,12 @@ class TestDevContainerAgentHostMainService extends DevContainerAgentHostMainServ protected override _createExec(): ISshExec { return async command => { this.execCommands.push(command); + if (command === 'id -u; id -g') { + return { stdout: '1000\n1000\n', stderr: '', code: 0 }; + } + if (command.includes('ln -sT') && this.cacheSetupError) { + throw this.cacheSetupError; + } if (command === 'command -v git >/dev/null 2>&1') { return { stdout: '', stderr: '', code: 0 }; } @@ -249,6 +271,9 @@ class TestDevContainerAgentHostMainService extends DevContainerAgentHostMainServ if (this._forceCliInstall && command.includes('--version &&')) { return { stdout: '', stderr: '', code: 1 }; } + if (this._forceCliInstall && command.startsWith('test -x ')) { + return { stdout: '', stderr: '', code: 1 }; + } if (command.includes('agent endpoints')) { this.endpointPolls++; return { @@ -529,14 +554,83 @@ suite('Dev Container Agent Host Main Service', () => { remoteWorkspaceFolder: '/workspaces/project', hostWorkspaceFolder: '/workspace', }, - devContainerArgs: [['up', '--log-level', 'debug', '--workspace-folder', '/workspace']], + devContainerArgs: [ + ['read-configuration', '--log-level', 'debug', '--workspace-folder', '/workspace', '--include-merged-configuration'], + ['up', '--log-level', 'debug', '--workspace-folder', '/workspace', '--mount', devContainerServerCacheMount], + ], relayCommand: '~/.vscode-server-oss/code-insiders --cli-data-dir ~/.vscode-server-oss/cli agent relay \'instance\' --user-data-dir \'/home/vscode/.config/Code\'', sent: ['{"jsonrpc":"2.0"}'], disposed: true, - output: ['connection:Starting Dev Container\n'], + output: ['connection:Starting Dev Container\n', 'connection:Using shared server cache at /vscode/vscode-server-oss/cli/servers/linux-x64\n'], }); }); + test('partitions the shared cache by container libc and configures it before running the CLI', async () => { + for (const [libc, platform] of [['', 'linux-x64'], ['musl', 'alpine-x64']]) { + const service = store.add(new TestDevContainerAgentHostMainService(libc)); + await service.connect({ connectionId: platform, workspaceFolder: '/workspace', name: 'Project' }); + const cacheCommandIndex = service.execCommands.findIndex(command => command.includes('ln -sT')); + assert.ok(cacheCommandIndex !== -1 && cacheCommandIndex < service.execCommands.findIndex(command => command.includes('agent endpoints'))); + assert.ok(service.execCommands[cacheCommandIndex].includes(`/vscode/vscode-server-oss/cli/servers/${platform}`)); + assert.ok(service.localCommands.some(command => command.command === 'docker' && command.args.slice(0, 4).join(' ') === 'exec --user root container-id')); + } + }); + + test('keeps the existing CLI cache with a visible warning when sharing is unavailable', async () => { + for (const reason of ['missing mount', 'existing private cache']) { + const log = new TestLogService(); + const service = store.add(new TestDevContainerAgentHostMainService('', false, undefined, process.env, true, [], new Set(), '/tmp', log)); + if (reason === 'missing mount') { + service.containerMounts = []; + } else { + service.cacheSetupError = new Error('Preserving existing private server cache'); + } + const output: string[] = []; + store.add(service.onDidOutput(event => output.push(event.data))); + await service.connect({ connectionId: reason, workspaceFolder: '/workspace', name: 'Project' }); + assert.deepStrictEqual({ + connected: service.relayCommand !== undefined, + warned: log.warnings.some(warning => warning.message.includes('Shared server cache unavailable')), + output: output.some(line => line.includes('keeping the existing CLI cache')), + }, { connected: true, warned: true, output: true }); + } + }); + + test('does not add a duplicate cache mount when the container configuration supplies it', async () => { + const service = store.add(new TestDevContainerAgentHostMainService()); + service.cacheMountConfigured = true; + await service.connect({ connectionId: 'configured-mount', workspaceFolder: '/workspace', name: 'Project' }); + assert.deepStrictEqual(service.devContainerArgs.filter(args => args[0] === 'up'), [['up', '--log-level', 'debug', '--workspace-folder', '/workspace']]); + }); + + test('caches bootstrap CLIs for each libc even when the server cache remains private', async () => { + for (const [libc, platform] of [['', 'linux-x64'], ['musl', 'alpine-x64']]) { + const service = store.add(new TestDevContainerAgentHostMainService(libc, true, undefined, process.env, true, [], new Set(), '/tmp', new NullLogService(), 'a'.repeat(40))); + service.cacheSetupError = new Error('Preserving existing private server cache'); + await service.connect({ connectionId: platform, workspaceFolder: '/workspace', name: 'Project' }); + const cacheCommand = service.execCommands.find(command => command.includes('flock 9')); + assert.deepStrictEqual({ + path: cacheCommand?.includes(`/vscode/vscode-server-oss/cli/bin/${platform}`), + copies: cacheCommand?.includes('cp "$entry/$archive" "$private_tmp/$archive"'), + privateDownloads: service.execCommands.filter(command => command.includes('curl') && !command.includes('flock 9')).length, + }, { path: true, copies: true, privateDownloads: 0 }); + } + }); + + test('failure to prepare the CLI cache does not disable the shared server cache', async () => { + const service = store.add(new TestDevContainerAgentHostMainService('', true, undefined, process.env, true, [], new Set(), '/tmp', new NullLogService(), 'a'.repeat(40))); + service.cliCacheSetupError = new Error('Read-only CLI cache'); + const output: string[] = []; + store.add(service.onDidOutput(event => output.push(event.data))); + await service.connect({ connectionId: 'private-cli', workspaceFolder: '/workspace', name: 'Project' }); + assert.deepStrictEqual({ + sharedServer: output.some(line => line.includes('Using shared server cache')), + warning: output.some(line => line.includes('Read-only CLI cache')), + cacheCommands: service.execCommands.filter(command => command.includes('flock 9')).length, + privateDownloads: service.execCommands.filter(command => command.includes('curl')).length, + }, { sharedServer: true, warning: true, cacheCommands: 0, privateDownloads: 1 }); + }); + test('forwards missing host Git identity without overwriting container identity', async () => { const forwarded = store.add(new TestDevContainerAgentHostMainService()); forwarded.hostGitConfig.set('user.name', 'Host User'); @@ -565,7 +659,7 @@ suite('Dev Container Agent Host Main Service', () => { }); assert.deepStrictEqual({ - hostCommands: forwarded.localCommands, + hostCommands: forwarded.localCommands.filter(command => command.command === 'git'), forwardedCommands: forwarded.execCommands.filter(command => command.includes('user.')), preservedCommands: preserved.execCommands.filter(command => command.includes('user.')), absentCommands: absent.execCommands.filter(command => command.includes('user.')), diff --git a/src/vs/platform/agentHost/test/node/devContainerServerCache.test.ts b/src/vs/platform/agentHost/test/node/devContainerServerCache.test.ts new file mode 100644 index 00000000000000..50f0879be09e04 --- /dev/null +++ b/src/vs/platform/agentHost/test/node/devContainerServerCache.test.ts @@ -0,0 +1,229 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { execFile } from 'child_process'; +import { chmod, lstat, mkdtemp, mkdir, readFile, readlink, readdir, rm, stat, symlink, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { promisify } from 'util'; +import { join } from '../../../../base/common/path.js'; +import { isLinux } from '../../../../base/common/platform.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { buildCreateDevContainerCacheCommand, buildLinkDevContainerServerCacheCommand, canAddDevContainerServerCacheMount, getDevContainerCliCachePath, getDevContainerServerCachePath } from '../../node/devContainerServerCache.js'; + +suite('Dev Container server cache', () => { + ensureNoDisposablesAreLeakedInTestSuite(); + + test('uses the product server folder and the container platform', () => { + assert.deepStrictEqual([ + getDevContainerServerCachePath('.vscode-server-insiders', { os: 'linux', arch: 'arm64' }), + getDevContainerServerCachePath('.vscode-server', { os: 'alpine', arch: 'x64' }), + getDevContainerServerCachePath('.vscode-server-oss', { os: 'linux', arch: 'armhf' }), + getDevContainerCliCachePath('.vscode-server-insiders', { os: 'linux', arch: 'arm64' }), + getDevContainerCliCachePath('.vscode-server', { os: 'alpine', arch: 'x64' }), + ], [ + '/vscode/vscode-server-insiders/cli/servers/linux-arm64', + '/vscode/vscode-server/cli/servers/alpine-x64', + '/vscode/vscode-server-oss/cli/servers/linux-armhf', + '/vscode/vscode-server-insiders/cli/bin/linux-arm64', + '/vscode/vscode-server/cli/bin/alpine-x64', + ]); + }); + + test('rejects unsafe path and ownership arguments', () => { + assert.throws(() => getDevContainerServerCachePath('../other', { os: 'linux', arch: 'x64' })); + assert.throws(() => getDevContainerServerCachePath('..', { os: 'linux', arch: 'x64' })); + assert.throws(() => getDevContainerServerCachePath('.vscode-server', { os: 'linux', arch: 'x64;false' })); + assert.throws(() => buildCreateDevContainerCacheCommand('/vscode/../other', '1000', '1000')); + assert.throws(() => buildCreateDevContainerCacheCommand('/vscode/cache', '1000;false', '1000')); + assert.throws(() => buildCreateDevContainerCacheCommand('/tmp/cache', '1000', '1000')); + assert.throws(() => buildCreateDevContainerCacheCommand('/tmp/cache', '1000', '1000', '/')); + assert.throws(() => buildCreateDevContainerCacheCommand('/tmp/cache', '1000', '1000', '/tmp/..')); + }); + + (isLinux ? test : test.skip)('creates cache directories concurrently and preserves existing ownership', async () => { + assert.ok(process.getuid && process.getgid); + const uid = process.getuid(); + const gid = process.getgid(); + const root = await mkdtemp(join(tmpdir(), 'vscode-cache-creation-')); + try { + const parents = ['product', 'product/cli', 'product/cli/servers'].map(path => join(root, path)); + const cache = join(parents[2], 'linux-arm64'); + const run = (owner = uid) => promisify(execFile)('/bin/sh', ['-c', buildCreateDevContainerCacheCommand(cache, String(owner), String(gid), root)]); + await Promise.all([run(), run(), run()]); + const created = await stat(cache); + await run(uid + 1); + const existing = await stat(cache); + assert.deepStrictEqual({ + created: { uid: created.uid, gid: created.gid, mode: created.mode & 0o777 }, + existing: { uid: existing.uid, gid: existing.gid, inode: existing.ino }, + parents: await Promise.all(parents.map(async path => (await stat(path)).isDirectory())), + }, { + created: { uid, gid, mode: 0o755 }, + existing: { uid, gid, inode: created.ino }, + parents: [true, true, true], + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + (isLinux ? test : test.skip)('cache directory creation rejects symlinked parents and cache directories', async () => { + assert.ok(process.getuid && process.getgid); + const uid = String(process.getuid()); + const gid = String(process.getgid()); + const root = await mkdtemp(join(tmpdir(), 'vscode-cache-creation-')); + try { + const outside = join(root, 'outside'); + const parent = join(root, 'product'); + const cache = join(parent, 'cli', 'servers', 'linux-arm64'); + const run = () => promisify(execFile)('/bin/sh', ['-c', buildCreateDevContainerCacheCommand(cache, uid, gid, root)]); + await mkdir(outside); + await symlink(outside, parent); + await assert.rejects(run()); + await rm(parent); + await mkdir(join(parent, 'cli', 'servers'), { recursive: true }); + await symlink(outside, cache); + await assert.rejects(run()); + assert.deepStrictEqual({ + target: await readlink(cache), + outsideEntries: await readdir(outside), + }, { target: outside, outsideEntries: [] }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + test('only adds a mount when it will not conflict with configured mounts', () => { + const canAdd = (configuration: object, mounts: readonly (string | { target: string })[] = []) => canAddDevContainerServerCacheMount(JSON.stringify({ configuration, mergedConfiguration: { mounts } })); + assert.deepStrictEqual([ + canAdd({}), + canAdd({}, ['type=volume,source=vscode,target=/vscode']), + canAdd({}, [{ target: '/vscode' }]), + canAdd({}, [{ target: '/vscode/other' }]), + canAdd({ runArgs: ['-v', 'custom:/vscode:ro'] }), + canAdd({ workspaceMount: 'type=bind,source=/workspace,target=/vscode' }), + canAdd({ dockerComposeFile: 'compose.yml' }), + canAdd({}, [{ target: '/vscode-other' }]), + ], [true, false, false, false, false, false, false, true]); + }); + + (isLinux ? test : test.skip)('shares extracted installs while leaving credentials private and supports concurrent setup', async () => { + const root = await mkdtemp(join(tmpdir(), 'vscode-server-cache-')); + try { + const cache = join(root, 'shared cache'); + const homes = [join(root, 'first'), join(root, 'second')]; + await mkdir(cache); + const run = async (home: string) => { + await promisify(execFile)('/bin/sh', ['-c', buildLinkDevContainerServerCacheCommand('.vscode-server-insiders', cache)], { env: { ...process.env, HOME: home } }); + }; + await Promise.all(homes.map(home => mkdir(join(home, '.vscode-server-insiders', 'cli'), { recursive: true }))); + await Promise.all([run(homes[0]), run(homes[0]), run(homes[1])]); + const cli = (home: string) => join(home, '.vscode-server-insiders', 'cli'); + await mkdir(join(cli(homes[0]), 'servers', 'Insiders-commit', 'server'), { recursive: true }); + await writeFile(join(cli(homes[0]), 'servers', 'Insiders-commit', 'server', 'product.json'), '{"commit":"commit"}'); + await writeFile(join(cli(homes[0]), 'token.json'), 'private'); + assert.deepStrictEqual({ + targets: await Promise.all(homes.map(home => readlink(join(cli(home), 'servers')))), + product: await readFile(join(cli(homes[1]), 'servers', 'Insiders-commit', 'server', 'product.json'), 'utf8'), + secondPrivateEntries: await readdir(cli(homes[1])), + cacheEntries: await readdir(cache), + }, { + targets: [cache, cache], + product: '{"commit":"commit"}', + secondPrivateEntries: ['servers'], + cacheEntries: ['Insiders-commit'], + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + (isLinux ? test : test.skip)('preserves existing private directories and different symlinks', async () => { + const root = await mkdtemp(join(tmpdir(), 'vscode-server-cache-')); + try { + const cache = join(root, 'shared'); + const cli = join(root, '.vscode-server', 'cli'); + const servers = join(cli, 'servers'); + await mkdir(cache); + await mkdir(servers, { recursive: true }); + await writeFile(join(servers, 'keep'), 'private'); + const run = () => promisify(execFile)('/bin/sh', ['-c', buildLinkDevContainerServerCacheCommand('.vscode-server', cache)], { env: { ...process.env, HOME: root } }); + await assert.rejects(run(), /Preserving existing private server cache/); + assert.strictEqual(await readFile(join(servers, 'keep'), 'utf8'), 'private'); + await rm(servers, { recursive: true }); + await symlink(join(root, 'other-cache'), servers); + await assert.rejects(run(), /Preserving existing servers symlink/); + assert.strictEqual(await readlink(servers), join(root, 'other-cache')); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + (isLinux ? test : test.skip)('removes only the same-target link when shared storage or metadata is unavailable', async () => { + const root = await mkdtemp(join(tmpdir(), 'vscode-cache-fallback-')); + try { + const cache = join(root, 'shared'); + const cli = join(root, '.vscode-server', 'cli'); + const servers = join(cli, 'servers'); + const run = () => promisify(execFile)('/bin/sh', ['-c', buildLinkDevContainerServerCacheCommand('.vscode-server', cache)], { env: { ...process.env, HOME: root } }); + await mkdir(cli, { recursive: true }); + await symlink(cache, servers); + await assert.rejects(run(), /Shared server cache is not writable/); + await mkdir(servers); + await writeFile(join(servers, 'private'), 'keep'); + await assert.rejects(run(), /Preserving existing private server cache/); + assert.strictEqual(await readFile(join(servers, 'private'), 'utf8'), 'keep'); + await rm(servers, { recursive: true }); + await symlink(join(root, 'other'), servers); + await assert.rejects(run(), /Preserving existing servers symlink/); + assert.strictEqual(await readlink(servers), join(root, 'other')); + await rm(servers); + await mkdir(cache); + await writeFile(join(cache, 'keep'), 'shared'); + for (const name of ['lru.json', '.locks']) { + await symlink(cache, servers); + await symlink(join(root, 'other'), join(cache, name)); + await assert.rejects(run(), /Shared server cache metadata is not writable/); + await mkdir(servers); + assert.deepStrictEqual({ + privateDirectory: (await lstat(servers)).isDirectory(), + sharedData: await readFile(join(cache, 'keep'), 'utf8'), + }, { privateDirectory: true, sharedData: 'shared' }); + await rm(servers, { recursive: true }); + await rm(join(cache, name)); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + (isLinux && process.getuid?.() !== 0 ? test : test.skip)('unwritable shared directories and metadata fall back to a writable private cache', async () => { + const root = await mkdtemp(join(tmpdir(), 'vscode-cache-permissions-')); + try { + const cache = join(root, 'shared'); + const servers = join(root, '.vscode-server', 'cli', 'servers'); + const run = () => promisify(execFile)('/bin/sh', ['-c', buildLinkDevContainerServerCacheCommand('.vscode-server', cache)], { env: { ...process.env, HOME: root } }); + await mkdir(cache); + await writeFile(join(cache, 'lru.json'), '[]'); + await mkdir(join(cache, '.locks')); + for (const path of [cache, join(cache, 'lru.json'), join(cache, '.locks')]) { + await run(); + await chmod(path, 0o555); + try { + await assert.rejects(run(), /Shared server cache.*is not writable/); + await mkdir(servers); + await writeFile(join(servers, 'private'), 'installed'); + assert.strictEqual(await readFile(join(servers, 'private'), 'utf8'), 'installed'); + await rm(servers, { recursive: true }); + } finally { + await chmod(path, 0o755); + } + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/vs/platform/agentHost/test/node/remoteAgentHostCliCache.test.ts b/src/vs/platform/agentHost/test/node/remoteAgentHostCliCache.test.ts new file mode 100644 index 00000000000000..4326bf031f10d1 --- /dev/null +++ b/src/vs/platform/agentHost/test/node/remoteAgentHostCliCache.test.ts @@ -0,0 +1,177 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { execFile } from 'child_process'; +import { mkdir, mkdtemp, readFile, readdir, rm, stat, symlink, utimes, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { promisify } from 'util'; +import { join } from '../../../../base/common/path.js'; +import { isLinux } from '../../../../base/common/platform.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { buildInstallRemoteCliFromCacheCommand } from '../../node/remoteAgentHostCliCache.js'; +import { shellEscape } from '../../node/sshRemoteAgentHostHelpers.js'; + +const exec = promisify(execFile); +const commit = 'a'.repeat(40); +const folder = '.vscode-server-insiders'; +const archive = 'code-insiders'; + +suite('Remote Agent Host CLI cache commands', () => { + ensureNoDisposablesAreLeakedInTestSuite(); + + test('rejects invalid cache keys and paths', () => { + assert.throws(() => buildInstallRemoteCliFromCacheCommand('relative', folder, 'insider', commit, 'url')); + assert.throws(() => buildInstallRemoteCliFromCacheCommand('/cache/../elsewhere', folder, 'insider', commit, 'url')); + assert.throws(() => buildInstallRemoteCliFromCacheCommand('/cache', folder, '../insider', commit, 'url')); + assert.throws(() => buildInstallRemoteCliFromCacheCommand('/cache', folder, 'insider', 'latest', 'url')); + }); +}); + +(isLinux ? suite : suite.skip)('Remote Agent Host CLI cache shell', () => { + ensureNoDisposablesAreLeakedInTestSuite(); + let root: string; + let cache: string; + let tools: string; + + setup(async () => { + root = await mkdtemp(join(tmpdir(), 'vscode-cli-cache-')); + cache = join(root, 'shared cache'); + tools = join(root, 'tools'); + await mkdir(cache); + await mkdir(tools); + await writeFile(join(tools, 'curl'), [ + '#!/bin/sh', + 'set -eu', + 'printf "download\\n" >> "$TEST_DOWNLOADS"', + 'test "$3" = "-o"', + 'if [ "${TEST_FAIL_DOWNLOAD:-}" = "yes" ]; then echo "download failed" >&2; exit 22; fi', + 'cp "$TEST_ARCHIVE" "$4"', + ].join('\n'), { mode: 0o755 }); + }); + + teardown(async () => { + await rm(root, { recursive: true, force: true }); + }); + + async function createArchive(version = commit): Promise { + const content = await mkdtemp(join(root, 'content-')); + await writeFile(join(content, archive), `#!/bin/sh\nprintf '%s\\n' 'code 1.0.0 (commit ${version})'\n`, { mode: 0o755 }); + const result = `${content}.tar.gz`; + await exec('tar', ['czf', result, '-C', content, archive]); + return result; + } + + function run(home: string, tarball: string, version = commit, env: NodeJS.ProcessEnv = {}, prefix = '') { + return exec('/bin/sh', ['-c', prefix + buildInstallRemoteCliFromCacheCommand(cache, folder, 'insider', version, 'https://example.invalid/cli')], { + env: { ...process.env, HOME: home, PATH: `${tools}:${process.env.PATH}`, TEST_ARCHIVE: tarball, TEST_DOWNLOADS: join(root, 'downloads'), ...env }, + }); + } + + const privateBin = (home: string, version = commit) => join(home, folder, `${archive}-${version}`); + const cachedBin = (directory: string, version = commit) => join(directory, `insider-${version}`, archive); + + test('concurrent cold installs download once and produce independent private copies', async () => { + const tarball = await createArchive(); + const homes = [join(root, 'first'), join(root, 'second')]; + await Promise.all(homes.map(home => run(home, tarball))); + const bins = [cachedBin(cache), ...homes.map(home => privateBin(home))]; + const inodes = await Promise.all(bins.map(async bin => (await stat(bin)).ino)); + await writeFile(privateBin(homes[0]), 'updated privately'); + const original = await readFile(cachedBin(cache), 'utf8'); + await rm(join(cache, `insider-${commit}`), { recursive: true }); + assert.deepStrictEqual({ + downloads: await readFile(join(root, 'downloads'), 'utf8'), + distinctInodes: new Set(inodes).size, + unaffectedCopy: await readFile(privateBin(homes[1]), 'utf8'), + leftovers: (await readdir(cache)).filter(name => name.endsWith('.staging')), + privateEntries: await readdir(join(homes[0], folder)), + }, { + downloads: 'download\n', + distinctInodes: 3, + unaffectedCopy: original, + leftovers: [], + privateEntries: [`${archive}-${commit}`], + }); + }); + + test('a warm cache does not invoke the downloader', async () => { + const tarball = await createArchive(); + await run(join(root, 'first'), tarball); + await run(join(root, 'second'), tarball, commit, { TEST_FAIL_DOWNLOAD: 'yes' }); + assert.strictEqual(await readFile(join(root, 'downloads'), 'utf8'), 'download\n'); + }); + + test('failed downloads and invalid executables are never published and can be retried', async () => { + const home = join(root, 'home'); + const tarball = await createArchive(); + await assert.rejects(run(home, tarball, commit, { TEST_FAIL_DOWNLOAD: 'yes' }), /download failed/); + await assert.rejects(run(home, await createArchive('b'.repeat(40))), /does not match the requested commit/); + assert.deepStrictEqual(await readdir(cache), ['.locks']); + await run(home, tarball); + assert.strictEqual(await readFile(join(root, 'downloads'), 'utf8'), 'download\ndownload\ndownload\n'); + }); + + test('recovers staging left by an interrupted owner without leaving stale locks', async () => { + const staging = join(cache, `.insider-${commit}.staging`); + await mkdir(join(cache, '.locks')); + await mkdir(staging); + await writeFile(join(staging, 'partial'), 'incomplete'); + await exec('/bin/sh', ['-c', 'exec 9>"$1"; flock 9', 'sh', join(cache, '.locks', `insider-${commit}`)]); + await run(join(root, 'home'), await createArchive()); + assert.deepStrictEqual((await readdir(cache)).sort(), ['.locks', `insider-${commit}`]); + }); + + test('rejects redirected entries and rejects a cached executable with a different commit', async () => { + const other = join(root, 'other'); + await mkdir(other); + await symlink(other, join(cache, `insider-${commit}`)); + await assert.rejects(run(join(root, 'home'), await createArchive())); + await rm(join(cache, `insider-${commit}`)); + await run(join(root, 'first'), await createArchive()); + await writeFile(cachedBin(cache), '#!/bin/sh\necho wrong-commit\n'); + await assert.rejects(run(join(root, 'second'), await createArchive()), /Cached CLI does not match/); + assert.deepStrictEqual(await readdir(other), []); + }); + + test('retention removes old cache entries without affecting installed copies', async () => { + const home = join(root, 'home'); + for (let index = 0; index < 6; index++) { + const version = index.toString().repeat(40); + await run(home, await createArchive(version), version); + await utimes(join(cache, `insider-${version}`), index + 1, index + 1); + } + assert.deepStrictEqual({ + cached: (await readdir(cache)).filter(name => name.startsWith('insider-')).length, + private: (await readdir(join(home, folder))).length, + firstCopy: (await exec(privateBin(home, '0'.repeat(40)), ['--version'])).stdout.trim(), + }, { + cached: 5, + private: 6, + firstCopy: `code 1.0.0 (commit ${'0'.repeat(40)})`, + }); + }); + + test('retention skips an entry locked by another installer', async () => { + const home = join(root, 'home'); + for (let index = 0; index < 5; index++) { + const version = index.toString().repeat(40); + await run(home, await createArchive(version), version); + await utimes(join(cache, `insider-${version}`), index + 1, index + 1); + } + const lockedVersion = '0'.repeat(40); + const lock = shellEscape(join(cache, '.locks', `insider-${lockedVersion}`)); + const prefix = `exec 8>${lock}\nflock 8\n`; + const sixth = '5'.repeat(40); + await run(home, await createArchive(sixth), sixth, {}, prefix); + assert.strictEqual((await readdir(cache)).filter(name => name.startsWith('insider-')).length, 6); + const seventh = '6'.repeat(40); + await run(home, await createArchive(seventh), seventh); + assert.deepStrictEqual({ + retained: (await readdir(cache)).filter(name => name.startsWith('insider-')).length, + privateCopy: (await exec(privateBin(home, lockedVersion), ['--version'])).stdout.trim(), + }, { retained: 5, privateCopy: `code 1.0.0 (commit ${lockedVersion})` }); + }); +}); diff --git a/src/vs/platform/agentHost/test/node/remoteAgentHostCliInstaller.test.ts b/src/vs/platform/agentHost/test/node/remoteAgentHostCliInstaller.test.ts new file mode 100644 index 00000000000000..ada6a3ecf5660a --- /dev/null +++ b/src/vs/platform/agentHost/test/node/remoteAgentHostCliInstaller.test.ts @@ -0,0 +1,88 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { CancellationError } from '../../../../base/common/errors.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { NullLogService } from '../../../log/common/log.js'; +import { ensureRemoteAgentHostCliInstalled } from '../../node/remoteAgentHostCliInstaller.js'; +import { ISshExec } from '../../node/sshRemoteAgentHostHelpers.js'; + +suite('Remote Agent Host CLI installer cache', () => { + const store = ensureNoDisposablesAreLeakedInTestSuite(); + const commit = 'a'.repeat(40); + const cliBin = `~/.vscode-server-insiders/code-insiders-${commit}`; + + function fixture(cacheError?: Error, exists = false, pinned = true, shared = true) { + const commands: string[] = []; + const messages: string[] = []; + const exec: ISshExec = async command => { + commands.push(command); + if (command.startsWith('test -x ')) { + return { code: exists ? 0 : 1, stdout: '', stderr: '' }; + } + if (command.includes('flock 9') && cacheError) { + throw cacheError; + } + return { code: 0, stdout: '', stderr: '' }; + }; + return { + commands, + messages, + run: () => ensureRemoteAgentHostCliInstalled(exec, { os: 'linux', arch: 'arm64' }, { + serverDataFolderName: '.vscode-server-insiders', + quality: 'insider', + commit: pinned ? commit : undefined, + cliCacheDir: shared ? '/vscode/vscode-server-insiders/cli/bin/linux-arm64' : undefined, + reportCacheStatus: message => messages.push(message), + reportInstalling: () => { }, + logService: store.add(new NullLogService()), + }), + }; + } + + test('installs a private copy from the cache instead of using the private downloader', async () => { + const test = fixture(); + const result = await test.run(); + assert.deepStrictEqual({ + result, + cacheAttempts: test.commands.filter(command => command.includes('flock 9')).length, + privateDownloads: test.commands.filter(command => command.includes('curl -fsSL') && !command.includes('flock 9')).length, + reportedCache: test.messages.some(message => message.startsWith('Installed private CLI copy')), + validated: test.commands.includes(`${cliBin} --version`), + }, { result: { cliBin, installed: true }, cacheAttempts: 1, privateDownloads: 0, reportedCache: true, validated: true }); + }); + + test('reports cache failure and falls back to the existing private installer', async () => { + const test = fixture(new Error('flock is unavailable')); + await test.run(); + assert.deepStrictEqual({ + privateDownloads: test.commands.filter(command => command.includes('curl -fsSL') && !command.includes('flock 9')).length, + messages: test.messages, + }, { privateDownloads: 1, messages: ['Shared CLI cache unavailable; downloading a private copy: flock is unavailable'] }); + }); + + test('does not fall back or continue downloading after cancellation', async () => { + const test = fixture(new CancellationError()); + await assert.rejects(test.run(), CancellationError); + assert.deepStrictEqual({ + lastCommandWasCache: test.commands.at(-1)?.includes('flock 9'), + messages: test.messages, + }, { lastCommandWasCache: true, messages: [] }); + }); + + test('preserves private reuse, unpinned updates, and installations without a shared cache', async () => { + const existing = fixture(undefined, true); + const unpinned = fixture(undefined, false, false); + const privateOnly = fixture(undefined, false, true, false); + await Promise.all([existing.run(), unpinned.run(), privateOnly.run()]); + assert.deepStrictEqual({ + cacheCommands: [...existing.commands, ...unpinned.commands, ...privateOnly.commands].filter(command => command.includes('flock 9')), + existingDownloads: existing.commands.filter(command => command.includes('curl')), + unpinnedUpdates: unpinned.commands.some(command => command.includes('code-insiders update')), + privateDownloads: privateOnly.commands.filter(command => command.includes('curl')).length, + }, { cacheCommands: [], existingDownloads: [], unpinnedUpdates: true, privateDownloads: 1 }); + }); +}); diff --git a/src/vs/sessions/contrib/providers/remoteAgentHost/REMOTE_AGENT_HOST_SESSIONS_PROVIDER.md b/src/vs/sessions/contrib/providers/remoteAgentHost/REMOTE_AGENT_HOST_SESSIONS_PROVIDER.md index 65ce4ccfea4c82..1b7a5b28a814e3 100644 --- a/src/vs/sessions/contrib/providers/remoteAgentHost/REMOTE_AGENT_HOST_SESSIONS_PROVIDER.md +++ b/src/vs/sessions/contrib/providers/remoteAgentHost/REMOTE_AGENT_HOST_SESSIONS_PROVIDER.md @@ -94,6 +94,10 @@ Focused tests live beside the remote provider and remote-host services. Tests ow VS Code bundles `@devcontainers/cli`; the workspace's host runs that pinned version, resolves Docker and related tools from its own environment, and owns the CLI processes and relays. For local workspaces this runs in the desktop shared process. For SSH, Tunnel, and WSL workspaces it runs in the connected source Agent Host through a capability-gated VS Code protocol extension. Older hosts do not offer container execution. WSL sources require Docker inside the selected distribution, for example through Docker Desktop's WSL integration. The connector runs `devcontainer up`, installs the matching VS Code remote CLI inside the container, and reuses or launches a dedicated standalone Agent Host. Its WebSocket protocol is relayed over `devcontainer exec` standard input/output and, for remote workspaces, over the existing source-host connection. +New image/Dockerfile containers receive the Docker daemon's shared `vscode` volume at `/vscode` unless their configuration already uses that path. Compose mounts remain configuration-owned. When that volume is available, the launcher links only the container user's CLI `servers` directory to `/vscode//cli/servers/-`, partitioned by the container's architecture and libc. The CLI owns the extracted installations, download locks, and eviction within that directory; credentials, endpoint registration, logs, and other runtime state remain private to each container. Existing private cache directories and different symlinks are never replaced. Cache directories retain their existing ownership, and unavailable or unwritable shared storage is reported before continuing with the existing CLI cache. The extension's older server-cache layout is not migrated. + +Commit-pinned bootstrap CLI downloads are cached separately under `/vscode//cli/bin/-/-`. The host-side installer owns this cache, without requiring a CLI to bootstrap itself. It downloads under an OS file lock, publishes validated entries atomically, and copies the executable into the container's private install location; self-update and cache eviction cannot change that private copy. Shared CLI retention keeps the five most recently used entries per quality and platform, skipping locked entries. Containers without writable shared storage or `flock` fall back to private downloads with diagnostics. Unpinned development builds retain their private, self-updating CLI installation. + Container entries retain the source host's VS Code authority and native workspace path. Open in VS Code encodes SSH and Tunnel sources with a parent authority; WSL sources instead encode the distribution and path as a Windows WSL UNC host path, as required by the Dev Containers extension. Container execution and detached-worktree operations continue to use the source distribution's Linux paths. The service persists the source-workspace identity once the connected provider publishes a session and keeps that `RemoteAgentHostSessionsProvider` registered independently of its live transport. On startup it reconstructs providers for persisted workspaces so their cached sessions remain visible; opening one of those sessions, using the provider's connect action, or another operation that requires the remote host starts the Dev Container and restores the transport on demand. The connection factory's `DevContainer` entry remains runtime-only because it carries the live connector and transport state. The shared remote Agent Host contribution observes connected transports and supplies connection-level filesystem, model, terminal, and log integration. Dev Container CLI output is streamed into one stable `Dev Container ()` Output channel per source workspace, which is reused across connection attempts.