From 3142a2f6541641488d572da453efba728a6c0539 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 11 Sep 2026 11:10:37 -0400 Subject: [PATCH 01/42] fix: fail denied input downloads once per batch (#177) --- api/src/download.test.ts | 70 +++++++++++++++++++++++++++++++++++++++- api/src/job.ts | 40 +++++++++++++++++++---- 2 files changed, 103 insertions(+), 7 deletions(-) diff --git a/api/src/download.test.ts b/api/src/download.test.ts index 5372a40c..2c7248e8 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll } from 'bun:test'; +import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, spyOn } from 'bun:test'; import * as fsp from 'fs/promises'; import * as path from 'path'; import * as os from 'os'; @@ -439,6 +439,74 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(contents).toBe('hi'); }); + it.each([401, 403])('does not retry an HTTP %i authorization denial', async status => { + const file: TFile = { id: 'denied', storage_session_id: 'previous', name: 'denied.txt' }; + let requests = 0; + routes.set('/sessions/previous/objects/denied', { + status, + onRequest: () => { requests++; }, + }); + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + + await expect(job.downloadAndWriteFile(file, 5, 1)).rejects.toThrow(`HTTP error: ${status}`); + expect(requests).toBe(1); + expect(await fsp.readdir(tmpDir)).toEqual([]); + }); + + it.each([404, 408, 429, 503])('still retries transient HTTP %i responses', async status => { + const file: TFile = { id: 'transient', storage_session_id: 'previous', name: 'ready.txt' }; + let requests = 0; + const route: Route = { + status, + body: 'ready', + onRequest: () => { if (++requests === 2) route.status = 200; }, + }; + routes.set('/sessions/previous/objects/transient', route); + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + + await expect(job.downloadAndWriteFile(file, 5, 1)).resolves.toBe('ready.txt'); + expect(requests).toBe(2); + expect(await fsp.readFile(path.join(tmpDir, 'ready.txt'), 'utf8')).toBe('ready'); + }); + + it('accounts for a denied 240-file batch once and stops queued downloads', async () => { + const files: TFile[] = Array.from({ length: 240 }, (_, index) => ({ + id: `file-${index}`, storage_session_id: 'previous', name: `file-${index}.txt`, + })); + let requests = 0; + routes.set('/sessions/previous/objects', { status: 200, body: '[]' }); + for (const file of files) { + routes.set(`/sessions/previous/objects/${file.id}`, { + status: 403, + delayMs: file.id === 'file-0' ? 0 : 30, + onRequest: () => { requests++; }, + }); + } + let dirty = false; + const job = makeJob(files, sessionWorkspaceAt(tmpDir, 'batch-test', () => { dirty = true; })); + const log = (job as unknown as { log: import('pino').Logger }).log; + const errorLog = spyOn(log, 'error'); + const originalConcurrency = config.prime_concurrency; + config.prime_concurrency = 8; + try { + await expect(job.prime()).rejects.toBeInstanceOf(SessionWorkspaceDirtyError); + expect(dirty).toBe(true); + expect(requests).toBeGreaterThan(0); + expect(requests).toBeLessThanOrEqual(8); + expect(errorLog).toHaveBeenCalledTimes(1); + expect(errorLog).toHaveBeenCalledWith(expect.objectContaining({ + inputCount: 240, completed: 0, failed: 1, cancelled: 7, notStarted: 232, + }), 'Input preparation batch failed'); + expect(await fsp.readdir(tmpDir)).toEqual([]); + } finally { + errorLog.mockRestore(); + config.prime_concurrency = originalConcurrency; + await job.cleanup(); + } + }); + it('fails when the server keeps 404-ing past the retry cap (no phantom write)', async () => { const file: TFile = { id: 'missing-id', diff --git a/api/src/job.ts b/api/src/job.ts index 610b748b..8121f51d 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -60,6 +60,14 @@ export { const AUTO_LOAD_DIRKEEP_TIMEOUT_MS = 10000; const AUTO_LOAD_DIRKEEP_RETRIES = 2; +/** Replaying the same sealed grant cannot repair an authorization denial. */ +class InputAuthorizationError extends Error { + constructor(status: number) { + super(`HTTP error: ${status}`); + this.name = 'InputAuthorizationError'; + } +} + /** * Bridges a `fetch` response body to a Node-stream Readable. The types at the * module boundary (Node's `stream/web` vs. lib.dom) don't overlap cleanly, @@ -993,11 +1001,18 @@ export class Job { submissionDir: this.submissionDir, identity: this.jobIdentity, }; + const startedAt = performance.now(); + let started = 0; + let completed = 0; + let cancelled = 0; let firstFailure: { error: unknown } | undefined; const runFileOperation = async (operation: () => Promise): Promise => { + started++; try { await operation(); + completed++; } catch (error) { + if (firstFailure) cancelled++; if (!firstFailure) { firstFailure = { error }; controller.abort(error); @@ -1031,6 +1046,15 @@ export class Job { Array.from({ length: workerCount }, () => runPrimeWorker()), ); if (firstFailure) { + this.log.error({ + inputCount: fileOps.length, + completed, + failed: 1, + cancelled, + notStarted: fileOps.length - started, + durationMs: Math.round(performance.now() - startedAt), + err: firstFailure.error, + }, 'Input preparation batch failed'); if (this.session) { /* A sibling may already have atomically replaced its destination. The * workspace now matches neither the previous checkpoint nor the full @@ -1302,6 +1326,9 @@ export class Job { if (!response.ok) { await response.body?.cancel().catch(() => {}); + if (response.status === 401 || response.status === 403) { + throw new InputAuthorizationError(response.status); + } throw new Error(`HTTP error: ${response.status}`); } @@ -1366,11 +1393,10 @@ export class Job { try { await fsp.unlink(tempPath); } catch { /* may not exist */ } throw abortReason(operation.signal); } - /* ValidationError is deterministic — a bad Content-Disposition - * filename will fail identically on every retry. Abort fast - * (cleanup + rethrow) instead of burning ~7.5s on exponential - * backoff and surfacing the error as a generic download failure. */ - if (error instanceof ValidationError) { + /* Invalid filenames and authorization denials cannot recover by + * replaying the same request. Abort the batch before exponential + * backoff amplifies the failure across its remaining files. */ + if (error instanceof ValidationError || error instanceof InputAuthorizationError) { try { await fsp.unlink(tempPath); } catch { /* may not exist */ } throw error; } @@ -1383,7 +1409,9 @@ export class Job { } } - this.log.error({ fileId: file.id, maxRetries, err: lastError }, 'Failed to download file'); + if (!context?.signal) { + this.log.error({ fileId: file.id, maxRetries, err: lastError }, 'Failed to download file'); + } try { await fsp.unlink(tempPath); } catch { /* may not exist */ } throw lastError ?? new Error(`Failed to download input ${file.id}`); } From 794df9e444acc2a8a2c6a0fbb65ddc0976c8ef41 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 11 Sep 2026 15:47:10 -0400 Subject: [PATCH 02/42] fix: preserve retries for transient egress ledger conflicts (#179) * fix: distinguish retryable ledger contention from scope denials * fix: preserve error classification through marker discovery and relay * test: exercise classified denials through gateway configuration * fix: honor bounded gateway retry hints during object downloads --- api/src/download.test.ts | 79 +++++++++++++++++++++++++++++- api/src/egress.ts | 1 + api/src/job.ts | 23 +++++++-- packages/code/src/relay.test.ts | 35 +++++++++++++ packages/code/src/relay.ts | 6 +++ service/src/egress-gateway.test.ts | 36 +++++++++++++- service/src/egress-gateway.ts | 4 ++ service/src/egress-grant.ts | 4 +- service/src/egress-ledger.ts | 2 +- 9 files changed, 182 insertions(+), 8 deletions(-) diff --git a/api/src/download.test.ts b/api/src/download.test.ts index 2c7248e8..bb8ec7e9 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -55,6 +55,7 @@ function makeRuntime(): Runtime { function makeJob(files: TFile[] = [], session?: SessionWorkspace): Job { return new Job({ session_id: 'test-session', + egress_grant: 'test-grant', runtime: makeRuntime(), files, args: [], @@ -440,10 +441,12 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { }); it.each([401, 403])('does not retry an HTTP %i authorization denial', async status => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; const file: TFile = { id: 'denied', storage_session_id: 'previous', name: 'denied.txt' }; let requests = 0; routes.set('/sessions/previous/objects/denied', { status, + headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' }, onRequest: () => { requests++; }, }); const job = makeJob([file]); @@ -454,7 +457,8 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(await fsp.readdir(tmpDir)).toEqual([]); }); - it.each([404, 408, 429, 503])('still retries transient HTTP %i responses', async status => { + it.each([403, 404, 408, 429, 503])('still retries transient HTTP %i responses', async status => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; const file: TFile = { id: 'transient', storage_session_id: 'previous', name: 'ready.txt' }; let requests = 0; const route: Route = { @@ -471,7 +475,79 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(await fsp.readFile(path.join(tmpDir, 'ready.txt'), 'utf8')).toBe('ready'); }); + it.each([false, true])('honors conflict retry hints with cancellation=%s', async cancel => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; + const controller = new AbortController(); + const timestamps: number[] = []; + let timer: ReturnType | undefined; + const route: Route = { + status: 503, body: 'ready', + headers: { 'X-CodeAPI-Error-Code': 'ledger_conflict', 'Retry-After': '1' }, + onRequest: () => { + timestamps.push(performance.now()); + if (timestamps.length === 2) route.status = 200; + else if (cancel) timer = setTimeout(() => controller.abort(new Error('cancelled retry')), 25); + }, + }; + routes.set('/sessions/previous/objects/retry-hint', route); + const file: TFile = { id: 'retry-hint', storage_session_id: 'previous', name: 'ready.txt' }; + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + try { + const result = job.downloadAndWriteFile(file, 5, 1, { + submissionDir: tmpDir, identity: fallbackSandboxIdentity(), signal: controller.signal, + }); + if (cancel) { + await expect(result).rejects.toThrow('cancelled retry'); + expect(timestamps).toHaveLength(1); + expect(await fsp.readdir(tmpDir)).toEqual([]); + } else { + await expect(result).resolves.toBe('ready.txt'); + expect(timestamps).toHaveLength(2); + expect(timestamps[1] - timestamps[0]).toBeGreaterThanOrEqual(900); + } + } finally { + clearTimeout(timer); + } + }); + + it('does not retry an unclassified direct file-server denial', async () => { + config.egress_gateway_url = ''; + let requests = 0; + const file: TFile = { id: 'denied', storage_session_id: 'previous', name: 'denied.txt' }; + routes.set('/sessions/previous/objects/denied', { + status: 403, onRequest: () => { requests++; }, + }); + const job = makeJob([file]); + asInternals(job).submissionDir = tmpDir; + await expect(job.downloadAndWriteFile(file, 5, 1)).rejects.toThrow('HTTP error: 403'); + expect(requests).toBe(1); + }); + + it.each(['legacy', 'classified', 'direct'])('handles %s marker denials before priming', async mode => { + config.egress_gateway_url = mode === 'direct' ? '' : `http://127.0.0.1:${serverPort}`; + let requests = 0; + const route: Route = { + status: 403, body: '[]', + headers: mode === 'classified' ? { 'X-CodeAPI-Error-Code': 'scope_mismatch' } : {}, + onRequest: () => { if (++requests === 2) route.status = 200; }, + }; + routes.set('/sessions/previous/objects', route); + const file: TFile = { id: 'ready', storage_session_id: 'previous', name: 'ready.txt' }; + routes.set('/sessions/previous/objects/ready', { status: 200, body: 'ready' }); + const job = makeJob([file], sessionWorkspaceAt(tmpDir, 'marker-retry')); + if (mode === 'legacy') { + await job.prime(); + expect(requests).toBe(2); + expect(await fsp.readFile(path.join(tmpDir, 'ready.txt'), 'utf8')).toBe('ready'); + } else { + await expect(job.prime()).rejects.toThrow('HTTP error loading .dirkeep markers: 403'); + expect(requests).toBe(1); + } + }); + it('accounts for a denied 240-file batch once and stops queued downloads', async () => { + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; const files: TFile[] = Array.from({ length: 240 }, (_, index) => ({ id: `file-${index}`, storage_session_id: 'previous', name: `file-${index}.txt`, })); @@ -480,6 +556,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { for (const file of files) { routes.set(`/sessions/previous/objects/${file.id}`, { status: 403, + headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' }, delayMs: file.id === 'file-0' ? 0 : 30, onRequest: () => { requests++; }, }); diff --git a/api/src/egress.ts b/api/src/egress.ts index 442872ab..69ecf6db 100644 --- a/api/src/egress.ts +++ b/api/src/egress.ts @@ -1 +1,2 @@ export const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; +export const EGRESS_ERROR_CODE_HEADER = 'X-CodeAPI-Error-Code'; diff --git a/api/src/job.ts b/api/src/job.ts index 8121f51d..bdcd5061 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -15,7 +15,7 @@ import { getRuntimes } from './runtime'; import { execute } from './nsjail'; import { config } from './config'; import { internalServiceHeaders } from './internal-service-auth'; -import { EGRESS_GRANT_HEADER } from './egress'; +import { EGRESS_GRANT_HEADER, EGRESS_ERROR_CODE_HEADER } from './egress'; import { injectTraceHeaders } from './telemetry'; import { applyReadOnlyInputPermissions, @@ -1196,6 +1196,11 @@ export class Job { } } + private isLegacyGatewayDenial(response: Response): boolean { + return !!config.egress_gateway_url && response.status === 403 && + !response.headers.has(EGRESS_ERROR_CODE_HEADER); + } + /** * Fetches normalized objects for one inherited session and returns the * `.dirkeep` markers belonging to exactly that session. Guards against: @@ -1219,7 +1224,7 @@ export class Job { signal: controller.signal, }, ); - if (res.status === 503 && attempt < AUTO_LOAD_DIRKEEP_RETRIES) { + if ((res.status === 503 || this.isLegacyGatewayDenial(res)) && attempt < AUTO_LOAD_DIRKEEP_RETRIES) { await res.body?.cancel().catch(() => {}); const retryAfterSeconds = Number(res.headers.get('retry-after')); await sleep( @@ -1326,7 +1331,10 @@ export class Job { if (!response.ok) { await response.body?.cancel().catch(() => {}); - if (response.status === 401 || response.status === 403) { + /* Older gateways also used 403 for transient ledger contention. + * Only classify 403 as permanent when the gateway distinguishes it. */ + if (response.status === 401 || + (response.status === 403 && !this.isLegacyGatewayDenial(response))) { throw new InputAuthorizationError(response.status); } throw new Error(`HTTP error: ${response.status}`); @@ -1402,7 +1410,14 @@ export class Job { } lastError = error instanceof Error ? error : new Error(String(error)); if (attempt < maxRetries) { - const delay = retryDelay * Math.pow(2, attempt - 1); + const backoff = retryDelay * Math.pow(2, attempt - 1); + const retryAfterSeconds = response?.status === 503 + ? Number(response.headers.get('retry-after')) : NaN; + /* Use the same bounded retry hint as marker discovery, without + * shortening exponential backoff or bypassing batch cancellation. */ + const delay = Number.isFinite(retryAfterSeconds) + ? Math.max(backoff, Math.min(1000, Math.max(0, retryAfterSeconds * 1000))) + : backoff; this.log.warn({ fileId: file.id, attempt, maxRetries, delay, err: lastError }, 'Download failed, retrying'); await sleep(delay, operation.signal); } diff --git a/packages/code/src/relay.test.ts b/packages/code/src/relay.test.ts index c0814408..a9b81fd0 100644 --- a/packages/code/src/relay.test.ts +++ b/packages/code/src/relay.test.ts @@ -381,3 +381,38 @@ test('file relay rejects plaintext remote upstreams', async () => { /HTTPS unless it is a local development host/, ); }); + +for (const [status, reason] of [[403, 'scope_mismatch'], [503, 'ledger_conflict']] as const) { + test(`file relay preserves ${reason} classification`, async () => { + const upstream = createServer((_req, res) => { + res.writeHead(status, { + 'X-CodeAPI-Error-Code': reason, + 'Retry-After': '1', + 'X-Internal-Secret': 'must-not-forward', + }).end('rejected'); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ + host: '127.0.0.1', port: 0, upstreamUrl, token: 'relay-secret', + maxBytes: 1024, timeoutMs: 1000, + }); + try { + const response = await fetch(`${relay.url}/sessions/storage-1/objects/file-1`, { + headers: { + 'X-LibreChat-Code-Relay-Token': 'relay-secret', + 'X-CodeAPI-Egress-Grant': 'grant-1', + }, + }); + assert.equal(response.status, status); + assert.equal(response.headers.get('x-codeapi-error-code'), reason); + assert.equal(response.headers.get('retry-after'), '1'); + assert.equal(response.headers.get('x-internal-secret'), null); + assert.equal(await response.text(), 'rejected'); + } finally { + await relay.close(); + await new Promise((resolve, reject) => + upstream.close(error => error ? reject(error) : resolve()), + ); + } + }); +} diff --git a/packages/code/src/relay.ts b/packages/code/src/relay.ts index 255845e9..14e91280 100644 --- a/packages/code/src/relay.ts +++ b/packages/code/src/relay.ts @@ -240,6 +240,12 @@ export async function startFileRelay( )!, } : {}), + ...(upstreamResponse.headers.has('x-codeapi-error-code') + ? { 'X-CodeAPI-Error-Code': upstreamResponse.headers.get('x-codeapi-error-code')! } + : {}), + ...(upstreamResponse.headers.has('retry-after') + ? { 'Retry-After': upstreamResponse.headers.get('retry-after')! } + : {}), 'Content-Length': String(body.length), }); response.end(body); diff --git a/service/src/egress-gateway.test.ts b/service/src/egress-gateway.test.ts index 99303850..717309d7 100644 --- a/service/src/egress-gateway.test.ts +++ b/service/src/egress-gateway.test.ts @@ -1,6 +1,6 @@ process.env.CODEAPI_EGRESS_GATEWAY_AUTOSTART = 'false'; -import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; +import { afterAll, beforeAll, beforeEach, describe, expect, test, spyOn } from 'bun:test'; import crypto from 'crypto'; import RedisMock from 'ioredis-mock'; import type { Server } from 'http'; @@ -435,6 +435,39 @@ describe('egress gateway routes', () => { } }); + test('reports exhausted ledger conflicts as retryable without forwarding the read', async () => { + const redis = new RedisMock(); + env.EGRESS_LEDGER_REQUIRED = true; + setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); + const duplicate = redis.duplicate.bind(redis); + const duplication = spyOn(redis, 'duplicate').mockImplementation(() => { + const connection = duplicate(); + const transaction = { + set: () => transaction, + exec: async () => null, + }; + spyOn(connection, 'multi').mockImplementation(() => transaction as never); + return connection; + }); + try { + await createEgressLedger(claims()); + const readSession = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const response = await gatewayFetch(`/sessions/${readSession}/objects?detail=normalized`, { + headers: grantHeader(), + }); + expect(response.status).toBe(503); + expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('ledger_conflict'); + expect(response.headers.get('Retry-After')).toBe('1'); + expect(upstreamCalls).toHaveLength(0); + expect((await assertEgressGrantActive(claims())).request_count).toBe(0); + } finally { + duplication.mockRestore(); + setEgressLedgerRedisForTest(null); + redis.disconnect(); + env.EGRESS_LEDGER_REQUIRED = false; + } + }); + test('lists only scoped objects and injects internal credentials', async () => { upstreamResponse = Response.json([ { id: 'file_123', name: 'inputs/data.csv', storage_session_id: 'sess_input' }, @@ -557,6 +590,7 @@ describe('egress gateway routes', () => { }); expect(response.status).toBe(403); + expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('scope_mismatch'); expect(upstreamCalls).toHaveLength(0); } finally { await redis.disconnect(); diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index d4d3e713..13a6de95 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -6,6 +6,7 @@ import { Readable } from 'stream'; import { env } from './config'; import { EGRESS_GRANT_HEADER, + EGRESS_ERROR_CODE_HEADER, EgressGrantError, egressGrantFromExecutionClaims, openEgressGrant, @@ -157,6 +158,7 @@ app.use((req: Request, res: Response, next: NextFunction) => { function errorStatus(error: EgressGrantError): number { if (error.reason === 'missing_secret' || error.reason === 'weak_secret') return 500; + if (error.reason === 'ledger_conflict') return 503; if (error.reason === 'malformed') return 400; if (error.reason === 'expired') return 401; return 403; @@ -165,6 +167,8 @@ function errorStatus(error: EgressGrantError): number { function sendEgressError(req: Request, res: Response, error: unknown): Response { if (error instanceof EgressGrantError) { const statusCode = errorStatus(error); + res.setHeader(EGRESS_ERROR_CODE_HEADER, error.reason); + if (error.reason === 'ledger_conflict') res.setHeader('Retry-After', '1'); logger.warn('Rejected egress gateway request', { requestId: requestId(res), reason: error.reason, diff --git a/service/src/egress-grant.ts b/service/src/egress-grant.ts index 8d8b1352..8148bec7 100644 --- a/service/src/egress-grant.ts +++ b/service/src/egress-grant.ts @@ -3,6 +3,7 @@ import type { ExecutionManifestClaims, ExecutionManifestInputFile } from './exec import type * as t from './types'; export const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; +export const EGRESS_ERROR_CODE_HEADER = 'X-CodeAPI-Error-Code'; export const EGRESS_GRANT_VERSION = 1; const TOKEN_PREFIX = 'ceg1'; @@ -19,7 +20,8 @@ export type EgressGrantErrorReason = | 'malformed' | 'expired' | 'wrong_type' - | 'scope_mismatch'; + | 'scope_mismatch' + | 'ledger_conflict'; export class EgressGrantError extends Error { readonly reason: EgressGrantErrorReason; diff --git a/service/src/egress-ledger.ts b/service/src/egress-ledger.ts index 24dda87c..b6bbf4fb 100644 --- a/service/src/egress-ledger.ts +++ b/service/src/egress-ledger.ts @@ -264,7 +264,7 @@ async function mutateRecord( }); releaseMutationConnection(client); } - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger update conflicted'); + throw new EgressGrantError('ledger_conflict', 'Egress grant ledger update conflicted'); } export async function assertEgressGrantActive(grant: EgressGrantClaims): Promise { From dead07bd466a17dab4bc5b2b3520312fa52b0e03 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 11 Sep 2026 16:36:29 -0400 Subject: [PATCH 03/42] perf: reuse authorized input versions and make egress accounting atomic (#180) * perf: reuse authorized input versions and make egress accounting atomic * perf: resolve authorized input manifests once per execution * test: preserve fetch signature in revocation fixture * fix: isolate shared-grant failures and prevent ledger replay --- .github/workflows/ci.yml | 3 + api/src/config.ts | 3 + api/src/download.test.ts | 44 +++ api/src/http-input-cache.test.ts | 163 +++++++++++ api/src/http-input-cache.ts | 177 +++++++++++ api/src/input-manifest.test.ts | 124 ++++++++ api/src/job.ts | 55 ++++ api/src/metrics.ts | 6 + api/src/session-inputs.ts | 36 ++- docs/INPUT_REUSE.md | 91 ++++++ .../templates/egress-gateway-deployment.yaml | 8 + .../templates/file-server-deployment.yaml | 4 + .../templates/worker-sandbox-deployment.yaml | 8 + helm/codeapi/values.yaml | 11 + launcher/src/main.rs | 4 + packages/code/src/relay.test.ts | 65 +++++ packages/code/src/relay.ts | 16 +- service/src/config.ts | 6 + service/src/egress-gateway.test.ts | 207 ++++++++++--- service/src/egress-gateway.ts | 97 +++++- service/src/egress-ledger-reconnect.test.ts | 63 ++++ service/src/egress-ledger-script.ts | 123 ++++++++ service/src/egress-ledger.test.ts | 81 ++++- service/src/egress-ledger.ts | 276 ++++-------------- service/src/file-download.test.ts | 49 ++++ service/src/file-download.ts | 27 ++ service/src/file-object-resolver.test.ts | 50 ++++ service/src/file-object-resolver.ts | 75 +++++ service/src/file-server.ts | 160 ++++------ service/src/test/redis.ts | 45 +++ 30 files changed, 1690 insertions(+), 387 deletions(-) create mode 100644 api/src/http-input-cache.test.ts create mode 100644 api/src/http-input-cache.ts create mode 100644 api/src/input-manifest.test.ts create mode 100644 docs/INPUT_REUSE.md create mode 100644 service/src/egress-ledger-reconnect.test.ts create mode 100644 service/src/egress-ledger-script.ts create mode 100644 service/src/file-download.test.ts create mode 100644 service/src/file-download.ts create mode 100644 service/src/file-object-resolver.test.ts create mode 100644 service/src/file-object-resolver.ts create mode 100644 service/src/test/redis.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71536caa..f041e2d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,9 @@ jobs: - name: Install dependencies run: bun ci + - name: Install Redis for ledger integration tests + run: sudo apt-get update && sudo apt-get install -y redis-server + - name: Build service run: bun run build diff --git a/api/src/config.ts b/api/src/config.ts index bbc2c184..f3b935ff 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -122,6 +122,9 @@ export const config = { /* Ceiling for the pushed input cache (session-inputs.ts). Eviction is * always safe — a miss simply re-pushes on the next probe — so this is a * disk guard, not a correctness knob. */ + http_input_cache_enabled: process.env.SANDBOX_HTTP_INPUT_CACHE_ENABLED === 'true', + http_input_cache_max_objects: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS, 4096), + http_input_cache_max_inflight: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT, 16), input_cache_max_bytes: safeInt( process.env.SANDBOX_INPUT_CACHE_MAX_BYTES, 512 * 1024 * 1024, diff --git a/api/src/download.test.ts b/api/src/download.test.ts index bb8ec7e9..da877dad 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -2,6 +2,8 @@ import { describe, it, expect, beforeEach, afterEach, beforeAll, afterAll, spyOn import * as fsp from 'fs/promises'; import * as path from 'path'; import * as os from 'os'; +import { createHash, randomUUID } from 'node:crypto'; +import { SESSION_INPUT_CACHE_DIR } from './session-inputs'; import * as semver from 'semver'; import { Job, SessionWorkspaceDirtyError, type TFile } from './job'; import type { Runtime } from './runtime'; @@ -511,6 +513,48 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { } }); + it('reuses versioned bytes in fresh workspaces without bypassing a later denial', async () => { + const previousCache = config.http_input_cache_enabled; + const version = randomUUID(); + const cacheKey = createHash('sha256').update(version).digest('hex'); + const otherDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'codeapi-cache-second-')); + config.http_input_cache_enabled = true; + config.egress_gateway_url = `http://127.0.0.1:${serverPort}`; + let reads = 0; + let checks = 0; + const meta: Route = { status: 200, body: JSON.stringify({ cacheable: true, cacheKey, version, size: 8, readOnly: false }), + onRequest: () => { checks++; }, + }; + routes.set('/sessions/previous/objects/cached/metadata', meta); + routes.set('/sessions/previous/objects/cached', { status: 200, body: 'original', + headers: { 'X-CodeAPI-Input-Version': version }, + onRequest: request => { reads++; expect(request.headers.get('x-codeapi-input-version')).toBe(version); }, + }); + const file: TFile = { id: 'cached', storage_session_id: 'previous', name: 'data.txt', input_cache_key: cacheKey }; + try { + const first = makeJob([file]); + asInternals(first).submissionDir = tmpDir; + await first.downloadAndWriteFile(file); + await fsp.writeFile(path.join(tmpDir, 'data.txt'), 'sandbox changed this'); + const second = makeJob([file]); + asInternals(second).submissionDir = otherDir; + await second.downloadAndWriteFile(file); + expect(await fsp.readFile(path.join(otherDir, 'data.txt'), 'utf8')).toBe('original'); + expect(reads).toBe(1); + expect(checks).toBe(2); + meta.status = 403; + meta.headers = { 'X-CodeAPI-Error-Code': 'scope_mismatch' }; + await expect(second.downloadAndWriteFile(file)).rejects.toThrow('HTTP error: 403'); + expect(checks).toBe(3); + expect(reads).toBe(1); + } finally { + config.http_input_cache_enabled = previousCache; + await fsp.rm(otherDir, { recursive: true, force: true }); + await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, cacheKey), { force: true }); + await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, `${cacheKey}.json`), { force: true }); + } + }); + it('does not retry an unclassified direct file-server denial', async () => { config.egress_gateway_url = ''; let requests = 0; diff --git a/api/src/http-input-cache.test.ts b/api/src/http-input-cache.test.ts new file mode 100644 index 00000000..674a03ab --- /dev/null +++ b/api/src/http-input-cache.test.ts @@ -0,0 +1,163 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { createHash, randomUUID } from 'node:crypto'; +import { rm } from 'node:fs/promises'; +import path from 'node:path'; +import { fetchCachedHttpInput } from './http-input-cache'; +import { hasCachedInput, SESSION_INPUT_CACHE_DIR } from './session-inputs'; + +const keys = new Set(); +afterEach(async () => { + for (const key of keys) { + await rm(path.join(SESSION_INPUT_CACHE_DIR, key), { force: true }); + await rm(path.join(SESSION_INPUT_CACHE_DIR, `${key}.json`), { force: true }); + } + keys.clear(); +}); +function fixture(body = 'input', principal = 'tenant/user') { + const version = randomUUID(); + const cacheKey = createHash('sha256').update(principal + version).digest('hex'); + keys.add(cacheKey); + const meta = { cacheable: true, version, cacheKey, size: Buffer.byteLength(body), readOnly: false, name: 'input.txt' }; + let reads = 0; + let authorizations = 0; + return { + meta, + counts: () => ({ reads, authorizations }), + args: { + maxBytes: 8192, maxObjects: 2, maxFileBytes: 8192, maxInflight: 4, + metadata: async () => { authorizations++; return Response.json(meta); }, + download: async (expected: string, _signal: AbortSignal) => { + reads++; + expect(expected).toBe(version); + return new Response(body, { headers: { 'X-CodeAPI-Input-Version': version } }); + }, + }, + }; +} + +function gate() { + let release!: () => void; + const promise = new Promise(resolve => { release = resolve; }); + return { promise, release }; +} + +describe('authorized HTTP input cache', () => { + test('fresh executions reuse bytes but authorize every hit', async () => { + const f = fixture(); + for (let i = 0; i < 3; i++) { + const response = await fetchCachedHttpInput(f.args); + expect(await response?.text()).toBe('input'); + expect(response?.headers.get('content-disposition')).toContain('input.txt'); + } + expect(f.counts()).toEqual({ reads: 1, authorizations: 3 }); + expect(await hasCachedInput('', '', f.meta.cacheKey)).toBe(false); // Cannot bypass preflight using a pushed key. + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(true); + const denied = await fetchCachedHttpInput({ ...f.args, + metadata: async () => new Response(null, { status: 403, headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' } }), + }); + expect(denied?.status).toBe(403); + expect(f.counts().reads).toBe(1); + }); + + test('new versions and principals never reuse an existing version key', async () => { + for (const [body, principal] of [['old', 'tenant/user'], ['new', 'tenant/user'], ['private', 'another-tenant/user']]) { + const f = fixture(body, principal); + expect(await (await fetchCachedHttpInput(f.args))?.text()).toBe(body); + expect(f.counts().reads).toBe(1); + } + }); + + test('coalesces misses while one cancelled caller leaves the remaining reader intact', async () => { + const f = fixture(); + const started = gate(); + const finish = gate(); + let downloads = 0; + let sharedSignal: AbortSignal | undefined; + const args = { ...f.args, download: async (version: string, signal: AbortSignal) => { + downloads++; sharedSignal = signal; started.release(); + await finish.promise; + return f.args.download(version, signal); + } }; + const controller = new AbortController(); + const first = fetchCachedHttpInput({ ...args, signal: controller.signal }); + const second = fetchCachedHttpInput(args); + await started.promise; + await new Promise(resolve => setTimeout(resolve, 10)); + controller.abort(new Error('first cancelled')); + await expect(first).rejects.toThrow('first cancelled'); + expect(sharedSignal?.aborted).toBe(false); + finish.release(); + expect(await (await second)?.text()).toBe('input'); + expect(downloads).toBe(1); + expect(f.counts().authorizations).toBe(2); + }); + + test('a shared fill does not propagate its creator grant denial to a valid waiter', async () => { + const f = fixture(); + const started = gate(); + const finish = gate(); + const denied = fetchCachedHttpInput({ ...f.args, download: async () => { + started.release(); + await finish.promise; + return new Response(null, { status: 403, headers: { 'X-CodeAPI-Error-Code': 'request_budget_exceeded' } }); + } }); + await started.promise; + const valid = fetchCachedHttpInput(f.args); + await Bun.sleep(10); + finish.release(); + expect((await denied)?.status).toBe(403); + // Job.fetchInputObject uses the waiter's own normal download on undefined. + expect(await valid).toBeUndefined(); + expect(await (await f.args.download(f.meta.version, new AbortController().signal)).text()).toBe('input'); + expect(f.counts().authorizations).toBe(2); + }); + + test('last-reader cancellation aborts the upstream fill without publishing', async () => { + const f = fixture(); + const started = gate(); + const aborted = gate(); + const controller = new AbortController(); + const pending = fetchCachedHttpInput({ ...f.args, signal: controller.signal, + download: async (_version, signal) => { + started.release(); + return new Promise((_resolve, reject) => signal.addEventListener('abort', () => { + aborted.release(); reject(signal.reason); + }, { once: true })); + }, + }); + await started.promise; + controller.abort(new Error('cancel fill')); + await expect(pending).rejects.toThrow('cancel fill'); + await aborted.promise; + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(false); + }); + + test('changed-version and oversized responses are never published', async () => { + const f = fixture(); + const changed = await fetchCachedHttpInput({ ...f.args, download: async () => new Response('changed', { + headers: { 'X-CodeAPI-Input-Version': randomUUID() }, + }) }); + expect(changed).toBeUndefined(); + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(false); + await expect(fetchCachedHttpInput({ ...f.args, download: async () => new Response('too many bytes', { + headers: { 'X-CodeAPI-Input-Version': f.meta.version }, + }) })).rejects.toThrow(); + expect(await hasCachedInput('', '', f.meta.cacheKey, 'http')).toBe(false); + }); + + test('cache quotas evict old entries and preserve an already-open reader', async () => { + const first = fixture('a'.repeat(4000)); + const second = fixture('b'.repeat(4000)); + const response = await fetchCachedHttpInput({ ...first.args, maxObjects: 1 }); + expect(await (await fetchCachedHttpInput({ ...second.args, maxObjects: 1 }))?.text()).toBe('b'.repeat(4000)); + expect(await hasCachedInput('', '', first.meta.cacheKey, 'http')).toBe(false); + expect(await response?.text()).toBe('a'.repeat(4000)); + }); + + test('legacy metadata protocols fall back without a cache read or fill', async () => { + const f = fixture(); + expect(await fetchCachedHttpInput({ ...f.args, metadata: async () => new Response(null, { status: 404 }) })).toBeUndefined(); + expect(await fetchCachedHttpInput({ ...f.args, metadata: async () => Response.json({ cacheable: false }) })).toBeUndefined(); + expect(f.counts().reads).toBe(0); + }); +}); diff --git a/api/src/http-input-cache.ts b/api/src/http-input-cache.ts new file mode 100644 index 00000000..c785bdf0 --- /dev/null +++ b/api/src/http-input-cache.ts @@ -0,0 +1,177 @@ +import { Readable } from 'node:stream'; +import { createGzip } from 'node:zlib'; +import { httpInputCacheEvents } from './metrics'; +import { cachedInputResponse, openCachedInput, storeCachedInputs } from './session-inputs'; + +type Metadata = { cacheable: true; cacheKey: string; version: string; size: number; name?: string; readOnly: boolean }; +type FillResult = { stored: boolean; status?: number; headers?: Headers }; +type Fill = { controller: AbortController; users: number; result: Promise }; +const fills = new Map(); + +function validMetadata(value: unknown, maxBytes: number): value is Metadata { + if (!value || typeof value !== 'object') return false; + const m = value as Metadata; + return m.cacheable === true && typeof m.cacheKey === 'string' && /^[0-9a-f]{64}$/.test(m.cacheKey) && + typeof m.version === 'string' && /^[0-9a-f-]{36}$/.test(m.version) && + Number.isSafeInteger(m.size) && m.size >= 0 && m.size + 1024 <= maxBytes && + typeof m.readOnly === 'boolean' && (m.name === undefined || (typeof m.name === 'string' && m.name.length <= 4096)); +} + +function tarHeader(name: string, bytes: number): Buffer { + const header = Buffer.alloc(512); + header.write(name, 0, 100, 'utf8'); + header.write('0000600\0', 100, 8, 'ascii'); + header.write(bytes.toString(8).padStart(11, '0') + '\0', 124, 12, 'ascii'); + header.fill(32, 148, 156); + header[156] = 48; + header.write('ustar\0', 257, 6, 'ascii'); + const checksum = header.reduce((sum, byte) => sum + byte, 0); + header.write(checksum.toString(8).padStart(6, '0') + '\0 ', 148, 8, 'ascii'); + return header; +} + +/** Reuse the pushed-cache writer's staging, quota, no-follow and atomic commit + * rules. No workspace pathname or sandbox-visible file is used as cache input. */ +async function fillCache(response: Response, meta: Metadata, maxBytes: number, maxObjects: number): Promise { + if (!response.body) throw new Error('Input response has no body'); + const body = response.body; + const sidecar = Buffer.from(JSON.stringify({ readOnly: meta.readOnly, source: 'http' })); + async function* archive(): AsyncGenerator { + yield tarHeader(meta.cacheKey, meta.size); + const reader = body.getReader(); + let bytes = 0; + try { + for (;;) { + const part = await reader.read(); + if (part.done) break; + bytes += part.value.byteLength; + if (bytes > meta.size) throw new Error('Input exceeded its authorized metadata size'); + yield Buffer.from(part.value); + } + if (bytes !== meta.size) throw new Error('Input size changed during preparation'); + } finally { + await reader.cancel().catch(() => {}); + reader.releaseLock(); + } + yield Buffer.alloc((512 - meta.size % 512) % 512); + yield tarHeader(`${meta.cacheKey}.json`, sidecar.length); + yield sidecar; + yield Buffer.alloc((512 - sidecar.length % 512) % 512); + yield Buffer.alloc(1024); + } + const source = Readable.from(archive()); + const compressed = createGzip(); + compressed.on('error', () => {}); // Queue admission checks an already-failed stream. + source.on('error', error => compressed.destroy(error)); + source.pipe(compressed); + try { + await storeCachedInputs(compressed, maxBytes, meta.size + sidecar.length, maxObjects); + } finally { + source.destroy(); + compressed.destroy(); + await body.cancel().catch(() => {}); + } +} + +async function waitForFill(fill: Fill, signal?: AbortSignal): Promise { + if (signal?.aborted) { + if (fill.users === 0) fill.controller.abort(signal.reason); + signal.throwIfAborted(); + } + fill.users++; + let abort: (() => void) | undefined; + try { + const cancelled = new Promise((_resolve, reject) => { + abort = () => reject(signal?.reason ?? new Error('Input preparation cancelled')); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + }); + return await Promise.race([fill.result, cancelled]); + } finally { + if (abort) signal?.removeEventListener('abort', abort); + if (--fill.users === 0) fill.controller.abort(new Error('No input-cache consumers remain')); + } +} + +/** Every caller performs its own scoped preflight, even for hits or shared fills. + * Only opaque version keys returned by that authorized gateway enter this cache. */ +export async function fetchCachedHttpInput(args: { + metadata(): Promise; + download(version: string, signal: AbortSignal): Promise; + signal?: AbortSignal; + maxBytes: number; + maxFileBytes: number; + maxInflight: number; + maxObjects: number; +}): Promise { + args.signal?.throwIfAborted(); + const preflight = await args.metadata(); + if (preflight.status === 404 || preflight.status === 405) { + await preflight.body?.cancel(); + httpInputCacheEvents.inc({ event: 'legacy_bypass' }); + return undefined; // Older gateway/relay: retain the uncached protocol. + } + if (!preflight.ok) { httpInputCacheEvents.inc({ event: 'preflight_failure' }); return preflight; } + const value: unknown = await preflight.json(); + if (!validMetadata(value, args.maxBytes) || value.size > args.maxFileBytes) { + httpInputCacheEvents.inc({ event: 'uncacheable' }); + return undefined; + } + const meta = value; + args.signal?.throwIfAborted(); + let cached = await openCachedInput('', '', meta.cacheKey, 'http'); + if (cached) httpInputCacheEvents.inc({ event: 'hit' }); + if (!cached) { + let fill = fills.get(meta.cacheKey); + const joinedExistingFill = fill !== undefined; + if (fill) httpInputCacheEvents.inc({ event: 'coalesced' }); + if (!fill) { + if (fills.size >= args.maxInflight) { + httpInputCacheEvents.inc({ event: 'capacity_bypass' }); + return undefined; + } + httpInputCacheEvents.inc({ event: 'fill' }); + const controller = new AbortController(); + fill = { controller, users: 0, result: Promise.resolve({ stored: false }) }; + const ownFill = fill; + fills.set(meta.cacheKey, ownFill); + ownFill.result = (async (): Promise => { + const response = await args.download(meta.version, controller.signal); + if (!response.ok) { + await response.body?.cancel(); + return { stored: false, status: response.status, headers: response.headers }; + } + if (response.headers.get('x-codeapi-input-version') !== meta.version || + (response.headers.get('x-read-only')?.toLowerCase() === 'true') !== meta.readOnly) { + await response.body?.cancel(); + return { stored: false }; // Old file server or inconsistent metadata: never publish. + } + await fillCache(response, meta, args.maxBytes, args.maxObjects); + return { stored: true }; + })().finally(() => { + if (fills.get(meta.cacheKey) === ownFill) fills.delete(meta.cacheKey); + }); + // A caller can cancel between admission and waiting; avoid an unhandled rejection. + void ownFill.result.catch(() => {}); + } + const result = await waitForFill(fill, args.signal); + if (result.status) { + // The transfer used the creator's grant. Its denial/budget must not reject + // another caller whose own preflight succeeded; use that caller's fetch. + if (joinedExistingFill) return undefined; + const headers = new Headers(result.headers); + headers.delete('content-length'); + return new Response(null, { status: result.status, headers }); + } + if (!result.stored) return undefined; + cached = await openCachedInput('', '', meta.cacheKey, 'http'); + } + if (!cached) return undefined; // Evicted between commit and open: normal download remains correct. + if (args.signal?.aborted) { + await cached.handle.close(); + args.signal.throwIfAborted(); + } + const response = cachedInputResponse(cached); + if (meta.name) response.headers.set('content-disposition', `attachment; filename*=UTF-8''${encodeURIComponent(meta.name)}`); + return response; +} diff --git a/api/src/input-manifest.test.ts b/api/src/input-manifest.test.ts new file mode 100644 index 00000000..db55ca4a --- /dev/null +++ b/api/src/input-manifest.test.ts @@ -0,0 +1,124 @@ +import { afterEach, expect, test } from 'bun:test'; +import { createHash, randomUUID } from 'node:crypto'; +import * as fsp from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { config } from './config'; +import { Job } from './job'; +import { SESSION_INPUT_CACHE_DIR } from './session-inputs'; +import { fallbackSandboxIdentity } from './workspace-isolation'; + +const originalFetch = globalThis.fetch; +const originalConfig = { http_input_cache_enabled: config.http_input_cache_enabled, egress_gateway_url: config.egress_gateway_url }; +const dirs: string[] = []; +const keys: string[] = []; +afterEach(async () => { + globalThis.fetch = originalFetch; + Object.assign(config, originalConfig); + await Promise.all(dirs.splice(0).map(dir => fsp.rm(dir, { recursive: true, force: true }))); + await Promise.all(keys.splice(0).flatMap(key => [key, `${key}.json`]).map(key => fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, key), { force: true }))); +}); + +async function fixture(count: number, mode: 'batch' | 'legacy' | 'race' = 'batch') { + config.http_input_cache_enabled = true; + config.egress_gateway_url = 'http://manifest.test'; + let manifests = 0, singlePreflights = 0, downloads = 0; + const version = randomUUID(); + const freshVersion = randomUUID(); + const metadata = (id: string, v = version) => { + const key = createHash('sha256').update(id + v).digest('hex'); + keys.push(key); + return { cacheable: true, cacheKey: key, version: v, size: 5, readOnly: false }; + }; + globalThis.fetch = (async (url: RequestInfo | URL, init?: RequestInit) => { + const pathname = new URL(String(url)).pathname; + if (pathname.endsWith('/objects')) return Response.json([]); + if (pathname === '/input-manifest') { + manifests++; + if (mode === 'legacy') return new Response(null, { status: 404 }); + const request = JSON.parse(init!.body as string) as { files: { objectHandle: string }[] }; + expect(request.files).toHaveLength(count); + return Response.json({ files: request.files.map(file => metadata(file.objectHandle)) }); + } + if (pathname.endsWith('/metadata')) { + singlePreflights++; + return Response.json(metadata(pathname.split('/').slice(-2)[0], mode === 'race' ? freshVersion : version)); + } + downloads++; + if (mode === 'race' && new Headers(init?.headers).get('X-CodeAPI-Input-Version') === version) { + return new Response(null, { status: 409 }); + } + return new Response('bytes', { headers: { 'X-CodeAPI-Input-Version': mode === 'race' ? freshVersion : version } }); + }) as typeof fetch; + async function prime(egressGrant = 'test-grant') { + const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'manifest-prime-')); + dirs.push(dir); + const session = { + runtimeSessionId: 'test', acquire: async () => ({ dir, workspaceId: 'test', identity: fallbackSandboxIdentity() }), + primedInputId: () => undefined, markPrimed: () => {}, markDirty: () => {}, + }; + const job = new Job({ + session_id: 'test', egress_grant: egressGrant, runtime: { language: 'bash', version: '5.0.0', aliases: [] }, + files: Array.from({ length: count }, (_, i) => ({ id: `f${i}`, storage_session_id: 's', name: `file${i}.txt` })), + args: [], stdin: '', timeouts: { run: 5000, compile: 5000 }, cpu_times: { run: 5000, compile: 5000 }, + memory_limits: { run: 128e6, compile: 128e6 }, session, + } as never); + (job as unknown as { log: { level: string } }).log.level = 'silent'; + await job.prime(); + expect(await fsp.readFile(path.join(dir, 'file0.txt'), 'utf8')).toBe('bytes'); + } + return { prime, counts: () => ({ manifests, singlePreflights, downloads }) }; +} + +test('240 inputs use one authorized manifest per fresh workspace and reuse only content', async () => { + const f = await fixture(240); + await f.prime(); + await f.prime(); + expect(f.counts()).toEqual({ manifests: 2, singlePreflights: 0, downloads: 240 }); +}, 30000); + +test('an older gateway falls back to independently authorized preflights', async () => { + const f = await fixture(2, 'legacy'); + await f.prime(); + expect(f.counts()).toEqual({ manifests: 1, singlePreflights: 2, downloads: 2 }); +}); + +test('a raced version consumes the batch entry and retries against fresh metadata', async () => { + const f = await fixture(1, 'race'); + await f.prime(); + expect(f.counts()).toEqual({ manifests: 1, singlePreflights: 1, downloads: 2 }); +}); + + +test('one execution grant denial cannot fail a coalesced execution with its own valid grant', async () => { + const f = await fixture(1); + const underlying = globalThis.fetch; + let release!: () => void; + let started!: () => void; + const blocked = new Promise(resolve => { release = resolve; }); + const creatorStarted = new Promise(resolve => { started = resolve; }); + let deniedDownloads = 0, validDownloads = 0; + globalThis.fetch = (async (url: RequestInfo | URL, init?: RequestInit) => { + if (String(url).endsWith('/objects/f0')) { + if (new Headers(init?.headers).get('X-CodeAPI-Egress-Grant') === 'denied') { + deniedDownloads++; + started(); await blocked; + return new Response(null, { status: 403, headers: { 'X-CodeAPI-Error-Code': 'scope_mismatch' } }); + } + validDownloads++; + } + return underlying(url, init); + }) as typeof fetch; + const creator = f.prime('denied'); + void creator.catch(() => {}); + await creatorStarted; + const waiter = f.prime('valid'); + try { + while (f.counts().manifests < 2) await Bun.sleep(1); + await Bun.sleep(20); + } finally { release(); } + const result = await Promise.allSettled([creator, waiter]); + expect(result.map(item => item.status)).toEqual(['rejected', 'fulfilled']); + expect(deniedDownloads).toBe(1); + expect(validDownloads).toBe(1); +}); diff --git a/api/src/job.ts b/api/src/job.ts index bdcd5061..d2504de9 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -42,6 +42,7 @@ import { validateFilePath, isValidFilePath, } from './validation'; +import { fetchCachedHttpInput } from './http-input-cache'; import { cachedInputResponse, inputCacheKey, openCachedInput } from './session-inputs'; export { @@ -733,6 +734,7 @@ export class Job { private sessionFiles: FileRef[] = []; private inheritedRefs: FileRef[] = []; private inputFileHashes = new Map(); + private inputManifest = new Map(); private inputDestinations = new Map(); private entryPointName: string | undefined; private chmoddedDirs = new Set(); @@ -938,6 +940,7 @@ export class Job { async prime(): Promise { this.inputDestinations.clear(); + this.inputManifest.clear(); const requestedDestinations = new Map(); for (const file of this.files) { validateFilePath(file.name, '/tmp/codeapi-request-validation'); @@ -990,6 +993,8 @@ export class Job { await this.autoLoadDirkeep(); } + await this.prepareInputManifest(); + /* Promise.all rejects as soon as one operation fails, while its siblings * keep running. The route's finally then calls cleanup(), which clears the * session path/identity. A delayed sibling used to resume afterward and @@ -1431,6 +1436,38 @@ export class Job { throw lastError ?? new Error(`Failed to download input ${file.id}`); } + private async prepareInputManifest(): Promise { + if (!config.http_input_cache_enabled || !config.egress_gateway_url) return; + const files = this.files.filter(file => file.id && file.storage_session_id); + if (!files.length) return; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), AUTO_LOAD_DIRKEEP_TIMEOUT_MS); + try { + const response = await fetch(`${this.fileEgressBaseUrl()}/input-manifest`, { + method: 'POST', headers: this.fileEgressHeaders({ 'Content-Type': 'application/json' }), + body: JSON.stringify({ files: files.map(file => ({ + sessionHandle: file.storage_session_id, objectHandle: file.id, + })) }), signal: controller.signal, + }); + if (!response.ok) { + await response.body?.cancel(); + return; // Older gateways/relays and transient failures use per-file preflight. + } + const manifest = await response.json() as { files?: unknown[] }; + if (!Array.isArray(manifest.files) || manifest.files.length !== files.length) return; + manifest.files.forEach((metadata, index) => { + if (metadata && typeof metadata === 'object' && !('retry' in metadata)) { + this.inputManifest.set(files[index], metadata); + } + }); + } catch { + // This is an optimization. Individual reads still authorize and report failures. + } finally { + clearTimeout(timeout); + controller.abort(); + } + } + /** * Resolves an input object's bytes, preferring the runner-local cache the * control plane pushes into on backends whose sandbox cannot reach the file @@ -1462,6 +1499,24 @@ export class Job { `Input ${file.id} was not delivered to the sandbox and no file server is reachable`, ); } + if (config.http_input_cache_enabled && config.egress_gateway_url) { + const response = await fetchCachedHttpInput({ + metadata: () => { + const metadata = this.inputManifest.get(file); + // A version-race retry must obtain a new authorized storage version. + this.inputManifest.delete(file); + return metadata === undefined + ? fetch(`${this.buildDownloadUrl(file)}/metadata`, { headers: this.fileEgressHeaders(), signal }) + : Promise.resolve(Response.json(metadata)); + }, + download: (version, sharedSignal) => fetch(this.buildDownloadUrl(file), { + headers: this.fileEgressHeaders({ 'X-CodeAPI-Input-Version': version }), signal: sharedSignal, + }), + signal, maxBytes: config.input_cache_max_bytes, maxFileBytes: config.max_file_size, + maxInflight: config.http_input_cache_max_inflight, maxObjects: config.http_input_cache_max_objects, + }); + if (response) return response; + } return fetch(this.buildDownloadUrl(file), { headers: this.fileEgressHeaders(), signal, diff --git a/api/src/metrics.ts b/api/src/metrics.ts index 2da33b27..328b4871 100644 --- a/api/src/metrics.ts +++ b/api/src/metrics.ts @@ -16,6 +16,12 @@ const httpRequestDuration = new Histogram({ buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], }); +export const httpInputCacheEvents = new Counter({ + name: 'codeapi_sandbox_http_input_cache_events_total', + help: 'Authorized HTTP input cache events; fills and failures may both occur for one input', + labelNames: ['event'] as const, +}); + export const sandboxExecutions = new Counter({ name: 'codeapi_sandbox_executions_total', help: 'Total number of sandbox execution attempts by outcome', diff --git a/api/src/session-inputs.ts b/api/src/session-inputs.ts index 0f1b7f3a..cc9f4b5a 100644 --- a/api/src/session-inputs.ts +++ b/api/src/session-inputs.ts @@ -147,6 +147,8 @@ export interface CachedInputMeta { * ref resolve to the first ref's path — which then overwrote a file the * sandbox had edited. */ readOnly: boolean; + /** HTTP entries require a fresh gateway preflight and cannot satisfy pushed-input probes. */ + source?: 'http'; } export interface CachedInput { @@ -176,7 +178,9 @@ function parseCachedInputMeta(raw: string): CachedInputMeta | null { ) { return null; } - return { readOnly: (parsed as { readOnly: boolean }).readOnly }; + const source = (parsed as { source?: unknown }).source; + if (source !== undefined && source !== 'http') return null; + return { readOnly: (parsed as { readOnly: boolean }).readOnly, ...(source === 'http' ? { source } : {}) }; } catch { return null; } @@ -186,8 +190,9 @@ export async function hasCachedInput( storageSessionId: string, id: string, cacheKey?: string, + source: 'push' | 'http' = 'push', ): Promise { - const opened = await openCachedInput(storageSessionId, id, cacheKey); + const opened = await openCachedInput(storageSessionId, id, cacheKey, source); if (!opened) return false; await opened.handle.close(); return true; @@ -197,6 +202,7 @@ export async function openCachedInput( storageSessionId: string, id: string, cacheKey?: string, + source: 'push' | 'http' = 'push', ): Promise { const key = cacheKey ?? inputCacheKey(storageSessionId, id); if (!/^[0-9a-f]{64}$/.test(key)) return null; @@ -229,7 +235,7 @@ export async function openCachedInput( await metaHandle?.close().catch(() => {}); } const meta = raw === null ? null : parseCachedInputMeta(raw); - if (!meta) { + if (!meta || (meta.source ?? 'push') !== source) { logger.warn({ key }, 'Ignoring session input with missing or invalid metadata'); await handle.close(); return null; @@ -325,7 +331,13 @@ async function extractInputArchive( }; compressedGuard.on('error', forwardCompressedError); body.once('error', forwardBodyError); - body.pipe(compressedGuard).pipe(gunzip); + const sourceState = body as NodeJS.ReadableStream & { errored?: Error; destroyed?: boolean }; + compressedGuard.pipe(gunzip); + if (sourceState.errored || sourceState.destroyed) { + forwardBodyError(sourceState.errored ?? new Error('Input stream was cancelled before extraction')); + } else { + body.pipe(compressedGuard); + } let buffered = Buffer.alloc(0); let current: @@ -517,7 +529,10 @@ async function storeCachedInputsOnce( body: NodeJS.ReadableStream, maxBytes = Number.MAX_SAFE_INTEGER, expectedBytes?: number, + maxObjects = Number.MAX_SAFE_INTEGER, ): Promise { + const readable = body as NodeJS.ReadableStream & { errored?: Error; destroyed?: boolean }; + if (readable.errored || readable.destroyed) throw readable.errored ?? new Error('Input stream was cancelled before cache admission'); await fsp.mkdir(SESSION_INPUT_CACHE_DIR, { recursive: true, mode: 0o700 }); const cacheStat = await fsp.lstat(SESSION_INPUT_CACHE_DIR); if (!cacheStat.isDirectory() || cacheStat.isSymbolicLink()) { @@ -571,6 +586,8 @@ async function storeCachedInputsOnce( } } + if (keys.length > maxObjects) throw new Error('Input batch exceeds cache object limit'); + await pruneInputCache(Math.max(0, maxBytes - stagedBytes), maxObjects - keys.length); let stored = 0; /* Commit sidecars before data. A new key remains a probe miss until both * exist; replacing an immutable key can only expose its new validated @@ -596,6 +613,7 @@ export async function storeCachedInputs( body: NodeJS.ReadableStream, maxBytes = Number.MAX_SAFE_INTEGER, expectedBytes?: number, + maxObjects = Number.MAX_SAFE_INTEGER, ): Promise { /* Concurrent pushes otherwise each budget only its own staging tree and can * collectively recreate the same transient disk spike. Queue extraction; @@ -607,7 +625,7 @@ export async function storeCachedInputs( }); await previous; try { - return await storeCachedInputsOnce(body, maxBytes, expectedBytes); + return await storeCachedInputsOnce(body, maxBytes, expectedBytes, maxObjects); } finally { release(); } @@ -615,7 +633,7 @@ export async function storeCachedInputs( /** Drops least-recently-used entries until the cache fits `maxBytes`. Eviction * is always safe: a miss simply re-pushes on the next probe. */ -export async function pruneInputCache(maxBytes: number): Promise { +export async function pruneInputCache(maxBytes: number, maxObjects = Number.MAX_SAFE_INTEGER): Promise { const names = await fsp.readdir(SESSION_INPUT_CACHE_DIR).catch(() => [] as string[]); const nameSet = new Set(names.filter(name => ENTRY_PATTERN.test(name))); const pairs: Array<{ key: string; size: number; atime: number }> = []; @@ -644,14 +662,16 @@ export async function pruneInputCache(maxBytes: number): Promise { await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, orphan), { force: true }).catch(() => {}); } } - if (total <= maxBytes) return; + let objects = pairs.length; + if (total <= maxBytes && objects <= maxObjects) return; pairs.sort((a, b) => a.atime - b.atime); for (const pair of pairs) { - if (total <= maxBytes) break; + if (total <= maxBytes && objects <= maxObjects) break; await fsp.rm(path.join(SESSION_INPUT_CACHE_DIR, pair.key), { force: true }).catch(() => {}); await fsp .rm(path.join(SESSION_INPUT_CACHE_DIR, `${pair.key}${META_SUFFIX}`), { force: true }) .catch(() => {}); total -= pair.size; + objects -= 1; } } diff --git a/docs/INPUT_REUSE.md b/docs/INPUT_REUSE.md new file mode 100644 index 00000000..40e277c4 --- /dev/null +++ b/docs/INPUT_REUSE.md @@ -0,0 +1,91 @@ +# Bounded input reuse for stateless executions + +Fresh execution workspaces can reuse input contents without retaining a mutable conversation sandbox. Each reader first authorizes a metadata request through the egress gateway. New file-server uploads carry a random `codeapi-version` metadata value that changes on every PUT, including overwrites with identical contents. The gateway binds a cache key to that version, storage identity, tenant, user, size, filename, and read-only flag. + +```mermaid +sequenceDiagram + participant R as Runner + participant G as Egress gateway + participant F as File server + participant C as Protected input cache + R->>G: POST bounded input manifest (one per execution) + G->>G: Verify grant, scope, expiry, revocation, budget + G->>F: Resolve current metadata with bounded concurrency + F-->>G: Current upload version and metadata + G-->>R: Ordered principal-scoped version keys + R->>C: Open authorized version + alt Cache miss + R->>G: Download with expected version + G->>G: Authorize and account download + G->>F: Forward expected version + F-->>R: Exact GET metadata and bytes, or 409 if changed + R->>C: Stage, validate size, atomically publish + end + R->>R: Copy into fresh workspace using existing priming rules +``` + +## Invariants + +- A cache hit never authorizes an input. Every execution performs its own preflight, including readers joining a shared fill. Denied or revoked grants cannot use cached data. +- Every manifest handle is scope-checked before storage access. Revocation is checked again before returning resolved metadata. A deadline and disconnect cancel storage work; failures and older gateways fall back to independently authorized per-file preflights. A version-race retry discards its manifest entry. +- HTTP entries are marked separately from pushed inputs. Supplying an HTTP key in `input_cache_key` cannot bypass preflight through the older pushed-cache path. +- Cache files stay outside execution workspaces and sandbox mounts. Priming copies bytes; it never hard-links a writable workspace to trusted cache contents. Existing no-follow, read-only, hashing, atomic rename, and descriptor-pinning behavior remains in use. +- Concurrent authorized misses for the same version can share one download. Cancelling one reader does not cancel remaining readers; cancelling the last reader aborts the shared request. The number of fills, cached bytes, and object count are bounded. +- The downloader checks the version from the **actual GET**, rather than labeling bytes with metadata from an earlier HEAD. A raced overwrite returns 409 and preparation retries from current metadata. Legacy objects without a version use the uncached path. +- All writers of input objects must assign a fresh version on every overwrite. The file server does so for both upload routes. Checkpoint storage uses a separate path. Direct bucket writes that preserve an old version marker are outside this protocol. +- The optional Redis object-key index stores only a locator hint. It is not an authorization or metadata cache. Preflights still read current storage metadata; indexed keys must match the exact session and object identity. +- Shared download errors belong to the initiating grant. A coalesced caller falls back to its own authorized download rather than inheriting that grant's denial or exhausted budget. +- Redis reconnects never replay unfulfilled ledger mutations. A lost reply fails closed and may leave a conservatively charged counter/reservation until grant expiry; automatically refunding an ambiguous mutation could over-credit its budget. +- Full grant policy is no longer returned to the gateway for each authorization check. Atomic Redis scripts serialize accounting with revocation. Duplicate releases cannot repeatedly refund unrelated counters. Newly created compact ledgers keep immutable policy separate from mutable counters. + +## Configuration + +| Helm value | Environment variable | Default | +|---|---|---| +| `egressGrant.ledgerCompact` | `CODEAPI_EGRESS_LEDGER_COMPACT` | `false` | +| `egressGrant.inputManifestMaxFiles` | `CODEAPI_INPUT_MANIFEST_MAX_FILES` | `512` | +| `egressGrant.inputManifestConcurrency` | `CODEAPI_INPUT_MANIFEST_CONCURRENCY` | `8` | +| `egressGrant.inputManifestTimeoutMs` | `CODEAPI_INPUT_MANIFEST_TIMEOUT_MS` | `10000` | +| `fileServer.objectIndexEnabled` | `CODEAPI_FILE_OBJECT_INDEX_ENABLED` | `false` | +| `fileServer.metadataConcurrency` | `CODEAPI_FILE_METADATA_CONCURRENCY` | `1` | +| `workerSandbox.sandbox.httpInputCacheEnabled` | `SANDBOX_HTTP_INPUT_CACHE_ENABLED` | `false` | +| `workerSandbox.sandbox.httpInputCacheMaxInflight` | `SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT` | `16` | +| `workerSandbox.sandbox.httpInputCacheMaxObjects` | `SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS` | `4096` | +| `workerSandbox.sandbox.inputCacheMaxBytes` | `SANDBOX_INPUT_CACHE_MAX_BYTES` | `536870912` | + +HTTP reuse requires a configured egress gateway. Cacheable objects are also bounded by the existing runner maximum file size. Cache capacity is local to each runner; eviction, restart, or routing to another runner causes a safe cache miss. The cache does not require persistent-session affinity. + +The manifest accepts at most 512 entries and a 4 MiB JSON body (protocol safety ceilings), with configured concurrency capped at 64. The runner bounds its opportunistic manifest request to 10 seconds, matching directory preparation, then uses per-file authorization if it cannot obtain a complete response. Oversized batches also fall back. Manifest requests remove repeated grant-header transfer and decoding, but still read current storage metadata for each file. + +Metadata listing concurrency preserves order and is capped at 64. A canary can use 8 after measuring storage load. This applies to directory-marker preparation as well; marker listings still happen and are not a retained conversation manifest. + +## Rollout and rollback + +1. Deploy the new binaries with feature flags off. Atomic accounting supports existing JSON ledgers, and legacy downloads retain metadata compatibility. The new file server stamps future uploads with versions. +2. Update **all** egress-gateway replicas before enabling compact ledgers. New binaries read both formats regardless of the creation flag. Older binaries cannot read compact hashes. To roll back to an older binary, disable compact creation, drain active grants, and wait their maximum TTL plus grace; never delete active ledgers to force a rollback. +3. Update all file-server writers before enabling the object-key index. Otherwise an older writer can change a locator without updating the index. Keep file-server replicas consistent during an indexed rollout. +4. Update the gateway, relay, runner, and launcher before enabling HTTP reuse on a small runner canary. Older gateway/relay metadata routes return 404/405 and fall back safely. Older files return `cacheable: false`. Keep the feature disabled for storage adapters that cannot return user metadata on GET. +5. Observe `codeapi_sandbox_http_input_cache_events_total` (bounded event labels, no identities), cold and warm preparation latency, storage/Redis operations, admission fairness, request budgets, and memory/disk pressure before widening the rollout. A successful manifest consumes one read request for the batch, matching the existing list-request accounting unit. Per-file compatibility preflights each consume a read request; each cold miss consumes an additional download request. Do not disable budget enforcement to accommodate a workload. +6. Disable HTTP reuse to return to normal downloads immediately. Cached files can age out normally; no workspace deletion or migration is needed. + +The creation flags default off. No deployment or object retention policy is changed by this code. Command grouping and persistent sessions remain independent options, not prerequisites for content reuse. Nothing deletes user inputs or infers shell dependencies. + +## Validation + +Ledger tests use an isolated real `redis-server` on a Unix socket with persistence disabled. Install Redis before running service tests. They cover legacy/compact formats, 240 concurrent reads against a strict budget, revocation, expiry, rejected uploads, duplicate releases, and format changes without resetting state. + +Focused commands: + +```sh +cd api +bun test src/input-manifest.test.ts src/http-input-cache.test.ts src/session-inputs.test.ts src/session-inputs.prime.test.ts src/download.test.ts src/inline-prime-atomicity.test.ts src/job-cleanup.test.ts +npx tsc --noEmit +``` + +```sh +cd service +bun test src/egress-ledger.test.ts src/egress-ledger-reconnect.test.ts src/egress-gateway.test.ts src/file-object-resolver.test.ts src/file-download.test.ts src/file-metadata.test.ts +npx tsc --noEmit +``` + +Run the code-package tests with Node (its supported test runner), plus launcher and deployment checks in CI. Cache regressions cover fresh-workspace reuse, cross-principal/version separation, denied preflights, pushed-key bypass prevention, coalesced cancellation, changed or oversized responses, and descriptor-safe eviction. Production latency targets must be validated with real regional storage latency and workload sizes; local synthetic results are not a production SLO. diff --git a/helm/codeapi/templates/egress-gateway-deployment.yaml b/helm/codeapi/templates/egress-gateway-deployment.yaml index 6e394909..49c865f0 100644 --- a/helm/codeapi/templates/egress-gateway-deployment.yaml +++ b/helm/codeapi/templates/egress-gateway-deployment.yaml @@ -40,6 +40,14 @@ spec: value: {{ .Values.hardenedSandboxMode | quote }} - name: CODEAPI_EGRESS_LEDGER_REQUIRED value: {{ .Values.egressGrant.ledgerRequired | quote }} + - name: CODEAPI_INPUT_MANIFEST_MAX_FILES + value: {{ .Values.egressGrant.inputManifestMaxFiles | quote }} + - name: CODEAPI_INPUT_MANIFEST_CONCURRENCY + value: {{ .Values.egressGrant.inputManifestConcurrency | quote }} + - name: CODEAPI_INPUT_MANIFEST_TIMEOUT_MS + value: {{ .Values.egressGrant.inputManifestTimeoutMs | quote }} + - name: CODEAPI_EGRESS_LEDGER_COMPACT + value: {{ .Values.egressGrant.ledgerCompact | quote }} - name: CODEAPI_EGRESS_LEDGER_TTL_GRACE_SECONDS value: {{ .Values.egressGrant.ledgerTtlGraceSeconds | quote }} - name: EGRESS_GATEWAY_PORT diff --git a/helm/codeapi/templates/file-server-deployment.yaml b/helm/codeapi/templates/file-server-deployment.yaml index 1dd95166..1a9e8531 100644 --- a/helm/codeapi/templates/file-server-deployment.yaml +++ b/helm/codeapi/templates/file-server-deployment.yaml @@ -51,6 +51,10 @@ spec: containerPort: {{ .Values.fileServer.service.port }} protocol: TCP env: + - name: CODEAPI_FILE_METADATA_CONCURRENCY + value: {{ .Values.fileServer.metadataConcurrency | quote }} + - name: CODEAPI_FILE_OBJECT_INDEX_ENABLED + value: {{ .Values.fileServer.objectIndexEnabled | quote }} {{ include "codeapi.otel.env" (dict "root" . "serviceName" "aiml-codeapi-file-server") | nindent 12 }} {{- if $useS3 }} # AWS S3 configuration (IRSA or static credentials) diff --git a/helm/codeapi/templates/worker-sandbox-deployment.yaml b/helm/codeapi/templates/worker-sandbox-deployment.yaml index 39c2e197..2819d998 100644 --- a/helm/codeapi/templates/worker-sandbox-deployment.yaml +++ b/helm/codeapi/templates/worker-sandbox-deployment.yaml @@ -314,6 +314,14 @@ spec: value: {{ (.Values.workerSandbox.sandbox.jobUidCount | default (.Values.workerSandbox.sandbox.maxConcurrentJobs | default (mul (.Values.workerSandbox.launcher.vcpus | default 2) 4))) | quote }} - name: SANDBOX_WORKSPACE_REAPER_MAX_AGE_SECONDS value: {{ (.Values.workerSandbox.sandbox.workspaceReaperMaxAgeSeconds | default 3600) | quote }} + - name: SANDBOX_HTTP_INPUT_CACHE_ENABLED + value: {{ .Values.workerSandbox.sandbox.httpInputCacheEnabled | quote }} + - name: SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT + value: {{ .Values.workerSandbox.sandbox.httpInputCacheMaxInflight | quote }} + - name: SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS + value: {{ .Values.workerSandbox.sandbox.httpInputCacheMaxObjects | quote }} + - name: SANDBOX_INPUT_CACHE_MAX_BYTES + value: {{ .Values.workerSandbox.sandbox.inputCacheMaxBytes | quote }} - name: SANDBOX_EXECUTE_BODY_LIMIT value: {{ .Values.workerSandbox.sandbox.executeBodyLimit | quote }} - name: SANDBOX_DISABLE_NETWORKING diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index ee997ce8..3ecfbef8 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -46,6 +46,11 @@ egressGrant: ttlSeconds: 900 ledgerRequired: true ledgerTtlGraceSeconds: 300 + # Enable only after all gateway replicas support compact ledgers. + ledgerCompact: false + inputManifestMaxFiles: 512 + inputManifestConcurrency: 8 + inputManifestTimeoutMs: 10000 # Worker signs sandbox execute requests with this private key; sandbox-runner # receives only the public verifier so a runner compromise cannot mint new @@ -268,6 +273,10 @@ workerSandbox: # Defaults to maxConcurrentJobs when unset. jobUidCount: null workspaceReaperMaxAgeSeconds: 3600 + httpInputCacheEnabled: false + httpInputCacheMaxInflight: 16 + httpInputCacheMaxObjects: 4096 + inputCacheMaxBytes: 536870912 # Language runtime package delivery packages: @@ -327,6 +336,8 @@ workerSandbox: # FILE SERVER (S3/MinIO integration, stateless) # ============================================================================= fileServer: + objectIndexEnabled: false + metadataConcurrency: 1 enabled: true replicaCount: 1 diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 02d26418..7771b9b9 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -421,6 +421,10 @@ fn is_allowed_guest_env_key(key: &str, egress_gateway_enabled: bool) -> bool { "SANDBOX_COMPILE_TIMEOUT", "SANDBOX_DATA_DIRECTORY", "SANDBOX_DISABLE_NETWORKING", + "SANDBOX_HTTP_INPUT_CACHE_ENABLED", + "SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT", + "SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS", + "SANDBOX_INPUT_CACHE_MAX_BYTES", "SANDBOX_EXECUTE_BODY_LIMIT", "SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY", "SANDBOX_FORWARD_TARGET", diff --git a/packages/code/src/relay.test.ts b/packages/code/src/relay.test.ts index a9b81fd0..c4d92288 100644 --- a/packages/code/src/relay.test.ts +++ b/packages/code/src/relay.test.ts @@ -416,3 +416,68 @@ for (const [status, reason] of [[403, 'scope_mismatch'], [503, 'ledger_conflict' } }); } + +test('file relay carries version preflights and download preconditions without opening metadata writes', async () => { + let requests = 0; + const upstream = createServer((req, res) => { + requests++; + if (req.url?.endsWith('/metadata')) { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ cacheable: true, version: 'opaque-version' })); + } else { + assert.equal(req.headers['x-codeapi-input-version'], 'opaque-version'); + res.writeHead(200, { 'X-CodeAPI-Input-Version': 'opaque-version' }).end('bytes'); + } + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ host: '127.0.0.1', port: 0, upstreamUrl, token: 'relay-secret', maxBytes: 1024, timeoutMs: 1000 }); + const headers = { 'X-LibreChat-Code-Relay-Token': 'relay-secret', 'X-CodeAPI-Egress-Grant': 'grant' }; + try { + const metadata = await fetch(`${relay.url}/sessions/s/objects/o/metadata`, { headers }); + assert.equal(metadata.status, 200); + assert.equal((await metadata.json() as { version: string }).version, 'opaque-version'); + const input = await fetch(`${relay.url}/sessions/s/objects/o`, { headers: { ...headers, 'X-CodeAPI-Input-Version': 'opaque-version' } }); + assert.equal(input.headers.get('x-codeapi-input-version'), 'opaque-version'); + assert.equal(await input.text(), 'bytes'); + const denied = await fetch(`${relay.url}/sessions/s/objects/o/metadata`, { method: 'PUT', headers, body: '' }); + assert.equal(denied.status, 404); + assert.equal(requests, 2); + } finally { + await relay.close(); + await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); + } +}); + + +test('file relay forwards bounded manifest POSTs and rejects alternate manifest methods', async () => { + let requests = 0; + const upstream = createServer(async (req, res) => { + requests++; + assert.equal(req.method, 'POST'); + assert.equal(req.url, '/input-manifest'); + assert.equal(req.headers['x-codeapi-egress-grant'], 'grant'); + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + assert.deepEqual(JSON.parse(Buffer.concat(chunks).toString()), { files: [] }); + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ files: [] })); + }); + const upstreamUrl = await listen(upstream); + const relay = await startFileRelay({ host: '127.0.0.1', port: 0, upstreamUrl, token: 'relay-secret', maxBytes: 128, timeoutMs: 1000 }); + const headers = { 'X-LibreChat-Code-Relay-Token': 'relay-secret', 'X-CodeAPI-Egress-Grant': 'grant', 'Content-Type': 'application/json' }; + try { + const response = await fetch(`${relay.url}/input-manifest`, { method: 'POST', headers, body: JSON.stringify({ files: [] }) }); + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), { files: [] }); + for (const method of ['GET', 'PUT']) { + const denied = await fetch(`${relay.url}/input-manifest`, { method, headers }); + assert.equal(denied.status, 404); + } + const oversized = await fetch(`${relay.url}/input-manifest`, { method: 'POST', headers, body: 'x'.repeat(129) }); + assert.equal(oversized.status, 413); + assert.equal(requests, 1); + } finally { + await relay.close(); + await new Promise((resolve, reject) => upstream.close(error => error ? reject(error) : resolve())); + } +}); diff --git a/packages/code/src/relay.ts b/packages/code/src/relay.ts index 14e91280..07aa8714 100644 --- a/packages/code/src/relay.ts +++ b/packages/code/src/relay.ts @@ -20,6 +20,7 @@ export interface FileRelayHandle { } const OBJECT_PATH = /^\/sessions\/[^/]+\/objects\/[^/]+$/; +const OBJECT_METADATA_PATH = /^\/sessions\/[^/]+\/objects\/[^/]+\/metadata$/; const OBJECT_LIST_PATH = /^\/sessions\/[^/]+\/objects$/; const MAX_RELAY_HEADER_BYTES = 512 * 1024; const LOCAL_HTTP_HOSTS = new Set([ @@ -160,16 +161,19 @@ export async function startFileRelay( response.end('{"status":"ok"}'); return; } + const manifestRequest = request.method === 'POST' && requestUrl.pathname === '/input-manifest' && requestUrl.search.length === 0; const objectRequest = OBJECT_PATH.test(requestUrl.pathname) && requestUrl.search.length === 0; + const metadataRequest = request.method === 'GET' && + OBJECT_METADATA_PATH.test(requestUrl.pathname) && requestUrl.search.length === 0; const normalizedListRequest = request.method === 'GET' && OBJECT_LIST_PATH.test(requestUrl.pathname) && requestUrl.searchParams.size === 1 && requestUrl.searchParams.get('detail') === 'normalized'; if ( - (request.method !== 'GET' && request.method !== 'PUT') || - (!objectRequest && !normalizedListRequest) + (request.method !== 'GET' && request.method !== 'PUT' && !manifestRequest) || + (!objectRequest && !normalizedListRequest && !metadataRequest && !manifestRequest) ) { response.writeHead(404).end(); return; @@ -191,7 +195,7 @@ export async function startFileRelay( }`; target.search = requestUrl.search; const requestBody = - request.method === 'PUT' + (request.method === 'PUT' || manifestRequest) ? await readRequestBody(request, options.maxBytes) : undefined; const upstreamResponse = await fetch(target, { @@ -200,7 +204,9 @@ export async function startFileRelay( ...(typeof grant === 'string' ? { 'X-CodeAPI-Egress-Grant': grant } : {}), - ...(request.method === 'PUT' + ...(typeof request.headers['x-codeapi-input-version'] === 'string' + ? { 'X-CodeAPI-Input-Version': request.headers['x-codeapi-input-version'] } : {}), + ...((request.method === 'PUT' || manifestRequest) ? { 'Content-Length': String(requestBody?.length ?? 0), ...(typeof request.headers['content-type'] === 'string' @@ -246,6 +252,8 @@ export async function startFileRelay( ...(upstreamResponse.headers.has('retry-after') ? { 'Retry-After': upstreamResponse.headers.get('retry-after')! } : {}), + ...(upstreamResponse.headers.has('x-codeapi-input-version') + ? { 'X-CodeAPI-Input-Version': upstreamResponse.headers.get('x-codeapi-input-version')! } : {}), 'Content-Length': String(body.length), }); response.end(body); diff --git a/service/src/config.ts b/service/src/config.ts index 94daf5a3..90df6b58 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -318,6 +318,12 @@ export const env = { EGRESS_GATEWAY_REQUEST_TIMEOUT_MS: Number(process.env.EGRESS_GATEWAY_REQUEST_TIMEOUT_MS) || 30_000, EGRESS_GATEWAY_REVOKE_TIMEOUT_MS: Number(process.env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS) || 5_000, EGRESS_LEDGER_REQUIRED: process.env.CODEAPI_EGRESS_LEDGER_REQUIRED === 'true' || process.env.CODEAPI_HARDENED_SANDBOX_MODE === 'true', + FILE_METADATA_CONCURRENCY: Math.min(64, Math.max(1, Math.floor(Number(process.env.CODEAPI_FILE_METADATA_CONCURRENCY) || 1))), + FILE_OBJECT_INDEX_ENABLED: process.env.CODEAPI_FILE_OBJECT_INDEX_ENABLED === 'true', + INPUT_MANIFEST_MAX_FILES: Math.min(512, Math.max(1, Math.floor(Number(process.env.CODEAPI_INPUT_MANIFEST_MAX_FILES) || 512))), + INPUT_MANIFEST_CONCURRENCY: Math.min(64, Math.max(1, Math.floor(Number(process.env.CODEAPI_INPUT_MANIFEST_CONCURRENCY) || 8))), + INPUT_MANIFEST_TIMEOUT_MS: Math.max(1, Math.floor(Number(process.env.CODEAPI_INPUT_MANIFEST_TIMEOUT_MS) || 10000)), + EGRESS_LEDGER_COMPACT: process.env.CODEAPI_EGRESS_LEDGER_COMPACT === 'true', EGRESS_LEDGER_TTL_GRACE_SECONDS: Number(process.env.CODEAPI_EGRESS_LEDGER_TTL_GRACE_SECONDS) || 300, EGRESS_GRANT_SECRET: process.env.CODEAPI_EGRESS_GRANT_SECRET ?? '', EGRESS_GRANT_TTL_SECONDS: resolveEgressGrantTtlSeconds(process.env.EGRESS_GRANT_TTL_SECONDS, defaultJobTimeoutMs), diff --git a/service/src/egress-gateway.test.ts b/service/src/egress-gateway.test.ts index 717309d7..c3cd4da4 100644 --- a/service/src/egress-gateway.test.ts +++ b/service/src/egress-gateway.test.ts @@ -1,14 +1,15 @@ process.env.CODEAPI_EGRESS_GATEWAY_AUTOSTART = 'false'; -import { afterAll, beforeAll, beforeEach, describe, expect, test, spyOn } from 'bun:test'; +import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; import crypto from 'crypto'; -import RedisMock from 'ioredis-mock'; +import { startTestRedis } from './test/redis'; import type { Server } from 'http'; import type { AddressInfo } from 'net'; import { env } from './config'; import { assertEgressGrantActive, createEgressLedger, + revokeEgressLedger, setEgressLedgerRedisForTest, } from './egress-ledger'; import { @@ -435,39 +436,171 @@ describe('egress gateway routes', () => { } }); - test('reports exhausted ledger conflicts as retryable without forwarding the read', async () => { - const redis = new RedisMock(); + test('batch preflight scopes every entry before reading and preserves order under a concurrency bound', async () => { + const files = Array.from({ length: 17 }, (_, i) => ({ id: `file_${i}`, session_id: 'sess_input', name: `file_${i}.csv` })); + const grant = claims({ input_files: files }); + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const body = { files: files.map(file => ({ sessionHandle: sid, objectHandle: objectHandle({ fileId: file.id, name: file.name }) })) }; + let active = 0, peak = 0, calls = 0; + const width = env.INPUT_MANIFEST_CONCURRENCY; + env.INPUT_MANIFEST_CONCURRENCY = 3; + globalThis.fetch = (async (input: RequestInfo | URL) => { + calls++; active++; peak = Math.max(peak, active); + await Bun.sleep(5); + active--; + const id = String(input).split('/').at(-2)!; + return Response.json({ version: crypto.randomUUID(), size: 5, originalFilename: `${id}.csv` }); + }) as typeof fetch; + const redis = await startTestRedis(); + setEgressLedgerRedisForTest(redis); env.EGRESS_LEDGER_REQUIRED = true; - setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); - const duplicate = redis.duplicate.bind(redis); - const duplication = spyOn(redis, 'duplicate').mockImplementation(() => { - const connection = duplicate(); - const transaction = { - set: () => transaction, - exec: async () => null, - }; - spyOn(connection, 'multi').mockImplementation(() => transaction as never); - return connection; + try { + await createEgressLedger(grant); + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(grant), 'Content-Type': 'application/json' }, body: JSON.stringify(body), + }); + expect(response.status).toBe(200); + const result = await response.json() as { files: { cacheKey: string; name: string }[] }; + expect(result.files.map(file => file.name)).toEqual(files.map(file => file.name)); + expect(result.files.every(file => /^[0-9a-f]{64}$/.test(file.cacheKey))).toBe(true); + expect(calls).toBe(17); + expect(peak).toBe(3); + expect((await assertEgressGrantActive(grant)).request_count).toBe(1); + body.files.push({ sessionHandle: sid, objectHandle: objectHandle({ fileId: 'outside_scope' }) }); + const denied = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(grant), 'Content-Type': 'application/json' }, body: JSON.stringify(body), + }); + expect(denied.status).toBe(403); + expect(calls).toBe(17); + } finally { + env.INPUT_MANIFEST_CONCURRENCY = width; + env.EGRESS_LEDGER_REQUIRED = false; + setEgressLedgerRedisForTest(null); + await redis.closeTestServer(); + } + }); + + test('batch preflight withholds resolved metadata if the grant is revoked during storage access', async () => { + const redis = await startTestRedis(); + setEgressLedgerRedisForTest(redis); + env.EGRESS_LEDGER_REQUIRED = true; + try { + const grant = claims(); + await createEgressLedger(grant); + globalThis.fetch = (async (_input: RequestInfo | URL) => { + await revokeEgressLedger(grant.grant_id!, 'test revocation'); + return Response.json({ version: crypto.randomUUID(), size: 5 }); + }) as typeof fetch; + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(grant), 'Content-Type': 'application/json' }, + body: JSON.stringify({ files: [{ sessionHandle: sessionHandle({ dir: 'read', sessionId: 'sess_input' }), objectHandle: objectHandle({}) }] }), + }); + expect(response.status).toBe(403); + expect(await response.text()).not.toContain('cacheKey'); + } finally { + env.EGRESS_LEDGER_REQUIRED = false; + setEgressLedgerRedisForTest(null); + await redis.closeTestServer(); + } + }); + + test('batch deadline aborts in-flight storage requests without starting queued inputs', async () => { + const timeout = env.INPUT_MANIFEST_TIMEOUT_MS; + const width = env.INPUT_MANIFEST_CONCURRENCY; + env.INPUT_MANIFEST_TIMEOUT_MS = 20; + env.INPUT_MANIFEST_CONCURRENCY = 1; + let started = 0, aborted = 0; + globalThis.fetch = (async (_input: RequestInfo | URL, init?: RequestInit) => { + started++; + return new Promise((_resolve, reject) => { + init!.signal!.addEventListener('abort', () => { aborted++; reject(init!.signal!.reason); }, { once: true }); + }); + }) as typeof fetch; + try { + const file = { sessionHandle: sessionHandle({ dir: 'read', sessionId: 'sess_input' }), objectHandle: objectHandle({}) }; + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(), 'Content-Type': 'application/json' }, + body: JSON.stringify({ files: [file, file] }), + }); + expect(response.ok).toBe(false); + expect(started).toBe(1); + expect(aborted).toBe(1); + } finally { + env.INPUT_MANIFEST_TIMEOUT_MS = timeout; + env.INPUT_MANIFEST_CONCURRENCY = width; + } + }); + + test('batch preflight rejects oversized and malformed manifests before storage access', async () => { + for (const files of [Array.from({ length: env.INPUT_MANIFEST_MAX_FILES + 1 }, () => ({})), [null], [{}]]) { + const response = await gatewayFetch('/input-manifest', { + method: 'POST', headers: { ...grantHeader(), 'Content-Type': 'application/json' }, body: JSON.stringify({ files }), + }); + expect(response.status).toBe(400); + } + expect(upstreamCalls).toHaveLength(0); + }); + + test('preflight authorizes scope and returns version keys scoped to the principal', async () => { + const version = crypto.randomUUID(); + upstreamResponse = Response.json({ version, size: 5, originalFilename: 'inputs/data.csv', readOnly: true }); + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const object = objectHandle({}); + const response = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { headers: grantHeader() }); + expect(response.status).toBe(200); + const metadata = await response.json() as { cacheKey: string; version: string; readOnly: boolean }; + expect(metadata.cacheKey).toMatch(/^[0-9a-f]{64}$/); + expect(metadata.version).toBe(version); + expect(metadata.readOnly).toBe(true); + expect(upstreamCalls[0].url).toEndWith('/sessions/sess_input/objects/file_123/metadata'); + const second = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { + headers: grantHeader(claims({ tenant_id: 'another_tenant' })), }); + expect((await second.json() as { cacheKey: string }).cacheKey).not.toBe(metadata.cacheKey); + const before = upstreamCalls.length; + const denied = await gatewayFetch(`/sessions/${sid}/objects/${objectHandle({ fileId: 'outside_scope' })}/metadata`, { + headers: grantHeader(), + }); + expect(denied.status).toBe(403); + expect(upstreamCalls).toHaveLength(before); + }); + + test('preflight denies revoked grants and old metadata stays uncached', async () => { + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const object = objectHandle({}); + upstreamResponse = Response.json({ size: 5 }); + const legacy = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { headers: grantHeader() }); + expect(await legacy.json()).toEqual({ cacheable: false }); + const redis = await startTestRedis(); + setEgressLedgerRedisForTest(redis); + env.EGRESS_LEDGER_REQUIRED = true; try { await createEgressLedger(claims()); - const readSession = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); - const response = await gatewayFetch(`/sessions/${readSession}/objects?detail=normalized`, { - headers: grantHeader(), - }); - expect(response.status).toBe(503); - expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('ledger_conflict'); - expect(response.headers.get('Retry-After')).toBe('1'); - expect(upstreamCalls).toHaveLength(0); - expect((await assertEgressGrantActive(claims())).request_count).toBe(0); + await redis.del(`codeapi:egress:grant:${claims().grant_id}`); + const before = upstreamCalls.length; + const denied = await gatewayFetch(`/sessions/${sid}/objects/${object}/metadata`, { headers: grantHeader() }); + expect(denied.status).toBe(403); + expect(upstreamCalls).toHaveLength(before); } finally { - duplication.mockRestore(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); - redis.disconnect(); env.EGRESS_LEDGER_REQUIRED = false; } }); + test('forwards the authorized input-version precondition on downloads', async () => { + const version = crypto.randomUUID(); + upstreamResponse = new Response('bytes', { headers: { 'X-CodeAPI-Input-Version': version } }); + const sid = sessionHandle({ dir: 'read', sessionId: 'sess_input' }); + const response = await gatewayFetch(`/sessions/${sid}/objects/${objectHandle({})}`, { + headers: { ...grantHeader(), 'X-CodeAPI-Input-Version': version }, + }); + expect(response.status).toBe(200); + expect(response.headers.get('x-codeapi-input-version')).toBe(version); + expect(new Headers(upstreamCalls[0].init.headers).get('x-codeapi-input-version')).toBe(version); + await response.text(); + }); + test('lists only scoped objects and injects internal credentials', async () => { upstreamResponse = Response.json([ { id: 'file_123', name: 'inputs/data.csv', storage_session_id: 'sess_input' }, @@ -495,7 +628,7 @@ describe('egress gateway routes', () => { }); test('accepts legacy rollout grants and handles while ledger-required mode is enabled', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); try { @@ -526,14 +659,14 @@ describe('egress gateway routes', () => { expect(record.max_output_files).toBe(50); expect(record.max_requests).toBe(1000); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('restores token-only legacy grants and creates ledger state before returning handles', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); try { @@ -569,14 +702,14 @@ describe('egress gateway routes', () => { expect(record.grant_id).toBe(legacyGrant.grant_id); expect(record.exec_id).toBe('exec_123'); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('rejects grantless handles for non-legacy grants in ledger-required mode', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); try { @@ -593,7 +726,7 @@ describe('egress gateway routes', () => { expect(response.headers.get('X-CodeAPI-Error-Code')).toBe('scope_mismatch'); expect(upstreamCalls).toHaveLength(0); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } @@ -849,7 +982,7 @@ describe('egress gateway routes', () => { }); test('rolls back upload reservations when upstream PUT throws', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); const grant = claims({ max_output_files: 1, max_requests: 3 }); @@ -894,14 +1027,14 @@ describe('egress gateway routes', () => { expect(retried.status).toBe(201); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('does not roll back ledger state when upload reservation is rejected', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { @@ -933,14 +1066,14 @@ describe('egress gateway routes', () => { expect(await upload('bbbbbbbbbbbbbbbbbbbbb')).toBe(403); expect(upstreamCalls).toHaveLength(1); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } }); test('enforces output budgets per turn when grants reuse an output session', async () => { - const redis = new RedisMock(); + const redis = await startTestRedis(); env.EGRESS_LEDGER_REQUIRED = true; setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { @@ -989,7 +1122,7 @@ describe('egress gateway routes', () => { expect(await upload(secondTurn, 'ddddddddddddddddddddd')).toBe(201); expect(await upload(secondTurn, 'eeeeeeeeeeeeeeeeeeeee')).toBe(403); } finally { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = false; } diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index 13a6de95..f86ce656 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -26,7 +26,7 @@ import { isSyntheticInternalRequestHeader, } from './internal-synthetic'; import { - assertEgressGrantActive, + checkEgressGrantActive, createEgressLedger, ensureEgressLedger, pingEgressLedger, @@ -44,6 +44,7 @@ import logger from './logger'; import { parseBoundedContentLength } from './http-limits'; import { validateEgressGatewayHardenedConfig } from './secure-startup'; import { isOpaqueObjectContentDisposition } from './file-metadata'; +import { mapObjectDetails } from './file-object-resolver'; export const app: Express = express(); app.disable('x-powered-by'); @@ -89,6 +90,8 @@ function routeFamily(req: Request): string { if (req.path === '/tool-call') return 'ptc-tool-call'; if (req.path.startsWith('/sessions/')) { if (req.method === 'PUT') return 'file-upload'; + if (req.method === 'POST' && req.path === '/input-manifest') return 'input-manifest'; + if (req.method === 'GET' && req.path.endsWith('/metadata')) return 'input-metadata'; if (req.method === 'GET' && req.path.includes('/objects/')) return 'file-download'; if (req.method === 'GET' && req.path.endsWith('/objects')) return 'file-list'; return 'file-unknown'; @@ -199,7 +202,7 @@ async function getGrant(req: Request, res: Response): Promise if (grant.legacy_grant) { await ensureEgressLedger(grant); } - await assertEgressGrantActive(grant); + await checkEgressGrantActive(grant); return grant; } @@ -438,7 +441,7 @@ async function restoreInternalSandboxResult(args: { if (grant.legacy_grant) { await ensureEgressLedger(grant); } - await assertEgressGrantActive(grant); + await checkEgressGrantActive(grant); const restored = restoreSandboxExecuteResult( args.result as Parameters[0], args.egressGrantToken, @@ -624,6 +627,88 @@ app.get('/sessions/:sessionHandle/objects', async (req, res) => { } }); +function inputMetadata( + metadata: { version?: unknown; size?: unknown; originalFilename?: unknown; readOnly?: unknown }, + grant: EgressGrantClaims, sessionId: string, objectId: string, +) { + if (typeof metadata.version !== 'string' || !/^[0-9a-f-]{36}$/.test(metadata.version) || + !Number.isSafeInteger(metadata.size) || (metadata.size as number) < 0) { + return { cacheable: false }; + } + const name = typeof metadata.originalFilename === 'string' ? metadata.originalFilename : undefined; + const readOnly = metadata.readOnly === true; + const cacheKey = crypto.createHash('sha256').update(JSON.stringify([ + 'authorized-http-input-v1', grant.tenant_id, grant.user_id, sessionId, objectId, + metadata.version, metadata.size, name, readOnly, + ])).digest('hex'); + return { cacheable: true, cacheKey, version: metadata.version, size: metadata.size, name, readOnly }; +} + +/** One budgeted HTTP read, with every handle checked before any storage access. + * Results belong only to this execution; the manifest is not an auth token. */ +app.post('/input-manifest', express.json({ limit: '4mb' }), async (req, res) => { + const controller = new AbortController(); + const cancel = () => controller.abort(); + const timeout = setTimeout(cancel, env.INPUT_MANIFEST_TIMEOUT_MS); + res.once('close', cancel); + try { + if (Object.keys(req.query).length || !Array.isArray(req.body?.files) || + req.body.files.length > env.INPUT_MANIFEST_MAX_FILES || + req.body.files.some((file: unknown) => !file || typeof file !== 'object' || + typeof (file as { sessionHandle?: unknown }).sessionHandle !== 'string' || + typeof (file as { objectHandle?: unknown }).objectHandle !== 'string')) { + return res.status(400).json({ error: 'Invalid input manifest' }); + } + const grant = await getGrant(req, res); + const files: Array<{ sessionId: string; objectId: string }> = req.body.files.map((file: { sessionHandle: string; objectHandle: string }) => { + const sessionId = openSessionParam(file.sessionHandle, grant, 'read'); + const object = openObjectParam(file.objectHandle, grant, sessionId); + return { sessionId, objectId: object.id }; + }); + await recordEgressRead(grant); + async function* inputs() { yield* files; } + const metadata = await mapObjectDetails(inputs(), async ({ sessionId, objectId }) => { + controller.signal.throwIfAborted(); + const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, + `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(objectId)}/metadata`), + { headers: injectTraceHeaders(internalServiceHeaders()), signal: controller.signal }); + if (!upstream.ok) { + await upstream.body?.cancel(); + // Recheck failures individually through the existing retry/classification path. + return { cacheable: false, retry: true }; + } + return inputMetadata(await upstream.json(), grant, sessionId, objectId); + }, env.INPUT_MANIFEST_CONCURRENCY); + // Do not publish a manifest after revocation/expiry during storage resolution. + await checkEgressGrantActive(grant); + return res.json({ files: metadata }); + } catch (error) { + return sendEgressError(req, res, error); + } finally { + clearTimeout(timeout); + res.removeListener('close', cancel); + controller.abort(); + } +}); + +/** Cache preflight is a scoped, budgeted read, never a reusable authorization grant. */ +app.get('/sessions/:sessionHandle/objects/:objectHandle/metadata', async (req, res) => { + try { + if (Object.keys(req.query).length) return res.status(400).json({ error: 'Metadata query parameters are not supported' }); + const grant = await getGrant(req, res); + const sessionId = openSessionParam(req.params.sessionHandle, grant, 'read'); + const object = openObjectParam(req.params.objectHandle, grant, sessionId); + await recordEgressRead(grant); + const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, + `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}/metadata`), + { headers: injectTraceHeaders(internalServiceHeaders()) }); + if (!upstream.ok) return pipeFetchResponse(upstream, res); + return res.json(inputMetadata(await upstream.json(), grant, sessionId, object.id)); + } catch (error) { + return sendEgressError(req, res, error); + } +}); + app.get('/sessions/:sessionHandle/objects/:objectHandle', async (req, res) => { try { if (Object.keys(req.query).length > 0) { @@ -633,12 +718,16 @@ app.get('/sessions/:sessionHandle/objects/:objectHandle', async (req, res) => { const sessionId = openSessionParam(req.params.sessionHandle, grant, 'read'); const object = openObjectParam(req.params.objectHandle, grant, sessionId); await recordEgressRead(grant); + const expectedVersion = req.header('x-codeapi-input-version'); + if (expectedVersion && !/^[0-9a-f-]{36}$/.test(expectedVersion)) { + return res.status(400).json({ error: 'Invalid input version' }); + } const upstream = await fetch( forwardUrl( env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}`, ), - { headers: injectTraceHeaders(internalServiceHeaders()) }, + { headers: injectTraceHeaders(internalServiceHeaders(expectedVersion ? { 'X-CodeAPI-Input-Version': expectedVersion } : {})) }, ); const headerOverrides = isOpaqueObjectContentDisposition( upstream.headers.get('content-disposition'), diff --git a/service/src/egress-ledger-reconnect.test.ts b/service/src/egress-ledger-reconnect.test.ts new file mode 100644 index 00000000..0cf4bb78 --- /dev/null +++ b/service/src/egress-ledger-reconnect.test.ts @@ -0,0 +1,63 @@ +import { expect, test } from 'bun:test'; +import { createConnection, createServer, type Socket, type AddressInfo } from 'node:net'; +import IORedis from 'ioredis'; +import { env } from './config'; +import { startTestRedis } from './test/redis'; +import { + EGRESS_LEDGER_REDIS_RETRY_OPTIONS, createEgressLedger, recordEgressRead, + assertEgressGrantActive, setEgressLedgerRedisForTest, +} from './egress-ledger'; +import type { EgressGrantClaims } from './egress-grant'; + +test('a lost Redis mutation reply rejects without replaying its applied counter after reconnect', async () => { + const redis = await startTestRedis(); + const sockets = new Set(); + let dropReply = false; + const proxy = createServer(downstream => { + const upstream = createConnection(redis.options.path!); + sockets.add(downstream); sockets.add(upstream); + downstream.pipe(upstream); + upstream.on('data', data => { + if (dropReply) { + dropReply = false; + downstream.destroy(); upstream.destroy(); + } else downstream.write(data); + }); + downstream.on('error', () => {}); + upstream.on('error', () => downstream.destroy()); + downstream.on('close', () => { sockets.delete(downstream); upstream.destroy(); }); + upstream.on('close', () => { sockets.delete(upstream); downstream.destroy(); }); + }); + await new Promise(resolve => proxy.listen(0, '127.0.0.1', resolve)); + const client = new IORedis({ host: '127.0.0.1', port: (proxy.address() as AddressInfo).port, + ...EGRESS_LEDGER_REDIS_RETRY_OPTIONS, retryStrategy: () => 10, lazyConnect: true }); + client.on('error', () => {}); + const required = env.EGRESS_LEDGER_REQUIRED; + const compact = env.EGRESS_LEDGER_COMPACT; + env.EGRESS_LEDGER_REQUIRED = true; + env.EGRESS_LEDGER_COMPACT = true; + setEgressLedgerRedisForTest(client); + try { + await client.connect(); + const now = Math.floor(Date.now() / 1000); + const grant: EgressGrantClaims = { v: 1, typ: 'grant', grant_id: 'reconnect', exec_id: 'exec', + tenant_id: 'tenant', user_id: 'user', session_key: 'session', input_files: [], read_sessions: [], + output_session_id: 'output', max_upload_bytes: 100, max_output_files: 10, max_requests: 10, iat: now, exp: now + 300 }; + await createEgressLedger(grant); + const reconnected = new Promise(resolve => client.once('ready', resolve)); + dropReply = true; + await expect(recordEgressRead(grant)).rejects.toThrow(); + await reconnected; + expect((await assertEgressGrantActive(grant)).request_count).toBe(1); + await recordEgressRead(grant); + expect((await assertEgressGrantActive(grant)).request_count).toBe(2); + } finally { + env.EGRESS_LEDGER_REQUIRED = required; + env.EGRESS_LEDGER_COMPACT = compact; + setEgressLedgerRedisForTest(null); + client.disconnect(); + for (const socket of sockets) socket.destroy(); + await new Promise(resolve => proxy.close(() => resolve())); + await redis.closeTestServer(); + } +}, 10000); diff --git a/service/src/egress-ledger-script.ts b/service/src/egress-ledger-script.ts new file mode 100644 index 00000000..c63346ad --- /dev/null +++ b/service/src/egress-ledger-script.ts @@ -0,0 +1,123 @@ +/** One-key transactions work on Redis Cluster and serialize admission with revocation. + * Legacy JSON is retained for mixed-version rollout; compact hashes avoid decoding + * the immutable input policy on the hot path. Never retry an ambiguous EVAL result: + * the operation may already have consumed its budget. */ +export const EGRESS_LEDGER_SCRIPT = ` +local key = KEYS[1] +local op = ARGV[1] +local kind = redis.call('TYPE', key) +if type(kind) == 'table' then kind = kind.ok end +local now = tonumber(ARGV[3]) +local function denied(message) return {'error', 'scope_mismatch', message} end +if op == 'create' then + if kind ~= 'none' then return {'ok'} end + local policy = cjson.decode(ARGV[4]) + if ARGV[5] == 'compact' then + redis.call('HSET', key, 'policy', ARGV[4], 'status', 'active', + 'exec_id', policy.exec_id, 'exp', policy.exp, + 'max_requests', policy.max_requests, 'max_upload_bytes', policy.max_upload_bytes, + 'max_output_files', policy.max_output_files, + 'request_count', 0, 'read_count', 0, 'upload_count', 0, 'tool_call_count', 0, 'uploaded_bytes', 0) + else + redis.call('SET', key, ARGV[4]) + end + redis.call('EXPIRE', key, tonumber(ARGV[6])) + return {'ok'} +end +if kind == 'none' then + if op == 'revoke' then return {'ok'} end + return denied('Egress grant ledger record is missing') +end +if kind ~= 'hash' and kind ~= 'string' then return denied('Invalid egress ledger representation') end +local compact = kind == 'hash' +local record = nil +if not compact then record = cjson.decode(redis.call('GET', key)) end +local function get(field) + if compact then return redis.call('HGET', key, field) end + return record[field] +end +local function number(field) return tonumber(get(field)) end +local function put(field, value) + if compact then redis.call('HSET', key, field, value) else record[field] = value end +end +local function add(field, value) + if compact then redis.call('HINCRBY', key, field, value) else record[field] = record[field] + value end +end +local function encodeRecord(value) + local encoded = cjson.encode(value) + -- Preserve the array contract for old gateways when cjson sees empty lists. + for _, field in ipairs({'input_files', 'read_sessions', 'output_file_ids'}) do + encoded = string.gsub(encoded, '"' .. field .. '":{}', '"' .. field .. '":[]') + end + return encoded +end +local function save() + if not compact then + -- Keep the original expiration, including a revocation tombstone's lifetime. + local ttl = redis.call('PTTL', key) + redis.call('SET', key, encodeRecord(record)) + if ttl >= 0 then redis.call('PEXPIRE', key, math.max(1, ttl)) end + end +end +if op == 'revoke' then + put('status', 'revoked') + put('revoked_at', now) + put('revoke_reason', ARGV[4]) + save() + return {'ok'} +end +if get('exec_id') ~= ARGV[2] then return denied('Egress grant ledger record does not match token') end +if get('status') ~= 'active' then return denied('Egress grant has been revoked') end +if number('exp') <= now then return {'error', 'expired', 'Egress grant is expired'} end +if op == 'check' then return {'ok'} end +if op == 'snapshot' then + if compact then + record = cjson.decode(redis.call('HGET', key, 'policy')) + for _, field in ipairs({'request_count', 'read_count', 'upload_count', 'tool_call_count', 'uploaded_bytes'}) do + record[field] = number(field) + end + record.output_file_ids = {} + local fields = redis.call('HKEYS', key) + for _, field in ipairs(fields) do + if string.sub(field, 1, 7) == 'output:' then table.insert(record.output_file_ids, string.sub(field, 8)) end + end + end + return {'ok', encodeRecord(record)} +end +local file = ARGV[4] +local bytes = tonumber(ARGV[5]) +local outputField = 'output:' .. file +local outputIndex = nil +if not compact and (op == 'reserve' or op == 'release') then + for i, id in ipairs(record.output_file_ids) do if id == file then outputIndex = i end end +end +local existing = compact and redis.call('HGET', key, outputField) or outputIndex +if op == 'release' then + -- A retried release must not refund another operation's request or byte budget. + if not existing then return {'ok'} end + local reservedBytes = compact and tonumber(existing) or bytes + if compact and reservedBytes ~= bytes then return denied('Upload release does not match reservation') end + add('uploaded_bytes', -math.min(number('uploaded_bytes'), reservedBytes)) + add('upload_count', -1) + add('request_count', -1) + if compact then redis.call('HDEL', key, outputField) else table.remove(record.output_file_ids, outputIndex) end +elseif op == 'reserve' or op == 'read' or op == 'tool' then + if number('request_count') >= number('max_requests') then return denied('Egress grant request budget exceeded') end + if op == 'reserve' then + local maxBytes = math.min(number('max_upload_bytes'), tonumber(ARGV[6])) + if not bytes or bytes < 0 or bytes ~= math.floor(bytes) or bytes > maxBytes then + return denied('Upload exceeds per-file egress byte limit') + end + if existing then return denied('Output file id has already been used for this grant') end + if number('upload_count') >= number('max_output_files') then return denied('Output file count budget exceeded') end + if number('uploaded_bytes') + bytes > maxBytes * number('max_output_files') then return denied('Aggregate upload byte budget exceeded') end + add('uploaded_bytes', bytes) + add('upload_count', 1) + if compact then redis.call('HSET', key, outputField, bytes) else table.insert(record.output_file_ids, file) end + elseif op == 'read' then add('read_count', 1) + else add('tool_call_count', 1) end + add('request_count', 1) +else return denied('Unknown egress ledger operation') end +save() +return {'ok'} +`; diff --git a/service/src/egress-ledger.test.ts b/service/src/egress-ledger.test.ts index 48b69d71..87fbec8d 100644 --- a/service/src/egress-ledger.test.ts +++ b/service/src/egress-ledger.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from 'bun:test'; -import RedisMock from 'ioredis-mock'; +import { startTestRedis } from './test/redis'; import { env } from './config'; import type { EgressGrantClaims } from './egress-grant'; import { EgressGrantError } from './egress-grant'; @@ -9,6 +9,9 @@ import { ensureEgressLedger, releaseEgressUpload, reserveEgressUpload, + recordEgressRead, + recordEgressToolCall, + checkEgressGrantActive, revokeEgressLedger, setEgressLedgerRedisForTest, } from './egress-ledger'; @@ -49,26 +52,30 @@ function expectEgressError(fn: () => Promise, reason: EgressGrantError[ ); } -describe('egress Redis ledger', () => { - let redis: InstanceType; +describe.each([false, true])('egress Redis ledger compact=%s', compact => { + let redis: Awaited>; let previousRequired: boolean; + let previousCompact: boolean; let previousMaxFileBytes: number; let previousTtlGraceSeconds: number; - beforeEach(() => { + beforeEach(async () => { previousRequired = env.EGRESS_LEDGER_REQUIRED; + previousCompact = env.EGRESS_LEDGER_COMPACT; + env.EGRESS_LEDGER_COMPACT = compact; previousMaxFileBytes = env.EGRESS_GATEWAY_MAX_FILE_BYTES; previousTtlGraceSeconds = env.EGRESS_LEDGER_TTL_GRACE_SECONDS; env.EGRESS_LEDGER_REQUIRED = true; env.EGRESS_GATEWAY_MAX_FILE_BYTES = 10; - redis = new RedisMock(); + redis = await startTestRedis(); setEgressLedgerRedisForTest(redis as unknown as Parameters[0]); }); afterEach(async () => { - await redis.disconnect(); + await redis.closeTestServer(); setEgressLedgerRedisForTest(null); env.EGRESS_LEDGER_REQUIRED = previousRequired; + env.EGRESS_LEDGER_COMPACT = previousCompact; env.EGRESS_GATEWAY_MAX_FILE_BYTES = previousMaxFileBytes; env.EGRESS_LEDGER_TTL_GRACE_SECONDS = previousTtlGraceSeconds; }); @@ -103,7 +110,7 @@ describe('egress Redis ledger', () => { ); }); - test('clears Redis WATCH after rejected mutations so later valid updates can proceed', async () => { + test('leaves counters unchanged after a rejected mutation', async () => { const claims = grant({ max_output_files: 2, max_requests: 5 }); await createEgressLedger(claims); @@ -143,7 +150,7 @@ describe('egress Redis ledger', () => { await expectEgressError(() => assertEgressGrantActive(claims), 'scope_mismatch'); }); - test('keeps concurrent WATCH mutations isolated on dedicated Redis connections', async () => { + test('accounts concurrent operations without WATCH connections', async () => { const claims = grant({ max_output_files: 16, max_requests: 16, @@ -170,7 +177,7 @@ describe('egress Redis ledger', () => { ); const record = await assertEgressGrantActive(claims); - expect(duplicateCount).toBe(8); + expect(duplicateCount).toBe(0); expect(record.request_count).toBe(12); expect(record.upload_count).toBe(12); expect(record.uploaded_bytes).toBe(12); @@ -184,4 +191,60 @@ describe('egress Redis ledger', () => { redis.duplicate = duplicate as typeof redis.duplicate; } }); + test('admits exactly the request budget under a 240-file burst', async () => { + const claims = grant({ max_requests: 100, input_files: Array.from({ length: 240 }, (_, i) => ({ + id: `file_${i}`, session_id: 'inputs', name: `${i}.txt`, + })) }); + await createEgressLedger(claims); + const results = await Promise.allSettled(Array.from({ length: 240 }, () => recordEgressRead(claims))); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(100); + expect((await assertEgressGrantActive(claims)).request_count).toBe(100); + expect((await assertEgressGrantActive(claims)).read_count).toBe(100); + }); + + test('rejects wrong execution, expired grants and mutations after revocation', async () => { + const claims = grant({ max_requests: 1000 }); + await createEgressLedger(claims); + await expectEgressError(() => recordEgressRead({ ...claims, exec_id: 'wrong' }), 'scope_mismatch'); + await recordEgressToolCall(claims.grant_id, claims.exec_id); + await Promise.all([ + ...Array.from({ length: 40 }, () => recordEgressRead(claims).catch(() => {})), + revokeEgressLedger(claims.grant_id, 'done'), + ]); + await createEgressLedger(claims); + await expectEgressError(() => checkEgressGrantActive(claims), 'scope_mismatch'); + await expectEgressError(() => recordEgressRead(claims), 'scope_mismatch'); + const expired = grant({ grant_id: 'expired', exp: nowSeconds() - 1 }); + await createEgressLedger(expired); + await expectEgressError(() => checkEgressGrantActive(expired), 'expired'); + }); + + test('duplicate releases cannot refund another upload or read', async () => { + const claims = grant({ max_requests: 10, max_output_files: 2 }); + await createEgressLedger(claims); + await recordEgressRead(claims); + await reserveEgressUpload({ grant: claims, fileId: 'a', bytes: 3 }); + await reserveEgressUpload({ grant: claims, fileId: 'b', bytes: 4 }); + await Promise.all(Array.from({ length: 10 }, () => releaseEgressUpload({ grant: claims, fileId: 'a', bytes: 3 }))); + expect(await assertEgressGrantActive(claims)).toMatchObject({ + request_count: 2, upload_count: 1, uploaded_bytes: 4, output_file_ids: ['b'], + }); + }); + + test('format selection affects new grants only and never resets existing state', async () => { + const claims = grant(); + await createEgressLedger(claims); + await recordEgressRead(claims); + env.EGRESS_LEDGER_COMPACT = !compact; + await ensureEgressLedger(claims); + await recordEgressRead(claims); + expect(await redis.type(`codeapi:egress:grant:${claims.grant_id}`)).toBe(compact ? 'hash' : 'string'); + expect((await assertEgressGrantActive(claims)).request_count).toBe(2); + if (!compact) { + const legacy = JSON.parse((await redis.get(`codeapi:egress:grant:${claims.grant_id}`))!); + expect(legacy.output_file_ids).toEqual([]); + expect(Array.isArray(legacy.input_files)).toBe(true); + } + }); + }); diff --git a/service/src/egress-ledger.ts b/service/src/egress-ledger.ts index b6bbf4fb..c635dcc4 100644 --- a/service/src/egress-ledger.ts +++ b/service/src/egress-ledger.ts @@ -6,6 +6,7 @@ import type { EgressGrantClaims } from './egress-grant'; import { EgressGrantError } from './egress-grant'; import logger from './logger'; import { redisKeepAliveOptions } from './redis-options'; +import { EGRESS_LEDGER_SCRIPT } from './egress-ledger-script'; type LedgerStatus = 'active' | 'revoked'; @@ -30,21 +31,17 @@ export interface EgressLedgerRecord { output_file_ids: string[]; } -let redis: IORedis | null = null; -const LEDGER_MUTATION_ATTEMPTS = 32; -const LEDGER_MUTATION_POOL_SIZE = Math.max(1, Number(process.env.CODEAPI_EGRESS_LEDGER_MUTATION_CONNECTIONS) || 32); - -type MutationConnectionWaiter = { - resolve: (client: IORedis) => void; - reject: (error: Error) => void; -}; - -const mutationConnections = new Set(); -let idleMutationConnections: IORedis[] = []; -let mutationConnectionWaiters: MutationConnectionWaiter[] = []; +/** A lost reply is ambiguous: never replay a possibly applied mutation. + * maxRetries=0 also rejects the pending promise on disconnect instead of leaving + * it unresolved when ioredis discards its unfulfilled-command queue. */ +export const EGRESS_LEDGER_REDIS_RETRY_OPTIONS = { + autoResendUnfulfilledCommands: false, + maxRetriesPerRequest: 0, +} as const; +let redis: IORedis | null = null; +const scriptClients = new WeakSet(); export function setEgressLedgerRedisForTest(client: IORedis | null): void { - resetMutationConnections(); redis = client; } @@ -66,7 +63,7 @@ function redisConnection(): IORedis { host: process.env.REDIS_HOST ?? 'redis', port: Number(process.env.REDIS_PORT) || 6379, password: process.env.REDIS_PASSWORD, - maxRetriesPerRequest: 1, + ...EGRESS_LEDGER_REDIS_RETRY_OPTIONS, retryStrategy, enableReadyCheck: true, connectTimeout: 10000, @@ -82,71 +79,6 @@ function redisConnection(): IORedis { return redis; } -function resetMutationConnections(): void { - const resetError = new Error('Egress ledger Redis connection reset'); - for (const waiter of mutationConnectionWaiters) { - waiter.reject(resetError); - } - mutationConnectionWaiters = []; - idleMutationConnections = []; - for (const client of mutationConnections) { - client.disconnect(); - } - mutationConnections.clear(); -} - -async function dedicatedMutationConnection(): Promise { - while (idleMutationConnections.length > 0) { - const client = idleMutationConnections.pop()!; - if (client.status !== 'end') { - return client; - } - mutationConnections.delete(client); - } - - if (mutationConnections.size < LEDGER_MUTATION_POOL_SIZE) { - return createMutationConnection(); - } - - return new Promise((resolve, reject) => { - mutationConnectionWaiters.push({ resolve, reject }); - }); -} - -function createMutationConnection(): IORedis { - const client = redisConnection().duplicate(); - mutationConnections.add(client); - client.on('error', error => logger.error('Egress ledger mutation Redis error', { error })); - return client; -} - -function releaseMutationConnection(client: IORedis): void { - if (!mutationConnections.has(client) || client.status === 'end') { - mutationConnections.delete(client); - const waiter = mutationConnectionWaiters.shift(); - if (waiter) { - try { - waiter.resolve(createMutationConnection()); - } catch (error) { - waiter.reject(error instanceof Error ? error : new Error(String(error))); - } - } - return; - } - - const waiter = mutationConnectionWaiters.shift(); - if (waiter) { - waiter.resolve(client); - return; - } - - idleMutationConnections.push(client); -} - -function sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); -} - export async function pingEgressLedger(): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return; await redisConnection().ping(); @@ -173,171 +105,79 @@ function recordFromGrant(grant: EgressGrantClaims): EgressLedgerRecord { }; } -export async function createEgressLedger(grant: EgressGrantClaims): Promise { - if (!grant.grant_id) { - throw new EgressGrantError('malformed', 'Egress grant id is required'); +async function executeLedger( + operation: string, + grantId: string, + executionId = '', + extra: Array = [], +): Promise { + const client = redisConnection() as IORedis & { + executeEgressLedger: (...args: Array) => Promise; + }; + if (!scriptClients.has(client)) { + client.defineCommand('executeEgressLedger', { numberOfKeys: 1, lua: EGRESS_LEDGER_SCRIPT }); + scriptClients.add(client); } - if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().set( - ledgerKey(grant.grant_id), - JSON.stringify(recordFromGrant(grant)), - 'EX', - ttlSeconds(grant.exp), - ); + const result = await client.executeEgressLedger( + ledgerKey(grantId), operation, executionId, + Math.floor(Date.now() / 1000), ...extra, + ) as string[]; + if (result[0] === 'error') { + throw new EgressGrantError(result[1] as EgressGrantError['reason'], result[2]); + } + return result[1]; } -export async function ensureEgressLedger(grant: EgressGrantClaims): Promise { - if (!grant.grant_id) { - throw new EgressGrantError('malformed', 'Egress grant id is required'); - } +export async function createEgressLedger(grant: EgressGrantClaims): Promise { + if (!grant.grant_id) throw new EgressGrantError('malformed', 'Egress grant id is required'); if (!env.EGRESS_LEDGER_REQUIRED) return; - await redisConnection().set( - ledgerKey(grant.grant_id), - JSON.stringify(recordFromGrant(grant)), - 'EX', - ttlSeconds(grant.exp), - 'NX', - ); + await executeLedger('create', grant.grant_id, grant.exec_id, [ + JSON.stringify(recordFromGrant(grant)), env.EGRESS_LEDGER_COMPACT ? 'compact' : 'legacy', ttlSeconds(grant.exp), + ]); } -async function loadRecord(grantId: string): Promise { - const raw = await redisConnection().get(ledgerKey(grantId)); - if (!raw) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record is missing'); - } - return JSON.parse(raw) as EgressLedgerRecord; -} +/** Admission is idempotent: neither replay nor rolling deployment resets budgets or revocation. */ +export const ensureEgressLedger = createEgressLedger; -function assertActive(record: EgressLedgerRecord, grant: Pick): void { - if (record.grant_id !== grant.grant_id || record.exec_id !== grant.exec_id) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record does not match token'); - } - if (record.status !== 'active') { - throw new EgressGrantError('scope_mismatch', 'Egress grant has been revoked'); - } - if (record.exp <= Math.floor(Date.now() / 1000)) { - throw new EgressGrantError('expired', 'Egress grant is expired'); - } -} - -async function mutateRecord( - grant: EgressGrantClaims, - mutate: (record: EgressLedgerRecord) => void, -): Promise { - if (!env.EGRESS_LEDGER_REQUIRED) { - return recordFromGrant(grant); - } - const client = await dedicatedMutationConnection(); - const key = ledgerKey(grant.grant_id); - try { - for (let i = 0; i < LEDGER_MUTATION_ATTEMPTS; i++) { - await client.watch(key); - let record: EgressLedgerRecord; - try { - const raw = await client.get(key); - if (!raw) { - throw new EgressGrantError('scope_mismatch', 'Egress grant ledger record is missing'); - } - record = JSON.parse(raw) as EgressLedgerRecord; - assertActive(record, grant); - mutate(record); - if (record.request_count > record.max_requests) { - throw new EgressGrantError('scope_mismatch', 'Egress grant request budget exceeded'); - } - } catch (error) { - await client.unwatch().catch(unwatchError => { - logger.warn('Failed to clear egress ledger WATCH after rejected mutation', { error: unwatchError }); - }); - throw error; - } - const result = await client.multi() - .set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)) - .exec(); - if (result) return record; - if (i < LEDGER_MUTATION_ATTEMPTS - 1) { - await sleep(Math.min(25, i + 1)); - } - } - } finally { - await client.unwatch().catch(error => { - logger.warn('Failed to clear egress ledger WATCH before returning mutation connection', { error }); - }); - releaseMutationConnection(client); - } - throw new EgressGrantError('ledger_conflict', 'Egress grant ledger update conflicted'); +/** Authorization hot path deliberately does not return the potentially large input policy. */ +export async function checkEgressGrantActive(grant: Pick): Promise { + if (!env.EGRESS_LEDGER_REQUIRED) return; + await executeLedger('check', grant.grant_id, grant.exec_id); } export async function assertEgressGrantActive(grant: EgressGrantClaims): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return recordFromGrant(grant); - const record = await loadRecord(grant.grant_id); - assertActive(record, grant); + const record = JSON.parse((await executeLedger('snapshot', grant.grant_id, grant.exec_id))!) as EgressLedgerRecord; + // Redis cjson represents empty Lua arrays as objects. + if (!Array.isArray(record.output_file_ids)) record.output_file_ids = []; + if (!Array.isArray(record.input_files)) record.input_files = []; + if (!Array.isArray(record.read_sessions)) record.read_sessions = []; return record; } export async function recordEgressRead(grant: EgressGrantClaims): Promise { - await mutateRecord(grant, record => { - record.request_count += 1; - record.read_count += 1; - }); + if (!env.EGRESS_LEDGER_REQUIRED) return; + await executeLedger('read', grant.grant_id, grant.exec_id, ['', 0]); } -export async function reserveEgressUpload(args: { - grant: EgressGrantClaims; - fileId: string; - bytes: number; -}): Promise { - await mutateRecord(args.grant, record => { - if (args.bytes > Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES)) { - throw new EgressGrantError('scope_mismatch', 'Upload exceeds per-file egress byte limit'); - } - if (record.output_file_ids.includes(args.fileId)) { - throw new EgressGrantError('scope_mismatch', 'Output file id has already been used for this grant'); - } - if (record.output_file_ids.length >= record.max_output_files) { - throw new EgressGrantError('scope_mismatch', 'Output file count budget exceeded'); - } - const aggregateLimit = Math.min(record.max_upload_bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES) * record.max_output_files; - if (record.uploaded_bytes + args.bytes > aggregateLimit) { - throw new EgressGrantError('scope_mismatch', 'Aggregate upload byte budget exceeded'); - } - record.request_count += 1; - record.upload_count += 1; - record.uploaded_bytes += args.bytes; - record.output_file_ids.push(args.fileId); - }); +export async function reserveEgressUpload(args: { grant: EgressGrantClaims; fileId: string; bytes: number }): Promise { + if (!env.EGRESS_LEDGER_REQUIRED) return; + if (!Number.isSafeInteger(args.bytes) || args.bytes < 0) throw new EgressGrantError('scope_mismatch', 'Invalid upload byte count'); + await executeLedger('reserve', args.grant.grant_id, args.grant.exec_id, [args.fileId, args.bytes, env.EGRESS_GATEWAY_MAX_FILE_BYTES]); } -export async function releaseEgressUpload(args: { - grant: EgressGrantClaims; - fileId: string; - bytes: number; -}): Promise { +export async function releaseEgressUpload(args: { grant: EgressGrantClaims; fileId: string; bytes: number }): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return; - await mutateRecord(args.grant, record => { - record.uploaded_bytes = Math.max(0, record.uploaded_bytes - args.bytes); - record.upload_count = Math.max(0, record.upload_count - 1); - record.request_count = Math.max(0, record.request_count - 1); - record.output_file_ids = record.output_file_ids.filter(id => id !== args.fileId); - }); + if (!Number.isSafeInteger(args.bytes) || args.bytes < 0) throw new EgressGrantError('scope_mismatch', 'Invalid upload byte count'); + await executeLedger('release', args.grant.grant_id, args.grant.exec_id, [args.fileId, args.bytes]); } export async function recordEgressToolCall(grantId: string | undefined, executionId: string): Promise { if (!env.EGRESS_LEDGER_REQUIRED || !grantId) return; - const grant = { grant_id: grantId, exec_id: executionId } as EgressGrantClaims; - await mutateRecord(grant, record => { - record.request_count += 1; - record.tool_call_count += 1; - }); + await executeLedger('tool', grantId, executionId, ['', 0]); } export async function revokeEgressLedger(grantId: string, reason: string): Promise { if (!env.EGRESS_LEDGER_REQUIRED) return; - const key = ledgerKey(grantId); - const raw = await redisConnection().get(key); - if (!raw) return; - const record = JSON.parse(raw) as EgressLedgerRecord; - record.status = 'revoked'; - record.revoked_at = Math.floor(Date.now() / 1000); - record.revoke_reason = reason; - await redisConnection().set(key, JSON.stringify(record), 'EX', ttlSeconds(record.exp)); + await executeLedger('revoke', grantId, '', [reason]); } diff --git a/service/src/file-download.test.ts b/service/src/file-download.test.ts new file mode 100644 index 00000000..cb32d771 --- /dev/null +++ b/service/src/file-download.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from 'bun:test'; +import { Readable, Writable } from 'node:stream'; +import { createServer } from 'node:http'; +import express from 'express'; +import { sendFileDownload } from './file-download'; + +async function serverFor(stream: Readable) { + const app = express(); + app.get('/', (req, res) => { void sendFileDownload(stream, res, req.header('x-codeapi-input-version')).catch(() => res.destroy()); }); + const server = createServer(app); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as { port: number }; + return { url: `http://127.0.0.1:${address.port}`, close: () => new Promise(resolve => server.close(() => resolve())) }; +} + +test('serves metadata from the downloaded version and rejects a stale preflight', async () => { + for (const expected of ['current', 'stale']) { + const stream = Object.assign(Readable.from(['bytes']), { headers: { + 'x-amz-meta-codeapi-version': 'current', 'x-amz-meta-read-only': 'true', + 'x-amz-meta-original-filename': 'verified.txt', + } }); + const server = await serverFor(stream); + try { + const response = await fetch(server.url, { headers: { 'X-CodeAPI-Input-Version': expected } }); + expect(response.status).toBe(expected === 'current' ? 200 : 409); + if (expected === 'current') { + expect(response.headers.get('x-read-only')).toBe('true'); + expect(response.headers.get('content-disposition')).toContain('verified.txt'); + expect(await response.text()).toBe('bytes'); + } else await response.text(); + } finally { await server.close(); } + } +}); + +test('downstream cancellation stops the storage stream under backpressure', async () => { + let produced = 0; + const stream = new Readable({ read() { if (++produced <= 1000) this.push(Buffer.alloc(64 * 1024)); else this.push(null); } }); + const response = Object.assign(new Writable({ + highWaterMark: 1, + write(_chunk, _encoding, callback) { setTimeout(callback, 10); }, + }), { setHeader() {} }); + const transfer = sendFileDownload(stream, response as unknown as express.Response); + const timer = setTimeout(() => response.destroy(), 25); + try { + await expect(transfer).rejects.toThrow(); + expect(stream.destroyed).toBe(true); + expect(produced).toBeLessThan(1000); + } finally { clearTimeout(timer); } +}); diff --git a/service/src/file-download.ts b/service/src/file-download.ts new file mode 100644 index 00000000..c46f80a1 --- /dev/null +++ b/service/src/file-download.ts @@ -0,0 +1,27 @@ +import type { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import type { Response } from 'express'; +import { contentDispositionForOriginalFilename, originalFilenameFromMetadata } from './file-metadata'; + +export async function sendFileDownload(dataStream: Readable, res: Response, expectedVersion?: string): Promise { + // MinIO returns the HTTP response stream. Read metadata from this exact GET, + // avoiding both a redundant HEAD and metadata/content races on overwrite. + const headers = (dataStream as Readable & { headers?: Record }).headers ?? {}; + if (expectedVersion && headers['x-amz-meta-codeapi-version'] !== expectedVersion) { + dataStream.destroy(); + res.status(409).json({ error: 'Input changed during preparation; retry with current metadata' }); + return; + } + const metadata: Record = {}; + for (const [key, value] of Object.entries(headers)) { + if (key.startsWith('x-amz-meta-')) metadata[key.slice(11)] = value; + } + res.setHeader('Content-Disposition', contentDispositionForOriginalFilename(originalFilenameFromMetadata(metadata))); + if (headers['content-type']) res.setHeader('Content-Type', headers['content-type']); + if (headers['content-length']) res.setHeader('Content-Length', headers['content-length']); + if (metadata['read-only'] === 'true') res.setHeader('X-Read-Only', 'true'); + if (metadata['codeapi-version']) res.setHeader('X-CodeAPI-Input-Version', metadata['codeapi-version']); + const cancel = (): void => { if (!res.writableFinished) dataStream.destroy(new Error('Download client disconnected')); }; + res.once('close', cancel); + try { await pipeline(dataStream, res); } finally { res.off('close', cancel); } +} diff --git a/service/src/file-object-resolver.test.ts b/service/src/file-object-resolver.test.ts new file mode 100644 index 00000000..deeb3265 --- /dev/null +++ b/service/src/file-object-resolver.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from 'bun:test'; +import { FileObjectResolver, mapObjectDetails } from './file-object-resolver'; +import type { BucketItemStat } from 'minio'; + +describe('storage object resolution', () => { + test('indexes exact identities while reading fresh version metadata on every request', async () => { + const index = new Map(); + let lists = 0; + let heads = 0; + let version = 'first'; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { lists++; yield { name: 's/identifier.txt' }; yield { name: 's/id.txt' }; }, + stat: async key => { heads++; expect(key).toBe('s/id.txt'); return { size: 5, etag: 'etag', lastModified: new Date(), metaData: { 'codeapi-version': version } } as BucketItemStat; }, + index: { get: async k => index.get(k) ?? null, set: async (k, v) => index.set(k, v), forget: async k => index.delete(k) }, + }); + expect((await resolver.metadata('s', 'id'))?.stat.metaData['codeapi-version']).toBe('first'); + version = 'second'; + expect((await resolver.metadata('s', 'id'))?.stat.metaData['codeapi-version']).toBe('second'); + expect(lists).toBe(1); + expect(heads).toBe(2); + }); + + test('ignores foreign-session index entries and does not cache absence', async () => { + let present = false; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: 's2/id.txt' }; if (present) yield { name: 's/id.txt' }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { get: async () => 's2/id.txt', set: async () => {}, forget: async () => {} }, + }); + expect(await resolver.resolve('s', 'id')).toBeUndefined(); + present = true; + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + }); +}); + +test('metadata listing stays bounded and ordered across 240 objects', async () => { + let active = 0; + let maximum = 0; + async function* objects() { for (let i = 0; i < 240; i++) yield i; } + const result = await mapObjectDetails(objects(), async value => { + maximum = Math.max(maximum, ++active); + await new Promise(resolve => setTimeout(resolve, value % 3)); + active--; + return value; + }, 8); + expect(maximum).toBe(8); + expect(result).toEqual(Array.from({ length: 240 }, (_, i) => i)); +}); diff --git a/service/src/file-object-resolver.ts b/service/src/file-object-resolver.ts new file mode 100644 index 00000000..a41e0dfc --- /dev/null +++ b/service/src/file-object-resolver.ts @@ -0,0 +1,75 @@ +import { createHash } from 'node:crypto'; +import path from 'node:path'; +import type { BucketItemStat } from 'minio'; + +export interface ObjectResolverDependencies { + bucket: string; + list(prefix: string): AsyncIterable<{ name?: string }>; + stat(key: string): Promise; + index?: { + get(key: string): Promise; + set(key: string, value: string, replace: boolean): Promise; + forget(key: string, value: string): Promise; + }; +} + +/** Storage-key index is a hint, never metadata or authorization. A fresh HEAD + * proves existence and supplies the current version even on index/cache hits. */ +export class FileObjectResolver { + constructor(private readonly deps: ObjectResolverDependencies) {} + + private indexKey(session: string, id: string): string { + return `codeapi:file-key:${createHash('sha256').update(JSON.stringify([this.deps.bucket, session, id])).digest('hex')}`; + } + + private matches(key: string, session: string, id: string): boolean { + return path.posix.dirname(key) === session && + (path.posix.basename(key) === id || path.posix.basename(key, path.posix.extname(key)) === id); + } + + async remember(session: string, id: string, key: string, replace = true): Promise { + if (!this.matches(key, session, id)) throw new Error('Object key does not match storage identity'); + await this.deps.index?.set(this.indexKey(session, id), key, replace); + } + + async resolve(session: string, id: string): Promise { + const cached = await this.deps.index?.get(this.indexKey(session, id)); + if (cached && this.matches(cached, session, id)) return cached; + for await (const object of this.deps.list(`${session}/${id}`)) { + if (object.name && this.matches(object.name, session, id)) { + await this.remember(session, id, object.name, false); + return object.name; + } + } + return undefined; + } + + async metadata(session: string, id: string): Promise<{ key: string; stat: BucketItemStat } | undefined> { + const key = await this.resolve(session, id); + if (!key) return undefined; + try { + return { key, stat: await this.deps.stat(key) }; + } catch (error) { + if (!['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? '')) throw error; + await this.deps.index?.forget(this.indexKey(session, id), key); + // Do not cache absence: a later upload can publish this identity again. + return undefined; + } + } +} + +/** Bound storage metadata requests while preserving listing order. */ +export async function mapObjectDetails(objects: AsyncIterable, describe: (object: T) => Promise, concurrency: number): Promise { + const results: R[] = []; + const batch: T[] = []; + const width = Math.max(1, Math.min(64, Math.floor(concurrency) || 1)); + for await (const object of objects) { + batch.push(object); + if (batch.length === width) { + results.push(...await Promise.all(batch.map(describe))); + batch.length = 0; + } + } + results.push(...await Promise.all(batch.map(describe))); + return results; +} diff --git a/service/src/file-server.ts b/service/src/file-server.ts index f9293e48..9b326ca2 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,4 +1,8 @@ import b from 'busboy'; +import { randomUUID } from 'node:crypto'; +import { mapObjectDetails } from './file-object-resolver'; +import { sendFileDownload } from './file-download'; +import { FileObjectResolver } from './file-object-resolver'; import path from 'path'; import IORedis from 'ioredis'; import express from 'express'; @@ -18,7 +22,6 @@ import logger from './fileServerLogger'; import { env } from './config'; import { redisKeepAliveOptions } from './redis-options'; import { - contentDispositionForOriginalFilename, decodeOriginalFilename, originalFilenameFromMetadata, } from './file-metadata'; @@ -144,6 +147,21 @@ redisClient.on('ready', () => { logger.info('Redis Client Ready'); }); +const objectResolver = new FileObjectResolver({ + bucket: bucketName, + list: prefix => minioClient.listObjects(bucketName, prefix, true), + stat: key => minioClient.statObject(bucketName, key), + ...(env.FILE_OBJECT_INDEX_ENABLED ? { index: { + get: (key: string) => redisClient.get(key), + set: (key: string, value: string, replace: boolean) => replace + ? redisClient.set(key, value, 'EX', env.SESSION_CACHE_TTL) + : redisClient.set(key, value, 'EX', env.SESSION_CACHE_TTL, 'NX'), + forget: (key: string, value: string) => redisClient.eval( + "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end return 0", 1, key, value, + ), + } } : {}), +}); + const minioRegion = process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; async function ensureBucketExists(retries = 10, delay = 1000): Promise { @@ -250,6 +268,8 @@ async function uploadFile( * `getObject` / `statObject` without a separate Redis lookup. */ const metaData: Record = { 'Content-Type': mimetype, + // New marker on every PUT, including same-ID overwrites and metadata changes. + 'X-Amz-Meta-Codeapi-Version': randomUUID(), 'X-Amz-Meta-Original-Filename': encodedFilename, 'X-Amz-Meta-Original-Filename-Encoded': 'base64', }; @@ -267,6 +287,7 @@ async function uploadFile( } else { await minioClient.putObject(bucketName, objectName, peeked.body, undefined, metaData); } + await objectResolver.remember(session_id, fileId, objectName); logger.info(`[${INSTANCE_ID}] File ID: ${fileId} | Filename: ${filename} | Session key: ${sessionKey}`); await redisClient.set(`upload:${sessionKey}${session_id}${fileId}`, 'true', 'EX', env.SESSION_CACHE_TTL); fileUploads.inc(); @@ -443,30 +464,14 @@ app.get('/sessions/:session_id/objects/:objectId/metadata', async (req, res) => const { session_id, objectId } = req.params; try { - const stream = minioClient.listObjects(bucketName, `${session_id}/${objectId}`, true); - let objectName = ''; - - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found', - session_id, - objectId, - }); - } - - const stat: Partial = await minioClient.statObject(bucketName, objectName); + const resolved = await objectResolver.metadata(session_id, objectId); + if (!resolved) return res.status(404).json({ error: 'File not found' }); + const { key: objectName, stat } = resolved; const originalFilename = originalFilenameFromMetadata(stat.metaData); return res.status(200).json({ name: objectName, + version: stat.metaData?.['codeapi-version'], ...(originalFilename ? { originalFilename } : {}), size: stat.size, lastModified: stat.lastModified, @@ -487,87 +492,33 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { const { session_id, objectId } = req.params; try { - // List objects to find the correct file with extension - const stream = minioClient.listObjects(bucketName, `${session_id}/${objectId}`, true); - let objectName = ''; - - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${objectId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { - logger.warn('File not found', { session_id, objectId, bucketName }); - return res.status(404).json({ - error: 'File not found', - details: 'No matching file found', - session_id, - objectId, - bucketName - }); - } - - logger.info(`[${INSTANCE_ID}] Attempting to download: ${objectName}`); - - const stat: Partial = await minioClient.statObject(bucketName, objectName); - - const originalFilename = originalFilenameFromMetadata(stat.metaData); - - logger.info(`[${INSTANCE_ID}] File found: ${objectName}`); - - // Explicitly remove problematic headers that might be duplicated - res.removeHeader('Transfer-Encoding'); - res.removeHeader('Date'); - - /* An object-key basename is only a storage identifier, not an original - * filename. If an S3-compatible backend drops user metadata, retain - * attachment semantics but omit the filename so the runner uses its - * caller-supplied destination. */ - res.setHeader('Content-Disposition', contentDispositionForOriginalFilename(originalFilename)); - if (stat.metaData?.['content-type'] != null) { - res.setHeader('Content-Type', stat.metaData['content-type']); - } - /* Surface the read-only flag on download so the sandbox can plumb it - * onto its in-memory file metadata without a separate metadata fetch. - * MinIO normalizes `X-Amz-Meta-Read-Only` to `read-only` in stat.metaData. */ - if (stat.metaData?.['read-only'] === 'true') { - res.setHeader('X-Read-Only', 'true'); - } - + const objectName = await objectResolver.resolve(session_id, objectId); + if (!objectName) return res.status(404).json({ error: 'File not found' }); const dataStream = await minioClient.getObject(bucketName, objectName); - fileDownloads.inc(); - - dataStream.on('data', (chunk) => { - res.write(chunk); - }); - - dataStream.on('end', () => { - res.end(); - }); - - dataStream.on('error', (err) => { - logger.error('Error streaming file:', { error: err, session_id, objectId, bucketName }); - // Only send error if headers haven't been sent yet - if (!res.headersSent) { - res.status(500).json({ - error: 'Error streaming file', - details: err.message - }); - } else { - res.end(); + try { + const headers = (dataStream as Readable & { headers?: Record }).headers ?? {}; + if (!headers['x-amz-meta-codeapi-version'] || !headers['x-amz-meta-original-filename']) { + // Preserve legacy/S3-compatible metadata behavior without promoting a + // later HEAD's version marker onto bytes from an earlier GET. + const stat = await minioClient.statObject(bucketName, objectName); + if (headers.etag?.replace(/^"|"$/g, '') !== stat.etag) { + return res.status(409).json({ error: 'Input changed during metadata lookup' }); + } + for (const [key, value] of Object.entries(stat.metaData ?? {})) { + if (key !== 'codeapi-version') headers[`x-amz-meta-${key}`] ??= value; + } } - }); + fileDownloads.inc(); + await sendFileDownload(dataStream, res, req.header('x-codeapi-input-version')); + } finally { + dataStream.destroy(); + } } catch (err) { - logger.error('Error downloading file:', { error: err, session_id, objectId, bucketName }); - return res.status(500).json({ - error: 'Error downloading file', - details: (err as Error | undefined)?.message, - session_id, - objectId, - bucketName - }); + logger.error('Error downloading file', { error: err, session_id, objectId }); + if (!res.headersSent && !res.destroyed) { + const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((err as { code?: string }).code ?? ''); + return res.status(missing ? 404 : 500).json({ error: 'Error downloading file' }); + } } }); @@ -585,7 +536,7 @@ function parseObjectName(objectName: string | undefined): { session_id: string; return { session_id, file_id }; } -const detailLevels: Record Promise> | undefined> = { +const detailLevels: Record Promise>> = { simple: async (obj: BucketItem): Promise> => obj.name ?? '', summary: async (obj: BucketItem): Promise> => ({ name: obj.name, @@ -639,14 +590,9 @@ app.get('/sessions/:session_id/objects', async (req, res) => { const { detail = 'simple' } = req.query; try { - const stream = minioClient.listObjects(bucketName, session_id, true); - const objects: (t.ObjectTypes | Partial | undefined)[] = []; - + const stream = minioClient.listObjects(bucketName, `${session_id}/`, true); const getDetail = detailLevels[detail as string] ?? detailLevels.simple; - - for await (const obj of stream) { - objects.push(await getDetail(obj)); - } + const objects = await mapObjectDetails(stream, getDetail, env.FILE_METADATA_CONCURRENCY); res.json(objects); } catch (err) { diff --git a/service/src/test/redis.ts b/service/src/test/redis.ts new file mode 100644 index 00000000..f243f657 --- /dev/null +++ b/service/src/test/redis.ts @@ -0,0 +1,45 @@ +import { spawn } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import IORedis from 'ioredis'; + +/** Real Lua semantics, isolated Unix socket, no TCP listener or durable data. */ +export async function startTestRedis(): Promise }> { + const dir = await mkdtemp(path.join(tmpdir(), 'codeapi-redis-')); + const socket = path.join(dir, 'redis.sock'); + const process = spawn('redis-server', [ + '--port', '0', '--unixsocket', socket, '--unixsocketperm', '700', + '--save', '', '--appendonly', 'no', '--dir', dir, + ], { stdio: 'ignore' }); + let failure: Error | undefined; + process.on('error', error => { failure = error; }); + const exited = new Promise(resolve => { + process.once('exit', () => resolve()); + process.once('error', () => resolve()); + }); + const client = new IORedis(socket, { lazyConnect: true, retryStrategy: () => null, maxRetriesPerRequest: 0 }); + client.on('error', () => {}); + const closeTestServer = async (): Promise => { + client.disconnect(); + process.kill('SIGTERM'); + await exited; + await rm(dir, { recursive: true, force: true }); + }; + try { + for (let attempt = 0; attempt < 100; attempt++) { + if (failure) throw failure; + try { + await client.connect(); + await client.ping(); + return Object.assign(client, { closeTestServer }); + } catch { + await new Promise(resolve => setTimeout(resolve, 20)); + } + } + throw new Error('Test Redis did not start; install redis-server'); + } catch (error) { + await closeTestServer(); + throw error; + } +} From a992ec0b4bf4c4a46caf9ef38b49f0a07010eb6b Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 06:45:37 -0400 Subject: [PATCH 04/42] =?UTF-8?q?=F0=9F=A7=B9=20fix:=20Evict=20Stale=20Fil?= =?UTF-8?q?e-Object=20Index=20Entries=20(#182)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: evict stale file-object index entries Forget cached locators after successful deletion and missing-object downloads so replacement keys resolve immediately. Reuse exact resolver matching in the delete route to avoid prefix collisions. Fixes #181 * fix: keep file-object deletion storage-authoritative * fix: retire superseded upload objects * fix: canonicalize replacement object keys * fix: collapse legacy object-key siblings * fix: recover reads from stale locators * fix: namespace canonical object identities --- service/src/file-object-resolver.test.ts | 143 ++++++++++++++++++++++- service/src/file-object-resolver.ts | 124 +++++++++++++++++--- service/src/file-server.ts | 74 +++++++++--- 3 files changed, 306 insertions(+), 35 deletions(-) diff --git a/service/src/file-object-resolver.test.ts b/service/src/file-object-resolver.test.ts index deeb3265..2342d87a 100644 --- a/service/src/file-object-resolver.test.ts +++ b/service/src/file-object-resolver.test.ts @@ -1,8 +1,27 @@ import { describe, expect, test } from 'bun:test'; -import { FileObjectResolver, mapObjectDetails } from './file-object-resolver'; +import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; import type { BucketItemStat } from 'minio'; describe('storage object resolution', () => { + test('replacement uploads converge on one stable object key', () => { + expect(storageKeyForUpload('s', 'id', '.csv', true)).toBe('s/.codeapi-objects/aWQ'); + expect(storageKeyForUpload('s', 'id', '.pdf', true)).toBe('s/.codeapi-objects/aWQ'); + expect(canonicalObjectId(storageKeyForUpload('s', 'report.csv', '', true))).toBe('report.csv'); + expect(storageKeyForUpload('s', 'generated', '.csv', false)).toBe('s/generated.csv'); + }); + + test('canonical dotted identities cannot match another legacy identity', async () => { + const dottedKey = storageKeyForUpload('s', 'report.csv', '', true); + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: dottedKey }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + }); + + expect(await resolver.listFresh('s', 'report.csv')).toEqual([dottedKey]); + expect(await resolver.listFresh('s', 'report')).toEqual([]); + }); + test('indexes exact identities while reading fresh version metadata on every request', async () => { const index = new Map(); let lists = 0; @@ -33,6 +52,128 @@ describe('storage object resolution', () => { present = true; expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); }); + + test('falls back to storage when the advisory index is unavailable', async () => { + const failures: string[] = []; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: 's/id.txt' }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + onIndexError: operation => failures.push(operation), + index: { + get: async () => { throw new Error('redis unavailable'); }, + set: async () => { throw new Error('redis unavailable'); }, + forget: async () => { throw new Error('redis unavailable'); }, + }, + }); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + expect(await resolver.listFresh('s', 'id')).toEqual(['s/id.txt']); + expect(failures).toEqual(['get', 'set']); + }); + + test('fresh listing ignores a stale locator and returns every exact sibling', async () => { + const index = new Map([['locator', 's/id.txt']]); + let lists = 0; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { + lists++; + yield { name: 's/identifier.txt' }; + yield { name: 's/id.csv' }; + yield { name: 's/id.pdf' }; + }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { + get: async () => index.get('locator') ?? null, + set: async (_key, value) => index.set('locator', value), + forget: async (_key, value) => { if (index.get('locator') === value) index.delete('locator'); }, + }, + }); + + expect(await resolver.listFresh('s', 'id')).toEqual(['s/id.csv', 's/id.pdf']); + expect(index.get('locator')).toBe('s/id.txt'); + expect(lists).toBe(2); + }); + + test('metadata re-resolves storage after a cached locator is missing', async () => { + const index = new Map([['locator', 's/id.txt']]); + let heads = 0; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { yield { name: 's/.codeapi-objects/aWQ' }; }, + stat: async key => { + heads++; + if (key === 's/id.txt') throw Object.assign(new Error('missing'), { code: 'NoSuchKey' }); + return { + size: 1, + etag: 'current', + lastModified: new Date(), + metaData: { 'codeapi-version': 'current' }, + } as BucketItemStat; + }, + index: { + get: async () => index.get('locator') ?? null, + set: async (_key, value) => index.set('locator', value), + forget: async (_key, value) => { if (index.get('locator') === value) index.delete('locator'); }, + }, + }); + + const metadata = await resolver.metadata('s', 'id'); + expect(metadata?.key).toBe('s/.codeapi-objects/aWQ'); + expect(metadata?.stat.metaData['codeapi-version']).toBe('current'); + expect(index.get('locator')).toBe('s/.codeapi-objects/aWQ'); + expect(heads).toBe(2); + }); + + test('forgets a deleted locator so a replacement key for the same identity resolves', async () => { + const index = new Map(); + const stored = new Set(['s/id.txt']); + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* (prefix) { for (const name of stored) if (name.startsWith(prefix)) yield { name }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { + get: async k => index.get(k) ?? null, + set: async (k, v, replace) => { if (replace || !index.has(k)) index.set(k, v); }, + forget: async (k, v) => { if (index.get(k) === v) index.delete(k); }, + }, + }); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + stored.delete('s/id.txt'); + await resolver.forget('s', 'id', 's/id.txt'); + expect(index.size).toBe(0); + + stored.add('s/id.csv'); + expect(await resolver.resolve('s', 'id')).toBe('s/id.csv'); + }); + + test('eviction is scoped to the identity and never drops a newer cached key', async () => { + const index = new Map(); + let lists = 0; + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* () { lists++; yield { name: 's/id.txt' }; }, + stat: async () => ({ metaData: {} } as BucketItemStat), + index: { + get: async k => index.get(k) ?? null, + set: async (k, v, replace) => { if (replace || !index.has(k)) index.set(k, v); }, + forget: async (k, v) => { if (index.get(k) === v) index.delete(k); }, + }, + }); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.txt'); + // A concurrent upload republished the identity before the delete evicted it. + await resolver.remember('s', 'id', 's/id.csv'); + await resolver.forget('s', 'id', 's/id.txt'); + // Keys outside the identity can never reach its entry. + await resolver.forget('s', 'id', 's2/id.txt'); + await resolver.forget('s', 'id', 's/other.txt'); + + expect(await resolver.resolve('s', 'id')).toBe('s/id.csv'); + expect(lists).toBe(1); + }); }); test('metadata listing stays bounded and ordered across 240 objects', async () => { diff --git a/service/src/file-object-resolver.ts b/service/src/file-object-resolver.ts index a41e0dfc..bc4b4ce5 100644 --- a/service/src/file-object-resolver.ts +++ b/service/src/file-object-resolver.ts @@ -2,10 +2,28 @@ import { createHash } from 'node:crypto'; import path from 'node:path'; import type { BucketItemStat } from 'minio'; +const CANONICAL_OBJECT_DIRECTORY = '.codeapi-objects'; + +export function canonicalObjectKey(session: string, id: string): string { + return `${session}/${CANONICAL_OBJECT_DIRECTORY}/${Buffer.from(id, 'utf8').toString('base64url')}`; +} + +export function canonicalObjectId(key: string): string | undefined { + const parts = key.split('/'); + if (parts.length !== 3 || parts[1] !== CANONICAL_OBJECT_DIRECTORY || parts[2] === '') return undefined; + try { + const id = Buffer.from(parts[2], 'base64url').toString('utf8'); + return canonicalObjectKey(parts[0], id) === key ? id : undefined; + } catch { + return undefined; + } +} + export interface ObjectResolverDependencies { bucket: string; list(prefix: string): AsyncIterable<{ name?: string }>; stat(key: string): Promise; + onIndexError?(operation: 'get' | 'set' | 'forget', error: unknown): void; index?: { get(key: string): Promise; set(key: string, value: string, replace: boolean): Promise; @@ -13,48 +31,122 @@ export interface ObjectResolverDependencies { }; } +/** Caller-supplied identities keep one stable storage key across replacement + * filenames. This gives concurrent PUTs one last-writer-wins S3 object without + * requiring a distributed lock or leaving extension-keyed siblings behind. */ +export function storageKeyForUpload( + session: string, + id: string, + extension: string, + replacing: boolean, +): string { + return replacing ? canonicalObjectKey(session, id) : `${session}/${id}${extension}`; +} + /** Storage-key index is a hint, never metadata or authorization. A fresh HEAD * proves existence and supplies the current version even on index/cache hits. */ export class FileObjectResolver { constructor(private readonly deps: ObjectResolverDependencies) {} + private reportIndexError(operation: 'get' | 'set' | 'forget', error: unknown): void { + this.deps.onIndexError?.(operation, error); + } + private indexKey(session: string, id: string): string { return `codeapi:file-key:${createHash('sha256').update(JSON.stringify([this.deps.bucket, session, id])).digest('hex')}`; } private matches(key: string, session: string, id: string): boolean { + if (key === canonicalObjectKey(session, id)) return true; return path.posix.dirname(key) === session && (path.posix.basename(key) === id || path.posix.basename(key, path.posix.extname(key)) === id); } async remember(session: string, id: string, key: string, replace = true): Promise { if (!this.matches(key, session, id)) throw new Error('Object key does not match storage identity'); - await this.deps.index?.set(this.indexKey(session, id), key, replace); + try { + await this.deps.index?.set(this.indexKey(session, id), key, replace); + } catch (error) { + this.reportIndexError('set', error); + } } - async resolve(session: string, id: string): Promise { - const cached = await this.deps.index?.get(this.indexKey(session, id)); - if (cached && this.matches(cached, session, id)) return cached; - for await (const object of this.deps.list(`${session}/${id}`)) { - if (object.name && this.matches(object.name, session, id)) { - await this.remember(session, id, object.name, false); - return object.name; + /** Evict a cached locator once its object is known to be gone. Scoped to the + * requested identity, and conditional on the stored value so a replacement + * key published concurrently for the same identity is never dropped. */ + async forget(session: string, id: string, key: string): Promise { + if (!this.matches(key, session, id)) return; + try { + await this.deps.index?.forget(this.indexKey(session, id), key); + } catch (error) { + this.reportIndexError('forget', error); + } + } + + private async cached(session: string, id: string): Promise { + try { + const key = await this.deps.index?.get(this.indexKey(session, id)); + return key && this.matches(key, session, id) ? key : undefined; + } catch (error) { + this.reportIndexError('get', error); + return undefined; + } + } + + private async findInStorage(session: string, id: string, replaceIndex: boolean): Promise { + for (const prefix of [canonicalObjectKey(session, id), `${session}/${id}`]) { + for await (const object of this.deps.list(prefix)) { + if (object.name && this.matches(object.name, session, id)) { + await this.remember(session, id, object.name, replaceIndex); + return object.name; + } } } return undefined; } + /** List every exact storage key for an identity without consulting its + * locator. Used to collapse legacy siblings and delete the whole identity. */ + async listFresh(session: string, id: string): Promise { + const keys = new Set(); + for (const prefix of [canonicalObjectKey(session, id), `${session}/${id}`]) { + for await (const object of this.deps.list(prefix)) { + if (object.name && this.matches(object.name, session, id)) keys.add(object.name); + } + } + return [...keys]; + } + + async resolve(session: string, id: string): Promise { + return await this.cached(session, id) ?? await this.findInStorage(session, id, false); + } + + /** Recover once a cached key is proven missing. Eviction and publication are + * advisory; the authoritative storage listing determines the replacement. */ + async recover(session: string, id: string, missingKey: string): Promise { + await this.forget(session, id, missingKey); + const [current] = await this.listFresh(session, id); + if (current) await this.remember(session, id, current); + return current; + } + async metadata(session: string, id: string): Promise<{ key: string; stat: BucketItemStat } | undefined> { - const key = await this.resolve(session, id); + let key = await this.resolve(session, id); if (!key) return undefined; - try { - return { key, stat: await this.deps.stat(key) }; - } catch (error) { - if (!['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? '')) throw error; - await this.deps.index?.forget(this.indexKey(session, id), key); - // Do not cache absence: a later upload can publish this identity again. - return undefined; + for (let attempt = 0; attempt < 2; attempt++) { + try { + return { key, stat: await this.deps.stat(key) }; + } catch (error) { + if (!['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? '')) throw error; + if (attempt === 1) { + await this.forget(session, id, key); + return undefined; + } + key = await this.recover(session, id, key); + if (!key) return undefined; + } } + return undefined; } } diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 9b326ca2..3dfa0a10 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,8 +1,7 @@ import b from 'busboy'; import { randomUUID } from 'node:crypto'; -import { mapObjectDetails } from './file-object-resolver'; +import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; import { sendFileDownload } from './file-download'; -import { FileObjectResolver } from './file-object-resolver'; import path from 'path'; import IORedis from 'ioredis'; import express from 'express'; @@ -151,6 +150,7 @@ const objectResolver = new FileObjectResolver({ bucket: bucketName, list: prefix => minioClient.listObjects(bucketName, prefix, true), stat: key => minioClient.statObject(bucketName, key), + onIndexError: (operation, error) => logger.warn('File-object index operation failed', { operation, error }), ...(env.FILE_OBJECT_INDEX_ENABLED ? { index: { get: (key: string) => redisClient.get(key), set: (key: string, value: string, replace: boolean) => replace @@ -162,6 +162,16 @@ const objectResolver = new FileObjectResolver({ } } : {}), }); +/** Index eviction is best effort: the index is only a hint, so a Redis failure + * must never turn a completed delete or a missing-object 404 into a 500. */ +async function forgetObjectKey(session_id: string, objectId: string, objectName: string): Promise { + try { + await objectResolver.forget(session_id, objectId, objectName); + } catch (error) { + logger.warn('Failed to evict file-object index entry', { error, session_id, objectId, objectName }); + } +} + const minioRegion = process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; async function ensureBucketExists(retries = 10, delay = 1000): Promise { @@ -257,7 +267,14 @@ async function uploadFile( ): Promise { const fileId = existingFileId ?? nanoid(); const fileExtension = path.extname(filename); - const objectName = `${session_id}/${fileId}${fileExtension}`; + // Caller-supplied identities use one canonical key, so concurrent writers + // converge on S3's last-writer semantics regardless of filename extension. + const objectName = storageKeyForUpload( + session_id, + fileId, + fileExtension, + existingFileId != null, + ); const encodedFilename = Buffer.from(filename).toString('base64'); @@ -287,6 +304,15 @@ async function uploadFile( } else { await minioClient.putObject(bucketName, objectName, peeked.body, undefined, metaData); } + if (existingFileId != null) { + // Retire every extension-keyed sibling left by older replacement behavior. + // Concurrent replacement writers share objectName and never delete it. + for (const sibling of await objectResolver.listFresh(session_id, fileId)) { + if (sibling === objectName) continue; + await minioClient.removeObject(bucketName, sibling); + await objectResolver.forget(session_id, fileId, sibling); + } + } await objectResolver.remember(session_id, fileId, objectName); logger.info(`[${INSTANCE_ID}] File ID: ${fileId} | Filename: ${filename} | Session key: ${sessionKey}`); await redisClient.set(`upload:${sessionKey}${session_id}${fileId}`, 'true', 'EX', env.SESSION_CACHE_TTL); @@ -490,11 +516,21 @@ app.get('/sessions/:session_id/objects/:objectId/metadata', async (req, res) => app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { const { session_id, objectId } = req.params; + let objectName: string | undefined; try { - const objectName = await objectResolver.resolve(session_id, objectId); + objectName = await objectResolver.resolve(session_id, objectId); if (!objectName) return res.status(404).json({ error: 'File not found' }); - const dataStream = await minioClient.getObject(bucketName, objectName); + let dataStream: Readable; + try { + dataStream = await minioClient.getObject(bucketName, objectName); + } catch (error) { + const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((error as { code?: string }).code ?? ''); + if (!missing) throw error; + objectName = await objectResolver.recover(session_id, objectId, objectName); + if (!objectName) return res.status(404).json({ error: 'File not found' }); + dataStream = await minioClient.getObject(bucketName, objectName); + } try { const headers = (dataStream as Readable & { headers?: Record }).headers ?? {}; if (!headers['x-amz-meta-codeapi-version'] || !headers['x-amz-meta-original-filename']) { @@ -515,8 +551,11 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { } } catch (err) { logger.error('Error downloading file', { error: err, session_id, objectId }); + const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((err as { code?: string }).code ?? ''); + // A locator that no longer names bytes must not shadow a replacement object + // published for the same identity until the index TTL expires. + if (missing && objectName) await forgetObjectKey(session_id, objectId, objectName); if (!res.headersSent && !res.destroyed) { - const missing = ['NoSuchKey', 'NotFound', 'NoSuchObject'].includes((err as { code?: string }).code ?? ''); return res.status(missing ? 404 : 500).json({ error: 'Error downloading file' }); } } @@ -527,6 +566,10 @@ app.get('/sessions/:session_id/objects/:objectId', async (req, res) => { */ function parseObjectName(objectName: string | undefined): { session_id: string; file_id: string } | null { if (objectName == null || objectName === '') return null; + const canonicalId = canonicalObjectId(objectName); + if (canonicalId != null) { + return { session_id: objectName.split('/', 1)[0], file_id: canonicalId }; + } const parts = objectName.split('/'); if (parts.length < 2) return null; const session_id = parts[0]; @@ -605,17 +648,9 @@ app.delete('/sessions/:session_id/objects/:fileId', async (req, res) => { const { session_id, fileId } = req.params; try { - const stream = minioClient.listObjects(bucketName, `${session_id}/${fileId}`, true); - let objectName = ''; + const objectNames = await objectResolver.listFresh(session_id, fileId); - for await (const obj of stream) { - if (obj.name.startsWith(`${session_id}/${fileId}`) === true) { - objectName = obj.name; - break; - } - } - - if (!objectName) { + if (objectNames.length === 0) { logger.warn('File not found for deletion', { session_id, fileId, bucketName }); return res.status(404).json({ error: 'File not found', @@ -626,8 +661,11 @@ app.delete('/sessions/:session_id/objects/:fileId', async (req, res) => { }); } - await minioClient.removeObject(bucketName, objectName); - logger.info(`[${INSTANCE_ID}] File deleted successfully: ${objectName}`); + for (const objectName of objectNames) { + await minioClient.removeObject(bucketName, objectName); + await forgetObjectKey(session_id, fileId, objectName); + } + logger.info(`[${INSTANCE_ID}] File identity deleted successfully`, { session_id, fileId, objectNames }); return res.status(200).json({ message: 'File deleted successfully', session_id, From c3fd558195c8c4513977b83963c6aad495a4874d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 06:45:48 -0400 Subject: [PATCH 05/42] perf: enable authorized input reuse by default (#183) --- api/src/config-defaults.test.ts | 21 +++++++++++++++++++++ api/src/config.ts | 3 ++- docs/INPUT_REUSE.md | 8 ++++---- helm/codeapi/values.yaml | 2 +- 4 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 api/src/config-defaults.test.ts diff --git a/api/src/config-defaults.test.ts b/api/src/config-defaults.test.ts new file mode 100644 index 00000000..281f5679 --- /dev/null +++ b/api/src/config-defaults.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from 'bun:test'; +import path from 'node:path'; + +function readHttpInputCacheDefault(value?: string): boolean { + const env = { ...process.env }; + if (value === undefined) delete env.SANDBOX_HTTP_INPUT_CACHE_ENABLED; + else env.SANDBOX_HTTP_INPUT_CACHE_ENABLED = value; + const script = [ + `const { config } = await import(${JSON.stringify(path.resolve(process.cwd(), 'src/config.ts'))})`, + 'process.stdout.write(JSON.stringify(config.http_input_cache_enabled))', + ].join(';'); + const result = Bun.spawnSync({ cmd: [process.execPath, '--eval', script], env }); + expect(result.exitCode).toBe(0); + return JSON.parse(result.stdout.toString()) as boolean; +} + +test('authorized HTTP input reuse defaults on and retains an explicit rollback switch', () => { + expect(readHttpInputCacheDefault()).toBe(true); + expect(readHttpInputCacheDefault('false')).toBe(false); + expect(readHttpInputCacheDefault('true')).toBe(true); +}); diff --git a/api/src/config.ts b/api/src/config.ts index f3b935ff..a5d0c993 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -122,7 +122,8 @@ export const config = { /* Ceiling for the pushed input cache (session-inputs.ts). Eviction is * always safe — a miss simply re-pushes on the next probe — so this is a * disk guard, not a correctness knob. */ - http_input_cache_enabled: process.env.SANDBOX_HTTP_INPUT_CACHE_ENABLED === 'true', + http_input_cache_enabled: + (process.env.SANDBOX_HTTP_INPUT_CACHE_ENABLED ?? 'true') === 'true', http_input_cache_max_objects: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS, 4096), http_input_cache_max_inflight: safeInt(process.env.SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT, 16), input_cache_max_bytes: safeInt( diff --git a/docs/INPUT_REUSE.md b/docs/INPUT_REUSE.md index 40e277c4..f87afa89 100644 --- a/docs/INPUT_REUSE.md +++ b/docs/INPUT_REUSE.md @@ -48,7 +48,7 @@ sequenceDiagram | `egressGrant.inputManifestTimeoutMs` | `CODEAPI_INPUT_MANIFEST_TIMEOUT_MS` | `10000` | | `fileServer.objectIndexEnabled` | `CODEAPI_FILE_OBJECT_INDEX_ENABLED` | `false` | | `fileServer.metadataConcurrency` | `CODEAPI_FILE_METADATA_CONCURRENCY` | `1` | -| `workerSandbox.sandbox.httpInputCacheEnabled` | `SANDBOX_HTTP_INPUT_CACHE_ENABLED` | `false` | +| `workerSandbox.sandbox.httpInputCacheEnabled` | `SANDBOX_HTTP_INPUT_CACHE_ENABLED` | `true` | | `workerSandbox.sandbox.httpInputCacheMaxInflight` | `SANDBOX_HTTP_INPUT_CACHE_MAX_INFLIGHT` | `16` | | `workerSandbox.sandbox.httpInputCacheMaxObjects` | `SANDBOX_HTTP_INPUT_CACHE_MAX_OBJECTS` | `4096` | | `workerSandbox.sandbox.inputCacheMaxBytes` | `SANDBOX_INPUT_CACHE_MAX_BYTES` | `536870912` | @@ -61,14 +61,14 @@ Metadata listing concurrency preserves order and is capped at 64. A canary can u ## Rollout and rollback -1. Deploy the new binaries with feature flags off. Atomic accounting supports existing JSON ledgers, and legacy downloads retain metadata compatibility. The new file server stamps future uploads with versions. +1. Deploy the new binaries with HTTP input reuse enabled by default. Mixed-version requests remain compatible: older gateways and relays fall back to normal downloads, while older unversioned objects return `cacheable: false`. The new file server stamps future uploads with versions. Set `workerSandbox.sandbox.httpInputCacheEnabled=false` only when a staged rollout requires the immediate rollback path. 2. Update **all** egress-gateway replicas before enabling compact ledgers. New binaries read both formats regardless of the creation flag. Older binaries cannot read compact hashes. To roll back to an older binary, disable compact creation, drain active grants, and wait their maximum TTL plus grace; never delete active ledgers to force a rollback. 3. Update all file-server writers before enabling the object-key index. Otherwise an older writer can change a locator without updating the index. Keep file-server replicas consistent during an indexed rollout. -4. Update the gateway, relay, runner, and launcher before enabling HTTP reuse on a small runner canary. Older gateway/relay metadata routes return 404/405 and fall back safely. Older files return `cacheable: false`. Keep the feature disabled for storage adapters that cannot return user metadata on GET. +4. Canary the default-on HTTP reuse path after updating the gateway, relay, runner, and launcher. Keep the feature explicitly disabled for storage adapters that cannot return user metadata on GET. 5. Observe `codeapi_sandbox_http_input_cache_events_total` (bounded event labels, no identities), cold and warm preparation latency, storage/Redis operations, admission fairness, request budgets, and memory/disk pressure before widening the rollout. A successful manifest consumes one read request for the batch, matching the existing list-request accounting unit. Per-file compatibility preflights each consume a read request; each cold miss consumes an additional download request. Do not disable budget enforcement to accommodate a workload. 6. Disable HTTP reuse to return to normal downloads immediately. Cached files can age out normally; no workspace deletion or migration is needed. -The creation flags default off. No deployment or object retention policy is changed by this code. Command grouping and persistent sessions remain independent options, not prerequisites for content reuse. Nothing deletes user inputs or infers shell dependencies. +HTTP input reuse defaults on. Compact-ledger creation and the object-key index remain off until their mixed-version rollout requirements are satisfied. No object retention policy is changed by this code. Command grouping and persistent sessions remain independent options, not prerequisites for content reuse. Nothing deletes user inputs or infers shell dependencies. ## Validation diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 3ecfbef8..bafed305 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -273,7 +273,7 @@ workerSandbox: # Defaults to maxConcurrentJobs when unset. jobUidCount: null workspaceReaperMaxAgeSeconds: 3600 - httpInputCacheEnabled: false + httpInputCacheEnabled: true httpInputCacheMaxInflight: 16 httpInputCacheMaxObjects: 4096 inputCacheMaxBytes: 536870912 From 76129e15e5f5a62d193074524459c80708c0d741 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 06:58:40 -0400 Subject: [PATCH 06/42] fix: honor requested input destinations (#184) --- api/openapi.yaml | 1 + api/src/download.test.ts | 111 +++++++++-------------- api/src/job-helpers.test.ts | 169 +++--------------------------------- api/src/job.ts | 86 ++++-------------- 4 files changed, 67 insertions(+), 300 deletions(-) diff --git a/api/openapi.yaml b/api/openapi.yaml index e4c04c14..23e1224c 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -67,6 +67,7 @@ components: properties: name: type: string + description: Relative path where the file is mounted in the sandbox. id: type: string content: diff --git a/api/src/download.test.ts b/api/src/download.test.ts index da877dad..d0b6ed6e 100644 --- a/api/src/download.test.ts +++ b/api/src/download.test.ts @@ -18,12 +18,8 @@ import { /** * Integration tests for `Job.downloadAndWriteFile` against a real HTTP - * listener. These exercise the cross-repo round-trip: the file-server - * (codeapi/service) emits `Content-Disposition: attachment; - * filename*=UTF-8''` for nested artifacts, and the - * sandbox-side parser must recover the path so the file lands at the same - * nested location on the next prime(). Hitting a real listener (not a - * mocked Response) catches anything fetch-level that a unit test would miss. + * listener. Hitting a real listener verifies that response metadata cannot + * redirect a caller-validated sandbox destination. */ interface DownloadInternals { @@ -176,21 +172,16 @@ afterEach(async () => { await fsp.rm(tmpDir, { recursive: true, force: true }); }); -describe('downloadAndWriteFile / RFC 5987 round-trip', () => { - it('writes a nested-path artifact at the encoded location', async () => { - /* Simulates the matplotlib-bug shape: codeapi previously returned a - * flat `file.name` and the original path was carried only by the - * server's `filename*=` header. The fix is: parser recovers the path, - * `mkdir { recursive: true }` creates the parent dir, file ends up - * where the user expects to `cat` it on the next turn. */ +describe('downloadAndWriteFile destinations', () => { + it('writes a nested-path artifact at the requested location', async () => { const file: TFile = { id: 'nested-id', storage_session_id: 'prev-session', - name: 'flat-fallback.txt', + name: 'proj/notes.txt', }; routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, - contentDisposition: "attachment; filename*=UTF-8''proj%2Fnotes.txt", + contentDisposition: "attachment; filename*=UTF-8''stored-original.txt", body: 'hello from a nested artifact\n', }); @@ -212,7 +203,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { const file: TFile = { id: 'opaque-object-handle', storage_session_id: 'opaque-session-handle', - name: 'gateway-fallback.txt', + name: 'gateway.txt', }; let sawGrantHeader = false; let sawRelayToken = false; @@ -275,14 +266,11 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect((await fsp.stat(path.join(tmpDir, 'readonly.txt'))).mode & 0o777).toBe(SANDBOX_READONLY_FILE_MODE); }); - it('falls back to the legacy filename= form when filename*= is absent', async () => { - /* Backwards-compat: older file-servers (or proxies that strip RFC - * 5987 extended-form headers) still send the legacy quoted form. The - * parser must still find a name and write the file. */ + it('downloads when a legacy filename header matches the requested name', async () => { const file: TFile = { id: 'legacy-id', storage_session_id: 'prev-session', - name: 'ignored.txt', + name: 'legacy.txt', }; routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, @@ -323,7 +311,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { expect(await fsp.stat(path.join(tmpDir, 'opaque-storage-id.xlsx')).catch(() => null)).toBeNull(); }); - it('resolves concurrent header destinations without provisional-name false conflicts', async () => { + it('keeps concurrent inputs at their requested destinations', async () => { const renamed: TFile = { id: 'renamed-id', storage_session_id: 'prev-session', @@ -338,8 +326,6 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { status: 200, contentDisposition: 'attachment; filename="actual.txt"', body: 'renamed bytes', - /* Make the other ref resolve `vacated.txt` while this ref's requested - * name would still be provisional under the old reservation scheme. */ delayMs: 75, }); routes.set(`/sessions/${encodeURIComponent(replacement.storage_session_id!)}/objects/${encodeURIComponent(replacement.id!)}`, { @@ -356,75 +342,62 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { await job.prime(); const submissionDir = asInternals(job).submissionDir; expect(await fsp.readFile(path.join(submissionDir, 'actual.txt'), 'utf8')) - .toBe('renamed bytes'); - expect(await fsp.readFile(path.join(submissionDir, 'vacated.txt'), 'utf8')) .toBe('replacement bytes'); + expect(await fsp.readFile(path.join(submissionDir, 'vacated.txt'), 'utf8')) + .toBe('renamed bytes'); } finally { config.prime_concurrency = originalPrimeConcurrency; await job.cleanup(); } }); - it('rejects concurrent refs that resolve to the same destination before either can overwrite', async () => { - const slower: TFile = { - id: 'same-slower-id', + it('keeps distinct requested names when stored objects share an original filename', async () => { + const original: TFile = { + id: 'original-id', storage_session_id: 'prev-session', - name: 'slower-fallback.txt', + name: 'data.xlsx', }; - const faster: TFile = { - id: 'same-faster-id', + const aliased: TFile = { + id: 'aliased-id', storage_session_id: 'prev-session', - name: 'faster-fallback.txt', + name: 'data-3f9a2c.xlsx', }; - routes.set(`/sessions/${encodeURIComponent(slower.storage_session_id!)}/objects/${encodeURIComponent(slower.id!)}`, { + routes.set(`/sessions/${encodeURIComponent(original.storage_session_id!)}/objects/${encodeURIComponent(original.id!)}`, { status: 200, - contentDisposition: 'attachment; filename="same.txt"', - body: 'slower bytes', + contentDisposition: 'attachment; filename="data.xlsx"', + body: 'original bytes', delayMs: 75, }); - routes.set(`/sessions/${encodeURIComponent(faster.storage_session_id!)}/objects/${encodeURIComponent(faster.id!)}`, { + routes.set(`/sessions/${encodeURIComponent(aliased.storage_session_id!)}/objects/${encodeURIComponent(aliased.id!)}`, { status: 200, - contentDisposition: 'attachment; filename="same.txt"', - body: 'faster bytes', + contentDisposition: 'attachment; filename="data.xlsx"', + body: 'aliased bytes', }); - let dirty = false; const job = makeJob( - [slower, faster], - sessionWorkspaceAt(tmpDir, 'rt_concurrent_same_destination', () => { dirty = true; }), + [original, aliased], + sessionWorkspaceAt(tmpDir, 'rt_shared_original_filename'), ); const originalPrimeConcurrency = config.prime_concurrency; config.prime_concurrency = 2; - let deadlockTimer: ReturnType | undefined; try { - const outcome = await Promise.race([ - job.prime().then( - () => ({ status: 'fulfilled' as const }), - error => ({ status: 'rejected' as const, error }), - ), - new Promise<{ status: 'timeout' }>(resolve => { - deadlockTimer = setTimeout(() => resolve({ status: 'timeout' }), 2_000); - }), - ]); - if (deadlockTimer) clearTimeout(deadlockTimer); - expect(outcome.status).toBe('rejected'); - if (outcome.status === 'rejected') { - expect(outcome.error).toBeInstanceOf(SessionWorkspaceDirtyError); - } - expect(dirty).toBe(true); - expect(await fsp.readFile(path.join(tmpDir, 'same.txt'), 'utf8')).toBe('faster bytes'); + await job.prime(); + const submissionDir = asInternals(job).submissionDir; + expect(await fsp.readFile(path.join(submissionDir, 'data.xlsx'), 'utf8')) + .toBe('original bytes'); + expect(await fsp.readFile(path.join(submissionDir, 'data-3f9a2c.xlsx'), 'utf8')) + .toBe('aliased bytes'); } finally { - if (deadlockTimer) clearTimeout(deadlockTimer); config.prime_concurrency = originalPrimeConcurrency; await job.cleanup(); } }); - it('decodes UTF-8 percent-encoded names with non-ASCII characters', async () => { + it('keeps a Unicode requested name when the header is percent encoded', async () => { const file: TFile = { id: 'utf8-id', storage_session_id: 'prev-session', - name: 'ignored.txt', + name: '你好.txt', }; routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, @@ -645,11 +618,7 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { await expect(fsp.access(path.join(tmpDir, 'should-not-exist.txt'))).rejects.toThrow(); }); - it('rejects a server-supplied filename that escapes the submission dir', async () => { - /* Companion guarantee for the path-preserving sanitizer on the - * LibreChat side: if a malicious / misconfigured server tries to - * smuggle a `..` traversal via Content-Disposition, the codeapi-side - * `validateFilePath` aborts before any write happens. */ + it('ignores a server-supplied filename that escapes the submission dir', async () => { const file: TFile = { id: 'evil-id', storage_session_id: 'prev-session', @@ -658,16 +627,14 @@ describe('downloadAndWriteFile / RFC 5987 round-trip', () => { routes.set(`/sessions/${encodeURIComponent(file.storage_session_id!)}/objects/${encodeURIComponent(file.id!)}`, { status: 200, contentDisposition: "attachment; filename*=UTF-8''..%2F..%2Fescape.txt", - body: 'should never be written', + body: 'safe bytes', }); const job = makeJob([file]); asInternals(job).submissionDir = tmpDir; - /* downloadAndWriteFile rethrows ValidationError fast (no retries) so - * `expect(...).rejects` is the right assertion. */ - await expect(job.downloadAndWriteFile(file)).rejects.toThrow(); - /* Defensive: nothing escaped to a parent dir. */ + await expect(job.downloadAndWriteFile(file)).resolves.toBe('innocent.txt'); + expect(await fsp.readFile(path.join(tmpDir, 'innocent.txt'), 'utf8')).toBe('safe bytes'); const parent = path.dirname(tmpDir); await expect(fsp.access(path.join(parent, 'escape.txt'))).rejects.toThrow(); }); diff --git a/api/src/job-helpers.test.ts b/api/src/job-helpers.test.ts index 0c31f9d3..460902da 100644 --- a/api/src/job-helpers.test.ts +++ b/api/src/job-helpers.test.ts @@ -3,7 +3,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { - resolveOriginalName, + resolveInputDestination, isNormalizedObjectForSession, markerConflictsWithExplicitFile, aggregateBashExtras, @@ -41,168 +41,19 @@ function makeRuntime(overrides: Partial & { language: string; pkgdir: s }; } -describe('resolveOriginalName', () => { - function responseWithHeader(value?: string): Response { - const headers = new Headers(); - if (value !== undefined) headers.set('content-disposition', value); - return new Response(null, { headers }); - } - - it('returns file.name when no Content-Disposition is present', () => { - expect( - resolveOriginalName(responseWithHeader(), { name: 'script.py', id: 'abc' }), - ).toBe('script.py'); - }); - - it('extracts quoted filename from Content-Disposition', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename="server-name.py"'), - { name: 'client-name.py', id: 'abc' }, - ), - ).toBe('server-name.py'); - }); - - it('extracts unquoted filename from Content-Disposition', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename=plain.txt'), - { name: 'ignored.txt', id: 'abc' }, - ), - ).toBe('plain.txt'); - }); - - it('falls back to file.id when file.name is empty and no header exists', () => { - expect( - resolveOriginalName(responseWithHeader(), { name: '', id: 'file-id-123' }), - ).toBe('file-id-123'); - }); - - it('falls back to file.name when header is malformed (no filename token)', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment'), - { name: 'fallback.py', id: 'abc' }, - ), - ).toBe('fallback.py'); - }); - - it('stops at the closing quote when the quoted filename is followed by more params', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename="foo.txt"; size=123'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo.txt'); - }); - - it('stops at a semicolon when the unquoted filename is followed by more params', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename=foo.txt; size=123'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo.txt'); - }); - - it('stops at whitespace when the unquoted filename is followed by whitespace-separated params', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename=foo.txt extra'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo.txt'); - }); - - it('returns empty string when both name and id are absent', () => { - expect(resolveOriginalName(responseWithHeader(), { name: '' })).toBe(''); +describe('resolveInputDestination', () => { + it('uses the caller-requested sandbox path', () => { + expect(resolveInputDestination({ name: 'nested/script.py', id: 'abc' })) + .toBe('nested/script.py'); }); - it('returns empty string when name is empty, id is absent, and header is malformed', () => { - expect( - resolveOriginalName(responseWithHeader('attachment'), { name: '' }), - ).toBe(''); - }); - - it('decodes RFC 5987 filename*= preserving slashes for nested artifact paths', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''test_folder%2Ftest_file.txt"), - { name: 'test_file.txt', id: 'abc' }, - ), - ).toBe('test_folder/test_file.txt'); - }); - - it('decodes RFC 5987 filename*= with a UTF-8 charset that includes a language tag', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8'en'foo%20bar.txt"), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('foo bar.txt'); - }); - - it('decodes RFC 5987 filename*= with non-ASCII characters', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''%E4%BD%A0%E5%A5%BD.txt"), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('你好.txt'); - }); - - it('tolerates a filename*= form missing the UTF-8 prefix', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename*=plain.txt'), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('plain.txt'); + it('falls back to the object id when no name exists', () => { + expect(resolveInputDestination({ name: '', id: 'file-id-123' })) + .toBe('file-id-123'); }); - it('falls through to legacy filename= when filename*= is malformed', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''bad%ZZ; filename=\"legacy.txt\""), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('legacy.txt'); - }); - - it('prefers filename*= over a legacy filename= present in the same header', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename=\"legacy.txt\"; filename*=UTF-8''nested%2Ffile.txt"), - { name: 'ignored', id: 'abc' }, - ), - ).toBe('nested/file.txt'); - }); - - it('keeps the requested name when an old file server advertises the opaque object basename', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''storage-id.xlsx"), - { name: 'Sample_-_Superstore.xlsx', id: 'storage-id' }, - ), - ).toBe('Sample_-_Superstore.xlsx'); - }); - - it('keeps the requested name for a legacy opaque filename header', () => { - expect( - resolveOriginalName( - responseWithHeader('attachment; filename="storage-id.csv"'), - { name: 'original.csv', id: 'storage-id' }, - ), - ).toBe('original.csv'); - }); - - it('keeps an authoritative nested filename even when its basename matches the object id', () => { - expect( - resolveOriginalName( - responseWithHeader("attachment; filename*=UTF-8''exports%2Fstorage-id.csv"), - { name: 'original.csv', id: 'storage-id' }, - ), - ).toBe('exports/storage-id.csv'); + it('returns an empty path when both name and id are absent', () => { + expect(resolveInputDestination({ name: '' })).toBe(''); }); }); diff --git a/api/src/job.ts b/api/src/job.ts index d2504de9..b7b82148 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -173,52 +173,12 @@ export function ensureNodeModulesSymlink( } /** - * Extracts the on-disk filename from a Content-Disposition response header, - * falling back to the request-supplied `file.name` (or `file.id` if no name - * was provided). Pure; exported for unit testing. - * - * Matches RFC 5987 / 8187 `filename*=UTF-8''` first because - * the file server emits that form for UTF-8-safe transport of arbitrary - * names — including paths with `/` separators that the legacy `filename=` - * form would mangle. Falls back to the legacy quoted (`filename="..."`) or - * unquoted (`filename=...`) forms, each stopping at the closing quote or - * the first whitespace/semicolon so trailing params like - * `attachment; filename="foo.txt"; size=123` correctly yield `foo.txt`. + * Resolves the on-disk destination for a by-reference input. The request owns + * the sandbox path; object response metadata must not redirect the write. + * Pure; exported for unit testing. */ -export function resolveOriginalName(response: Response, file: TFile): string { - const fallback = file.name || (file.id ?? ''); - const header = response.headers.get('content-disposition'); - if (!header) return fallback; - - const preferRequestedName = (candidate: string): string => { - /* Older file servers advertised path.basename(objectName) when an - * S3-compatible backend omitted original-filename user metadata. That - * basename is ``, so it is a storage identifier rather - * than an authoritative destination. Preserve the caller's requested name - * during rolling upgrades instead of exposing the opaque id in /mnt/data. */ - const opaqueStem = path.basename(candidate, path.extname(candidate)); - const isFlatObjectBasename = candidate === path.basename(candidate); - return file.name && file.id && isFlatObjectBasename && opaqueStem === file.id - ? file.name - : candidate; - }; - - const star = header.match(/filename\*=(?:UTF-8'[^']*')?([^;]+)/i); - if (star) { - const raw = star[1].trim(); - try { - return preferRequestedName(decodeURIComponent(raw)); - } catch { - /* Malformed percent-encoding (e.g. `%ZZ`) — fall through to the legacy - * forms. The same header may emit both `filename*=` and a legacy - * `filename=` per RFC 5987 §4.3, so a corrupt extended form should - * not poison a valid fallback. */ - } - } - - const match = header.match(/filename="([^"]+)"/i) - ?? header.match(/filename=([^\s;]+)/i); - return match ? preferRequestedName(match[1]) : fallback; +export function resolveInputDestination(file: TFile): string { + return file.name || (file.id ?? ''); } /** @@ -959,12 +919,9 @@ export class Job { ); } requestedDestinations.set(file.name, file); - /* Inline destinations are final, so keep them reserved while reference - * downloads resolve their authoritative Content-Disposition names. - * A ref's requested name is only a fallback, not a real destination yet: - * reserving every ref here makes concurrent swaps/order-dependent - * renames falsely conflict before the owning response has resolved. */ - if (!file.id) this.inputDestinations.set(file.name, file); + /* The request owns every sandbox destination. Reserve it before parallel + * priming begins so object metadata cannot redirect a later write. */ + this.inputDestinations.set(file.name, file); } if (this.session) { @@ -1083,10 +1040,8 @@ export class Job { ): Promise { throwIfAborted(context.signal); if (this.session && file.id && (await this.reusePrimedInput(file, context))) { - /* Reuse has no response header to pass through downloadAndWriteFile, so - * its requested name becomes authoritative only after the on-disk copy - * has been verified. Reserve it before another concurrent ref can claim - * and overwrite that path. */ + /* Inherited markers are registered after prime's initial reservation + * pass, so reserve the verified requested path here as well. */ this.reserveInputDestination(file, file.name); return; } @@ -1345,10 +1300,10 @@ export class Job { throw new Error(`HTTP error: ${response.status}`); } - const originalName = resolveOriginalName(response, file); - validateFilePath(originalName, operation.submissionDir); - this.reserveInputDestination(file, originalName); - const finalPath = path.join(operation.submissionDir, originalName); + const destination = resolveInputDestination(file); + validateFilePath(destination, operation.submissionDir); + this.reserveInputDestination(file, destination); + const finalPath = path.join(operation.submissionDir, destination); const finalParent = path.dirname(finalPath); /* Persistent-session workspaces can hold a prior turn's symlink, so build * ancestors no-follow; a fresh per-job workspace can use plain mkdir -p. */ @@ -1376,7 +1331,7 @@ export class Job { operation.signal, ); const readOnly = response.headers.get('x-read-only')?.toLowerCase() === 'true'; - this.inputFileHashes.set(originalName, { + this.inputFileHashes.set(destination, { originalId: file.id, originalSessionId: file.storage_session_id!, hash, @@ -1389,15 +1344,8 @@ export class Job { await applyReadOnlyInputPermissions(finalPath); } - /* Keep the in-memory TFile in sync with the on-disk name so that - * inputByName lookups in handleSessionFiles match walkDir's - * path.relative() output. Otherwise a Content-Disposition override - * would leave file.name pointing at the client-submitted name while - * the file lives under originalName on disk. */ - if (originalName !== file.name) file.name = originalName; - - this.log.info({ file: originalName, hash: hash.substring(0, 8) }, 'Downloaded file'); - return originalName; + this.log.info({ file: destination, hash: hash.substring(0, 8) }, 'Downloaded file'); + return destination; } catch (error: unknown) { if (response?.body && !response.bodyUsed) { await response.body.cancel().catch(() => {}); From 9d3936fa73e1b447f2e9a3a3961cdef8b411da18 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 08:39:20 -0400 Subject: [PATCH 07/42] fix: disambiguate legacy dotted object identities (#186) --- service/src/file-object-resolver.test.ts | 37 +++++++++++++++++++++++- service/src/file-object-resolver.ts | 15 ++++++++-- service/src/file-server.ts | 13 ++++++--- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/service/src/file-object-resolver.test.ts b/service/src/file-object-resolver.test.ts index 2342d87a..69ad3c2f 100644 --- a/service/src/file-object-resolver.test.ts +++ b/service/src/file-object-resolver.test.ts @@ -1,5 +1,12 @@ import { describe, expect, test } from 'bun:test'; -import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; +import { + canonicalObjectId, + canonicalObjectKey, + FileObjectResolver, + legacyObjectId, + mapObjectDetails, + storageKeyForUpload, +} from './file-object-resolver'; import type { BucketItemStat } from 'minio'; describe('storage object resolution', () => { @@ -22,6 +29,34 @@ describe('storage object resolution', () => { expect(await resolver.listFresh('s', 'report')).toEqual([]); }); + test('legacy extension keys map to exactly one dotted or undotted identity', async () => { + const canonicalDotted = canonicalObjectKey('s', 'report.csv'); + const objects = new Set([ + 's/report.csv', + 's/report.csv.txt', + canonicalDotted, + ]); + const resolver = new FileObjectResolver({ + bucket: 'files', + list: async function* (prefix) { + for (const name of objects) if (name.startsWith(prefix)) yield { name }; + }, + stat: async () => ({ metaData: {} } as BucketItemStat), + }); + + expect(legacyObjectId('s/report.csv', 's')).toBe('report'); + expect(legacyObjectId('s/report.csv.txt', 's')).toBe('report.csv'); + expect(legacyObjectId('s/report', 's')).toBe('report'); + expect(legacyObjectId('other/report.csv', 's')).toBeUndefined(); + await expect(resolver.remember('s', 'report.csv', 's/report.csv')) + .rejects.toThrow('Object key does not match storage identity'); + expect(await resolver.listFresh('s', 'report')).toEqual(['s/report.csv']); + expect(await resolver.listFresh('s', 'report.csv')).toEqual([ + canonicalDotted, + 's/report.csv.txt', + ]); + }); + test('indexes exact identities while reading fresh version metadata on every request', async () => { const index = new Map(); let lists = 0; diff --git a/service/src/file-object-resolver.ts b/service/src/file-object-resolver.ts index bc4b4ce5..1665941a 100644 --- a/service/src/file-object-resolver.ts +++ b/service/src/file-object-resolver.ts @@ -19,6 +19,18 @@ export function canonicalObjectId(key: string): string | undefined { } } +/** Legacy objects were stored as `/`. + * Derive exactly one identity by removing that final extension when present. + * In particular, `session/report.csv` belongs to `report`, while a legacy + * `report.csv` identity would be stored as e.g. `session/report.csv.txt`. + * Dotted identities without a filename extension use canonical storage. */ +export function legacyObjectId(key: string, session: string): string | undefined { + if (path.posix.dirname(key) !== session) return undefined; + const basename = path.posix.basename(key); + const extension = path.posix.extname(basename); + return extension === '' ? basename : basename.slice(0, -extension.length); +} + export interface ObjectResolverDependencies { bucket: string; list(prefix: string): AsyncIterable<{ name?: string }>; @@ -58,8 +70,7 @@ export class FileObjectResolver { private matches(key: string, session: string, id: string): boolean { if (key === canonicalObjectKey(session, id)) return true; - return path.posix.dirname(key) === session && - (path.posix.basename(key) === id || path.posix.basename(key, path.posix.extname(key)) === id); + return legacyObjectId(key, session) === id; } async remember(session: string, id: string, key: string, replace = true): Promise { diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 3dfa0a10..22302042 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -1,6 +1,12 @@ import b from 'busboy'; import { randomUUID } from 'node:crypto'; -import { canonicalObjectId, FileObjectResolver, mapObjectDetails, storageKeyForUpload } from './file-object-resolver'; +import { + canonicalObjectId, + FileObjectResolver, + legacyObjectId, + mapObjectDetails, + storageKeyForUpload, +} from './file-object-resolver'; import { sendFileDownload } from './file-download'; import path from 'path'; import IORedis from 'ioredis'; @@ -573,9 +579,8 @@ function parseObjectName(objectName: string | undefined): { session_id: string; const parts = objectName.split('/'); if (parts.length < 2) return null; const session_id = parts[0]; - const fileNameWithExt = parts[1]; - // Remove extension to get file_id - const file_id = fileNameWithExt.replace(/\.[^.]+$/, ''); + const file_id = legacyObjectId(objectName, session_id); + if (file_id == null) return null; return { session_id, file_id }; } From 3946ffb6e8664c152a589f37dd8c0feb4dccb6f8 Mon Sep 17 00:00:00 2001 From: "Ignaz \"Ian\" Kraft" Date: Sat, 12 Sep 2026 20:26:48 +0200 Subject: [PATCH 08/42] fix: helm egress deployment getting stuck on install (#176) * fix: helm egress deployment getting stuck on install * only wait for redis if the ledger is required * Update helm/codeapi/templates/egress-gateway-deployment.yaml Co-authored-by: Danny Avila --------- Co-authored-by: Danny Avila --- helm/codeapi/templates/egress-gateway-deployment.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/helm/codeapi/templates/egress-gateway-deployment.yaml b/helm/codeapi/templates/egress-gateway-deployment.yaml index 49c865f0..941844ce 100644 --- a/helm/codeapi/templates/egress-gateway-deployment.yaml +++ b/helm/codeapi/templates/egress-gateway-deployment.yaml @@ -24,6 +24,12 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- if or .Values.egressGrant.ledgerRequired .Values.hardenedSandboxMode }} + initContainers: + - name: wait-for-redis + image: busybox:1.36.1 + command: ['sh', '-c', 'until nc -z {{ include "codeapi.redis.host" . }} {{ include "codeapi.redis.port" . }}; do echo waiting for redis; sleep 2; done'] + {{- end }} containers: - name: egress-gateway image: "{{ .Values.egressGateway.image.repository }}:{{ .Values.egressGateway.image.tag }}" From 8a90f6d1ba01c97ed5c4e01f835a783958e581b7 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 16:17:36 -0400 Subject: [PATCH 09/42] feat: Add Trusted VM Command Policy (#187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🛰️ feat: Add trusted VM command policy * docs: clarify trusted VM socket boundary --- docs/adr/001-stateful-code-environments.md | 13 +++- packages/code/README.md | 35 +++++++++ packages/code/package.json | 4 + packages/code/src/cli.test.ts | 40 ++++++++++ packages/code/src/cli.ts | 24 +++++- packages/code/src/index.ts | 1 + packages/code/src/native-policy.test.ts | 63 ++++++++++++++++ packages/code/src/native-policy.ts | 86 ++++++++++++++++++++++ packages/code/src/native-process.test.ts | 30 ++++++++ packages/code/src/native-process.ts | 2 + packages/code/src/native-sandbox.test.ts | 50 ++++++++++++- packages/code/src/native-sandbox.ts | 29 ++++++-- 12 files changed, 364 insertions(+), 13 deletions(-) create mode 100644 packages/code/src/native-policy.test.ts create mode 100644 packages/code/src/native-policy.ts diff --git a/docs/adr/001-stateful-code-environments.md b/docs/adr/001-stateful-code-environments.md index 8b9830df..cfa44bf4 100644 --- a/docs/adr/001-stateful-code-environments.md +++ b/docs/adr/001-stateful-code-environments.md @@ -56,8 +56,14 @@ worker replacement; the UI and operator documentation must not imply otherwise. revocation. - Pairing codes and credentials are stored by digest where lookup permits. - One configured worker has at most one active fenced assignment. -- Sandbox isolation and default-deny egress remain mandatory; pairing secures - the transport identity but does not make the host a sandbox. +- Sandbox isolation and default-deny egress remain the mandatory default; + pairing secures the transport identity but does not make the host a sandbox. + An operator may explicitly delegate network and local-socket restrictions to + an approved outer VM boundary through a named, digested worker policy. That + delegation retains direct workspace filesystem rules, cancellation, and + resource limits. The operator is responsible for preventing permitted host + services (for example, a privileged container socket) from bypassing those + rules and exposing worker identity or credential material. - A compromised worker can lie about advertised capabilities. Capability labels and policy digests are audit signals until enforcement is coupled to an attested sandbox or trusted host policy. @@ -65,7 +71,8 @@ worker replacement; the UI and operator documentation must not imply otherwise. ## Consequences - `@librechat/code` owns the provider-neutral protocol, identity handling, and - worker CLI; Code API owns enrollment, scheduling, and execution policy. + worker CLI, including machine-local execution-policy presets; Code API owns + enrollment, scheduling, and execution policy. - LibreChat owns environment persistence, ownership, RBAC, and user experience. - The Agents SDK keeps only its adapter until a second concrete consumer proves which coding-tool abstractions are genuinely provider neutral. diff --git a/packages/code/README.md b/packages/code/README.md index 819d9f5c..e46d6fd6 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -211,6 +211,41 @@ LIBRECHAT_CODE_COMMAND_SANDBOX=native-srt librechat-code run \ --worker-dir /path/to/project --allow-workspace-commands ``` +### Trusted VM command policy + +The native SRT backend can be made intentionally permissive when the selected +machine already supplies an administrator-approved outer security boundary. +The `trusted-vm` preset keeps SRT's direct filesystem rules, credential +masking, private scratch storage, cancellation, time limits, and output limits, +while allowing unmatched outbound destinations, local port binding, and Unix +sockets: + +```bash +librechat-code run \ + --worker-dir /home/ubuntu/src \ + --allow-workspace-writes \ + --allow-workspace-commands \ + --command-policy-preset trusted-vm +``` + +`LIBRECHAT_CODE_COMMAND_POLICY_PRESET=trusted-vm` is the environment equivalent. +The default is `restricted`, which preserves the default-deny network policy. +The preset configures `native-srt`; it is not an unsandboxed host-shell +backend. It is rejected unless native workspace commands are enabled. Its +normalized effective controls are included in the worker policy digest, and +the worker advertises `anthropic-srt:trusted-vm` unless an operator supplied a +custom sandbox profile label. + +Treat this preset as delegation to the machine's outer security controls. Any +outbound destination can receive workspace data, local listeners can accept +connections reachable under host policy, and Unix socket access may expose +powerful host services such as a container daemon. A socket that grants host +privilege can bypass SRT's filesystem rules and reach worker or GitHub identity +material; the outer VM boundary must prevent that path or explicitly accept +that trust. Register only the intended source root. Worker identity, +mutation-quarantine state, and configured GitHub App key files must remain +outside it. + ## Docker runtime supervisor (optional hardened adapter) `DockerRuntimeSupervisor` is the first self-contained local OCI adapter. It diff --git a/packages/code/package.json b/packages/code/package.json index a819af9d..452a8be7 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -35,6 +35,10 @@ "types": "./dist/native-sandbox.d.ts", "import": "./dist/native-sandbox.js" }, + "./native-policy": { + "types": "./dist/native-policy.d.ts", + "import": "./dist/native-policy.js" + }, "./github": { "types": "./dist/github.d.ts", "import": "./dist/github.js" diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index f978fce5..18456072 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -94,6 +94,46 @@ test('CLI rejects an unknown command sandbox before entering the run loop', () = ); }); +test('CLI rejects an unknown native SRT command policy preset', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'host-shell', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /must be restricted or trusted-vm/); +}); + +test('CLI refuses a permissive policy when native commands are unavailable', () => { + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'https://code.example/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm', + }, + }, + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /requires native-srt workspace commands/); +}); + test('CLI rejects incomplete GitHub App authentication before worker registration', () => { const result = spawnSync( process.execPath, diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 27a8d2d9..fdd28e9f 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -29,6 +29,10 @@ import { import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { + resolveNativeSrtCommandPolicy, + serializeNativeSrtCommandPolicy, +} from './native-policy.js'; import { workspaceMutationGuard } from './workspace-guards.js'; import type { NativeProcessSandboxOptions } from './native-process.js'; import type { LocalWorkspaceConfig } from './workspace.js'; @@ -404,6 +408,19 @@ async function run( 'LIBRECHAT_CODE_COMMAND_SANDBOX must be native-srt or runtime', ); } + const commandPolicy = resolveNativeSrtCommandPolicy( + option(args, '--command-policy-preset') ?? + process.env.LIBRECHAT_CODE_COMMAND_POLICY_PRESET?.trim().toLowerCase() ?? + 'restricted', + ); + if ( + commandPolicy.preset !== 'restricted' && + (!allowWorkspaceCommands || commandSandboxMode !== 'native-srt') + ) { + throw new Error( + 'A permissive command policy preset requires native-srt workspace commands', + ); + } const github = runtimeSessionId == null ? githubCredentials() @@ -756,6 +773,7 @@ async function run( }); const nativeOptions: NativeProcessSandboxOptions = { workspaceRoot: canonicalWorkerDirectory!, + commandPolicy, protectedPaths: [ identityPath, ...rootQuarantinePaths.values(), @@ -820,7 +838,9 @@ async function run( sandboxProfile: process.env.LIBRECHAT_CODE_SANDBOX_PROFILE ?? (allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? 'anthropic-srt' + ? commandPolicy.preset === 'restricted' + ? 'anthropic-srt' + : `anthropic-srt:${commandPolicy.preset}` : runtimeMode.startsWith('docker') ? 'oci-docker' : 'nsjail'), @@ -829,7 +849,7 @@ async function run( .update(policy) .update( allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? `\0native-srt\0${commandAllowedDomains.join('\0')}\0${github.policyIdentity}` + ? `\0native-srt\0${serializeNativeSrtCommandPolicy(commandPolicy)}\0${commandAllowedDomains.join('\0')}\0${github.policyIdentity}` : '', ) .digest('hex'), diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 712e7487..5363f0c0 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -5,6 +5,7 @@ export * from './storage.js'; export * from './runtime.js'; export * from './workspace.js'; export * from './workspace-runtime.js'; +export * from './native-policy.js'; export * from './native-sandbox.js'; export * from './native-process.js'; export * from './github.js'; diff --git a/packages/code/src/native-policy.test.ts b/packages/code/src/native-policy.test.ts new file mode 100644 index 00000000..4e9f4691 --- /dev/null +++ b/packages/code/src/native-policy.test.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + normalizeNativeSrtCommandPolicy, + resolveNativeSrtCommandPolicy, + serializeNativeSrtCommandPolicy, +} from './native-policy.js'; + +test('restricted remains the default native SRT command policy', () => { + assert.deepEqual(resolveNativeSrtCommandPolicy(), { + version: 1, + preset: 'restricted', + network: { + outbound: 'allowlist', + allowLocalBinding: false, + allowAllUnixSockets: false, + }, + }); +}); + +test('trusted-vm resolves to explicit permissive network controls', () => { + assert.deepEqual(resolveNativeSrtCommandPolicy('trusted-vm'), { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }); +}); + +test('unknown and forged native policies fail closed', () => { + assert.throws( + () => resolveNativeSrtCommandPolicy('host-shell'), + /must be restricted or trusted-vm/, + ); + assert.throws( + () => + normalizeNativeSrtCommandPolicy({ + ...resolveNativeSrtCommandPolicy('restricted'), + network: { + ...resolveNativeSrtCommandPolicy('restricted').network, + allowAllUnixSockets: true, + }, + }), + /does not match its preset/, + ); +}); + +test('serialized policy is stable and includes effective controls', () => { + const first = serializeNativeSrtCommandPolicy( + resolveNativeSrtCommandPolicy('trusted-vm'), + ); + const second = serializeNativeSrtCommandPolicy( + resolveNativeSrtCommandPolicy('trusted-vm'), + ); + assert.equal(first, second); + assert.match(first, /"outbound":"unrestricted"/); + assert.match(first, /"allowLocalBinding":true/); + assert.match(first, /"allowAllUnixSockets":true/); +}); diff --git a/packages/code/src/native-policy.ts b/packages/code/src/native-policy.ts new file mode 100644 index 00000000..5d289bfb --- /dev/null +++ b/packages/code/src/native-policy.ts @@ -0,0 +1,86 @@ +export const NATIVE_SRT_COMMAND_POLICY_PRESETS = [ + 'restricted', + 'trusted-vm', +] as const; + +export type NativeSrtCommandPolicyPreset = + typeof NATIVE_SRT_COMMAND_POLICY_PRESETS[number]; + +export interface NativeSrtCommandPolicy { + version: 1; + preset: NativeSrtCommandPolicyPreset; + network: { + outbound: 'allowlist' | 'unrestricted'; + allowLocalBinding: boolean; + allowAllUnixSockets: boolean; + }; +} + +const PRESETS: Record = { + restricted: { + version: 1, + preset: 'restricted', + network: { + outbound: 'allowlist', + allowLocalBinding: false, + allowAllUnixSockets: false, + }, + }, + 'trusted-vm': { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }, +}; + +function isPreset(value: unknown): value is NativeSrtCommandPolicyPreset { + return ( + typeof value === 'string' && + NATIVE_SRT_COMMAND_POLICY_PRESETS.some((preset) => preset === value) + ); +} + +/** Resolve a named convenience preset into the explicit policy SRT enforces. */ +export function resolveNativeSrtCommandPolicy( + preset: unknown = 'restricted', +): NativeSrtCommandPolicy { + if (!isPreset(preset)) { + throw new Error( + 'Native SRT command policy preset must be restricted or trusted-vm', + ); + } + const policy = PRESETS[preset]; + return { ...policy, network: { ...policy.network } }; +} + +/** Validate a programmatic policy and return canonical preset-owned values. */ +export function normalizeNativeSrtCommandPolicy( + policy?: NativeSrtCommandPolicy, +): NativeSrtCommandPolicy { + const normalized = resolveNativeSrtCommandPolicy( + policy?.preset ?? 'restricted', + ); + if ( + policy !== undefined && + (policy.version !== normalized.version || + policy.network?.outbound !== normalized.network.outbound || + policy.network?.allowLocalBinding !== + normalized.network.allowLocalBinding || + policy.network?.allowAllUnixSockets !== + normalized.network.allowAllUnixSockets) + ) { + throw new Error('Native SRT command policy does not match its preset'); + } + return normalized; +} + +/** Stable policy material used in the bridge capability digest. */ +export function serializeNativeSrtCommandPolicy( + policy: NativeSrtCommandPolicy, +): string { + return JSON.stringify(normalizeNativeSrtCommandPolicy(policy)); +} diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index d77ec559..8dcfd4c1 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -108,6 +108,36 @@ test('executor bootstrap excludes bridge credentials and Node injection variable await sandbox.close(); }); +test('executor forwards the resolved command policy without worker credentials', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: '/workspace', + commandPolicy: { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }, + }, + fake.fork, + ); + await sandbox.prepare(); + assert.deepEqual(fake.messages[0].options.commandPolicy, { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }); + await sandbox.close(); +}); + test('executor hands credentials over IPC only for the current command', async () => { const fake = fixture(); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index fe4e1aa8..96d89355 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -182,6 +182,7 @@ export class NativeProcessWorkspaceCommandSandbox child.on('disconnect', lost); const { workspaceRoot, + commandPolicy, protectedPaths, allowedDomains, homeDirectory, @@ -192,6 +193,7 @@ export class NativeProcessWorkspaceCommandSandbox { options: { workspaceRoot, + commandPolicy, protectedPaths, allowedDomains, homeDirectory, diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index e2dc4bda..cf790965 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -19,7 +19,10 @@ import { join } from 'node:path'; import { PassThrough } from 'node:stream'; import test from 'node:test'; -import type { SandboxRuntimeConfig } from '@anthropic-ai/sandbox-runtime'; +import type { + SandboxAskCallback, + SandboxRuntimeConfig, +} from '@anthropic-ai/sandbox-runtime'; import type { ChildProcessWithoutNullStreams } from 'node:child_process'; import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; @@ -46,6 +49,7 @@ function fakeManager( } = {}, ) { let config: SandboxRuntimeConfig | undefined; + let askCallback: SandboxAskCallback | undefined; let reset = false; let credentialSeenDuringWrap: string | undefined; let gitLfsRequiredSeenDuringWrap: string | undefined; @@ -55,8 +59,12 @@ function fakeManager( async checkDependenciesAsync() { return { warnings: [], errors: options.dependencyErrors ?? [] }; }, - async initialize(value: SandboxRuntimeConfig) { + async initialize( + value: SandboxRuntimeConfig, + callback?: SandboxAskCallback, + ) { config = value; + askCallback = callback; if (options.initializeError) throw options.initializeError; }, async wrapWithSandboxArgv(command: string) { @@ -107,6 +115,9 @@ function fakeManager( get config() { return config; }, + get askCallback() { + return askCallback; + }, get reset() { return reset; }, @@ -276,6 +287,8 @@ test('initializes SRT with a default-deny network and scrubbed worker credential assert.deepEqual(fake.config?.network.allowedDomains, []); assert.equal(fake.config?.network.strictAllowlist, true); assert.equal(fake.config?.network.allowAllUnixSockets, false); + assert.equal(fake.config?.network.allowLocalBinding, false); + assert.equal(fake.askCallback, undefined); assert.deepEqual(fake.config?.filesystem.allowRead, [ canonicalRoot, scratchDirectory, @@ -305,6 +318,39 @@ test('initializes SRT with a default-deny network and scrubbed worker credential await assert.rejects(access(scratchDirectory!)); }); +test('trusted-vm permits unmatched egress and local development sockets', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + commandPolicy: { + version: 1, + preset: 'trusted-vm', + network: { + outbound: 'unrestricted', + allowLocalBinding: true, + allowAllUnixSockets: true, + }, + }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + + await sandbox.prepare(); + + assert.equal(fake.config?.network.strictAllowlist, false); + assert.equal(fake.config?.network.allowLocalBinding, true); + assert.equal(fake.config?.network.allowAllUnixSockets, true); + assert.equal( + await fake.askCallback?.({ host: 'packages.example', port: 443 }), + true, + ); + assert.deepEqual(fake.config?.filesystem.allowWrite.slice(0, 1), [ + await realpath(root), + ]); +}); + test('provides an isolated scratch directory to commands and restores the host environment', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 139a10d6..171ea663 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -41,7 +41,12 @@ import type { ChildProcessWithoutNullStreams, SpawnOptionsWithoutStdio, } from 'node:child_process'; -import type { SandboxRuntimeConfig } from '@anthropic-ai/sandbox-runtime'; +import type { + SandboxAskCallback, + SandboxRuntimeConfig, +} from '@anthropic-ai/sandbox-runtime'; +import { normalizeNativeSrtCommandPolicy } from './native-policy.js'; +import type { NativeSrtCommandPolicy } from './native-policy.js'; import type { WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, @@ -124,7 +129,10 @@ const HOST_TEMPORARY_ROOT = tmpdir(); interface NativeSandboxManager { isSupportedPlatform(): boolean; checkDependenciesAsync(): Promise<{ warnings: string[]; errors: string[] }>; - initialize(config: SandboxRuntimeConfig): Promise; + initialize( + config: SandboxRuntimeConfig, + sandboxAskCallback?: SandboxAskCallback, + ): Promise; wrapWithSandboxArgv( command: string, binShell?: string, @@ -153,6 +161,7 @@ type SpawnCommand = ( export interface NativeSrtWorkspaceCommandSandboxOptions { workspaceRoot: string; + commandPolicy?: NativeSrtCommandPolicy; /** Trusted worker files that must never become workspace-readable or writable. */ protectedPaths?: string[]; allowedDomains?: string[]; @@ -369,13 +378,18 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'REGISTRATION_INVALID', ); } + const commandPolicy = normalizeNativeSrtCommandPolicy( + this.options.commandPolicy, + ); + const unrestrictedNetwork = + commandPolicy.network.outbound === 'unrestricted'; const config: SandboxRuntimeConfig = { network: { allowedDomains: [...(this.options.allowedDomains ?? [])], deniedDomains: [], - strictAllowlist: true, - allowAllUnixSockets: false, - allowLocalBinding: false, + strictAllowlist: !unrestrictedNetwork, + allowAllUnixSockets: commandPolicy.network.allowAllUnixSockets, + allowLocalBinding: commandPolicy.network.allowLocalBinding, ...(this.options.maskedEnvironment ? { tlsTerminate: {} } : {}), }, filesystem: { @@ -439,7 +453,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox enableWeakerNetworkIsolation: false, git: { safeDirectories: [root] }, }; - await this.manager.initialize(config); + await this.manager.initialize( + config, + unrestrictedNetwork ? async () => true : undefined, + ); this.canonicalRoot = root; } From 31def177aa3ca2b4651bc413f96cf63d6522d707 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 20:31:55 -0400 Subject: [PATCH 10/42] fix: Retry Clean Cancelled BYOM Settlements Through Stop Grace (#188) A clean atomic workspace mutation rejection was settled with retries cut off at the original execution deadline. When Stop arrived near that deadline, process-tree termination finished after it, so the first settlement attempt was aborted immediately while Code API was still draining the cancellation. The client received ASSIGNMENT_EXPIRED and the durable mutation guard stayed armed. Route clean workspace mutation rejections through the known-clean rejection recovery path: a transient-retrying heartbeat and settlement retries through the rejection acknowledgement grace, floored at the bridge cancellation settlement grace. Worker shutdown still fails closed. Share the grace constant from the protocol module so the bridge and worker stay aligned. Closes #173 --- packages/code/src/protocol.ts | 2 + packages/code/src/worker.ts | 26 ++- packages/code/src/workspace-worker.test.ts | 202 +++++++++++++++++++++ service/src/bridge/store.ts | 4 +- 4 files changed, 227 insertions(+), 7 deletions(-) diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 44cfdfd0..c1dad949 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -20,6 +20,8 @@ export const BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS = 5 * 60_000; export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES = 256 * 1024; export const BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH = 32; +/** How long Code API drains a clean rejection after Stop cancels a workspace mutation. */ +export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index fde84768..19f219da 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -1,6 +1,7 @@ import { randomBytes } from 'node:crypto'; import { + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS, BRIDGE_PROTOCOL_VERSION, BridgeProtocolError, bridgeWorkerPath, @@ -1619,7 +1620,13 @@ export class BridgeWorker { assignment.runtimeSessionId != null && settlement.status === 'rejected' && (!sandboxStarted || sandboxRejectedExecution); - if (knownCleanStatefulRejection) { + // An armed mutation reaches settlement as rejected only after an atomic + // failure that does not require quarantine. Code API accepts that + // rejection after expiry and drains it for its own grace after Stop, so a + // Stop near the deadline must not cut off retries at the deadline. + const knownCleanWorkspaceRejection = + workspaceMutationArmed && settlement.status === 'rejected'; + if (knownCleanStatefulRejection || knownCleanWorkspaceRejection) { heartbeatController.abort(); await heartbeat; const recoveryHeartbeatController = new AbortController(); @@ -1627,15 +1634,24 @@ export class BridgeWorker { recoveryHeartbeatController.signal, true, ).catch(() => undefined); + const rejectionAckGraceMs = Math.max( + 0, + this.options.rejectionAckGraceMs ?? REJECTION_ACK_GRACE_MS, + ); try { await this.settleWithRetry( assignment, settlement, localDeadlineAtMs + - Math.max( - 0, - this.options.rejectionAckGraceMs ?? REJECTION_ACK_GRACE_MS, - ), + (knownCleanStatefulRejection + ? rejectionAckGraceMs + : Math.max( + rejectionAckGraceMs, + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS, + )), + // Stateful rejections outlive shutdown; workspace guards still + // fail closed when the worker itself stops. + knownCleanStatefulRejection ? undefined : signal, ); } finally { recoveryHeartbeatController.abort(); diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 14bf2d61..8f9fcd45 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -1480,6 +1480,208 @@ test('worker clears quarantine after a command cancellation confirms process ter assert.deepEqual(lifecycle, ['arm', 'execute', 'settle', 'clear']); }); +test('worker retries a clean Stop rejection near its deadline through the cancellation grace', async () => { + const lifecycle: string[] = []; + const settlements: Array> = []; + const remainingMs = 100; + const startedAt = Date.now(); + const baseCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', operations: ['read_file' as const] }], + }; + const workspaceTools = new SandboxWorkspaceTools({ + workspaceTools: { + capabilities: baseCapabilities, + mutationFailuresAreAtomic: true, + async execute() { throw new Error('base executor must not run'); }, + }, + commandWorkspaces: ['primary'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute(_request, signal) { + lifecycle.push('execute'); + await new Promise((resolve) => { + if (signal?.aborted) return resolve(); + signal?.addEventListener('abort', () => resolve(), { once: true }); + }); + lifecycle.push('stop'); + // Process-group termination is confirmed after the original deadline. + await new Promise((resolve) => setTimeout(resolve, remainingMs)); + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + false, + ); + }, + }, + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceTools.capabilities, + }, + workspaceTools, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + cancellationPollIntervalMs: 5, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/cancellation')) { + return Response.json({ + protocolVersion: 1, + cancelled: Date.now() >= startedAt + remainingMs / 2, + }); + } + if (!String(input).endsWith('/settle')) { + return Response.json({ protocolVersion: 1, accepted: true }); + } + lifecycle.push('settle'); + settlements.push({ + ...(JSON.parse(String(init?.body)) as Record), + attemptedAt: Date.now(), + }); + // Settlement delivery takes a real transport turn and honors its deadline. + await new Promise((resolve, reject) => { + const timer = setTimeout(resolve, 10); + init?.signal?.addEventListener( + 'abort', + () => { + clearTimeout(timer); + reject(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + }); + if (settlements.length === 1) { + return Response.json( + { error: 'Bridge settlement temporarily unavailable' }, + { status: 503 }, + ); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-command-stopped-near-deadline', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(startedAt + remainingMs).toISOString(), + remainingMs, + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'sleep 30; touch delayed.txt', + }, + }); + + assert.deepEqual(lifecycle, [ + 'arm', + 'execute', + 'stop', + 'settle', + 'settle', + 'clear', + ]); + assert.ok(Number(settlements[0]?.attemptedAt) > startedAt + remainingMs); + assert.equal(settlements[1]?.status, 'rejected'); + assert.equal(settlements[1]?.errorCode, 'EXECUTION_ABORTED'); +}); + +test('worker keeps quarantine armed when shutdown interrupts a clean command rejection', async () => { + const lifecycle: string[] = []; + const controller = new AbortController(); + const baseCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', operations: ['read_file' as const] }], + }; + const workspaceTools = new SandboxWorkspaceTools({ + workspaceTools: { + capabilities: baseCapabilities, + mutationFailuresAreAtomic: true, + async execute() { throw new Error('base executor must not run'); }, + }, + commandWorkspaces: ['primary'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute() { + lifecycle.push('execute'); + controller.abort(new Error('shutdown')); + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + false, + ); + }, + }, + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: workspaceTools.capabilities, + }, + workspaceTools, + workspaceMutationQuarantine: mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + fetchImpl: async () => { + lifecycle.push('settle'); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await assert.rejects( + worker.executeAndSettle( + { + protocolVersion: 1, + assignmentId: 'assignment-command-shutdown-cleanly', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'sleep 30', + }, + }, + controller.signal, + ), + /shutdown/, + ); + assert.deepEqual(lifecycle, ['arm', 'execute']); +}); + test('worker retains quarantine when an atomic executor cannot confirm durability', async () => { const lifecycle: string[] = []; const workspaceCapabilities = { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 32205571..5e4249a9 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -11,6 +11,7 @@ import type { } from '../../../packages/code/src/protocol'; import { + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS, BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, @@ -23,7 +24,6 @@ import { BridgeWorkspaceSlots } from './slots'; const PREFIX = 'codeapi:bridge:v1'; const POLL_INTERVAL_MS = 100; -const CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; const DEFAULT_WORKER_TTL_SECONDS = 60; const DEFAULT_REDIS_COMMAND_TIMEOUT_MS = 1_000; @@ -1924,7 +1924,7 @@ export class RedisBridgeStore { // Give Stop its own grace so a near-timeout cancellation is not // misclassified as an ambiguous timeout. const cancellationDeadlineAtMs = - Date.now() + CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS; + Date.now() + BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS; let cancellationPollMs = POLL_INTERVAL_MS; while (Date.now() < cancellationDeadlineAtMs) { const raw = await boundedCommand( From 118eb9b3ad704e752acc6ffa4b50fb9f1b3bb0ee Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 12 Sep 2026 21:15:39 -0400 Subject: [PATCH 11/42] fix: Release Unassigned Workspace Slots When Dispatch Cleanup Fails (#189) Closes #170 --- service/src/bridge/concurrent-store.test.ts | 40 +++++++++++++++++++++ service/src/bridge/store.ts | 40 +++++++++++++++------ 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index fe53c44f..736b7216 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -384,6 +384,46 @@ test('queued cancellation never leases and does not block another root', async ( ).toBeUndefined(); }); +test('unassigned slot releases even when dispatch cleanup fails', async () => { + await register(); + const originalIncr = redis.incr.bind(redis); + const originalSet = redis.set.bind(redis); + const set = originalSet as (...args: unknown[]) => unknown; + redis.incr = ((key: string) => + key.endsWith(':generation') + ? Promise.reject(new Error('injected generation outage')) + : originalIncr(key)) as typeof redis.incr; + redis.set = ((key: string, ...args: unknown[]) => + key.endsWith(':cancelled') + ? Promise.reject(new Error('injected cancellation outage')) + : set(key, ...args)) as typeof redis.set; + try { + // The reservation succeeds, then dispatch fails before storing an assignment. + await expect(dispatch('a')).rejects.toThrow('injected cancellation outage'); + } finally { + redis.incr = originalIncr; + redis.set = originalSet; + } + expect( + await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`), + ).toBe(0); + expect( + await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`), + ).toBeNull(); + const next = dispatch('a'); + const assignment = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + expect(assignment.request).toMatchObject({ workspaceId: 'a' }); + await settle(assignment); + await expect(next).resolves.toMatchObject({ status: 'rejected' }); +}); + test('late quarantine releases its slot after caller cancellation and retains only its root fence', async () => { await register(); const controller = new AbortController(); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 5e4249a9..b09d3792 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -1060,20 +1060,15 @@ export class RedisBridgeStore { // already committed result rather than turning cleanup availability // into a client-visible failure that could prompt duplicate work. } + } else if (workspaceSlots != null && assignment == null) { + await this.cleanupUnassignedSlot( + args.workerId, + lockIncarnationId, + assignmentId, + ); } else { await this.cleanupDispatch(args.workerId, assignmentId, assignment); } - if (workspaceSlots != null && assignment == null) { - await boundedCommand( - workspaceSlots.release( - args.workerId, - lockIncarnationId, - assignmentId, - ), - this.redisCommandTimeoutMs, - 'Bridge unassigned slot cleanup', - ); - } } } @@ -2138,6 +2133,29 @@ export class RedisBridgeStore { ]); } + private async cleanupUnassignedSlot( + workerId: string, + incarnationId: string, + assignmentId: string, + ): Promise { + // No stored assignment owns this reservation, so a cancellation outage + // must not leave the slot and its root busy until TTL expiry. + const [cleanup, release] = await Promise.allSettled([ + this.cleanupDispatch(workerId, assignmentId, undefined), + boundedCommand( + new BridgeWorkspaceSlots(this.redis).release( + workerId, + incarnationId, + assignmentId, + ), + this.redisCommandTimeoutMs, + 'Bridge unassigned slot cleanup', + ), + ]); + if (cleanup.status === 'rejected') throw cleanup.reason; + if (release.status === 'rejected') throw release.reason; + } + private async commitPendingWorkspace( assignment: StoredAssignment, settlement: AnyCodeBridgeSettlement, From 8764d019ecb4a503d6be34d8b8234f031b1f77e5 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 13 Sep 2026 04:05:29 -0400 Subject: [PATCH 12/42] fix: Exit Cleanly When Native Executor Shuts Down Concurrently (#191) * fix: Exit Cleanly When Native Executor Shuts Down Concurrently A native BYOM worker under systemd KillMode=control-group receives SIGTERM at the same time as its forked SRT executor. The child ignores IPC once it is shutting down, so the parent's close handshake is left pending until the child exits, which rejects it with 'Native executor is unavailable'. That rejection escaped the CLI finally block and turned an idle administrative stop into exit status 1. Treat the close handshake as best-effort: the executor is terminated in finally regardless, and the active command has already drained, so a lost or stalled reply carries no mutation risk. Also make the child report exit status 0 when its own SRT teardown succeeded. Closes #190 * fix: Surface Explicit Executor Cleanup Failures During Close Only a lost, refused, or stalled close handshake is benign at shutdown. A negative close reply from the executor is a real cleanup failure and still rejects so pool shutdown can aggregate it. --- .../code/src/native-process-child.test.ts | 2 +- packages/code/src/native-process-child.ts | 5 +- packages/code/src/native-process.test.ts | 70 +++++++++++++++++++ packages/code/src/native-process.ts | 22 ++++-- 4 files changed, 91 insertions(+), 8 deletions(-) diff --git a/packages/code/src/native-process-child.test.ts b/packages/code/src/native-process-child.test.ts index 1d4866ae..8aca3652 100644 --- a/packages/code/src/native-process-child.test.ts +++ b/packages/code/src/native-process-child.test.ts @@ -33,7 +33,7 @@ for (const signal of ['SIGINT', 'SIGHUP', 'SIGTERM'] as const) { // installation finished without requiring platform SRT dependencies. child.once('message', () => child.kill(signal)); child.send({ id: 'startup-probe', type: 'probe' }); - assert.deepEqual(await exited, { code: 1, signal: null }); + assert.deepEqual(await exited, { code: 0, signal: null }); }, ); } diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index b5221731..05ffedfe 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -25,7 +25,10 @@ const shutdown = () => { if (shuttingDown) return; shuttingDown = true; active?.controller.abort(); - void (sandbox?.close() ?? Promise.resolve()).finally(() => process.exit(1)); + void (sandbox?.close() ?? Promise.resolve()).then( + () => process.exit(0), + () => process.exit(1), + ); setTimeout(() => process.exit(1), 5000); }; process.on('disconnect', shutdown); diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 8dcfd4c1..800dcf11 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -301,6 +301,76 @@ test('executor close drains an active command before closing IPC', async () => { await assert.rejects(sandbox.execute(request), /unavailable/); }); +test('executor close resolves when the child exits during the close handshake', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await sandbox.prepare(); + Object.assign(fake.child, { + send(message: Record, callback: (error: null) => void) { + fake.messages.push(message); + callback(null); + queueMicrotask(() => { + Object.assign(fake.child, { connected: false }); + fake.child.emit('exit', 1, null); + fake.child.emit('disconnect'); + }); + return true; + }, + }); + await sandbox.close(); + assert.equal(fake.messages.filter((m) => m.type === 'close').length, 1); + await assert.rejects(sandbox.execute(request), /unavailable/); +}); + +test('executor close still reports a cleanup failure the child replies with', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await sandbox.prepare(); + Object.assign(fake.child, { + send(message: Record, callback: (error: null) => void) { + fake.messages.push(message); + callback(null); + queueMicrotask(() => + fake.child.emit('message', { + id: message.id, + ok: false, + code: 'COMMAND_UNAVAILABLE', + errorMessage: 'scratch cleanup failed', + mutation: false, + requiresQuarantine: false, + }), + ); + return true; + }, + }); + await assert.rejects(sandbox.close(), /scratch cleanup failed/); + assert.equal(fake.killCalls, 1); + await assert.rejects(sandbox.execute(request), /unavailable/); +}); + +test('executor close skips the handshake once the child is already lost', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/workspace' }, + fake.fork, + ); + await sandbox.prepare(); + Object.assign(fake.child, { connected: false }); + fake.child.emit('exit', 1, null); + await sandbox.close(); + assert.equal( + fake.messages.some((m) => m.type === 'close'), + false, + ); + await assert.rejects(sandbox.execute(request), /unavailable/); +}); + test('executor startup loss is not reported as an applied mutation', async () => { const fake = fixture(); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 96d89355..e1a02498 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -73,6 +73,15 @@ export function nativeExecutorEnvironment( ); } +/** The executor process was lost, refused a send, or stalled past its + * deadline, as opposed to a failure the executor reported explicitly. */ +class NativeExecutorUnavailableError extends WorkspaceToolError { + constructor(mutation: boolean) { + super('Native executor is unavailable', 'COMMAND_UNAVAILABLE', mutation); + this.name = 'NativeExecutorUnavailableError'; + } +} + /** One persistent, process-isolated SRT manager per workspace. No automatic * restart/replay: losing IPC after execution starts is an ambiguous mutation. */ export class NativeProcessWorkspaceCommandSandbox @@ -109,11 +118,7 @@ export class NativeProcessWorkspaceCommandSandbox } private unavailable(mutation: boolean): WorkspaceToolError { - return new WorkspaceToolError( - 'Native executor is unavailable', - 'COMMAND_UNAVAILABLE', - mutation, - ); + return new NativeExecutorUnavailableError(mutation); } private async start(): Promise { @@ -340,12 +345,17 @@ export class NativeProcessWorkspaceCommandSandbox return this.closing; } + /** An executor that exits, disconnects, or stalls while closing is + * terminated in `finally` regardless, and the active command has already + * drained, so only a failure the executor reports explicitly is surfaced. */ private async stop(): Promise { await this.active?.catch(() => undefined); await this.ready?.catch(() => undefined); try { if (this.child?.connected && !this.failed) - await this.rpc('close', {}, 10_000, false); + await this.rpc('close', {}, 10_000, false).catch((error: unknown) => { + if (!(error instanceof NativeExecutorUnavailableError)) throw error; + }); } finally { this.failed = true; this.terminate(); From 25f3841c64422c289f30743520cb3ca88cc471fb Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 13 Sep 2026 23:21:49 -0400 Subject: [PATCH 13/42] fix: authenticate GitHub App Git operations (#192) --- packages/code/src/github.test.ts | 9 +++++++-- packages/code/src/github.ts | 9 ++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index bd62acfd..78d9751d 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -137,12 +137,17 @@ test('builds process-scoped Git HTTPS authorization without embedding credential const provider = new StaticGitHubCredentialProvider( 'github_pat_abcdefghijklmnopqrstuvwxyz', ); + const encodedCredential = Buffer.from( + 'x-access-token:github_pat_abcdefghijklmnopqrstuvwxyz', + 'utf8', + ).toString('base64'); assert.deepEqual( gitHubCredentialEnvironment(await provider.getCredential()), { - [GITHUB_CREDENTIAL_ENV_NAME]: 'github_pat_abcdefghijklmnopqrstuvwxyz', + [GITHUB_CREDENTIAL_ENV_NAME]: encodedCredential, }, ); + assert.ok(!encodedCredential.includes('github_pat_')); }); test('composes the masked credential with SRT Git configuration inside the sandbox', () => { @@ -155,7 +160,7 @@ test('composes the masked credential with SRT Git configuration inside the sandb assert.match(wrapped, /http\.https:\/\/github\.com\/\.extraheader/); assert.match(wrapped, /\$\{LIBRECHAT_CODE_GITHUB_AUTHORIZATION\}/); assert.match(wrapped, /unset LIBRECHAT_CODE_GITHUB_AUTHORIZATION/); - assert.equal(wrapped.match(/Authorization: Bearer/g)?.length, 1); + assert.equal(wrapped.match(/Authorization: Basic/g)?.length, 1); assert.ok(!wrapped.includes('github_pat_')); }); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 7b1f12b1..b8046897 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -197,7 +197,10 @@ export function gitHubCredentialEnvironment( credential: GitHubCredential, ): Record { return { - [GITHUB_CREDENTIAL_ENV_NAME]: credential.value, + [GITHUB_CREDENTIAL_ENV_NAME]: Buffer.from( + `x-access-token:${credential.value}`, + 'utf8', + ).toString('base64'), }; } @@ -250,14 +253,14 @@ export function wrapGitHubCredentialCommand( return [ 'set "GIT_CONFIG_GLOBAL=NUL"', 'set "GIT_CONFIG_NOSYSTEM=1"', - `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Bearer %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, + `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, `set "${GITHUB_CREDENTIAL_ENV_NAME}="`, command, ].join(' && '); } return [ 'export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1', - `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Bearer \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, + `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, `unset ${GITHUB_CREDENTIAL_ENV_NAME}`, command, ].join(';\n'); From 1c7af888c774a52d3a8d4170590c6c409b03ae6e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 01:17:13 -0400 Subject: [PATCH 14/42] fix: isolate file deletion rate limits (#193) --- service/src/config.ts | 6 +++ service/src/middleware/limits.test.ts | 56 ++++++++++++++++++++++++ service/src/middleware/limits.ts | 11 +++++ service/src/service/exec-timeout.test.ts | 2 +- service/src/service/router.ts | 12 +++-- 5 files changed, 83 insertions(+), 4 deletions(-) diff --git a/service/src/config.ts b/service/src/config.ts index 90df6b58..d025831e 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -346,6 +346,12 @@ export const env = { // Files List Rate Limits FETCH_LIMIT_WINDOW: Number(process.env.FETCH_LIMIT_WINDOW) || 60 * 1000, // 1 minute FETCH_MAX_REQUESTS: Number(process.env.FETCH_MAX_REQUESTS) || 120, // 120 requests per minute + // File Delete Rate Limits. Fall back to the fetch settings so existing + // deployments keep their current limits while using an independent bucket. + DELETE_LIMIT_WINDOW: + Number(process.env.DELETE_LIMIT_WINDOW) || Number(process.env.FETCH_LIMIT_WINDOW) || 60 * 1000, + DELETE_MAX_REQUESTS: + Number(process.env.DELETE_MAX_REQUESTS) || Number(process.env.FETCH_MAX_REQUESTS) || 120, // Redis Key Cache Config SESSION_CACHE_TTL: Number(process.env.SESSION_CACHE_TTL) || 86400, /** TTL for the durable `session-owner:` record that backs diff --git a/service/src/middleware/limits.test.ts b/service/src/middleware/limits.test.ts index e7c1c46f..12f3daf4 100644 --- a/service/src/middleware/limits.test.ts +++ b/service/src/middleware/limits.test.ts @@ -111,6 +111,40 @@ async function startRateLimitedApp(max: number, windowMs: number): Promise { + const redis = new TestRedisRateLimitStore(); + setRateLimitRedisForTests(redis); + + const app = express(); + app.use((req, _res, next) => { + applyPrincipal(req as AuthenticatedRequest, { + userId: 'user-a', + tenantId: 'tenant-a', + principalSource: 'librechat_jwt', + }); + next(); + }); + app.get( + '/v1/files/session-a', + createRateLimiter('test-fetch', windowMs, max, { message: 'Too many file list requests.' }), + (_req, res) => res.status(200).json({ ok: true }), + ); + app.delete( + '/v1/files/session-a/file-a', + createRateLimiter('test-delete', windowMs, max, { + message: 'Too many file deletion requests.', + structuredBody: true, + }), + (_req, res) => res.status(200).json({ ok: true }), + ); + + const server = app.listen(0, '127.0.0.1'); + servers.push(server); + await once(server, 'listening'); + const address = server.address() as AddressInfo; + return `http://127.0.0.1:${address.port}`; +} + function postExec(url: string, headers: Record = {}): Promise { return fetch(`${url}/v1/exec`, { method: 'POST', @@ -228,3 +262,25 @@ describe('execution rate limiting', () => { expect((await postExec(url)).status).toBe(200); }); }); + +describe('file operation rate limiting', () => { + test('keeps deletion traffic out of the file-list bucket and returns structured retry guidance', async () => { + const url = await startIndependentFileLimiterApp(1, 30_000); + + expect((await fetch(`${url}/v1/files/session-a`)).status).toBe(200); + expect((await fetch(`${url}/v1/files/session-a/file-a`, { method: 'DELETE' })).status).toBe(200); + + const rejectedDelete = await fetch(`${url}/v1/files/session-a/file-a`, { method: 'DELETE' }); + const body = await rejectedDelete.json() as ReturnType; + expect(rejectedDelete.status).toBe(429); + expect(rejectedDelete.headers.get('retry-after')).not.toBeNull(); + expect(body.error).toBe('rate_limited'); + expect(body.message).toContain('Too many file deletion requests.'); + + const rejectedList = await fetch(`${url}/v1/files/session-a`); + expect(rejectedList.status).toBe(429); + expect(await rejectedList.json()).toEqual({ + error: expect.stringContaining('Too many file list requests.'), + }); + }); +}); diff --git a/service/src/middleware/limits.ts b/service/src/middleware/limits.ts index b16ed196..099261a1 100644 --- a/service/src/middleware/limits.ts +++ b/service/src/middleware/limits.ts @@ -202,3 +202,14 @@ export const fetchLimiter = createRateLimiter( env.FETCH_MAX_REQUESTS, { message: 'Too many file list requests.' } ); + +export const deleteLimiter = createRateLimiter( + 'delete', + env.DELETE_LIMIT_WINDOW, + env.DELETE_MAX_REQUESTS, + { + message: 'Too many file deletion requests.', + structuredBody: true, + logRejections: true, + } +); diff --git a/service/src/service/exec-timeout.test.ts b/service/src/service/exec-timeout.test.ts index 70e00d17..b10a94c6 100644 --- a/service/src/service/exec-timeout.test.ts +++ b/service/src/service/exec-timeout.test.ts @@ -11,7 +11,7 @@ test('/exec validates timeout before enqueue and forwards its cap to both langua mock.module('./src/middleware/auth', () => ({ sessionAuth: passthrough })); mock.module('./src/middleware/limits', () => ({ executionLimiter: passthrough, uploadLimiter: passthrough, - downloadLimiter: passthrough, fetchLimiter: passthrough, + downloadLimiter: passthrough, fetchLimiter: passthrough, deleteLimiter: passthrough, })); mock.module('./src/lifecycle', () => ({ checkServiceStartUp: () => false, checkServiceShutDown: () => false, diff --git a/service/src/service/router.ts b/service/src/service/router.ts index 2c42f60c..43d0ff4b 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -7,7 +7,13 @@ import { Readable } from 'stream'; import type * as t from '../types'; import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; import { sessionAuth } from '../middleware/auth'; -import { executionLimiter, uploadLimiter, downloadLimiter, fetchLimiter } from '../middleware/limits'; +import { + executionLimiter, + uploadLimiter, + downloadLimiter, + fetchLimiter, + deleteLimiter, +} from '../middleware/limits'; import { internalServiceHeaders } from '../internal-service-auth'; import { resolveSessionKey, resolveOutputBucketSessionKey, SessionKeyResolutionError, parseUploadSessionKeyInput, type SessionKeyInput } from '../session-key'; import { pyQueue, otherQueue, pyQueueEvents, otherQueueEvents, queueNames, connection } from '../queue'; @@ -997,7 +1003,7 @@ const deleteSessionObject = async (req: t.AuthenticatedRequest, res: Response) = } }; -router.delete('/files/:session_id/:fileId', fetchLimiter, sessionAuth, deleteSessionObject); +router.delete('/files/:session_id/:fileId', deleteLimiter, sessionAuth, deleteSessionObject); /** * Alias of the route above, on the path LibreChat's `deleteCodeEnvFile` @@ -1013,6 +1019,6 @@ router.delete('/files/:session_id/:fileId', fetchLimiter, sessionAuth, deleteSes * * GET on this same path is the metadata proxy above. */ -router.delete('/sessions/:session_id/objects/:fileId', fetchLimiter, sessionAuth, deleteSessionObject); +router.delete('/sessions/:session_id/objects/:fileId', deleteLimiter, sessionAuth, deleteSessionObject); export default router; From 737f498ebedc3b9b26da3e5f206d7e0c49b4e901 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 01:27:39 -0400 Subject: [PATCH 15/42] feat: broker GitHub CLI authentication (#194) --- packages/code/README.md | 19 +++++---- packages/code/src/cli.ts | 15 +++---- packages/code/src/github.test.ts | 73 ++++++++++++++++++++++++++++++++ packages/code/src/github.ts | 40 +++++++++++++++++ 4 files changed, 129 insertions(+), 18 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index e46d6fd6..d06fbf9a 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -180,14 +180,17 @@ verification are implemented; use macOS, Linux, or WSL2. This also applies to Gi private keys. Git receives authentication through process-scoped `GIT_CONFIG_*` variables. -The same isolated config supplies the standard Git LFS filters; hosts using LFS -must install `git-lfs`, and checkout fails instead of silently leaving pointer -files when it is unavailable. -SRT replaces only the bearer-token portion with a sentinel inside the sandbox -and substitutes the real value in its host proxy only for `github.com` HTTPS -traffic. TLS termination is enabled for that substitution. The worker restores -the parent environment immediately after constructing the sandbox command; it -never writes credentials into the repository, a remote URL, or Git config. +When the GitHub CLI is installed, `gh api`, pull-request, issue, and workflow +commands receive the same installation scope through `GH_TOKEN` (or +`GH_ENTERPRISE_TOKEN` for GHES). The same isolated Git config supplies the +standard Git LFS filters; hosts using LFS must install `git-lfs`, and checkout +fails instead of silently leaving pointer files when it is unavailable. +SRT replaces each real credential with a sentinel inside the sandbox and +substitutes the real value in its host proxy only for the corresponding Git or +GitHub API host. TLS termination is enabled for that substitution. The worker +restores the parent environment immediately after constructing the sandbox +command; it never writes credentials into the repository, a remote URL, Git +config, or the GitHub CLI credential store. GitHub's required domains are added to the command egress allowlist only when authentication is configured. The worker identity, GitHub App key path, token source variables, and mutation-quarantine record remain denied to sandboxed diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index fdd28e9f..a87f1a5a 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -38,11 +38,11 @@ import type { NativeProcessSandboxOptions } from './native-process.js'; import type { LocalWorkspaceConfig } from './workspace.js'; import { GITHUB_ALLOWED_DOMAINS, - GITHUB_CREDENTIAL_ENV_NAME, GitHubAppCredentialProvider, + gitHubCommandCredentialEnvironment, + gitHubMaskedCredentialVariables, StaticGitHubCredentialProvider, gitHubAuthenticationPolicyIdentity, - gitHubCredentialEnvironment, normalizeGitHubHost, wrapGitHubCredentialCommand, } from './github.js'; @@ -783,16 +783,11 @@ async function run( ...(github.provider ? { maskedEnvironment: { - variables: [ - { - name: GITHUB_CREDENTIAL_ENV_NAME, - extract: '^(.+)$', - injectHosts: [github.host], - }, - ], + variables: gitHubMaskedCredentialVariables(github.host), async resolve(signal?: AbortSignal) { - return gitHubCredentialEnvironment( + return gitHubCommandCredentialEnvironment( await github.provider!.getCredential(signal), + github.host, ); }, wrapCommand(command: string, platform: NodeJS.Platform) { diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 78d9751d..4b028239 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -17,6 +17,9 @@ import { GitHubAppCredentialProvider, StaticGitHubCredentialProvider, gitHubAuthenticationPolicyIdentity, + gitHubCliTokenEnvironmentName, + gitHubCommandCredentialEnvironment, + gitHubMaskedCredentialVariables, GITHUB_CREDENTIAL_ENV_NAME, gitHubCredentialEnvironment, normalizeGitHubHost, @@ -150,6 +153,65 @@ test('builds process-scoped Git HTTPS authorization without embedding credential assert.ok(!encodedCredential.includes('github_pat_')); }); +test('adds a GitHub CLI token only to the command-sandbox credential bundle', async () => { + const provider = new StaticGitHubCredentialProvider( + 'github_pat_abcdefghijklmnopqrstuvwxyz', + ); + const credential = await provider.getCredential(); + assert.deepEqual(gitHubCommandCredentialEnvironment(credential), { + [GITHUB_CREDENTIAL_ENV_NAME]: Buffer.from( + 'x-access-token:github_pat_abcdefghijklmnopqrstuvwxyz', + 'utf8', + ).toString('base64'), + GH_TOKEN: 'github_pat_abcdefghijklmnopqrstuvwxyz', + }); + assert.deepEqual( + gitHubCommandCredentialEnvironment(credential, 'github.example.test'), + { + [GITHUB_CREDENTIAL_ENV_NAME]: Buffer.from( + 'x-access-token:github_pat_abcdefghijklmnopqrstuvwxyz', + 'utf8', + ).toString('base64'), + GH_ENTERPRISE_TOKEN: 'github_pat_abcdefghijklmnopqrstuvwxyz', + }, + ); +}); + +test('selects the GitHub CLI token variable for public and enterprise hosts', () => { + assert.equal(gitHubCliTokenEnvironmentName('github.com'), 'GH_TOKEN'); + assert.equal( + gitHubCliTokenEnvironmentName('github.example.test'), + 'GH_ENTERPRISE_TOKEN', + ); +}); + +test('restricts Git and GitHub CLI credential substitution to their respective hosts', () => { + assert.deepEqual(gitHubMaskedCredentialVariables('github.com'), [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: ['github.com'], + }, + { + name: 'GH_TOKEN', + extract: '^(.+)$', + injectHosts: ['api.github.com'], + }, + ]); + assert.deepEqual(gitHubMaskedCredentialVariables('github.example.test'), [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: ['github.example.test'], + }, + { + name: 'GH_ENTERPRISE_TOKEN', + extract: '^(.+)$', + injectHosts: ['github.example.test'], + }, + ]); +}); + test('composes the masked credential with SRT Git configuration inside the sandbox', () => { const wrapped = wrapGitHubCredentialCommand( 'git push', @@ -160,10 +222,21 @@ test('composes the masked credential with SRT Git configuration inside the sandb assert.match(wrapped, /http\.https:\/\/github\.com\/\.extraheader/); assert.match(wrapped, /\$\{LIBRECHAT_CODE_GITHUB_AUTHORIZATION\}/); assert.match(wrapped, /unset LIBRECHAT_CODE_GITHUB_AUTHORIZATION/); + assert.doesNotMatch(wrapped, /unset GH_TOKEN/); assert.equal(wrapped.match(/Authorization: Basic/g)?.length, 1); assert.ok(!wrapped.includes('github_pat_')); }); +test('targets GitHub CLI at an enterprise host without exposing its token', () => { + const wrapped = wrapGitHubCredentialCommand( + 'gh pr create', + 'github.example.test', + 'linux', + ); + assert.match(wrapped, /GH_HOST=github\.example\.test/); + assert.doesNotMatch(wrapped, /GH_ENTERPRISE_TOKEN=/); +}); + test('rejects an insecure GitHub App API endpoint before reading the private key', () => { assert.throws( () => diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index b8046897..c727e70d 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -204,6 +204,43 @@ export function gitHubCredentialEnvironment( }; } +export function gitHubCommandCredentialEnvironment( + credential: GitHubCredential, + host = 'github.com', +): Record { + return { + ...gitHubCredentialEnvironment(credential), + [gitHubCliTokenEnvironmentName(host)]: credential.value, + }; +} + +export function gitHubCliTokenEnvironmentName(host: string): string { + return host === 'github.com' ? 'GH_TOKEN' : 'GH_ENTERPRISE_TOKEN'; +} + +export function gitHubApiHost(host: string): string { + return host === 'github.com' ? 'api.github.com' : host; +} + +export function gitHubMaskedCredentialVariables(host: string): Array<{ + name: string; + injectHosts: string[]; + extract: string; +}> { + return [ + { + name: GITHUB_CREDENTIAL_ENV_NAME, + extract: '^(.+)$', + injectHosts: [host], + }, + { + name: gitHubCliTokenEnvironmentName(host), + extract: '^(.+)$', + injectHosts: [gitHubApiHost(host)], + }, + ]; +} + export function gitHubAuthenticationPolicyIdentity(options: { mode?: 'app' | 'token'; host: string; @@ -249,10 +286,12 @@ export function wrapGitHubCredentialCommand( platform: NodeJS.Platform = process.platform, ): string { const key = `http.https://${host}/.extraheader`; + const cliHost = host === 'github.com' ? undefined : host; if (platform === 'win32') { return [ 'set "GIT_CONFIG_GLOBAL=NUL"', 'set "GIT_CONFIG_NOSYSTEM=1"', + ...(cliHost ? [`set "GH_HOST=${cliHost}"`] : []), `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, `set "${GITHUB_CREDENTIAL_ENV_NAME}="`, command, @@ -260,6 +299,7 @@ export function wrapGitHubCredentialCommand( } return [ 'export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1', + ...(cliHost ? [`export GH_HOST=${cliHost}`] : []), `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, `unset ${GITHUB_CREDENTIAL_ENV_NAME}`, command, From 03e2fc11951fa3faf2fb5943afc445346177434e Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 13:58:14 -0400 Subject: [PATCH 16/42] feat: Run PTC in Selected BYOM Workspaces (#195) * feat: run PTC in selected BYOM workspaces * fix: harden native workspace PTC replay * fix: preserve replay isolation and bridge limits * test: tolerate hosts without filesystem cloning * test: surface copy-on-write clone faults * fix: harden native workspace PTC admission * fix: close native replay effect and finalization boundaries --- docs/remote-bridge/README.md | 19 +- packages/code/README.md | 38 +- packages/code/src/cli.ts | 14 + packages/code/src/index.ts | 1 + packages/code/src/native-pool.test.ts | 25 + packages/code/src/native-pool.ts | 76 +- packages/code/src/native-process-child.ts | 35 +- packages/code/src/native-process.test.ts | 166 +++- packages/code/src/native-process.ts | 249 +++++- .../code/src/native-programmatic-live.test.ts | 41 + packages/code/src/native-programmatic.test.ts | 608 +++++++++++++ packages/code/src/native-programmatic.ts | 816 ++++++++++++++++++ packages/code/src/native-sandbox.test.ts | 357 ++++++-- packages/code/src/native-sandbox.ts | 448 ++++++++-- packages/code/src/protocol.test.ts | 184 ++++ packages/code/src/protocol.ts | 454 ++++++++-- packages/code/src/worker-slots.test.ts | 48 ++ packages/code/src/worker.ts | 159 +++- packages/code/src/workspace-worker.test.ts | 82 ++ packages/code/src/workspace.ts | 5 + service/src/bridge/router.ts | 1 + service/src/bridge/selection.ts | 1 + service/src/bridge/store.ts | 112 ++- service/src/bridge/workspace-store.test.ts | 117 +++ service/src/egress-gateway.ts | 37 +- service/src/egress-grant.test.ts | 8 + service/src/preamble-bash.test.ts | 226 ++++- service/src/preamble-bash.ts | 125 ++- service/src/preamble.test.ts | 46 +- service/src/preamble.ts | 276 ++++-- service/src/ptc-constants.test.ts | 9 + service/src/ptc-constants.ts | 7 +- .../src/sandbox-backend/remote-bridge.test.ts | 28 + service/src/sandbox-backend/remote-bridge.ts | 1 + service/src/sandbox-backend/types.ts | 3 + service/src/sandbox-dispatch.test.ts | 67 +- service/src/sandbox-dispatch.ts | 33 +- service/src/sandbox-egress.ts | 29 + service/src/service/programmatic-router.ts | 477 +++++++--- .../src/service/programmatic-state.test.ts | 2 + service/src/service/programmatic-state.ts | 2 + service/src/service/replay-state.ts | 2 + service/src/types/service.ts | 29 +- service/src/workers.ts | 11 + 44 files changed, 4850 insertions(+), 624 deletions(-) create mode 100644 packages/code/src/native-programmatic-live.test.ts create mode 100644 packages/code/src/native-programmatic.test.ts create mode 100644 packages/code/src/native-programmatic.ts create mode 100644 service/src/ptc-constants.test.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index ab7e2f64..99a0dece 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -149,6 +149,17 @@ implementation and an allowlist of workspace IDs, preserves per-workspace operation restrictions, validates bounded results, and treats an unknown command failure as an uncertain mutation. +Selected attached workspaces on macOS, Linux, and WSL2 can also advertise Bash +Programmatic Tool Calling. Native Windows workers do not advertise Bash PTC. +Code API then runs each replay iteration through the same workspace-scoped +native SRT executor. Source code operates in the selected local root, while +replay metadata, injected skills and attachments, and generated artifacts are +staged in an execution-private data directory and removed after settlement. +Only authorized file references and returned artifacts cross the relay; the +repository is never uploaded to Code API. This capability is advertised only +when native SRT commands and a file-relay upstream are both configured, so +older or partially configured workers continue to fail closed. + Native SRT is the MVP and default command backend on a user's chosen laptop or VM. It uses Seatbelt on macOS, bubblewrap/seccomp on Linux, and the SRT restricted-account helper on Windows. It confines writes to the registered @@ -157,9 +168,11 @@ credentials, and denies network egress by default. Startup fails closed when the platform dependencies are unavailable; there is no unsandboxed fallback. Use `LIBRECHAT_CODE_COMMAND_ALLOWED_DOMAINS` for an explicit comma-separated egress allowlist. -Linux hosts must provide Bash at `/bin/bash`, `bubblewrap`, `socat`, and -`ripgrep`; macOS uses system facilities. Windows requires SRT's one-time -restricted-account setup. +Linux hosts must provide `bubblewrap`, `socat`, and `ripgrep`; macOS uses +system facilities. Bash Programmatic Tool Calling additionally requires Bash +5.2 or newer and `jq` on `PATH` on macOS, Linux, and WSL2. The worker resolves +the compatible shell from `PATH` rather than assuming `/bin/bash`. Windows +requires SRT's one-time restricted-account setup. The optional `docker-nsjail` adapter enables a stronger container boundary with `--allow-workspace-commands` (or diff --git a/packages/code/README.md b/packages/code/README.md index d06fbf9a..87d9dded 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -143,14 +143,48 @@ policy. This matches the personal-machine SRT trust model; use the Docker/NsJail backend or a dedicated VM boundary when hard teardown of adversarial process trees is required. -Linux hosts need Bash at `/bin/bash`, `bubblewrap`, `socat`, and `ripgrep`; macOS uses system -facilities. Follow SRT's one-time restricted-account setup when using Windows. +Linux hosts need `bubblewrap`, `socat`, and `ripgrep`; macOS uses system +facilities. Bash Programmatic Tool Calling additionally requires Bash 5.2 or +newer and `jq` on `PATH` on macOS, Linux, and WSL2. The worker resolves that +shell explicitly instead of assuming `/bin/bash`, which remains Bash 3.2 on +many macOS hosts. Follow SRT's one-time restricted-account setup when using Windows. An operator may allow explicit egress destinations with the comma-separated `LIBRECHAT_CODE_COMMAND_ALLOWED_DOMAINS` setting. Treat that as a security policy: an allowed destination can receive workspace data. The normalized allowlist is included in the worker policy digest. Tool approval hooks remain the user-facing allow/deny boundary for each invocation. +When Code API negotiates `bash` programmatic execution for a selected +workspace, the same native SRT executor also supports replay-mode Programmatic +Tool Calling on macOS, Linux, and WSL2 workers. Native Windows does not +advertise this Bash capability. The repository remains the command working directory. Generated +PTC scripts, replay history, skill files, chat attachments, and returned +artifacts use an owner-only per-execution directory under the worker's private +SRT scratch root, exposed to code as `LIBRECHAT_CODE_DATA_DIR`. That directory +is removed after every iteration and is never placed in the repository. + +Replay probes run against a disposable copy-on-write snapshot with network and +socket access denied, including under `trusted-vm`. External effects must not +repeat while discovering pending tools. Use registered tools for network-dependent +replay control flow; the final commit pass runs once under the configured policy. +Each probe's SRT proxy session is revoked before restoring the commit policy; +per-command network overrides alone do not restrict SRT's session-level proxies. +Probe failures do not quarantine the real workspace. Once the commit pass starts, +its fence remains until result restoration succeeds; uncertain finalization +quarantines only that workspace. + +Reference inputs and artifact outputs travel only through the configured +`LIBRECHAT_CODE_FILE_RELAY_UPSTREAM`, using Code API's execution-scoped opaque +egress grant. The worker rejects redirects and bounds each transfer to 10 MiB, +each execution to 100 files and 100 MiB total, and transfer concurrency to four. +Caller inputs are limited to 98 files, reserving two for the script and replay +history. Code API reserves one third of the job budget for all transfer batches +and negotiates each transfer's deadline before signing the request. +Its parent process keeps a 64-entry/32-MiB LRU input cache keyed by a stable, +Code-API-authorized digest; sandboxed commands cannot read that cache. Requests +against one workspace remain serialized, while negotiated lease slots allow +different registered roots to execute concurrently. + The native sandbox preserves standard `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and `NO_PROXY` names (including lowercase forms), plus Windows process and profile variables on Windows. SRT remains responsible for the final sandbox environment diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index a87f1a5a..289b0dc4 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -408,6 +408,11 @@ async function run( 'LIBRECHAT_CODE_COMMAND_SANDBOX must be native-srt or runtime', ); } + const nativeProgrammaticEnabled = + allowWorkspaceCommands && + commandSandboxMode === 'native-srt' && + process.platform !== 'win32' && + (fileRelayUpstream?.length ?? 0) > 0; const commandPolicy = resolveNativeSrtCommandPolicy( option(args, '--command-policy-preset') ?? process.env.LIBRECHAT_CODE_COMMAND_POLICY_PRESET?.trim().toLowerCase() ?? @@ -780,6 +785,9 @@ async function run( github.privateKeyPath, ].filter((path): path is string => path != null), allowedDomains: commandAllowedDomains, + ...(nativeProgrammaticEnabled + ? { programmaticFileUpstream: fileRelayUpstream } + : {}), ...(github.provider ? { maskedEnvironment: { @@ -819,6 +827,9 @@ async function run( workspaceTools = new SandboxWorkspaceTools({ workspaceTools, commandWorkspaces: roots.map((root) => root.id), + ...(nativeProgrammaticEnabled + ? { programmaticLanguages: ['bash'] } + : {}), commandSandbox: nativeCommandSandbox ?? new RuntimeWorkspaceCommandSandbox({ @@ -878,6 +889,9 @@ async function run( runtimeSupervisor, capabilities, workspaceTools, + ...(nativeProgrammaticEnabled && nativeCommandSandbox + ? { workspaceProgrammatic: nativeCommandSandbox } + : {}), ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { workspaceQuarantines: new Map( diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 5363f0c0..6f94b190 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -7,6 +7,7 @@ export * from './workspace.js'; export * from './workspace-runtime.js'; export * from './native-policy.js'; export * from './native-sandbox.js'; +export * from './native-programmatic.js'; export * from './native-process.js'; export * from './github.js'; export * from './worker.js'; diff --git a/packages/code/src/native-pool.test.ts b/packages/code/src/native-pool.test.ts index 3a24db34..cad30a52 100644 --- a/packages/code/src/native-pool.test.ts +++ b/packages/code/src/native-pool.test.ts @@ -13,6 +13,31 @@ const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ workspaceId, command: 'fixture', }); + +test('native pool preflights every registered root with bounded concurrency', async () => { + const prepared: string[] = []; + let active = 0; + let peak = 0; + const pool = new NativeWorkspaceCommandPool(roots, 2, (options) => ({ + async prepare() { + active += 1; + peak = Math.max(peak, active); + await new Promise(resolve => setTimeout(resolve, 5)); + prepared.push(options.workspaceRoot); + active -= 1; + }, + async close() {}, + async execute() { + throw new Error('unreachable'); + }, + })); + + await pool.prepare(); + assert.deepEqual(prepared.sort(), ['/fixture/a', '/fixture/b', '/fixture/c']); + assert.equal(peak, 2); + await pool.close(); +}); + test('a known-clean executor failure is retired without replaying the command', async () => { let created = 0; let executed = 0; diff --git a/packages/code/src/native-pool.ts b/packages/code/src/native-pool.ts index 28f155e2..766409b1 100644 --- a/packages/code/src/native-pool.ts +++ b/packages/code/src/native-pool.ts @@ -2,6 +2,7 @@ import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; import type { NativeProcessSandboxOptions } from './native-process.js'; import type { + BridgeWorkspaceProgrammaticRequest, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, } from './protocol.js'; @@ -10,7 +11,10 @@ interface Entry { sandbox: Pick< NativeProcessWorkspaceCommandSandbox, 'prepare' | 'execute' | 'close' - >; + > & + Partial< + Pick + >; busy: boolean; } @@ -92,12 +96,25 @@ export class NativeWorkspaceCommandPool { } async prepare(): Promise { - const entry = await this.allocate(this.roots.keys().next().value!); - try { - await entry.sandbox.prepare(); - } finally { - entry.busy = false; - } + const workspaceIds = [...this.roots.keys()]; + let next = 0; + await Promise.all( + Array.from( + { length: Math.min(this.capacity, workspaceIds.length) }, + async () => { + for (;;) { + const index = next++; + if (index >= workspaceIds.length) return; + const entry = await this.allocate(workspaceIds[index]!); + try { + await entry.sandbox.prepare(); + } finally { + entry.busy = false; + } + } + }, + ), + ); } async execute( @@ -136,6 +153,51 @@ export class NativeWorkspaceCommandPool { } } + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + const entry = await this.allocate(workspaceId); + let enteredExecutor = false; + try { + if (signal?.aborted) + throw new WorkspaceToolError( + 'Programmatic execution cancelled before dispatch', + 'EXECUTION_ABORTED', + ); + enteredExecutor = true; + if (!entry.sandbox.executeProgrammatic) { + throw new WorkspaceToolError( + 'Native programmatic executor is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + return await entry.sandbox.executeProgrammatic( + workspaceId, + request, + signal, + ); + } catch (error) { + if ( + enteredExecutor && + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted + ) { + try { + await entry.sandbox.close(); + if (this.entries.get(workspaceId) === entry) + this.entries.delete(workspaceId); + } catch { + /* Retain ownership for subsequent cleanup/shutdown. */ + } + } + throw error; + } finally { + entry.busy = false; + } + } + async close(): Promise { this.closing = true; await this.allocation; diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index 05ffedfe..2e8beb38 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -1,11 +1,16 @@ import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { NativeWorkspaceProgrammaticExecutor } from './native-programmatic.js'; import { WorkspaceToolError } from './workspace.js'; import type { NativeSrtWorkspaceCommandSandboxOptions } from './native-sandbox.js'; -import type { WorkspaceExecuteCommandRequest } from './protocol.js'; +import type { + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandRequest, +} from './protocol.js'; // This entrypoint is private to a forked trusted executor. No HTTP listener, // argv credentials, bridge token, or persisted pairing material is required. let sandbox: NativeSrtWorkspaceCommandSandbox | undefined; +let programmaticExecutor: NativeWorkspaceProgrammaticExecutor | undefined; let active: { id: string; controller: AbortController } | undefined; let busy = false; let credentials: Record = {}; @@ -44,11 +49,14 @@ process.on('message', async (raw: unknown) => { NativeSrtWorkspaceCommandSandboxOptions, 'maskedEnvironment' > & { + programmaticFileUpstream?: string; variables?: NonNullable< NativeSrtWorkspaceCommandSandboxOptions['maskedEnvironment'] >['variables']; }; request: WorkspaceExecuteCommandRequest; + programmaticRequest?: BridgeWorkspaceProgrammaticRequest; + workspaceId?: string; credentials?: Record; wrappedCommand?: string; }; @@ -62,7 +70,8 @@ process.on('message', async (raw: unknown) => { try { let result: unknown; if (message.type === 'prepare' && !sandbox) { - const { variables, ...options } = message.options; + const { variables, programmaticFileUpstream, ...options } = + message.options; sandbox = new NativeSrtWorkspaceCommandSandbox({ ...options, ...(variables @@ -80,11 +89,33 @@ process.on('message', async (raw: unknown) => { : {}), }); await sandbox.prepare(); + programmaticExecutor = programmaticFileUpstream + ? new NativeWorkspaceProgrammaticExecutor({ + sandbox, + upstreamUrl: programmaticFileUpstream, + }) + : undefined; + await programmaticExecutor?.prepare(); } else if (message.type === 'execute' && sandbox) { active = { id: message.id, controller: new AbortController() }; credentials = message.credentials ?? {}; wrappedCommand = message.wrappedCommand; result = await sandbox.execute(message.request, active.controller.signal); + } else if ( + message.type === 'programmatic' && + sandbox && + programmaticExecutor && + message.programmaticRequest && + typeof message.workspaceId === 'string' + ) { + active = { id: message.id, controller: new AbortController() }; + credentials = message.credentials ?? {}; + wrappedCommand = message.wrappedCommand; + result = await programmaticExecutor.execute( + message.programmaticRequest, + message.workspaceId, + active.controller.signal, + ); } else if (message.type === 'close' && sandbox) { await sandbox.close(); } else throw new Error('Invalid executor state'); diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 800dcf11..e96f1e65 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -1,13 +1,29 @@ import assert from 'node:assert/strict'; import { EventEmitter } from 'node:events'; import test from 'node:test'; +import { mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import type { ChildProcess, ForkOptions } from 'node:child_process'; import { NativeProcessWorkspaceCommandSandbox, nativeExecutorEnvironment, + trustedProgrammaticExecutable, } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; +test('preflight rejects relative and workspace-controlled executables including symlinks', async t => { + const root = await mkdtemp(join(tmpdir(), 'native-ptc-path-')); + const outside = await mkdtemp(join(tmpdir(), 'native-ptc-link-')); + t.after(async () => { await rm(root, { recursive: true, force: true }); await rm(outside, { recursive: true, force: true }); }); + const executable = join(root, 'bash'); + await writeFile(executable, '#!/bin/sh\nexit 0\n', { mode: 0o700 }); + await symlink(executable, join(outside, 'bash')); + await assert.rejects(trustedProgrammaticExecutable('./bash', root), /absolute/); + await assert.rejects(trustedProgrammaticExecutable(executable, root), /outside the workspace/); + await assert.rejects(trustedProgrammaticExecutable(join(outside, 'bash'), root), /outside the workspace/); +}); + const request = { protocolVersion: 1 as const, operation: 'execute_command' as const, @@ -43,13 +59,19 @@ function fixture( queueMicrotask(() => { if (message.type === 'prepare' && prepare) return prepare(child, message); - if (message.type === 'execute' && execute) + if ( + (message.type === 'execute' || message.type === 'programmatic') && + execute + ) return execute(child, message); if (message.type === 'cancel') return; child.emit('message', { id: message.id, ok: true, - ...(message.type === 'execute' ? { result } : {}), + ...(message.type === 'execute' || + message.type === 'programmatic' + ? { result } + : {}), }); }); return true; @@ -171,8 +193,88 @@ test('executor hands credentials over IPC only for the current command', async ( await sandbox.close(); }); +test('programmatic executor resolves and scopes credentials to its command', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + maskedEnvironment: { + variables: [{ name: 'TOKEN', injectHosts: ['github.com'] }], + async resolve() { + return { TOKEN: 'per-programmatic-secret' }; + }, + wrapCommand(command) { + return `wrapped ${command}`; + }, + }, + }, + fake.fork, + ); + const programmaticRequest = { + headers: {}, + body: { + language: 'bash' as const, + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'git status' }], + }, + }; + await sandbox.executeProgrammatic('primary', programmaticRequest); + assert.equal( + JSON.stringify(fake.options).includes('per-programmatic-secret'), + false, + ); + const message = fake.messages.find( + candidate => candidate.type === 'programmatic', + )!; + assert.deepEqual(message.credentials, { TOKEN: 'per-programmatic-secret' }); + assert.equal( + message.wrappedCommand, + 'wrapped exec "$LIBRECHAT_CODE_BASH_PATH" "$LIBRECHAT_CODE_DATA_DIR/main.sh"', + ); + await sandbox.close(); +}); + +test('programmatic executor preserves a child-reported pre-dispatch failure', async () => { + const fake = fixture((child, message) => + child.emit('message', { + id: message.id, + ok: false, + code: 'COMMAND_UNAVAILABLE', + errorMessage: 'Programmatic input download failed', + mutation: false, + requiresQuarantine: false, + }), + ); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }), + (error: unknown) => + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted && + !error.requiresQuarantine, + ); + await sandbox.close(); +}); + test('executor loss after dispatch is an uncertain mutation and is never replayed', async () => { - const fake = fixture((child) => child.emit('exit', 1)); + const fake = fixture(child => child.emit('exit', 1)); const sandbox = new NativeProcessWorkspaceCommandSandbox( { workspaceRoot: '/workspace' }, fake.fork, @@ -180,16 +282,17 @@ test('executor loss after dispatch is an uncertain mutation and is never replaye await assert.rejects( sandbox.execute(request), (error: unknown) => - error instanceof WorkspaceToolError && error.mutationMayHaveCommitted, + error instanceof WorkspaceToolError && + error.mutationMayHaveCommitted, ); await assert.rejects(sandbox.execute(request), /unavailable/); - assert.equal(fake.messages.filter((m) => m.type === 'execute').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'execute').length, 1); await sandbox.close(); }); test('executor cancellation targets the active request and preserves mutation certainty', async () => { let dispatched!: () => void; - const dispatch = new Promise((resolve) => { + const dispatch = new Promise(resolve => { dispatched = resolve; }); const fake = fixture(() => dispatched()); @@ -202,7 +305,7 @@ test('executor cancellation targets the active request and preserves mutation ce await dispatch; await assert.rejects(sandbox.execute(request), /unavailable/); controller.abort(); - const command = fake.messages.find((m) => m.type === 'execute')!; + const command = fake.messages.find(m => m.type === 'execute')!; assert.deepEqual(fake.messages.at(-1), { type: 'cancel', id: command.id }); fake.child.emit('message', { id: command.id, @@ -224,7 +327,7 @@ test('executor cancellation targets the active request and preserves mutation ce test('executor ignores a cleanup exemption on non-cancellation failures', async () => { let dispatched!: () => void; - const dispatch = new Promise((resolve) => { + const dispatch = new Promise(resolve => { dispatched = resolve; }); const fake = fixture(() => dispatched()); @@ -234,7 +337,7 @@ test('executor ignores a cleanup exemption on non-cancellation failures', async ); const execution = sandbox.execute(request); await dispatch; - const command = fake.messages.find((message) => message.type === 'execute')!; + const command = fake.messages.find(message => message.type === 'execute')!; fake.child.emit('message', { id: command.id, ok: false, @@ -269,7 +372,8 @@ test('executor rejects mismatched results as uncertain and fences subsequent com await assert.rejects( sandbox.execute(request), (error: unknown) => - error instanceof WorkspaceToolError && error.mutationMayHaveCommitted, + error instanceof WorkspaceToolError && + error.mutationMayHaveCommitted, ); await assert.rejects(sandbox.execute(request), /unavailable/); await sandbox.close(); @@ -277,7 +381,7 @@ test('executor rejects mismatched results as uncertain and fences subsequent com test('executor close drains an active command before closing IPC', async () => { let dispatched!: () => void; - const dispatch = new Promise((resolve) => { + const dispatch = new Promise(resolve => { dispatched = resolve; }); const fake = fixture(() => dispatched()); @@ -288,16 +392,16 @@ test('executor close drains an active command before closing IPC', async () => { const execution = sandbox.execute(request); await dispatch; const closing = sandbox.close(); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal( - fake.messages.some((m) => m.type === 'close'), + fake.messages.some(m => m.type === 'close'), false, ); - const command = fake.messages.find((m) => m.type === 'execute')!; + const command = fake.messages.find(m => m.type === 'execute')!; fake.child.emit('message', { id: command.id, ok: true, result }); assert.deepEqual(await execution, result); await closing; - assert.equal(fake.messages.filter((m) => m.type === 'close').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'close').length, 1); await assert.rejects(sandbox.execute(request), /unavailable/); }); @@ -321,7 +425,7 @@ test('executor close resolves when the child exits during the close handshake', }, }); await sandbox.close(); - assert.equal(fake.messages.filter((m) => m.type === 'close').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'close').length, 1); await assert.rejects(sandbox.execute(request), /unavailable/); }); @@ -365,7 +469,7 @@ test('executor close skips the handshake once the child is already lost', async fake.child.emit('exit', 1, null); await sandbox.close(); assert.equal( - fake.messages.some((m) => m.type === 'close'), + fake.messages.some(m => m.type === 'close'), false, ); await assert.rejects(sandbox.execute(request), /unavailable/); @@ -377,17 +481,20 @@ test('executor startup loss is not reported as an applied mutation', async () => { workspaceRoot: '/workspace' }, (path, args, options) => { const child = fake.fork(path, args, options); - queueMicrotask(() => child.emit('error', new Error('startup failed'))); + queueMicrotask(() => + child.emit('error', new Error('startup failed')), + ); return child; }, ); await assert.rejects( sandbox.execute(request), (error: unknown) => - error instanceof WorkspaceToolError && !error.mutationMayHaveCommitted, + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted, ); assert.equal( - fake.messages.some((m) => m.type === 'execute'), + fake.messages.some(m => m.type === 'execute'), false, ); await sandbox.close(); @@ -409,7 +516,7 @@ test('executor shutdown receipt fences reuse before the OS exit event', async () ); await assert.rejects(sandbox.execute(request)); await assert.rejects(sandbox.execute(request), /unavailable/); - assert.equal(fake.messages.filter((m) => m.type === 'execute').length, 1); + assert.equal(fake.messages.filter(m => m.type === 'execute').length, 1); await sandbox.close(); }); @@ -431,7 +538,8 @@ test('executor preserves bounded startup diagnostics and conventional host setti ok: false, mutation: false, code: 'COMMAND_UNAVAILABLE', - errorMessage: 'Native sandbox dependencies are unavailable: bubblewrap', + errorMessage: + 'Native sandbox dependencies are unavailable: bubblewrap', }), ); const sandbox = new NativeProcessWorkspaceCommandSandbox( @@ -465,7 +573,10 @@ test('executor matches POSIX names exactly and folds names only on Windows', () https_proxy: 'http://proxy:8080', }); assert.deepEqual( - nativeExecutorEnvironment({ Path: 'C:\\bin', Temp: 'C:\\temp' }, 'win32'), + nativeExecutorEnvironment( + { Path: 'C:\\bin', Temp: 'C:\\temp' }, + 'win32', + ), { Path: 'C:\\bin', Temp: 'C:\\temp' }, ); }); @@ -486,7 +597,8 @@ test('executor classifies every pre-dispatch setup failure as mutation-atomic', return {}; }, wrapCommand(command) { - if (failure === 'wrapper') throw new Error('wrapper failed'); + if (failure === 'wrapper') + throw new Error('wrapper failed'); return command; }, }, @@ -503,10 +615,12 @@ test('executor classifies every pre-dispatch setup failure as mutation-atomic', error instanceof WorkspaceToolError && !error.mutationMayHaveCommitted && error.code === - (failure === 'abort' ? 'EXECUTION_ABORTED' : 'COMMAND_UNAVAILABLE'), + (failure === 'abort' + ? 'EXECUTION_ABORTED' + : 'COMMAND_UNAVAILABLE'), ); assert.equal( - fake.messages.some((m) => m.type === 'execute'), + fake.messages.some(m => m.type === 'execute'), false, ); await sandbox.close(); diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index e1a02498..d2ea0d72 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -1,11 +1,21 @@ -import { fork } from 'node:child_process'; +import { execFile, fork } from 'node:child_process'; import { randomUUID } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { access, realpath } from 'node:fs/promises'; +import { isAbsolute, join, relative, sep } from 'node:path'; +import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; -import { isWorkspaceToolRequest, isWorkspaceToolResult } from './protocol.js'; +import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; +import { + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + isWorkspaceToolRequest, + isWorkspaceToolResult, +} from './protocol.js'; import type { ChildProcess, ForkOptions } from 'node:child_process'; import type { NativeSrtWorkspaceCommandSandboxOptions } from './native-sandbox.js'; import type { WorkspaceCommandSandbox } from './workspace.js'; import type { + BridgeWorkspaceProgrammaticRequest, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, } from './protocol.js'; @@ -13,7 +23,84 @@ import type { export type NativeProcessSandboxOptions = Omit< NativeSrtWorkspaceCommandSandboxOptions, 'manager' | 'spawnCommand' | 'platform' ->; +> & { + /** Hardened Code API egress gateway used for execution-scoped files. */ + programmaticFileUpstream?: string; +}; + +const execFileAsync = promisify(execFile); + +async function systemProgrammaticExecutable( + name: string, + workspaceRoot: string, +): Promise { + // Preflight runs outside SRT. Never execute a workspace-controlled PATH + // entry (including cwd, node_modules/.bin, or a symlink to another root). + for (const directory of ['/opt/homebrew/bin', '/usr/local/bin', '/usr/bin', '/bin', '/home/linuxbrew/.linuxbrew/bin']) { + const candidate = join(directory, name); + try { + const canonical = await trustedProgrammaticExecutable(candidate, workspaceRoot); + if (!['/opt/homebrew/', '/usr/local/', '/usr/bin/', '/bin/', '/home/linuxbrew/.linuxbrew/'].some(root => canonical.startsWith(root))) continue; + return canonical; + } catch { + // Continue through the bounded PATH entries. + } + } +} + +export async function trustedProgrammaticExecutable(candidate: string, workspaceRoot: string): Promise { + if (!isAbsolute(candidate)) throw new Error('Programmatic executable must be absolute'); + const [canonical, root] = await Promise.all([realpath(candidate), realpath(workspaceRoot)]); + const path = relative(root, canonical); + if (path === '' || (!isAbsolute(path) && path !== '..' && !path.startsWith(`..${sep}`))) { + throw new Error('Programmatic executable must be outside the workspace'); + } + await access(canonical, fsConstants.X_OK); + return canonical; +} + +async function resolveProgrammaticShell( + options: NativeProcessSandboxOptions, +): Promise { + const environment = options.environment ?? process.env; + const shellPath = + options.shellPath != null + ? await trustedProgrammaticExecutable(options.shellPath, options.workspaceRoot) + : await systemProgrammaticExecutable('bash', options.workspaceRoot); + const jqPath = await systemProgrammaticExecutable('jq', options.workspaceRoot); + if (!shellPath || !jqPath) { + throw new WorkspaceToolError( + 'Native programmatic execution requires trusted host installations of Bash 5.2 or newer and jq', + 'COMMAND_UNAVAILABLE', + ); + } + try { + const [{ stdout: bashVersion }] = await Promise.all([ + execFileAsync(shellPath, ['--version'], { + env: nativeExecutorEnvironment(environment), + timeout: 5_000, + }), + execFileAsync(jqPath, ['--version'], { + env: nativeExecutorEnvironment(environment), + timeout: 5_000, + }), + ]); + const match = /version\s+(\d+)\.(\d+)/i.exec(bashVersion); + if ( + !match || + Number(match[1]) < 5 || + (Number(match[1]) === 5 && Number(match[2]) < 2) + ) { + throw new Error('unsupported Bash version'); + } + } catch { + throw new WorkspaceToolError( + 'Native programmatic execution requires trusted host installations of Bash 5.2 or newer and jq', + 'COMMAND_UNAVAILABLE', + ); + } + return shellPath; +} /** Only OS discovery and conventional proxy settings cross into the executor. * In particular, never inherit NODE_OPTIONS, bridge identity, or app secrets. */ @@ -77,20 +164,22 @@ export function nativeExecutorEnvironment( * deadline, as opposed to a failure the executor reported explicitly. */ class NativeExecutorUnavailableError extends WorkspaceToolError { constructor(mutation: boolean) { - super('Native executor is unavailable', 'COMMAND_UNAVAILABLE', mutation); + super( + 'Native executor is unavailable', + 'COMMAND_UNAVAILABLE', + mutation, + ); this.name = 'NativeExecutorUnavailableError'; } } /** One persistent, process-isolated SRT manager per workspace. No automatic * restart/replay: losing IPC after execution starts is an ambiguous mutation. */ -export class NativeProcessWorkspaceCommandSandbox - implements WorkspaceCommandSandbox -{ +export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSandbox { readonly mutationFailuresAreAtomic = true as const; private child?: ChildProcess; private ready?: Promise; - private active?: Promise; + private active?: Promise; private closing?: Promise; private failed = false; private terminationTimer?: ReturnType; @@ -122,12 +211,17 @@ export class NativeProcessWorkspaceCommandSandbox } private async start(): Promise { + const programmaticShellPath = this.options.programmaticFileUpstream + ? await resolveProgrammaticShell(this.options) + : this.options.shellPath; const child = this.forkExecutor( new URL('./native-process-child.js', import.meta.url), [], { execArgv: [], - env: nativeExecutorEnvironment(this.options.environment ?? process.env), + env: nativeExecutorEnvironment( + this.options.environment ?? process.env, + ), stdio: ['ignore', 'ignore', 'ignore', 'ipc'], serialization: 'json', }, @@ -165,6 +259,8 @@ export class NativeProcessWorkspaceCommandSandbox const processTerminationConfirmed = code === 'EXECUTION_ABORTED' && message.requiresQuarantine === false; + const mutationMayHaveCommitted = + pending.mutation && message.mutation !== false; pending.reject( new WorkspaceToolError( typeof message.errorMessage === 'string' && @@ -172,8 +268,8 @@ export class NativeProcessWorkspaceCommandSandbox ? message.errorMessage : 'Native executor request failed', code, - pending.mutation && message.mutation !== false, - pending.mutation && !processTerminationConfirmed, + mutationMayHaveCommitted, + mutationMayHaveCommitted && !processTerminationConfirmed, ), ); } @@ -192,6 +288,7 @@ export class NativeProcessWorkspaceCommandSandbox allowedDomains, homeDirectory, shellPath, + programmaticFileUpstream, } = this.options; await this.rpc( 'prepare', @@ -202,13 +299,14 @@ export class NativeProcessWorkspaceCommandSandbox protectedPaths, allowedDomains, homeDirectory, - shellPath, + shellPath: programmaticShellPath ?? shellPath, + programmaticFileUpstream, variables: this.options.maskedEnvironment?.variables, }, }, 30_000, false, - ).catch((error) => { + ).catch(error => { this.failed = true; this.terminate(); throw error; @@ -223,7 +321,10 @@ export class NativeProcessWorkspaceCommandSandbox !isWorkspaceToolRequest(request) || request.operation !== 'execute_command' ) { - throw new WorkspaceToolError('Invalid native command', 'INVALID_REQUEST'); + throw new WorkspaceToolError( + 'Invalid native command', + 'INVALID_REQUEST', + ); } if (this.active || this.closing || this.failed) throw this.unavailable(false); @@ -236,12 +337,114 @@ export class NativeProcessWorkspaceCommandSandbox } } + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + if (this.active || this.closing || this.failed) + throw this.unavailable(false); + if (!this.options.programmaticFileUpstream) { + throw new WorkspaceToolError( + 'Native programmatic file transport is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const active = this.executeProgrammaticOnce( + request, + workspaceId, + signal, + ); + this.active = active; + try { + return await active; + } finally { + this.active = undefined; + } + } + + private async executeProgrammaticOnce( + request: BridgeWorkspaceProgrammaticRequest, + workspaceId: string, + signal?: AbortSignal, + ): Promise { + if (signal?.aborted) + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + let credentials: Record | undefined; + let wrappedCommand: string | undefined; + try { + await this.prepare(); + if (signal?.aborted) throw new Error('aborted'); + credentials = await this.options.maskedEnvironment?.resolve(signal); + if (signal?.aborted) throw new Error('aborted'); + wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( + NATIVE_PROGRAMMATIC_COMMAND, + process.platform, + ); + if (signal?.aborted) throw new Error('aborted'); + } catch (error) { + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + throw error instanceof WorkspaceToolError + ? new WorkspaceToolError(error.message, error.code, false) + : new WorkspaceToolError( + 'Native programmatic executor setup failed before dispatch', + 'COMMAND_UNAVAILABLE', + ); + } + const result = await this.rpc( + 'programmatic', + { + programmaticRequest: request, + workspaceId, + credentials, + wrappedCommand, + }, + (request.body.run_timeout ?? 30_000) * + ((request.body.replay_tool_count ?? 0) > 0 ? 2 : 1) + + (Math.ceil( + request.body.files.filter(file => 'id' in file).length / 4, + ) + + Math.ceil( + (request.body.max_output_files ?? + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / 4, + )) * + (request.body.transfer_timeout_ms ?? 30_000) + + 5_000, + true, + signal, + ); + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + true, + ); + } + if (typeof result !== 'object' || result === null) { + this.failed = true; + this.terminate(); + throw this.unavailable(true); + } + return result; + } + private async executeOnce( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, ): Promise { if (signal?.aborted) - throw new WorkspaceToolError('Command aborted', 'EXECUTION_ABORTED'); + throw new WorkspaceToolError( + 'Command aborted', + 'EXECUTION_ABORTED', + ); let credentials: Record | undefined; let wrappedCommand: string | undefined; try { @@ -258,7 +461,10 @@ export class NativeProcessWorkspaceCommandSandbox // No execute RPC has been sent: setup, token refresh and wrapping cannot // have mutated the workspace. Do not quarantine it for setup failures. if (signal?.aborted) - throw new WorkspaceToolError('Command aborted', 'EXECUTION_ABORTED'); + throw new WorkspaceToolError( + 'Command aborted', + 'EXECUTION_ABORTED', + ); throw error instanceof WorkspaceToolError ? new WorkspaceToolError(error.message, error.code, false) : new WorkspaceToolError( @@ -324,7 +530,7 @@ export class NativeProcessWorkspaceCommandSandbox reject(this.unavailable(mutation)); }; try { - child.send({ type, id, ...payload }, (error) => { + child.send({ type, id, ...payload }, error => { if (error) sendFailed(); }); } catch { @@ -353,9 +559,12 @@ export class NativeProcessWorkspaceCommandSandbox await this.ready?.catch(() => undefined); try { if (this.child?.connected && !this.failed) - await this.rpc('close', {}, 10_000, false).catch((error: unknown) => { - if (!(error instanceof NativeExecutorUnavailableError)) throw error; - }); + await this.rpc('close', {}, 10_000, false).catch( + (error: unknown) => { + if (!(error instanceof NativeExecutorUnavailableError)) + throw error; + }, + ); } finally { this.failed = true; this.terminate(); diff --git a/packages/code/src/native-programmatic-live.test.ts b/packages/code/src/native-programmatic-live.test.ts new file mode 100644 index 00000000..2bc22356 --- /dev/null +++ b/packages/code/src/native-programmatic-live.test.ts @@ -0,0 +1,41 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import type { AddressInfo } from 'node:net'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; +import { resolveNativeSrtCommandPolicy } from './native-policy.js'; + +test('real SRT prevents speculative network effects under trusted-vm', { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 30_000, +}, async () => { + const root = await mkdtemp(join(tmpdir(), 'native-ptc-effects-')); + let effects = 0; + const server = createServer((_req, res) => { effects += 1; res.end('ok'); }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const port = (server.address() as AddressInfo).port; + const executor = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: root, + commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), + programmaticFileUpstream: `http://127.0.0.1:${port}`, + }); + try { + await executor.prepare(); + const result = await executor.executeProgrammatic('primary', { headers: {}, body: { + language: 'bash', version: '5.2.0', session_id: 'isolated-canary', replay_tool_count: 1, + run_timeout: 5000, + files: [{ name: 'main.sh', content: `curl --noproxy '*' --connect-timeout 1 --max-time 2 -s -X POST http://127.0.0.1:${port}/effect >/dev/null\nprintf once >> commit.txt\n` }], + } }) as { run: { code: number } }; + assert.equal(result.run.code, 0); + assert.equal(effects, 1, 'probe must not emit a network effect'); + assert.equal(await readFile(join(root, 'commit.txt'), 'utf8'), 'once'); + } finally { + try { await executor.close(); } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(root, { recursive: true, force: true }); + } + } +}); diff --git a/packages/code/src/native-programmatic.test.ts b/packages/code/src/native-programmatic.test.ts new file mode 100644 index 00000000..82650f95 --- /dev/null +++ b/packages/code/src/native-programmatic.test.ts @@ -0,0 +1,608 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { NativeWorkspaceProgrammaticExecutor } from './native-programmatic.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { AddressInfo } from 'node:net'; +import type { BridgeWorkspaceProgrammaticRequest } from './protocol.js'; + +test('stages skill files privately and returns generated artifacts', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-test-')); + const uploads = new Map(); + let downloadCount = 0; + const server = createServer(async (req, res) => { + assert.equal(req.headers['x-codeapi-egress-grant'], 'grant'); + if (req.method === 'GET') { + downloadCount += 1; + assert.match( + req.url ?? '', + /\/sessions\/input-session\/objects\/skill-file$/, + ); + res.end('skill-value'); + return; + } + assert.equal(req.method, 'PUT'); + assert.equal(req.headers['content-type'], 'text/plain'); + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + uploads.set( + decodeURIComponent(req.headers['x-original-filename'] as string), + Buffer.concat(chunks), + ); + res.statusCode = 200; + res.end(); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + let observedDataDirectory = ''; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + observedDataDirectory = dataDirectory; + assert.equal( + await readFile( + join(dataDirectory, 'skills/example/reference.txt'), + 'utf8', + ), + 'skill-value', + ); + await writeFile(join(dataDirectory, 'result.txt'), 'artifact'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: 'done\n', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + const request: BridgeWorkspaceProgrammaticRequest = { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + execution_id: 'execution-one', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [ + { name: 'main.sh', content: 'printf done' }, + { name: '_ptc_history.json', content: '{}' }, + { + name: 'skills/example/reference.txt', + id: 'skill-file', + storage_session_id: 'input-session', + input_cache_key: createHash('sha256') + .update('stable-authorized-input-identity') + .digest('hex'), + }, + ], + }, + }; + try { + const result = await executor.execute(request, 'primary'); + const replay = await executor.execute(request, 'primary'); + assert.equal(result.run.stdout, 'done\n'); + assert.equal(replay.run.stdout, 'done\n'); + assert.equal(result.session_id, 'output-session'); + assert.equal(downloadCount, 1); + await executor.execute( + { + ...request, + body: { ...request.body, execution_id: 'execution-two' }, + }, + 'primary', + ); + assert.equal(downloadCount, 2); + assert.equal(result.files.length, 1); + assert.equal(result.files[0]?.name, 'result.txt'); + assert.equal(uploads.get('result.txt')?.toString(), 'artifact'); + assert.deepEqual( + await readdir(observedDataDirectory).catch(() => []), + [], + ); + } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('reports unsupported and rejected artifacts without invalidating a completed command', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-artifact-test-')); + const uploads = new Map(); + const server = createServer(async (req, res) => { + const name = decodeURIComponent(req.headers['x-original-filename'] as string); + uploads.set(name, req.headers['content-type']); + for await (const _chunk of req) { + // Drain the bounded request body before responding. + } + res.statusCode = name === 'image.png' ? 503 : 200; + res.end(); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(_request, dataDirectory) { + await writeFile(join(dataDirectory, '_ptc_report.csv'), 'a,b\n1,2\n'); + await writeFile(join(dataDirectory, 'image.png'), 'not-a-real-png'); + await writeFile(join(dataDirectory, 'model.bin'), 'unsupported'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: 'done\n', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [{ name: 'main.sh', content: 'printf done' }], + }, + }, + 'primary', + ); + assert.equal(result.run.code, 0); + assert.deepEqual(result.files.map(file => file.name), ['_ptc_report.csv']); + assert.deepEqual(result.artifact_delivery, { + code: 'artifact_delivery_failed', + status: 'partial', + attempted: 3, + delivered: 1, + failed: 2, + }); + assert.equal(uploads.get('_ptc_report.csv'), 'text/csv'); + assert.equal(uploads.get('image.png'), 'image/png'); + assert.equal(uploads.has('model.bin'), false); + } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('reports artifact transport failure without quarantining a completed command', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-artifact-transport-test-')); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + fetchImpl: async () => { + throw new TypeError('transport unavailable'); + }, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(_request, dataDirectory) { + await writeFile(join(dataDirectory, 'result.txt'), 'artifact'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: 'done\n', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [{ name: 'main.sh', content: 'printf done' }], + }, + }, + 'primary', + ); + assert.equal(result.run.code, 0); + assert.deepEqual(result.files, []); + assert.deepEqual(result.artifact_delivery, { + code: 'artifact_delivery_failed', + status: 'failed', + attempted: 1, + delivered: 0, + failed: 1, + }); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('preflights copy-on-write isolation and removes its private snapshot', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-preflight-test-')); + let executionDirectory = ''; + let probes = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + executionDirectory = await mkdtemp(join(scratch, 'execution-')); + return executionDirectory; + }, + async createProgrammaticProbeWorkspace(directory) { + probes += 1; + const workspace = join(directory, 'workspace'); + await mkdir(workspace); + return workspace; + }, + async executeProgrammatic() { + throw new Error('unreachable'); + }, + }, + }); + try { + await executor.prepare(); + assert.equal(probes, 1); + assert.deepEqual(await readdir(executionDirectory).catch(() => []), []); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('keeps replay probes read-only and commits the script exactly once', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-probe-test-')); + const phases: boolean[] = []; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async createProgrammaticProbeWorkspace(executionDirectory) { + const workspace = join(executionDirectory, 'workspace'); + await mkdir(workspace); + return workspace; + }, + async executeProgrammatic( + _request, + dataDirectory, + _signal, + options, + ) { + phases.push(options?.probe === true); + if (options?.probe === true) { + assert.match(options.workspaceRoot ?? '', /\/workspace$/); + } + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: options?.probe ? 1 : 0, + stdout: options?.probe ? 'probe\n' : 'commit\n', + stderr: options?.probe ? 'expected probe denial\n' : '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + execution_id: 'probe-then-commit', + replay_tool_count: 1, + session_id: 'execution-session', + files: [ + { name: 'main.sh', content: 'printf done' }, + { name: '_ptc_history.json', content: '{}' }, + ], + }, + }, + 'primary', + ); + assert.deepEqual(phases, [true, false]); + assert.equal(result.run.stdout, 'commit\n'); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('returns pending calls from the private control file even when stdout truncates', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-control-test-')); + let phases = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async createProgrammaticProbeWorkspace(executionDirectory) { + const workspace = join(executionDirectory, 'workspace'); + await mkdir(workspace); + return workspace; + }, + async executeProgrammatic( + _request, + dataDirectory, + _signal, + options, + ) { + assert.equal(options?.probe, true); + phases += 1; + await writeFile( + join(dataDirectory, '_ptc_pending_result.json'), + JSON.stringify({ + pending: [ + { + call_id: 'call_001', + tool_name: 'lookup', + input: {}, + }, + ], + }), + ); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: 'x'.repeat(256 * 1024), + stderr: '', + truncated: true, + timedOut: false, + }; + }, + }, + }); + try { + const result = await executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + execution_id: 'truncated-control', + replay_tool_count: 1, + session_id: 'execution-session', + files: [ + { name: 'main.sh', content: 'lookup "{}"' }, + { name: '_ptc_history.json', content: '{}' }, + ], + }, + }, + 'primary', + ); + assert.equal(phases, 1); + assert.equal(result.run.stdout, ''); + assert.equal(result.run.stderr, ''); + assert.deepEqual(JSON.parse(result.pending_tool_calls_payload ?? ''), { + pending: [{ call_id: 'call_001', tool_name: 'lookup', input: {} }], + }); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +test('rejects traversal before creating execution state', async () => { + let allocated = false; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + allocated = true; + return '/unused'; + }, + async executeProgrammatic() { + throw new Error('unreachable'); + }, + }, + }); + await assert.rejects( + executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + files: [{ name: '../main.sh', content: 'echo unsafe' }], + }, + }, + 'primary', + ), + /Invalid selected-workspace programmatic request/, + ); + assert.equal(allocated, false); +}); + +test('rejects artifacts above the negotiated byte ceiling before upload', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-output-limit-')); + let uploads = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + fetchImpl: async () => { + uploads += 1; + return new Response(); + }, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + await writeFile(join(dataDirectory, 'artifact.txt'), 'too large'); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + try { + await assert.rejects( + executor.execute( + { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + max_output_file_bytes: 4, + files: [{ name: 'main.sh', content: 'printf done' }], + }, + }, + 'primary', + ), + /exceeds the file limit/, + ); + assert.equal(uploads, 0); + } finally { + await rm(scratch, { recursive: true, force: true }); + } +}); + +for (const failure of ['truncated', 'process-error']) test(`a failed speculative probe does not quarantine the real workspace (${failure})`, async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-failed-probe-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { return await mkdtemp(join(scratch, 'execution-')); }, + async createProgrammaticProbeWorkspace(directory) { const root = join(directory, 'workspace'); await mkdir(root); return root; }, + async executeProgrammatic(request, _directory, _signal, options) { + assert.equal(options?.probe, true); + if (failure === 'process-error') throw new WorkspaceToolError('probe output exceeded its limit', 'COMMAND_UNAVAILABLE', true, true); + return { protocolVersion: 1, operation: 'execute_command', workspaceId: request.workspaceId, + exitCode: 0, stdout: '', stderr: '', truncated: true, timedOut: false }; + }, + }, + }); + await assert.rejects(executor.execute({ headers: {}, body: { + language: 'bash', version: '5.2.0', session_id: 'session', replay_tool_count: 1, + files: [{ name: 'main.sh', content: 'true' }], + } }, 'primary'), (error: unknown) => { + assert.match(String(error), /probe output exceeded/); + assert.equal((error as { requiresQuarantine: boolean }).requiresQuarantine, false); + assert.equal((error as { mutationMayHaveCommitted: boolean }).mutationMayHaveCommitted, false); + return true; + }); +}); + +test('unchanged inputs do not consume the negotiated output budget', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-unchanged-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { return await mkdtemp(join(scratch, 'execution-')); }, + async executeProgrammatic(request) { + return { protocolVersion: 1, operation: 'execute_command', workspaceId: request.workspaceId, + exitCode: 0, stdout: '', stderr: '', truncated: false, timedOut: false }; + }, + }, + }); + const result = await executor.execute({ headers: {}, body: { + language: 'bash', version: '5.2.0', session_id: 'session', max_output_file_bytes: 1, + files: [{ name: 'main.sh', content: 'true' }, { name: 'input.txt', content: 'unchanged input' }], + } }, 'primary'); + assert.deepEqual(result.files, []); +}); + +test('stops admitting downloads and drains in-flight transfers before cleanup', async () => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-transfer-test-')); + let executionDirectory = ''; + let requestCount = 0; + const server = createServer((req, res) => { + requestCount += 1; + if (requestCount === 1) { + res.statusCode = 503; + res.end(); + return; + } + setTimeout(() => res.end('in-flight'), 25); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + executionDirectory = await mkdtemp(join(scratch, 'execution-')); + return executionDirectory; + }, + async executeProgrammatic() { + throw new Error('unreachable'); + }, + }, + }); + const request: BridgeWorkspaceProgrammaticRequest = { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + output_session_id: 'output-session', + egress_grant: 'grant', + files: [ + ...Array.from({ length: 8 }, (_, index) => ({ + name: `inputs/${index}.txt`, + id: `input-${index}`, + storage_session_id: 'input-session', + })), + { name: 'main.sh', content: 'printf done' }, + ], + }, + }; + try { + const startedAt = performance.now(); + await assert.rejects( + executor.execute(request, 'primary'), + /Programmatic input download failed with HTTP 503/, + ); + assert.ok(performance.now() - startedAt >= 20); + assert.ok(requestCount <= 4); + assert.deepEqual(await readdir(executionDirectory).catch(() => []), []); + } finally { + await new Promise(resolve => server.close(() => resolve())); + await rm(scratch, { recursive: true, force: true }); + } +}); diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts new file mode 100644 index 00000000..42974b79 --- /dev/null +++ b/packages/code/src/native-programmatic.ts @@ -0,0 +1,816 @@ +import { createHash, randomBytes } from 'node:crypto'; +import { constants } from 'node:fs'; +import { cp, mkdir, open, readdir, rm, writeFile } from 'node:fs/promises'; +import { dirname, join, relative, sep } from 'node:path'; + +import { + BRIDGE_PROTOCOL_VERSION, + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES, + bridgeArtifactMediaType, + isBridgeWorkspaceProgrammaticRequest, + isSafePortableRelativePath, + isSupportedBridgeArtifactName, +} from './protocol.js'; +import { validateFileRelayUpstream } from './relay.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { + BridgeProgrammaticPayloadFile, + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandResult, +} from './protocol.js'; +import type { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; + +const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; +const EXECUTION_MAIN_FILE = 'main.sh'; +const EXECUTION_HISTORY_FILE = '_ptc_history.json'; +const EXECUTION_CONTROL_FILE = '_ptc_pending_result.json'; +export const NATIVE_PROGRAMMATIC_COMMAND = + 'exec "$LIBRECHAT_CODE_BASH_PATH" "$LIBRECHAT_CODE_DATA_DIR/main.sh"'; +const TRANSFER_TIMEOUT_MS = 30_000; +const TRANSFER_CONCURRENCY = 4; +const MAX_WALK_ENTRIES = 2_000; +const INPUT_CACHE_MAX_ENTRIES = 64; +const INPUT_CACHE_MAX_BYTES = 32 * 1024 * 1024; +const CONTROL_PAYLOAD_MAX_BYTES = 512 * 1024; + +type ProgrammaticFileResult = { + id: string; + name: string; + storage_session_id: string; + modified_from?: { id: string; storage_session_id: string }; +}; + +type ProgrammaticResult = { + language: 'bash'; + version: string; + session_id: string; + files: ProgrammaticFileResult[]; + artifact_delivery?: { + code: 'artifact_delivery_failed'; + status: 'partial' | 'failed'; + attempted: number; + delivered: number; + failed: number; + }; + pending_tool_calls_payload?: string; + run: { + stdout: string; + stderr: string; + code: number | null; + signal: string | null; + output: string; + memory: null; + message: string | null; + status: string | null; + cpu_time: null; + wall_time: number; + }; +}; + +type InputBaseline = { + sha256: string; + source?: { id: string; storage_session_id: string }; +}; + +function sha256(value: Uint8Array): string { + return createHash('sha256').update(value).digest('hex'); +} + +function outputFileId(): string { + return randomBytes(18).toString('base64url').slice(0, 21); +} + +function localPath(root: string, name: string): string { + if (!isSafePortableRelativePath(name)) { + throw new WorkspaceToolError( + 'Invalid programmatic file path', + 'INVALID_PATH', + ); + } + const path = join(root, ...name.split('/')); + const child = relative(root, path); + if (child === '' || child === '..' || child.startsWith(`..${sep}`)) { + throw new WorkspaceToolError( + 'Invalid programmatic file path', + 'INVALID_PATH', + ); + } + return path; +} + +async function readBoundedResponse( + response: Response, + signal: AbortSignal, +): Promise { + const declaredLength = response.headers.get('content-length'); + if ( + declaredLength != null && + (!/^\d+$/.test(declaredLength) || + Number(declaredLength) > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES) + ) { + await response.body?.cancel(); + throw new WorkspaceToolError( + 'Programmatic input exceeds the file limit', + 'READ_LIMIT_EXCEEDED', + ); + } + if (!response.body) return Buffer.alloc(0); + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let bytes = 0; + try { + for (;;) { + if (signal.aborted) throw signal.reason; + const { done, value } = await reader.read(); + if (done) break; + bytes += value.byteLength; + if (bytes > BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES) { + throw new WorkspaceToolError( + 'Programmatic input exceeds the file limit', + 'READ_LIMIT_EXCEEDED', + ); + } + chunks.push(Buffer.from(value)); + } + } finally { + await reader.cancel().catch(() => undefined); + } + return Buffer.concat(chunks, bytes); +} + +async function mapConcurrent( + values: readonly T[], + concurrency: number, + action: (value: T) => Promise, +): Promise { + const results = new Array(values.length); + let next = 0; + let failed = false; + let failure: unknown; + await Promise.all( + Array.from( + { length: Math.min(concurrency, values.length) }, + async () => { + for (;;) { + if (failed) return; + const index = next++; + if (index >= values.length) return; + try { + results[index] = await action(values[index]!); + } catch (error) { + if (!failed) { + failed = true; + failure = error; + } + return; + } + } + }, + ), + ); + if (failed) throw failure; + return results; +} + +async function listRegularFiles(root: string): Promise { + const files: string[] = []; + const pending = ['']; + let entries = 0; + while (pending.length > 0) { + const directory = pending.pop()!; + for (const entry of await readdir(join(root, directory), { + withFileTypes: true, + })) { + if (++entries > MAX_WALK_ENTRIES) { + throw new WorkspaceToolError( + 'Programmatic output contains too many entries', + 'WRITE_LIMIT_EXCEEDED', + ); + } + const name = directory ? `${directory}/${entry.name}` : entry.name; + if (!isSafePortableRelativePath(name)) continue; + if (entry.isSymbolicLink()) continue; + if (entry.isDirectory()) pending.push(name); + else if (entry.isFile()) files.push(name); + } + } + return files.sort(); +} + +export interface NativeWorkspaceProgrammaticOptions { + sandbox: Pick< + NativeSrtWorkspaceCommandSandbox, + 'createExecutionDirectory' | 'executeProgrammatic' + > & + Partial< + Pick + >; + upstreamUrl: string; + fetchImpl?: typeof fetch; +} + +/** + * Executes one replay-mode Bash PTC iteration in an attached workspace. + * Program code and injected files live in a private SRT scratch directory; + * the selected repository remains the command cwd and is never used as a + * transport cache. + */ +export class NativeWorkspaceProgrammaticExecutor { + private readonly upstream: URL; + private readonly fetchImpl: typeof fetch; + /** Parent-process cache: sandboxed children cannot inspect this memory. */ + private readonly inputCache = new Map< + string, + { bytes: Buffer; lastUsed: number } + >(); + private inputCacheBytes = 0; + + constructor(private readonly options: NativeWorkspaceProgrammaticOptions) { + this.upstream = validateFileRelayUpstream(options.upstreamUrl); + this.fetchImpl = options.fetchImpl ?? fetch; + } + + /** + * Prove copy-on-write isolation before the worker advertises Bash PTC. + * The probe uses the exact registered root and private scratch path that a + * real replay will use, then removes the snapshot before registration. + */ + async prepare(signal?: AbortSignal): Promise { + const createProbeWorkspace = + this.options.sandbox.createProgrammaticProbeWorkspace; + if (createProbeWorkspace == null) { + throw new WorkspaceToolError( + 'Selected-workspace PTC probe isolation is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const executionDirectory = + await this.options.sandbox.createExecutionDirectory(); + try { + await createProbeWorkspace.call( + this.options.sandbox, + executionDirectory, + signal, + ); + } finally { + await rm(executionDirectory, { recursive: true, force: true }); + } + } + + private cacheKey( + executionId: string | undefined, + file: Extract, + ): string | undefined { + return executionId && file.input_cache_key + ? `${executionId}:${file.input_cache_key}` + : undefined; + } + + private cachedInput(key: string): Buffer | undefined { + const cached = this.inputCache.get(key); + if (!cached) return undefined; + cached.lastUsed = Date.now(); + return cached.bytes; + } + + private cacheInput(key: string, bytes: Buffer): void { + if (bytes.byteLength > INPUT_CACHE_MAX_BYTES) return; + const existing = this.inputCache.get(key); + if (existing) this.inputCacheBytes -= existing.bytes.byteLength; + while ( + this.inputCache.size >= INPUT_CACHE_MAX_ENTRIES || + this.inputCacheBytes + bytes.byteLength > INPUT_CACHE_MAX_BYTES + ) { + let oldestKey: string | undefined; + let oldestAt = Number.POSITIVE_INFINITY; + for (const [candidate, value] of this.inputCache) { + if (value.lastUsed < oldestAt) { + oldestAt = value.lastUsed; + oldestKey = candidate; + } + } + if (!oldestKey) break; + this.inputCacheBytes -= + this.inputCache.get(oldestKey)!.bytes.byteLength; + this.inputCache.delete(oldestKey); + } + this.inputCache.set(key, { bytes, lastUsed: Date.now() }); + this.inputCacheBytes += bytes.byteLength; + } + + private async downloadInput( + file: Extract, + grant: string, + executionId: string | undefined, + signal?: AbortSignal, + transferTimeoutMs = TRANSFER_TIMEOUT_MS, + ): Promise { + const key = this.cacheKey(executionId, file); + const cached = key ? this.cachedInput(key) : undefined; + if (cached) return cached; + const controller = new AbortController(); + const abort = (): void => controller.abort(signal?.reason); + signal?.addEventListener('abort', abort, { once: true }); + const timer = setTimeout(() => controller.abort(), transferTimeoutMs); + try { + const response = await this.fetchImpl( + new URL( + `sessions/${encodeURIComponent(file.storage_session_id)}/objects/${encodeURIComponent(file.id)}`, + `${this.upstream.toString().replace(/\/+$/, '')}/`, + ), + { + headers: { [EGRESS_GRANT_HEADER]: grant }, + redirect: 'error', + signal: controller.signal, + }, + ); + if (!response.ok) { + await response.body?.cancel(); + throw new WorkspaceToolError( + `Programmatic input download failed with HTTP ${response.status}`, + 'COMMAND_UNAVAILABLE', + ); + } + const bytes = await readBoundedResponse( + response, + controller.signal, + ); + if (key) this.cacheInput(key, bytes); + return bytes; + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + } + } + + async execute( + request: BridgeWorkspaceProgrammaticRequest, + workspaceId: string, + signal?: AbortSignal, + ): Promise { + if (!isBridgeWorkspaceProgrammaticRequest(request)) { + throw new WorkspaceToolError( + 'Invalid selected-workspace programmatic request', + 'INVALID_REQUEST', + ); + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + const grant = request.body.egress_grant; + const refFiles = request.body.files.filter( + ( + file, + ): file is Extract => + 'id' in file, + ); + if (refFiles.length > 0 && !grant) { + throw new WorkspaceToolError( + 'Programmatic input grant is unavailable', + 'INVALID_REQUEST', + ); + } + const executionDirectory = + await this.options.sandbox.createExecutionDirectory(); + const inputDirectory = join(executionDirectory, 'inputs'); + let dataDirectory = join(executionDirectory, 'final'); + const baselines = new Map(); + let totalInputBytes = 0; + const startedAt = performance.now(); + let commandDispatched = false; + try { + await mkdir(inputDirectory, { mode: 0o700 }); + await mapConcurrent( + request.body.files, + TRANSFER_CONCURRENCY, + async (file): Promise => { + const bytes = + 'content' in file + ? Buffer.from(file.content) + : await this.downloadInput( + file, + grant!, + request.body.execution_id, + signal, + request.body.transfer_timeout_ms, + ); + totalInputBytes += bytes.byteLength; + if ( + totalInputBytes > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES + ) { + throw new WorkspaceToolError( + 'Programmatic inputs exceed the total byte limit', + 'READ_LIMIT_EXCEEDED', + ); + } + const path = localPath(inputDirectory, file.name); + await mkdir(dirname(path), { + recursive: true, + mode: 0o700, + }); + await writeFile(path, bytes, { flag: 'wx', mode: 0o600 }); + baselines.set(file.name, { + sha256: sha256(bytes), + ...('id' in file + ? { + source: { + id: file.id, + storage_session_id: + file.storage_session_id, + }, + } + : {}), + }); + }, + ); + + const run = async ( + directory: string, + probe: boolean, + workspaceRoot?: string, + ): Promise => { + await cp(inputDirectory, directory, { + recursive: true, + force: false, + errorOnExist: true, + mode: constants.COPYFILE_FICLONE, + }); + if (!probe) commandDispatched = true; + return await this.options.sandbox.executeProgrammatic( + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command', + workspaceId, + command: NATIVE_PROGRAMMATIC_COMMAND, + timeoutMs: Math.min( + request.body.run_timeout ?? + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + ), + maxOutputBytes: + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES, + }, + directory, + signal, + { probe, workspaceRoot }, + ); + }; + + const readPending = async ( + directory: string, + ): Promise => { + try { + const path = join(directory, EXECUTION_CONTROL_FILE); + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const metadata = await handle.stat(); + if ( + !metadata.isFile() || + metadata.size === 0 || + metadata.size > CONTROL_PAYLOAD_MAX_BYTES + ) { + throw new WorkspaceToolError( + 'Native programmatic control frame is invalid', + 'COMMAND_UNAVAILABLE', + ); + } + return await handle.readFile('utf8'); + } finally { + await handle.close(); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') + return; + throw error; + } + }; + + if ((request.body.replay_tool_count ?? 0) > 0) { + const probeDirectory = join(executionDirectory, 'probe'); + const createProbeWorkspace = + this.options.sandbox.createProgrammaticProbeWorkspace; + if (createProbeWorkspace == null) { + throw new WorkspaceToolError( + 'Selected-workspace PTC probe isolation is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const probeWorkspace = await createProbeWorkspace.call( + this.options.sandbox, + executionDirectory, + signal, + ); + const probeResult = await run( + probeDirectory, + true, + probeWorkspace, + ); + const pending = await readPending(probeDirectory); + if (pending) { + return this.result( + request, + { + ...probeResult, + /** Probe output is speculative and the script will + * run once under its real policy after tool + * resolution. Never duplicate it or expose + * expected read-only policy denials to callers. */ + stdout: '', + stderr: '', + }, + [], + performance.now() - startedAt, + pending, + ); + } + if (probeResult.truncated) { + throw new WorkspaceToolError( + 'Native programmatic probe output exceeded its limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + if ( + probeResult.timedOut || + probeResult.signal + ) { + return this.result( + request, + probeResult, + [], + performance.now() - startedAt, + ); + } + /** A read-only probe commonly exits non-zero after it reaches + * an intentional workspace write denial. With no pending call, + * run the script once under its real policy so ordinary writes + * and their resulting exit status are evaluated exactly once. */ + } + + const commandResult = await run(dataDirectory, false); + if (await readPending(dataDirectory)) { + throw new WorkspaceToolError( + 'Native programmatic commit pass requested an unexpected replay tool', + 'COMMAND_UNAVAILABLE', + true, + true, + ); + } + if (commandResult.truncated) { + throw new WorkspaceToolError( + 'Native programmatic output exceeded its limit', + 'WRITE_LIMIT_EXCEEDED', + true, + true, + ); + } + + const outputSessionId = request.body.output_session_id; + const outputNames = (await listRegularFiles(dataDirectory)).filter( + name => + name !== EXECUTION_MAIN_FILE && + name !== EXECUTION_HISTORY_FILE && + name !== EXECUTION_CONTROL_FILE && + !name.startsWith('skills/'), + ); + const changed: Array<{ + name: string; + bytes: Buffer; + source?: { id: string; storage_session_id: string }; + }> = []; + let totalOutputBytes = 0; + for (const name of outputNames) { + const path = localPath(dataDirectory, name); + const baseline = baselines.get(name); + const maxOutputFileBytes = Math.min( + request.body.max_output_file_bytes ?? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + ); + let bytes: Buffer; + const handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const metadata = await handle.stat(); + if (!metadata.isFile()) continue; + // An unchanged input is not an output. It may legitimately + // exceed the negotiated output ceiling, but never the + // protocol's bounded input limit. + if (metadata.size > (baseline ? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES : maxOutputFileBytes)) { + throw new WorkspaceToolError( + 'Programmatic output exceeds the file limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + bytes = await handle.readFile(); + } finally { + await handle.close(); + } + if (baseline?.sha256 === sha256(bytes)) continue; + if (bytes.byteLength > maxOutputFileBytes) { + throw new WorkspaceToolError( + 'Programmatic output exceeds the file limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + totalOutputBytes += bytes.byteLength; + if ( + totalOutputBytes > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES + ) { + throw new WorkspaceToolError( + 'Programmatic outputs exceed the total byte limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } + changed.push({ name, bytes, source: baseline?.source }); + } + const maxOutputFiles = Math.min( + request.body.max_output_files ?? + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + ); + if (changed.length > maxOutputFiles) { + throw new WorkspaceToolError( + 'Programmatic output contains too many files', + 'WRITE_LIMIT_EXCEEDED', + ); + } + const uploadable = changed.filter(({ name }) => + isSupportedBridgeArtifactName(name), + ); + if (uploadable.length > 0 && (!grant || !outputSessionId)) { + throw new WorkspaceToolError( + 'Programmatic output grant is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + const uploadResults = await mapConcurrent( + uploadable, + TRANSFER_CONCURRENCY, + async ({ name, bytes, source }): Promise => { + const id = outputFileId(); + const controller = new AbortController(); + const abort = (): void => controller.abort(signal?.reason); + signal?.addEventListener('abort', abort, { once: true }); + const timer = setTimeout( + () => controller.abort(), + request.body.transfer_timeout_ms ?? TRANSFER_TIMEOUT_MS, + ); + try { + let response: Response; + try { + response = await this.fetchImpl( + new URL( + `sessions/${encodeURIComponent(outputSessionId!)}/objects/${id}`, + `${this.upstream.toString().replace(/\/+$/, '')}/`, + ), + { + method: 'PUT', + headers: { + [EGRESS_GRANT_HEADER]: grant!, + 'Content-Type': bridgeArtifactMediaType(name), + 'Content-Length': String(bytes.byteLength), + 'X-Original-Filename': encodeURIComponent(name), + }, + body: new Uint8Array(bytes), + redirect: 'error', + signal: controller.signal, + }, + ); + } catch (error) { + if (signal?.aborted) throw error; + return undefined; + } + await response.body?.cancel(); + if (!response.ok) { + return undefined; + } + return { + id, + name, + storage_session_id: outputSessionId!, + ...(source ? { modified_from: source } : {}), + }; + } finally { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + } + }, + ); + const files = uploadResults.filter( + (file): file is ProgrammaticFileResult => file != null, + ); + const artifactDelivery = + files.length < changed.length + ? { + code: 'artifact_delivery_failed' as const, + status: files.length > 0 ? ('partial' as const) : ('failed' as const), + attempted: changed.length, + delivered: files.length, + failed: changed.length - files.length, + } + : undefined; + return this.result( + request, + commandResult, + files, + performance.now() - startedAt, + undefined, + artifactDelivery, + ); + } catch (error) { + if (!commandDispatched) { + // The low-level command runner classifies any launched process as a + // possible mutation. A probe can only mutate its disposable snapshot, + // so translate that classification at this ownership boundary. + throw new WorkspaceToolError( + error instanceof Error ? error.message : 'Programmatic preparation failed', + error instanceof WorkspaceToolError ? error.code : 'COMMAND_UNAVAILABLE', + false, + false, + ); + } + if (error instanceof WorkspaceToolError) { + if ( + error.mutationMayHaveCommitted || + error.requiresQuarantine + ) { + throw error; + } + throw new WorkspaceToolError( + error.message, + error.code, + true, + true, + ); + } + throw new WorkspaceToolError( + 'Native programmatic execution failed after dispatch', + 'COMMAND_UNAVAILABLE', + true, + true, + ); + } finally { + try { + await rm(executionDirectory, { recursive: true, force: true }); + } catch { + throw new WorkspaceToolError( + 'Native programmatic execution cleanup failed', + 'COMMAND_UNAVAILABLE', + commandDispatched, + commandDispatched, + ); + } + } + } + + private result( + request: BridgeWorkspaceProgrammaticRequest, + command: WorkspaceExecuteCommandResult, + files: ProgrammaticFileResult[], + elapsedMs: number, + pendingToolCallsPayload?: string, + artifactDelivery?: ProgrammaticResult['artifact_delivery'], + ): ProgrammaticResult { + return { + language: 'bash', + version: request.body.version, + // Code API masks the execution session separately from the writable + // output bucket. Sandbox results must identify the output bucket so the + // gateway can restore it to the caller-owned session after upload. + session_id: + request.body.output_session_id ?? request.body.session_id, + files, + ...(artifactDelivery ? { artifact_delivery: artifactDelivery } : {}), + ...(pendingToolCallsPayload + ? { pending_tool_calls_payload: pendingToolCallsPayload } + : {}), + run: { + stdout: command.stdout, + stderr: command.stderr, + code: command.exitCode, + signal: command.signal ?? null, + output: `${command.stdout}${command.stderr}`, + memory: null, + message: command.timedOut ? 'Execution timed out' : null, + status: command.timedOut ? 'timeout' : null, + cpu_time: null, + wall_time: elapsedMs / 1000, + }, + }; + } +} diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index cf790965..3252ee50 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -7,6 +7,7 @@ import { mkdtemp, mkdir, open, + readFile, realpath, rename, rm, @@ -25,7 +26,10 @@ import type { } from '@anthropic-ai/sandbox-runtime'; import type { ChildProcessWithoutNullStreams } from 'node:child_process'; -import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { + CopyOnWriteCloneUnavailableError, + NativeSrtWorkspaceCommandSandbox, +} from './native-sandbox.js'; import { restoreScratchTraversal } from './native-scratch.js'; import { WorkspaceToolError } from './workspace.js'; @@ -54,6 +58,8 @@ function fakeManager( let credentialSeenDuringWrap: string | undefined; let gitLfsRequiredSeenDuringWrap: string | undefined; let scratchSelectorSeenDuringWrap: string | undefined; + let networkSeenDuringWrap: SandboxRuntimeConfig['network'] | undefined; + let customConfigSeenDuringWrap: Partial | undefined; const manager = { isSupportedPlatform: () => true, async checkDependenciesAsync() { @@ -67,19 +73,30 @@ function fakeManager( askCallback = callback; if (options.initializeError) throw options.initializeError; }, - async wrapWithSandboxArgv(command: string) { + updateConfig(value: SandboxRuntimeConfig) { config = value; }, + async wrapWithSandboxArgv( + command: string, + _binShell?: string, + customConfig?: Partial, + ) { await options.beforeWrap?.(); - credentialSeenDuringWrap = process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; + networkSeenDuringWrap = config?.network; + customConfigSeenDuringWrap = customConfig; + credentialSeenDuringWrap = + process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; gitLfsRequiredSeenDuringWrap = process.env.GIT_CONFIG_VALUE_3; scratchSelectorSeenDuringWrap = process.env.CLAUDE_CODE_TMPDIR; const ambientGitEnvironment = Object.fromEntries( Object.entries(process.env).filter( - ([name, value]) => name.startsWith('GIT_CONFIG_') && value != null, + ([name, value]) => + name.startsWith('GIT_CONFIG_') && value != null, ), ); let gitEnvironment = ambientGitEnvironment; if (options.appendGitSafeDirectory) { - const index = Number(ambientGitEnvironment.GIT_CONFIG_COUNT ?? '0'); + const index = Number( + ambientGitEnvironment.GIT_CONFIG_COUNT ?? '0', + ); gitEnvironment = { ...(options.inheritedGitEnvironment ?? {}), GIT_CONFIG_COUNT: String(index + 1), @@ -130,10 +147,118 @@ function fakeManager( get scratchSelectorSeenDuringWrap() { return scratchSelectorSeenDuringWrap; }, + get networkSeenDuringWrap() { return networkSeenDuringWrap; }, + get customConfigSeenDuringWrap() { + return customConfigSeenDuringWrap; + }, }; } -test('exclusive lifecycle rejects a second workspace sharing an SRT manager', async (t) => { +for (const trustedVm of [false, true]) test(`programmatic probe denies real-workspace writes and external effects (trusted=${trustedVm})`, async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + allowedDomains: ['api.example.com'], + ...(trustedVm ? { commandPolicy: { version: 1 as const, preset: 'trusted-vm' as const, + network: { outbound: 'unrestricted' as const, allowLocalBinding: true, allowAllUnixSockets: true }, + } } : {}), + }); + const dataDirectory = await sandbox.createExecutionDirectory(); + await sandbox.executeProgrammatic(request, dataDirectory, undefined, { + probe: true, + }); + assert.deepEqual(fake.customConfigSeenDuringWrap?.network, { + allowedDomains: [], + deniedDomains: [], + strictAllowlist: true, + allowUnixSockets: [], + allowAllUnixSockets: false, + allowLocalBinding: false, + }); + assert.deepEqual(fake.networkSeenDuringWrap, fake.customConfigSeenDuringWrap?.network); + assert.equal(fake.config?.network.strictAllowlist, !trustedVm); + assert.equal(fake.reset, true, 'probe proxy session must be revoked before restoring policy'); + assert.deepEqual(fake.customConfigSeenDuringWrap?.filesystem?.allowWrite, [ + await realpath(dataDirectory), + ]); + assert.equal( + fake.scratchSelectorSeenDuringWrap, + await realpath(dataDirectory), + ); + assert.ok( + fake.customConfigSeenDuringWrap?.filesystem?.denyWrite?.includes( + await realpath(root), + ), + ); + await sandbox.close(); +}); + +test('probe network cleanup failure fences executor reuse', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-probe-cleanup-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ workspaceRoot: root, manager: fake.manager }); + const directory = await sandbox.createExecutionDirectory(); + const reset = fake.manager.reset; + fake.manager.reset = async () => { throw new Error('proxy shutdown failed'); }; + await assert.rejects(sandbox.executeProgrammatic(request, directory, undefined, { probe: true }), /probe network cleanup failed/); + await assert.rejects(sandbox.execute(request)); + fake.manager.reset = reset; + await sandbox.close(); +}); + +test('programmatic probes use a copy-on-write workspace without mutating the project', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'state.txt'), 'original'); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + const executionDirectory = await sandbox.createExecutionDirectory(); + let snapshot: string; + try { + snapshot = await sandbox.createProgrammaticProbeWorkspace(executionDirectory); + } catch (error) { + if (error instanceof CopyOnWriteCloneUnavailableError) { + t.skip('host filesystem does not support copy-on-write cloning'); + return; + } + throw error; + } + await writeFile(join(snapshot, 'state.txt'), 'probe-only'); + assert.equal(await readFile(join(root, 'state.txt'), 'utf8'), 'original'); + assert.equal(await readFile(join(snapshot, 'state.txt'), 'utf8'), 'probe-only'); +}); + +test('programmatic probes do not hide clone implementation failures as unsupported filesystems', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + spawnCommand() { + throw Object.assign(new Error('spawn /bin/cp ENOENT'), { code: 'ENOENT' }); + }, + }); + t.after(() => sandbox.close()); + const executionDirectory = await sandbox.createExecutionDirectory(); + + await assert.rejects( + sandbox.createProgrammaticProbeWorkspace(executionDirectory), + (error: unknown) => + error instanceof WorkspaceToolError && + !(error instanceof CopyOnWriteCloneUnavailableError) && + error.message === 'Copy-on-write workspace clone failed unexpectedly', + ); +}); + +test('exclusive lifecycle rejects a second workspace sharing an SRT manager', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -164,15 +289,15 @@ test('exclusive lifecycle rejects a second workspace sharing an SRT manager', as assert.equal((await second.execute(request)).stdout, 'hello'); }); -test('exclusive lifecycle rejects overlapping commands and waits before resetting', async (t) => { +test('exclusive lifecycle rejects overlapping commands and waits before resetting', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); let entered!: () => void; - const wrapping = new Promise((resolve) => { + const wrapping = new Promise(resolve => { entered = resolve; }); let release!: () => void; - const gate = new Promise((resolve) => { + const gate = new Promise(resolve => { release = resolve; }); const fake = fakeManager({ @@ -191,7 +316,7 @@ test('exclusive lifecycle rejects overlapping commands and waits before resettin await assert.rejects(sandbox.execute(request), /active command/); const closing = sandbox.close(); const secondClose = sandbox.close(); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal(fake.reset, false); await assert.rejects(sandbox.prepare(), /closing/); release(); @@ -200,7 +325,7 @@ test('exclusive lifecycle rejects overlapping commands and waits before resettin assert.equal(fake.reset, true); }); -test('exclusive lifecycle retains ownership after a failed reset until cleanup succeeds', async (t) => { +test('exclusive lifecycle retains ownership after a failed reset until cleanup succeeds', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -226,16 +351,16 @@ test('exclusive lifecycle retains ownership after a failed reset until cleanup s await second.close(); }); -test('exclusive lifecycle waits for initialization before resetting the manager', async (t) => { +test('exclusive lifecycle waits for initialization before resetting the manager', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); let entered!: () => void; - const initializing = new Promise((resolve) => { + const initializing = new Promise(resolve => { entered = resolve; }); let release!: () => void; - const gate = new Promise((resolve) => { + const gate = new Promise(resolve => { release = resolve; }); fake.manager.initialize = async () => { @@ -249,7 +374,7 @@ test('exclusive lifecycle waits for initialization before resetting the manager' const preparing = sandbox.prepare(); await initializing; const closing = sandbox.close(); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal(fake.reset, false); release(); await preparing; @@ -257,7 +382,7 @@ test('exclusive lifecycle waits for initialization before resetting the manager' assert.equal(fake.reset, true); }); -test('initializes SRT with a default-deny network and scrubbed worker credentials', async (t) => { +test('initializes SRT with a default-deny network and scrubbed worker credentials', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const identity = join(tmpdir(), 'librechat-code-identity.json'); t.after(() => rm(root, { recursive: true, force: true })); @@ -301,7 +426,7 @@ test('initializes SRT with a default-deny network and scrubbed worker credential assert.ok(fake.config?.filesystem.denyRead.includes(canonicalHome)); assert.ok(fake.config?.filesystem.denyWrite.includes(canonicalIdentity)); assert.ok( - fake.config?.filesystem.denyWrite.some((path) => + fake.config?.filesystem.denyWrite.some(path => path.endsWith('/tmp/claude'), ), ); @@ -318,7 +443,7 @@ test('initializes SRT with a default-deny network and scrubbed worker credential await assert.rejects(access(scratchDirectory!)); }); -test('trusted-vm permits unmatched egress and local development sockets', async (t) => { +test('trusted-vm permits unmatched egress and local development sockets', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -351,7 +476,7 @@ test('trusted-vm permits unmatched egress and local development sockets', async ]); }); -test('provides an isolated scratch directory to commands and restores the host environment', async (t) => { +test('provides an isolated scratch directory to commands and restores the host environment', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const originalTmpdir = process.env.TMPDIR; @@ -380,7 +505,7 @@ test('provides an isolated scratch directory to commands and restores the host e await assert.rejects(access(result.stdout)); }); -test('removes scratch storage when SRT initialization fails', async (t) => { +test('removes scratch storage when SRT initialization fails', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager({ initializeError: new Error('init failed') }); @@ -396,7 +521,7 @@ test('removes scratch storage when SRT initialization fails', async (t) => { assert.equal(fake.reset, true); }); -test('rejects workspaces nested inside SRT shared scratch storage', async (t) => { +test('rejects workspaces nested inside SRT shared scratch storage', async t => { if (process.platform === 'win32') return; const sharedRoot = '/tmp/claude'; await mkdir(sharedRoot, { recursive: true }); @@ -431,17 +556,17 @@ test('rejects a workspace that contains worker scratch storage', async () => { await sandbox.close(); }); -test('keeps concurrent sandbox scratch directories independent', async (t) => { +test('keeps concurrent sandbox scratch directories independent', async t => { const firstRoot = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const secondRoot = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(firstRoot, { recursive: true, force: true })); t.after(() => rm(secondRoot, { recursive: true, force: true })); let releaseWrap!: () => void; let wrapStarted!: () => void; - const wrapStartedPromise = new Promise((resolve) => { + const wrapStartedPromise = new Promise(resolve => { wrapStarted = resolve; }); - const holdWrap = new Promise((resolve) => { + const holdWrap = new Promise(resolve => { releaseWrap = resolve; }); const firstFake = fakeManager({ @@ -478,7 +603,7 @@ test('keeps concurrent sandbox scratch directories independent', async (t) => { await secondSandbox.close(); }); -test('removes scratch storage after a command revokes traversal permissions', async (t) => { +test('removes scratch storage after a command revokes traversal permissions', async t => { if (process.platform === 'win32') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -498,7 +623,7 @@ test('removes scratch storage after a command revokes traversal permissions', as await assert.rejects(access(result.stdout)); }); -test('scratch traversal never follows a descendant replaced after inspection', async (t) => { +test('scratch traversal never follows a descendant replaced after inspection', async t => { if (process.platform === 'win32') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-race-')); const outside = await mkdtemp(join(tmpdir(), 'librechat-code-outside-')); @@ -529,7 +654,7 @@ test('scratch traversal never follows a descendant replaced after inspection', a assert.equal((await stat(outsideChild)).mode & 0o777, 0o711); }); -test('scratch traversal removes command-created Darwin ACLs', async (t) => { +test('scratch traversal removes command-created Darwin ACLs', async t => { if (process.platform !== 'darwin') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -548,7 +673,7 @@ test('scratch traversal removes command-created Darwin ACLs', async (t) => { await assert.rejects(access(result.stdout)); }); -test('scratch traversal bounds descriptors and work across a deep tree', async (t) => { +test('scratch traversal bounds descriptors and work across a deep tree', async t => { if (process.platform === 'win32') return; const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-depth-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -565,12 +690,17 @@ test('scratch traversal bounds descriptors and work across a deep tree', async ( await restoreScratchTraversal(rootHandle); - assert.equal((await stat(directories[directories.length - 1])).mode & 0o777, 0o700); + assert.equal( + (await stat(directories[directories.length - 1])).mode & 0o777, + 0o700, + ); }); -test('scratch traversal rejects trees beyond its recovery depth limit', async (t) => { +test('scratch traversal rejects trees beyond its recovery depth limit', async t => { if (process.platform === 'win32') return; - const root = await mkdtemp(join(tmpdir(), 'librechat-code-scratch-depth-limit-')); + const root = await mkdtemp( + join(tmpdir(), 'librechat-code-scratch-depth-limit-'), + ); t.after(() => rm(root, { recursive: true, force: true })); let directory = root; for (let depth = 0; depth < 129; depth += 1) { @@ -586,7 +716,7 @@ test('scratch traversal rejects trees beyond its recovery depth limit', async (t ); }); -test('does not replace scratch state while cleanup remains pending', async (t) => { +test('does not replace scratch state while cleanup remains pending', async t => { if (process.platform === 'win32') return; const workspace = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const retained = await mkdtemp(join(tmpdir(), 'librechat-code-retained-')); @@ -642,7 +772,7 @@ const windowsEnvironment = { }; for (const platform of ['darwin', 'linux', 'win32'] as const) { - test(`preserves required ${platform} environment names without allowing credentials`, async (t) => { + test(`preserves required ${platform} environment names without allowing credentials`, async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -658,23 +788,43 @@ for (const platform of ['darwin', 'linux', 'win32'] as const) { LD_PRELOAD: '/host/private.so', }; const sandbox = new NativeSrtWorkspaceCommandSandbox({ - workspaceRoot: root, platform, allowedDomains: ['github.com'], + workspaceRoot: root, + platform, + allowedDomains: ['github.com'], environment: { - ...proxyEnvironment, ...windowsEnvironment, ...credentials, + ...proxyEnvironment, + ...windowsEnvironment, + ...credentials, HtTp_PrOxY: 'http://mixed-case.invalid:8080', - PATH: '/usr/bin', LC_ALL: 'C.UTF-8', + PATH: '/usr/bin', + LC_ALL: 'C.UTF-8', }, manager: fake.manager, }); t.after(() => sandbox.close()); await sandbox.prepare(); - const denied = new Set(fake.config?.credentials?.envVars - ?.filter(({ mode }) => mode === 'deny').map(({ name }) => name)); - for (const name of [...Object.keys(proxyEnvironment), 'PATH', 'LC_ALL']) { - assert.equal(denied.has(name), false, `${name} must remain available`); + const denied = new Set( + fake.config?.credentials?.envVars + ?.filter(({ mode }) => mode === 'deny') + .map(({ name }) => name), + ); + for (const name of [ + ...Object.keys(proxyEnvironment), + 'PATH', + 'LC_ALL', + ]) { + assert.equal( + denied.has(name), + false, + `${name} must remain available`, + ); } for (const name of Object.keys(windowsEnvironment)) { - assert.equal(denied.has(name), platform !== 'win32', `${name} must be platform-specific`); + assert.equal( + denied.has(name), + platform !== 'win32', + `${name} must be platform-specific`, + ); } assert.equal(denied.has('HtTp_PrOxY'), platform !== 'win32'); for (const name of Object.keys(credentials)) { @@ -685,12 +835,14 @@ for (const platform of ['darwin', 'linux', 'win32'] as const) { }); } -test('uses SRT proxy values without restoring inherited proxies or credentials', async (t) => { +test('uses SRT proxy values without restoring inherited proxies or credentials', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const wrappedEnvironment = { - HTTP_PROXY: 'http://localhost:3128', HTTPS_PROXY: 'http://localhost:3128', - ALL_PROXY: 'http://localhost:3128', NO_PROXY: 'localhost', + HTTP_PROXY: 'http://localhost:3128', + HTTPS_PROXY: 'http://localhost:3128', + ALL_PROXY: 'http://localhost:3128', + NO_PROXY: 'localhost', }; const fake = fakeManager({ wrappedEnvironment }); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -700,14 +852,19 @@ test('uses SRT proxy values without restoring inherited proxies or credentials', }); t.after(() => sandbox.close()); const result = await sandbox.execute({ - ...request, maxOutputBytes: 256, - command: 'printf "%s|%s|%s|%s|%s" "$HTTP_PROXY" "$HTTPS_PROXY" "$ALL_PROXY" "$NO_PROXY" "${GITHUB_TOKEN-unset}"', + ...request, + maxOutputBytes: 256, + command: + 'printf "%s|%s|%s|%s|%s" "$HTTP_PROXY" "$HTTPS_PROXY" "$ALL_PROXY" "$NO_PROXY" "${GITHUB_TOKEN-unset}"', }); assert.equal(result.exitCode, 0); - assert.equal(result.stdout, `${Object.values(wrappedEnvironment).join('|')}|unset`); + assert.equal( + result.stdout, + `${Object.values(wrappedEnvironment).join('|')}|unset`, + ); }); -test('masks a host credential for only its injection host and restores the parent environment', async (t) => { +test('masks a host credential for only its injection host and restores the parent environment', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -731,7 +888,8 @@ test('masks a host credential for only its injection host and restores the paren ], async resolve() { return { - LIBRECHAT_CODE_TEST_CREDENTIAL: 'Authorization: Bearer real-secret', + LIBRECHAT_CODE_TEST_CREDENTIAL: + 'Authorization: Bearer real-secret', }; }, }, @@ -759,7 +917,7 @@ test('masks a host credential for only its injection host and restores the paren }); }); -test('serializes credential handoff across concurrent sandbox instances', async (t) => { +test('serializes credential handoff across concurrent sandbox instances', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const original = process.env.LIBRECHAT_CODE_TEST_CREDENTIAL; @@ -770,11 +928,11 @@ test('serializes credential handoff across concurrent sandbox instances', async else process.env.LIBRECHAT_CODE_TEST_CREDENTIAL = original; }); let firstEntered!: () => void; - const firstEnteredPromise = new Promise((resolve) => { + const firstEnteredPromise = new Promise(resolve => { firstEntered = resolve; }); let releaseFirst!: () => void; - const firstGate = new Promise((resolve) => { + const firstGate = new Promise(resolve => { releaseFirst = resolve; }); let secondEntered = false; @@ -817,7 +975,7 @@ test('serializes credential handoff across concurrent sandbox instances', async const secondExecution = sandbox(second.manager, 'second-secret').execute( request, ); - await new Promise((resolve) => setImmediate(resolve)); + await new Promise(resolve => setImmediate(resolve)); assert.equal(secondEntered, false); releaseFirst(); await firstExecution; @@ -828,7 +986,7 @@ test('serializes credential handoff across concurrent sandbox instances', async assert.equal(process.env.LIBRECHAT_CODE_TEST_CREDENTIAL, undefined); }); -test('isolates Git from host-level global and system configuration', async (t) => { +test('isolates Git from host-level global and system configuration', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -844,7 +1002,7 @@ test('isolates Git from host-level global and system configuration', async (t) = assert.equal(result.stdout, '/dev/null|1'); }); -test('restores trusted Git LFS filters without reading host Git configuration', async (t) => { +test('restores trusted Git LFS filters without reading host Git configuration', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager({ @@ -884,7 +1042,7 @@ test('restores trusted Git LFS filters without reading host Git configuration', assert.ok(!denied?.includes('GIT_CONFIG_VALUE_0')); }); -test('filters environment names case-insensitively only on Windows', async (t) => { +test('filters environment names case-insensitively only on Windows', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager(); @@ -923,7 +1081,7 @@ test('filters environment names case-insensitively only on Windows', async (t) = assert.ok(!denied?.includes('git_config_count')); }); -test('fails closed when the configured POSIX shell is unavailable', async (t) => { +test('fails closed when the configured POSIX shell is unavailable', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -942,7 +1100,7 @@ test('fails closed when the configured POSIX shell is unavailable', async (t) => ); }); -test('fails closed when SRT dependencies are unavailable', async (t) => { +test('fails closed when SRT dependencies are unavailable', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const fake = fakeManager({ dependencyErrors: ['bubblewrap missing'] }); @@ -960,7 +1118,7 @@ test('fails closed when SRT dependencies are unavailable', async (t) => { ); }); -test('refuses workspace roots that expose worker home or control files', async (t) => { +test('refuses workspace roots that expose worker home or control files', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const controlDirectory = join(root, '.control'); await mkdir(controlDirectory); @@ -985,7 +1143,7 @@ test('refuses workspace roots that expose worker home or control files', async ( ); }); -test('executes in the canonical workspace and bounds aggregate output', async (t) => { +test('executes in the canonical workspace and bounds aggregate output', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); await mkdir(join(root, 'src')); t.after(() => rm(root, { recursive: true, force: true })); @@ -1015,7 +1173,7 @@ test('executes in the canonical workspace and bounds aggregate output', async (t ); }); -test('rejects an escaping or unavailable command working directory', async (t) => { +test('rejects an escaping or unavailable command working directory', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1026,11 +1184,12 @@ test('rejects an escaping or unavailable command working directory', async (t) = await assert.rejects( sandbox.execute({ ...request, cwd: '..' }), (error: unknown) => - error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + error instanceof WorkspaceToolError && + error.code === 'INVALID_REQUEST', ); }); -test('terminates detached command descendants before returning', async (t) => { +test('terminates detached command descendants before returning', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1043,15 +1202,15 @@ test('terminates detached command descendants before returning', async (t) => { command: '(sleep 0.2; printf late > late.txt) >/dev/null 2>&1 &', }); assert.equal(result.exitCode, 0); - await new Promise((resolve) => setTimeout(resolve, 350)); + await new Promise(resolve => setTimeout(resolve, 350)); await assert.rejects(access(join(root, 'late.txt'))); }); -test('reports cancellation after command start as a potentially committed mutation', async (t) => { +test('reports cancellation after command start as a potentially committed mutation', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); let commandStarted!: () => void; - const commandStartedPromise = new Promise((resolve) => { + const commandStartedPromise = new Promise(resolve => { commandStarted = resolve; }); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1081,7 +1240,7 @@ test('reports cancellation after command start as a potentially committed mutati ); }); -test('closes stdin immediately when the command protocol provides no input', async (t) => { +test('closes stdin immediately when the command protocol provides no input', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const sandbox = new NativeSrtWorkspaceCommandSandbox({ @@ -1098,7 +1257,7 @@ test('closes stdin immediately when the command protocol provides no input', asy assert.equal(result.timedOut, false); }); -test('maps platform-native exit statuses into the bridge protocol range', async (t) => { +test('maps platform-native exit statuses into the bridge protocol range', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); const spawnCommand = () => { @@ -1123,7 +1282,7 @@ test('maps platform-native exit statuses into the bridge protocol range', async assert.equal(result.exitCode, 1); }); -test('cleans allocated command state exactly once on every execution exit', async (t) => { +test('cleans allocated command state exactly once on every execution exit', async t => { for (const outcome of [ 'abort-before-spawn', 'spawn-throw', @@ -1134,8 +1293,12 @@ test('cleans allocated command state exactly once on every execution exit', asyn 'wrap-throw', ] as const) { for (const cleanupThrows of [false, true]) { - await t.test(`${outcome}, cleanup throws: ${cleanupThrows}`, async (t) => { - const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + await t.test( + `${outcome}, cleanup throws: ${cleanupThrows}`, + async t => { + const root = await mkdtemp( + join(tmpdir(), 'librechat-code-native-'), + ); t.after(() => rm(root, { recursive: true, force: true })); const controller = new AbortController(); let cleanupCalls = 0; @@ -1143,9 +1306,11 @@ test('cleans allocated command state exactly once on every execution exit', asyn let allocated = false; const fake = fakeManager({ async beforeWrap() { - if (outcome === 'wrap-throw') throw new Error('wrap failed'); + if (outcome === 'wrap-throw') + throw new Error('wrap failed'); allocated = true; - if (outcome === 'abort-before-spawn') controller.abort(); + if (outcome === 'abort-before-spawn') + controller.abort(); }, }); fake.manager.cleanupAfterCommand = () => { @@ -1160,13 +1325,17 @@ test('cleans allocated command state exactly once on every execution exit', asyn spawnCommand() { spawnCalls += 1; assert.equal(allocated, true); - if (outcome === 'spawn-throw') throw new Error('spawn failed'); - const child = new EventEmitter() as ChildProcessWithoutNullStreams; + if (outcome === 'spawn-throw') + throw new Error('spawn failed'); + const child = + new EventEmitter() as ChildProcessWithoutNullStreams; let closeQueued = false; const close = () => { if (!closeQueued) { closeQueued = true; - queueMicrotask(() => child.emit('close', null, 'SIGKILL')); + queueMicrotask(() => + child.emit('close', null, 'SIGKILL'), + ); } return true; }; @@ -1180,7 +1349,10 @@ test('cleans allocated command state exactly once on every execution exit', asyn queueMicrotask(() => { assert.equal(cleanupCalls, 0); if (outcome === 'error') { - child.emit('error', new Error('spawn failed')); + child.emit( + 'error', + new Error('spawn failed'), + ); } else if (outcome === 'abort-after-spawn') { controller.abort(); } else if (outcome === 'close') { @@ -1196,28 +1368,43 @@ test('cleans allocated command state exactly once on every execution exit', asyn ); if (outcome === 'close' || outcome === 'timeout') { const result = await execution; - assert.equal(result.exitCode, outcome === 'close' ? 0 : null); + assert.equal( + result.exitCode, + outcome === 'close' ? 0 : null, + ); assert.equal(result.timedOut, outcome === 'timeout'); } else { - await assert.rejects(execution, (error: unknown) => + await assert.rejects( + execution, + (error: unknown) => error instanceof WorkspaceToolError && - error.code === (outcome.startsWith('abort') + error.code === + (outcome.startsWith('abort') ? 'EXECUTION_ABORTED' : 'COMMAND_UNAVAILABLE') && - error.mutationMayHaveCommitted === (outcome === 'abort-after-spawn') && + error.mutationMayHaveCommitted === + (outcome === 'abort-after-spawn') && error.requiresQuarantine === - (outcome === 'abort-after-spawn' && process.platform === 'win32'), + (outcome === 'abort-after-spawn' && + process.platform === 'win32'), ); } assert.equal( spawnCalls, - outcome === 'abort-before-spawn' || outcome === 'wrap-throw' ? 0 : 1, + outcome === 'abort-before-spawn' || + outcome === 'wrap-throw' + ? 0 + : 1, + ); + assert.equal( + cleanupCalls, + outcome === 'wrap-throw' ? 0 : 1, ); - assert.equal(cleanupCalls, outcome === 'wrap-throw' ? 0 : 1); assert.equal(allocated, false); await sandbox.close(); assert.equal(fake.reset, true); - }); + }, + ); } } }); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 171ea663..8d150d9f 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -11,14 +11,7 @@ import { sep, } from 'node:path'; import { constants as fsConstants } from 'node:fs'; -import { - access, - mkdtemp, - open, - realpath, - rm, - stat, -} from 'node:fs/promises'; +import { access, mkdtemp, open, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { SandboxManager } from '@anthropic-ai/sandbox-runtime'; @@ -141,8 +134,12 @@ interface NativeSandboxManager { cwd?: string, options?: { commandId?: string; commandText?: string }, ): Promise<{ argv: string[]; env: NodeJS.ProcessEnv }>; - annotateStderrWithSandboxFailures(commandId: string, stderr: string): string; + annotateStderrWithSandboxFailures( + commandId: string, + stderr: string, + ): string; cleanupAfterCommand(): void; + updateConfig?(config: SandboxRuntimeConfig): void; reset(): Promise; } @@ -224,13 +221,16 @@ function deniedEnvironmentNames( platform: NodeJS.Platform, ): string[] { return Object.keys(environment) - .filter((name) => { + .filter(name => { const normalized = platform === 'win32' ? name.toUpperCase() : name; return ( normalized.startsWith('LIBRECHAT_CODE_') || (!SAFE_CHILD_ENV_NAMES.has(normalized) && !PROXY_CHILD_ENV_NAMES.has(normalized) && - !(platform === 'win32' && WINDOWS_CHILD_ENV_NAMES.has(normalized)) && + !( + platform === 'win32' && + WINDOWS_CHILD_ENV_NAMES.has(normalized) + ) && !normalized.startsWith('LC_')) ); }) @@ -244,6 +244,36 @@ function normalizedEnvironmentName( return platform === 'win32' ? name.toUpperCase() : name; } +/** Distinguishes an unsupported host filesystem from an implementation fault. */ +export class CopyOnWriteCloneUnavailableError extends WorkspaceToolError { + constructor() { + super( + 'Selected-workspace PTC requires copy-on-write filesystem cloning', + 'COMMAND_UNAVAILABLE', + ); + this.name = 'CopyOnWriteCloneUnavailableError'; + } +} + +function isCopyOnWriteUnsupported( + error: unknown, + platform: NodeJS.Platform, +): boolean { + if (platform === 'win32') return true; + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + (error.code === 'ENOTSUP' || error.code === 'EOPNOTSUPP') + ) { + return true; + } + return ( + error instanceof Error && + error.message.toLowerCase().includes('operation not supported') + ); +} + export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox { readonly mutationFailuresAreAtomic = true as const; private readonly manager: NativeSandboxManager; @@ -252,6 +282,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private readonly platform: NodeJS.Platform; private initialized?: Promise; private canonicalRoot?: string; + private runtimeConfig?: SandboxRuntimeConfig; + private denyReadPaths: string[] = []; + private denyWritePaths: string[] = []; private scratchDirectory?: string; private scratchHandle?: FileHandle; private execution?: Promise; @@ -288,7 +321,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } managerOwners.set(this.manager, this); - this.initialized = this.initializeOnce().catch(async (error) => { + this.initialized = this.initializeOnce().catch(async error => { await this.manager.reset().catch(() => { this.resetFailed = true; }); @@ -314,7 +347,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'COMMAND_UNAVAILABLE', ); } - const home = await canonicalPath(this.options.homeDirectory ?? homedir()); + const home = await canonicalPath( + this.options.homeDirectory ?? homedir(), + ); if (isWithin(root, home)) { throw new WorkspaceToolError( 'Native sandbox workspace cannot contain the worker home directory', @@ -324,7 +359,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const protectedPaths = await Promise.all( (this.options.protectedPaths ?? []).map(canonicalPath), ); - if (protectedPaths.some((path) => isWithin(root, path))) { + if (protectedPaths.some(path => isWithin(root, path))) { throw new WorkspaceToolError( 'Native sandbox workspace cannot contain worker control files', 'REGISTRATION_INVALID', @@ -338,13 +373,16 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const inheritedWritablePaths = [ ...sharedScratchPaths, ...(await Promise.all( - [join(home, '.npm', '_logs'), join(home, '.claude', 'debug')].map( - canonicalPath, - ), + [ + join(home, '.npm', '_logs'), + join(home, '.claude', 'debug'), + ].map(canonicalPath), )), ]; - const deniedInheritedWritablePaths = [...new Set(inheritedWritablePaths)]; - if (deniedInheritedWritablePaths.some((path) => isWithin(path, root))) { + const deniedInheritedWritablePaths = [ + ...new Set(inheritedWritablePaths), + ]; + if (deniedInheritedWritablePaths.some(path => isWithin(path, root))) { throw new WorkspaceToolError( 'Native sandbox workspace cannot be inside an inherited writable path', 'REGISTRATION_INVALID', @@ -359,7 +397,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } if (this.platform !== 'win32') { try { - await access(this.options.shellPath ?? '/bin/bash', fsConstants.X_OK); + await access( + this.options.shellPath ?? '/bin/bash', + fsConstants.X_OK, + ); } catch { throw new WorkspaceToolError( `Native sandbox shell is unavailable: ${this.options.shellPath ?? '/bin/bash'}`, @@ -383,35 +424,40 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); const unrestrictedNetwork = commandPolicy.network.outbound === 'unrestricted'; - const config: SandboxRuntimeConfig = { - network: { + const network: SandboxRuntimeConfig['network'] = { allowedDomains: [...(this.options.allowedDomains ?? [])], deniedDomains: [], strictAllowlist: !unrestrictedNetwork, allowAllUnixSockets: commandPolicy.network.allowAllUnixSockets, allowLocalBinding: commandPolicy.network.allowLocalBinding, ...(this.options.maskedEnvironment ? { tlsTerminate: {} } : {}), - }, + }; + const config: SandboxRuntimeConfig = { + network, filesystem: { denyRead: [ home, - ...sharedScratchPaths.filter((path) => + ...sharedScratchPaths.filter(path => deniedInheritedWritablePaths.includes(path), ), ], allowRead: [ root, - ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), + ...(canonicalScratchDirectory + ? [canonicalScratchDirectory] + : []), ], allowWrite: [ root, - ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), + ...(canonicalScratchDirectory + ? [canonicalScratchDirectory] + : []), ], denyWrite: [...protectedPaths, ...deniedInheritedWritablePaths], allowGitConfig: false, }, credentials: { - files: protectedPaths.map((path) => ({ + files: protectedPaths.map(path => ({ path, mode: 'deny' as const, })), @@ -424,15 +470,18 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox }, this.platform, ) - .filter((name) => { + .filter(name => { const normalized = normalizedEnvironmentName( name, this.platform, ); return ( - !Object.hasOwn(TRUSTED_GIT_ENVIRONMENT, normalized) && + !Object.hasOwn( + TRUSTED_GIT_ENVIRONMENT, + normalized, + ) && !this.options.maskedEnvironment?.variables.some( - (variable) => + variable => normalizedEnvironmentName( variable.name, this.platform, @@ -440,12 +489,16 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ) ); }) - .map((name) => ({ name, mode: 'deny' as const })), - ...(this.options.maskedEnvironment?.variables.map((variable) => ({ + .map(name => ({ name, mode: 'deny' as const })), + ...(this.options.maskedEnvironment?.variables.map( + variable => ({ ...variable, mode: 'mask' as const, - ...(variable.extract ? { onExtractNoMatch: 'error' as const } : {}), - })) ?? []), + ...(variable.extract + ? { onExtractNoMatch: 'error' as const } + : {}), + }), + ) ?? []), ], }, allowAppleEvents: false, @@ -458,6 +511,17 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox unrestrictedNetwork ? async () => true : undefined, ); this.canonicalRoot = root; + this.runtimeConfig = config; + this.denyReadPaths = [ + home, + ...sharedScratchPaths.filter(path => + deniedInheritedWritablePaths.includes(path), + ), + ]; + this.denyWritePaths = [ + ...protectedPaths, + ...deniedInheritedWritablePaths, + ]; } async execute( @@ -479,9 +543,248 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } } + /** + * Allocate an owner-only execution directory that is already inside this + * sandbox's allowlist. The caller must remove the returned directory after + * the execution settles. It is intentionally unavailable on native Windows + * until the restricted-account TEMP directory can be opened and verified by + * the trusted parent process. + */ + async createExecutionDirectory(): Promise { + await this.initialize(); + if (!this.scratchDirectory || this.platform === 'win32') { + throw new WorkspaceToolError( + 'Native programmatic execution storage is unavailable', + 'COMMAND_UNAVAILABLE', + ); + } + return await mkdtemp(join(this.scratchDirectory, 'execution-')); + } + + /** + * Clone the current workspace into private scratch for a side-effect- + * equivalent replay probe. Platform clone flags are intentionally strict: + * silently falling back to a byte copy would make every tool-bearing run + * consume time and disk proportional to the repository size. + */ + async createProgrammaticProbeWorkspace( + executionDirectory: string, + signal?: AbortSignal, + ): Promise { + await this.initialize(); + const scratchDirectory = this.scratchDirectory; + const root = this.canonicalRoot; + let parent: string; + try { + parent = await realpath(executionDirectory); + if ( + !scratchDirectory || + !root || + !isWithin(scratchDirectory, parent) || + !(await stat(parent)).isDirectory() + ) { + throw new Error('invalid execution directory'); + } + } catch { + throw new WorkspaceToolError( + 'Programmatic execution directory is unavailable', + 'INVALID_PATH', + ); + } + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + const destination = join(parent, 'workspace'); + try { + if (this.platform === 'win32') { + throw new Error('copy-on-write cloning is unavailable on Windows'); + } + const args = + this.platform === 'darwin' + ? ['-cR', root, destination] + : ['--archive', '--reflink=always', root, destination]; + await new Promise((resolveCopy, rejectCopy) => { + const child = this.spawnCommand('/bin/cp', args, { + env: { + PATH: this.environment.PATH, + LANG: this.environment.LANG, + LC_ALL: this.environment.LC_ALL, + }, + signal, + }); + let stderr = Buffer.alloc(0); + child.stderr.on('data', (chunk: Buffer) => { + if (stderr.byteLength < 4_096) { + stderr = Buffer.concat([stderr, chunk]).subarray(0, 4_096); + } + }); + child.once('error', rejectCopy); + child.once('close', code => { + if (code === 0) resolveCopy(); + else { + rejectCopy( + new Error( + `copy-on-write clone failed (${code ?? 'signal'}): ${boundedUtf8(stderr, 4_096)}`, + ), + ); + } + }); + }); + return await realpath(destination); + } catch (error) { + await rm(destination, { recursive: true, force: true }).catch( + () => undefined, + ); + if (signal?.aborted) { + throw new WorkspaceToolError( + 'Programmatic execution aborted', + 'EXECUTION_ABORTED', + ); + } + if (isCopyOnWriteUnsupported(error, this.platform)) { + throw new CopyOnWriteCloneUnavailableError(); + } + throw new WorkspaceToolError( + 'Copy-on-write workspace clone failed unexpectedly', + 'COMMAND_UNAVAILABLE', + ); + } + } + + /** Run a generated program from a verified private execution directory. */ + async executeProgrammatic( + request: WorkspaceExecuteCommandRequest, + dataDirectory: string, + signal?: AbortSignal, + options?: { probe?: boolean; workspaceRoot?: string }, + ): Promise { + if (this.execution || this.closing) { + throw new WorkspaceToolError( + 'Native sandbox already has an active command or is closing', + 'COMMAND_UNAVAILABLE', + ); + } + await this.initialize(); + const scratchDirectory = this.scratchDirectory; + let canonicalDataDirectory: string; + let canonicalWorkspaceRoot: string | undefined; + try { + canonicalDataDirectory = await realpath(dataDirectory); + canonicalWorkspaceRoot = options?.workspaceRoot + ? await realpath(options.workspaceRoot) + : undefined; + if ( + !scratchDirectory || + !isWithin(scratchDirectory, canonicalDataDirectory) || + !(await stat(canonicalDataDirectory)).isDirectory() || + (canonicalWorkspaceRoot != null && + (!isWithin(scratchDirectory, canonicalWorkspaceRoot) || + !(await stat(canonicalWorkspaceRoot)).isDirectory())) + ) { + throw new Error('invalid execution directory'); + } + } catch { + throw new WorkspaceToolError( + 'Programmatic execution directory is unavailable', + 'INVALID_PATH', + ); + } + const execute = () => this.executeExclusive( + request, + signal, + { + LIBRECHAT_CODE_DATA_DIR: canonicalDataDirectory, + LIBRECHAT_CODE_CONTROL_PATH: join( + canonicalDataDirectory, + '_ptc_pending_result.json', + ), + LIBRECHAT_CODE_BASH_PATH: this.options.shellPath ?? '/bin/bash', + PTC_HISTORY_PATH: join( + canonicalDataDirectory, + '_ptc_history.json', + ), + TMPDIR: canonicalDataDirectory, + }, + options?.probe + ? { + filesystem: { + allowRead: [ + canonicalWorkspaceRoot ?? this.canonicalRoot!, + canonicalDataDirectory, + ], + allowWrite: [ + ...(canonicalWorkspaceRoot != null + ? [canonicalWorkspaceRoot] + : []), + canonicalDataDirectory, + ], + denyRead: this.denyReadPaths, + denyWrite: [ + this.canonicalRoot!, + ...this.denyWritePaths, + ], + }, + network: { + // A probe is speculative, even on a trusted VM. + // Copy-on-write protects files, not remote mutations. + allowedDomains: [], + deniedDomains: [], + strictAllowlist: true, + allowUnixSockets: [], + allowAllUnixSockets: false, + allowLocalBinding: false, + }, + } + : undefined, + canonicalDataDirectory, + canonicalWorkspaceRoot, + ); + const execution = options?.probe ? this.withProbeNetwork(execute) : execute(); + this.execution = execution; + try { + return await execution; + } finally { + this.execution = undefined; + } + } + + private async withProbeNetwork(execute: () => Promise): Promise { + const config = this.runtimeConfig; + if (!config || !this.manager.updateConfig) { + throw new WorkspaceToolError('Native probe network isolation is unavailable', 'COMMAND_UNAVAILABLE'); + } + // SRT's proxies and Unix/local socket rules read session configuration, + // not wrapWithSandboxArgv's per-command override. + this.manager.updateConfig({ ...config, network: { + allowedDomains: [], deniedDomains: [], strictAllowlist: true, + allowUnixSockets: [], allowAllUnixSockets: false, allowLocalBinding: false, + } }); + try { + return await execute(); + } finally { + try { + // Revoke the probe's proxy endpoints and credentials before restoring + // network access. A lingering probe must never inherit the commit's + // permissive proxy session through a live updateConfig. + await this.manager.reset(); + await this.manager.initialize(config, config.network.strictAllowlist ? undefined : async () => true); + } catch { + this.resetFailed = true; + throw new WorkspaceToolError('Native probe network cleanup failed', 'COMMAND_UNAVAILABLE'); + } + } + } + private async executeExclusive( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, + trustedEnvironment?: NodeJS.ProcessEnv, + customConfig?: Partial, + sandboxScratchDirectory?: string, + workspaceRoot?: string, ): Promise { if ( !isWorkspaceToolRequest(request) || @@ -499,7 +802,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } await this.initialize(); - const root = this.canonicalRoot!; + const root = workspaceRoot ?? this.canonicalRoot!; let cwd: string; try { cwd = await realpath(resolve(root, request.cwd ?? '.')); @@ -528,7 +831,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox { ...TRUSTED_GIT_ENVIRONMENT, ...(credentialEnvironment ?? {}), - ...this.scratchSelectorEnvironment(), + ...this.scratchSelectorEnvironment(sandboxScratchDirectory), }, () => this.manager.wrapWithSandboxArgv( @@ -536,7 +839,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox this.platform === 'win32' ? undefined : (this.options.shellPath ?? '/bin/bash'), - undefined, + customConfig, signal, cwd, { commandId, commandText: request.command }, @@ -561,7 +864,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'EXECUTION_ABORTED', ); } - return await this.runWrapped(request, wrapped, cwd, commandId, signal); + return await this.runWrapped( + request, + wrapped, + cwd, + commandId, + signal, + trustedEnvironment, + ); } finally { // A successful wrap owns command state even when no child is spawned. try { @@ -578,7 +888,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ): Promise { const previousMutation = hostEnvironmentMutationQueue; let releaseMutation!: () => void; - hostEnvironmentMutationQueue = new Promise((resolve) => { + hostEnvironmentMutationQueue = new Promise(resolve => { releaseMutation = resolve; }); await previousMutation; @@ -604,31 +914,41 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox cwd: string, commandId: string, signal?: AbortSignal, + trustedEnvironment?: NodeJS.ProcessEnv, ): Promise { const outputLimit = - request.maxOutputBytes ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; + request.maxOutputBytes ?? + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; const timeoutMs = request.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS; return await new Promise( (resolvePromise, reject) => { let child: ChildProcessWithoutNullStreams; try { - child = this.spawnCommand(wrapped.argv[0], wrapped.argv.slice(1), { + child = this.spawnCommand( + wrapped.argv[0], + wrapped.argv.slice(1), + { cwd, env: { ...wrapped.env, ...this.scratchEnvironment(), + ...trustedEnvironment, ...TRUSTED_GIT_CONFIG_ENTRIES, GIT_CONFIG_COUNT: - wrapped.env.GIT_CONFIG_COUNT ?? TRUSTED_GIT_CONFIG_COUNT, + wrapped.env.GIT_CONFIG_COUNT ?? + TRUSTED_GIT_CONFIG_COUNT, GIT_CONFIG_GLOBAL: - this.platform === 'win32' ? 'NUL' : '/dev/null', + this.platform === 'win32' + ? 'NUL' + : '/dev/null', GIT_CONFIG_NOSYSTEM: '1', }, detached: this.platform !== 'win32', shell: false, windowsHide: true, - }); + }, + ); child.stdin.end(); } catch { reject( @@ -654,10 +974,15 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const accepted = chunk.subarray(0, remaining); target.push(accepted); outputBytes += accepted.byteLength; - if (accepted.byteLength !== chunk.byteLength) truncated = true; + if (accepted.byteLength !== chunk.byteLength) + truncated = true; }; - child.stdout.on('data', (chunk: Buffer) => append(stdout, chunk)); - child.stderr.on('data', (chunk: Buffer) => append(stderr, chunk)); + child.stdout.on('data', (chunk: Buffer) => + append(stdout, chunk), + ); + child.stderr.on('data', (chunk: Buffer) => + append(stderr, chunk), + ); const abort = (): void => { if (settled) return; this.killCommandTree(child); @@ -706,7 +1031,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); return; } - const stdoutValue = boundedUtf8(Buffer.concat(stdout), outputLimit); + const stdoutValue = boundedUtf8( + Buffer.concat(stdout), + outputLimit, + ); const stderrBudget = Math.max( 0, outputLimit - Buffer.byteLength(stdoutValue), @@ -714,7 +1042,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const rawStderr = Buffer.concat(stderr).toString('utf8'); let annotatedStderr = rawStderr; try { - annotatedStderr = this.manager.annotateStderrWithSandboxFailures( + annotatedStderr = + this.manager.annotateStderrWithSandboxFailures( commandId, rawStderr, ); @@ -730,12 +1059,15 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox operation: 'execute_command', workspaceId: request.workspaceId, exitCode: - timedOut || childSignal ? null : this.protocolExitCode(code), + timedOut || childSignal + ? null + : this.protocolExitCode(code), ...(childSignal ? { signal: childSignal } : {}), stdout: stdoutValue, stderr: stderrValue, truncated: - truncated || Buffer.byteLength(annotatedStderr) > stderrBudget, + truncated || + Buffer.byteLength(annotatedStderr) > stderrBudget, timedOut, }); }); @@ -775,7 +1107,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const canonicalTemporaryRoot = await canonicalPath(HOST_TEMPORARY_ROOT); - const sharedScratchRoot = sharedScratchPaths.find((path) => + const sharedScratchRoot = sharedScratchPaths.find(path => isWithin(path, canonicalTemporaryRoot), ); const scratchDirectory = await mkdtemp( @@ -798,7 +1130,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox true, ); if (((await scratchHandle.stat()).mode & 0o777) !== 0o700) { - throw new Error('Native sandbox scratch directory is not private'); + throw new Error( + 'Native sandbox scratch directory is not private', + ); } this.scratchHandle = scratchHandle; } catch (error) { @@ -829,11 +1163,13 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox : { TMPDIR: scratchDirectory }; } - private scratchSelectorEnvironment(): NodeJS.ProcessEnv { - const scratchDirectory = this.scratchDirectory; + private scratchSelectorEnvironment( + selectedDirectory = this.scratchDirectory, + ): NodeJS.ProcessEnv { + const scratchDirectory = selectedDirectory; if (!scratchDirectory) return {}; return Object.fromEntries( - SRT_SCRATCH_SELECTOR_NAMES.map((name) => [name, scratchDirectory]), + SRT_SCRATCH_SELECTOR_NAMES.map(name => [name, scratchDirectory]), ); } diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index fd426783..08cba1c7 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -1,8 +1,11 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { + bridgeArtifactMediaType, bridgeWorkerPath, comparePortableRelativePaths, + isBridgeWorkspaceProgrammaticRequest, + isSupportedBridgeArtifactName, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, isWorkspaceToolRequest, @@ -13,6 +16,23 @@ import type { WorkspacePreviewEditRequest, } from './protocol.js'; +test('accepts gateway directory markers as artifacts', () => { + assert.equal(isSupportedBridgeArtifactName('.dirkeep'), true); + assert.equal(isSupportedBridgeArtifactName('nested/.dirkeep'), true); + assert.equal(isSupportedBridgeArtifactName('nested/.dirkeep.exe'), false); +}); + +test('rejects caller-supplied programmatic control payloads', () => { + for (const name of ['_ptc_pending_result.json', '_PTC_PENDING_RESULT.JSON', 'nested/_ptc_pending_result.json']) { + assert.equal(isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { language: 'bash', version: '5.2.0', session_id: 'session', files: [ + { name: 'main.sh', content: 'true' }, { name, content: '{}' }, + ] }, + }), false); + } +}); + const validSingleEditRequest: WorkspaceEditFileRequest = { protocolVersion: 1, operation: 'edit_file', @@ -592,3 +612,167 @@ test('workspace capabilities allow per-workspace operation restrictions', () => false, ); }); + +test('workspace programmatic capability is closed to Bash command roots', () => { + const workspaceTools = { + protocolVersion: 1, + operations: ['execute_command'], + programmaticLanguages: ['bash'], + workspaces: [{ id: 'project-a' }], + }; + assert.equal( + isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools, + }), + true, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { ...workspaceTools, operations: ['read_file'] }, + }), + false, + ); + assert.equal( + isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { ...workspaceTools, programmaticLanguages: ['python'] }, + }), + false, + ); +}); + +test('workspace programmatic requests accept only stable input cache identities', () => { + const request = { + headers: {}, + body: { + language: 'bash', + version: '5.2', + execution_id: 'execution_1', + replay_tool_count: 2, + max_output_files: 50, + max_output_file_bytes: 10_000_000, + session_id: 'session-1', + files: [ + { name: 'main.sh', content: 'echo ready' }, + { + name: 'skills/example.txt', + id: 'file-1', + storage_session_id: 'storage-1', + input_cache_key: 'a'.repeat(64), + }, + ], + }, + }; + assert.equal(isBridgeWorkspaceProgrammaticRequest(request), true); + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + ...request, + body: { + ...request.body, + files: [request.body.files[0], { ...request.body.files[1], input_cache_key: '../cache' }], + }, + }), + false, + ); + for (const body of [ + { ...request.body, execution_id: '../execution' }, + { ...request.body, replay_tool_count: -1 }, + { ...request.body, replay_tool_count: 257 }, + { ...request.body, max_output_files: -1 }, + { ...request.body, max_output_files: 101 }, + { ...request.body, max_output_file_bytes: 0 }, + { ...request.body, max_output_file_bytes: 10 * 1024 * 1024 + 1 }, + ]) { + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ ...request, body }), + false, + ); + } +}); + +test('workspace programmatic history can use the bounded replay aggregate budget', () => { + const history = 'h'.repeat(10 * 1024 * 1024 + 1); + const body = { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [ + { name: 'main.sh', content: 'echo ready' }, + { name: '_ptc_history.json', content: history }, + ], + }; + assert.equal(isBridgeWorkspaceProgrammaticRequest({ headers: {}, body }), true); + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { + ...body, + files: [ + { name: 'main.sh', content: history }, + { name: '_ptc_history.json', content: '{}' }, + ], + }, + }), + false, + ); +}); + +test('workspace programmatic requests reject non-canonical file paths', () => { + for (const name of ['./main.sh', 'scripts//main.sh', 'scripts/./main.sh', '.']) { + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [ + { name: 'main.sh', content: 'echo ready' }, + { name, content: 'data' }, + ], + }, + }), + false, + name, + ); + } +}); + +test('workspace programmatic requests reject ancestor-descendant input conflicts', () => { + for (const names of [ + ['main.sh', 'main.sh/data.txt'], + ['main.sh', 'assets', 'assets/logo.png'], + ['main.sh', 'deep/path/file.txt', 'deep'], + ]) { + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: names.map(name => ({ name, content: 'data' })), + }, + }), + false, + names.join(', '), + ); + } +}); + +test('bridge artifact policy and media types match the hardened gateway contract', () => { + assert.equal(isSupportedBridgeArtifactName('reports/result.json'), true); + assert.equal(isSupportedBridgeArtifactName('preview.png'), true); + assert.equal(isSupportedBridgeArtifactName('model.bin'), false); + assert.equal(bridgeArtifactMediaType('preview.png'), 'image/png'); + assert.equal(bridgeArtifactMediaType('reports/result.json'), 'application/json'); + assert.equal(bridgeArtifactMediaType('Dockerfile'), 'application/octet-stream'); +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index c1dad949..b92d21ac 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -20,9 +20,144 @@ export const BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS = 5 * 60_000; export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES = 256 * 1024; export const BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH = 32; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES = 100; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES = BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES - 2; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY = 4; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS = 30_000; + +/** Reserve a bounded share for all input/output batches, not per-file grants. */ +export function programmaticTransferReserveMs(jobTimeoutMs: number): number { + return Math.max(1, Math.floor(jobTimeoutMs / 3)); +} +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES = 10 * 1024 * 1024; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_HISTORY_BYTES = 40_000_000; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES = 100 * 1024 * 1024; /** How long Code API drains a clean rejection after Stop cancels a workspace mutation. */ export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; +/** + * Artifact names accepted by the hardened egress gateway. Keep this policy in + * the bridge protocol package so a remote worker can reject unsupported output + * locally instead of discovering the mismatch only after mutating a workspace. + */ +const BRIDGE_ARTIFACT_EXTENSIONS = new Set([ + '.c', '.cs', '.cpp', '.go', '.java', '.js', '.kt', '.kts', '.lua', + '.php', '.pl', '.ps1', '.py', '.r', '.rb', '.rs', '.scala', '.sh', + '.sql', '.swift', '.ts', '.jsx', '.tsx', '.groovy', + '.css', '.htm', '.html', '.less', '.sass', '.scss', '.svg', '.svelte', '.vue', + '.adoc', '.asciidoc', '.md', '.rst', '.tex', '.txt', '.wiki', + '.csv', '.json', '.bson', '.json5', '.jsonl', '.parquet', '.tsv', + '.xml', '.yaml', '.yml', + '.ics', '.ical', '.ifb', '.icalendar', + '.conf', '.env', '.gitignore', '.ini', '.properties', '.toml', + '.doc', '.docx', '.pdf', '.ppt', '.pptx', '.xls', '.xlsx', + '.odt', '.ods', '.odp', '.rtf', + '.avif', '.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', + '.tif', '.tiff', '.webp', + '.eot', '.ttf', '.woff', '.woff2', + '.7z', '.bz2', '.gz', '.gzip', '.rar', '.tar', '.zip', + '.tf', '.tfvars', '.tfstate', '.hcl', + '.dockerfile', '.Dockerfile', '.dockerignore', + '.helmignore', '.helmfile', '.jenkinsfile', '.vagrantfile', + '.eslintrc', '.prettierrc', '.editorconfig', '.nomad', + '.bat', '.cmd', '.deb', '.log', '.rpm', '.vbs', +]); + +function portableBasename(name: string): string { + return name.slice(name.lastIndexOf('/') + 1); +} + +/** Apply the gateway's extension allowlist without importing service code. */ +export function isSupportedBridgeArtifactName(name: string): boolean { + const basename = portableBasename(name); + if (basename === '.dirkeep') return true; + const dot = basename.lastIndexOf('.'); + const extension = dot > 0 ? basename.slice(dot).toLowerCase() : ''; + const dottedBasename = `.${basename}`; + return ( + (extension !== '' && BRIDGE_ARTIFACT_EXTENSIONS.has(extension)) || + BRIDGE_ARTIFACT_EXTENSIONS.has(basename) || + BRIDGE_ARTIFACT_EXTENSIONS.has(basename.toLowerCase()) || + (extension === '' && + (BRIDGE_ARTIFACT_EXTENSIONS.has(dottedBasename) || + BRIDGE_ARTIFACT_EXTENSIONS.has(dottedBasename.toLowerCase()))) + ); +} + +const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { + '.avif': 'image/avif', + '.bmp': 'image/bmp', + '.bz2': 'application/x-bzip2', + '.c': 'text/x-c', + '.conf': 'text/plain', + '.cpp': 'text/x-c++src', + '.css': 'text/css', + '.csv': 'text/csv', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.gif': 'image/gif', + '.gz': 'application/gzip', + '.gzip': 'application/gzip', + '.htm': 'text/html', + '.html': 'text/html', + '.ico': 'image/x-icon', + '.ics': 'text/calendar', + '.ifb': 'text/calendar', + '.ical': 'text/calendar', + '.icalendar': 'text/calendar', + '.ini': 'text/plain', + '.java': 'text/x-java-source', + '.jpeg': 'image/jpeg', + '.jpg': 'image/jpeg', + '.js': 'text/javascript', + '.json': 'application/json', + '.json5': 'application/json5', + '.jsonl': 'application/x-ndjson', + '.jsx': 'text/jsx', + '.log': 'text/plain', + '.md': 'text/markdown', + '.odt': 'application/vnd.oasis.opendocument.text', + '.ods': 'application/vnd.oasis.opendocument.spreadsheet', + '.odp': 'application/vnd.oasis.opendocument.presentation', + '.parquet': 'application/vnd.apache.parquet', + '.pdf': 'application/pdf', + '.png': 'image/png', + '.ppt': 'application/vnd.ms-powerpoint', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.py': 'text/x-python', + '.rst': 'text/x-rst', + '.rtf': 'application/rtf', + '.sh': 'application/x-sh', + '.sql': 'application/sql', + '.svg': 'image/svg+xml', + '.tar': 'application/x-tar', + '.tex': 'application/x-tex', + '.tif': 'image/tiff', + '.tiff': 'image/tiff', + '.toml': 'application/toml', + '.ts': 'text/typescript', + '.tsx': 'text/tsx', + '.tsv': 'text/tab-separated-values', + '.txt': 'text/plain', + '.webp': 'image/webp', + '.woff': 'font/woff', + '.woff2': 'font/woff2', + '.xls': 'application/vnd.ms-excel', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.xml': 'application/xml', + '.yaml': 'application/yaml', + '.yml': 'application/yaml', + '.zip': 'application/zip', +}; + +/** Infer a safe response media type from an already-validated artifact name. */ +export function bridgeArtifactMediaType(name: string): string { + const basename = portableBasename(name).toLowerCase(); + const dot = basename.lastIndexOf('.'); + const extension = dot > 0 ? basename.slice(dot) : basename; + return BRIDGE_ARTIFACT_MEDIA_TYPES[extension] ?? 'application/octet-stream'; +} + export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; export type BridgeWorkspaceToolOperation = @@ -38,6 +173,7 @@ export type WorkspaceWriteFileMode = 'replace' | 'create'; export type WorkspaceEditFileMode = 'single' | 'batch'; export type WorkspaceEditFileFeature = 'expected_base_sha256'; export type WorkspaceListFileFeature = 'after_path'; +export type WorkspaceProgrammaticLanguage = 'bash'; export interface BridgeWorkspaceDescriptor { id: string; @@ -58,6 +194,8 @@ export interface BridgeWorkspaceToolCapabilities { editFileFeatures?: WorkspaceEditFileFeature[]; /** Omitted by workers that cannot continue a bounded file listing. */ listFileFeatures?: WorkspaceListFileFeature[]; + /** Languages that can execute PTC replay inside a selected workspace. */ + programmaticLanguages?: WorkspaceProgrammaticLanguage[]; } export interface WorkspaceReadFileRequest { @@ -153,8 +291,7 @@ interface WorkspaceEditFileRequestBase { expectedBaseSha256?: string; } -export interface WorkspaceSingleEditFileRequest - extends WorkspaceEditFileRequestBase { +export interface WorkspaceSingleEditFileRequest extends WorkspaceEditFileRequestBase { /** Legacy single-edit form. */ oldText: string; /** Legacy single-edit form. */ @@ -162,8 +299,7 @@ export interface WorkspaceSingleEditFileRequest edits?: never; } -export interface WorkspaceBatchEditFileRequest - extends WorkspaceEditFileRequestBase { +export interface WorkspaceBatchEditFileRequest extends WorkspaceEditFileRequestBase { /** Ordered exact replacements applied atomically as one file mutation. */ edits: WorkspaceTextEdit[]; oldText?: never; @@ -171,8 +307,7 @@ export interface WorkspaceBatchEditFileRequest } export type WorkspaceEditFileRequest = - | WorkspaceSingleEditFileRequest - | WorkspaceBatchEditFileRequest; + WorkspaceSingleEditFileRequest | WorkspaceBatchEditFileRequest; export interface WorkspaceTextEdit { oldText: string; @@ -195,23 +330,20 @@ interface WorkspacePreviewEditRequestBase { path: string; } -export interface WorkspaceSinglePreviewEditRequest - extends WorkspacePreviewEditRequestBase { +export interface WorkspaceSinglePreviewEditRequest extends WorkspacePreviewEditRequestBase { oldText: string; newText: string; edits?: never; } -export interface WorkspaceBatchPreviewEditRequest - extends WorkspacePreviewEditRequestBase { +export interface WorkspaceBatchPreviewEditRequest extends WorkspacePreviewEditRequestBase { edits: WorkspaceTextEdit[]; oldText?: never; newText?: never; } export type WorkspacePreviewEditRequest = - | WorkspaceSinglePreviewEditRequest - | WorkspaceBatchPreviewEditRequest; + WorkspaceSinglePreviewEditRequest | WorkspaceBatchPreviewEditRequest; export interface WorkspacePreviewEditResult { protocolVersion: BridgeProtocolVersion; @@ -392,12 +524,7 @@ const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ 'truncated', 'timedOut', ]); -const WORKSPACE_SEARCH_MATCH_KEYS = new Set([ - 'path', - 'line', - 'column', - 'text', -]); +const WORKSPACE_SEARCH_MATCH_KEYS = new Set(['path', 'line', 'column', 'text']); export interface BridgeWorkerCapabilities { /** Opt-in protocol: maximum concurrently leased independent workspace roots. */ @@ -437,6 +564,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceEditFileFeatures?: WorkspaceEditFileFeature[]; /** Listing features this Code API can safely route to a capability-aware worker. */ supportedWorkspaceListFileFeatures?: WorkspaceListFileFeature[]; + /** PTC languages this Code API can safely route into a selected workspace. */ + supportedWorkspaceProgrammaticLanguages?: WorkspaceProgrammaticLanguage[]; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -469,6 +598,37 @@ export interface BridgeSandboxRequest { headers: Record; } +export type BridgeProgrammaticPayloadFile = + | { name: string; content: string } + | { + name: string; + id: string; + storage_session_id: string; + input_cache_key?: string; + }; + +export interface BridgeWorkspaceProgrammaticBody { + language: 'bash'; + version: string; + /** Stable identity shared by every replay iteration of one execution. */ + execution_id?: string; + /** Declared replay tools; zero allows the worker to skip the probe pass. */ + replay_tool_count?: number; + run_timeout?: number; + transfer_timeout_ms?: number; + /** Manifest-bound upload ceiling negotiated by Code API. */ + max_output_files?: number; + /** Effective per-file upload ceiling negotiated by Code API. */ + max_output_file_bytes?: number; + files: BridgeProgrammaticPayloadFile[]; + session_id: string; + output_session_id?: string; + egress_grant?: string; +} + +export type BridgeWorkspaceProgrammaticRequest = + BridgeSandboxRequest; + export interface BridgeAssignment { workspaceLeaseSlot?: number; protocolVersion: BridgeProtocolVersion; @@ -481,7 +641,9 @@ export interface BridgeAssignment { /** Server-calculated execution budget at lease time; avoids VM clock skew. */ remainingMs?: number; runtimeSessionId?: string; - executionKind?: 'sandbox' | 'workspace_tool'; + executionKind?: 'sandbox' | 'workspace_tool' | 'workspace_programmatic'; + /** Selected workspace for workspace-scoped programmatic execution. */ + workspaceId?: string; request: BridgeSandboxRequest | WorkspaceToolRequest; } @@ -589,6 +751,140 @@ export function isValidBridgeWorkerId(workerId: string): boolean { return BRIDGE_WORKER_ID_PATTERN.test(workerId); } +export function isBridgeWorkspaceProgrammaticRequest( + value: unknown, +): value is BridgeWorkspaceProgrammaticRequest { + if (typeof value !== 'object' || value === null) return false; + const request = value as Record; + if ( + typeof request.headers !== 'object' || + request.headers === null || + !Object.values(request.headers).every( + entry => typeof entry === 'string', + ) || + typeof request.body !== 'object' || + request.body === null + ) { + return false; + } + const body = request.body as Record; + if ( + body.language !== 'bash' || + typeof body.version !== 'string' || + body.version.length === 0 || + body.version.length > BRIDGE_RUNTIME_MAX_LENGTH || + (body.execution_id !== undefined && + (typeof body.execution_id !== 'string' || + !/^[A-Za-z0-9_-]{1,128}$/.test(body.execution_id))) || + (body.replay_tool_count !== undefined && + (!Number.isSafeInteger(body.replay_tool_count) || + Number(body.replay_tool_count) < 0 || + Number(body.replay_tool_count) > 256)) || + (body.max_output_files !== undefined && + (!Number.isSafeInteger(body.max_output_files) || + Number(body.max_output_files) < 0 || + Number(body.max_output_files) > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES)) || + (body.max_output_file_bytes !== undefined && + (!Number.isSafeInteger(body.max_output_file_bytes) || + Number(body.max_output_file_bytes) < 1 || + Number(body.max_output_file_bytes) > + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES)) || + typeof body.session_id !== 'string' || + body.session_id.length === 0 || + body.session_id.length > 32_768 || + /[\0\r\n]/.test(body.session_id) || + (body.output_session_id !== undefined && + (typeof body.output_session_id !== 'string' || + body.output_session_id.length === 0 || + body.output_session_id.length > 32_768 || + /[\0\r\n]/.test(body.output_session_id))) || + (body.egress_grant !== undefined && + (typeof body.egress_grant !== 'string' || + body.egress_grant.length === 0 || + body.egress_grant.length > 256 * 1024)) || + (body.transfer_timeout_ms !== undefined && + (!Number.isSafeInteger(body.transfer_timeout_ms) || + Number(body.transfer_timeout_ms) < 1 || + Number(body.transfer_timeout_ms) > BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS)) || + (body.run_timeout !== undefined && + (!Number.isSafeInteger(body.run_timeout) || + Number(body.run_timeout) < 1 || + Number(body.run_timeout) > + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS)) || + !Array.isArray(body.files) || + body.files.length < 1 || + body.files.length > BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES + ) { + return false; + } + let inlineBytes = 0; + const names = new Set(); + for (const rawFile of body.files) { + if (typeof rawFile !== 'object' || rawFile === null) return false; + const file = rawFile as Record; + if ( + !isSafePortableRelativePath(file.name) || + file.name === '.' || + portableBasename(file.name).toLowerCase() === '_ptc_pending_result.json' || + normalizePortableRelativePath(file.name) !== file.name || + names.has(file.name) + ) { + return false; + } + names.add(file.name); + if (typeof file.content === 'string') { + inlineBytes += Buffer.byteLength(file.content); + if ( + Object.keys(file).some( + key => key !== 'name' && key !== 'content', + ) || + Buffer.byteLength(file.content) > + (file.name === '_ptc_history.json' + ? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_HISTORY_BYTES + : BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES) + ) { + return false; + } + continue; + } + if ( + typeof file.id !== 'string' || + file.id.length === 0 || + file.id.length > 32_768 || + /[\0\r\n]/.test(file.id) || + typeof file.storage_session_id !== 'string' || + file.storage_session_id.length === 0 || + file.storage_session_id.length > 32_768 || + /[\0\r\n]/.test(file.storage_session_id) || + Object.keys(file).some( + key => + key !== 'name' && + key !== 'id' && + key !== 'storage_session_id' && + key !== 'input_cache_key', + ) || + (file.input_cache_key !== undefined && + (typeof file.input_cache_key !== 'string' || + !/^[a-f0-9]{64}$/.test(file.input_cache_key))) + ) { + return false; + } + } + for (const name of names) { + const segments = name.split('/'); + let ancestor = ''; + for (let index = 0; index < segments.length - 1; index += 1) { + ancestor = ancestor ? `${ancestor}/${segments[index]}` : segments[index]!; + if (names.has(ancestor)) return false; + } + } + return ( + names.has('main.sh') && + inlineBytes <= BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES + ); +} + export function isSafePortableRelativePath(value: unknown): value is string { if ( typeof value !== 'string' || @@ -602,20 +898,23 @@ export function isSafePortableRelativePath(value: unknown): value is string { ) { return false; } - return value.split('/').every((segment) => segment !== '..'); + return value.split('/').every(segment => segment !== '..'); } function normalizePortableRelativePath(value: string): string { return ( value .split('/') - .filter((segment) => segment.length > 0 && segment !== '.') + .filter(segment => segment.length > 0 && segment !== '.') .join('/') || '.' ); } /** Compare path segments in ripgrep's sorted, depth-first traversal order. */ -export function comparePortableRelativePaths(left: string, right: string): number { +export function comparePortableRelativePaths( + left: string, + right: string, +): number { const encoder = new TextEncoder(); const leftSegments = left.split('/'); const rightSegments = right.split('/'); @@ -645,9 +944,14 @@ function isWithinRequestedPath(candidate: string, requested?: string): boolean { ); } -function isValidWorkspaceEditRequest(request: Record): boolean { +function isValidWorkspaceEditRequest( + request: Record, +): boolean { const hasBatch = request.edits !== undefined; - if (hasBatch && (request.oldText !== undefined || request.newText !== undefined)) { + if ( + hasBatch && + (request.oldText !== undefined || request.newText !== undefined) + ) { return false; } const edits = hasBatch @@ -665,7 +969,10 @@ function isValidWorkspaceEditRequest(request: Record): boolean if ( typeof edit !== 'object' || edit === null || - !hasOnlyKeys(edit as Record, WORKSPACE_TEXT_EDIT_KEYS) + !hasOnlyKeys( + edit as Record, + WORKSPACE_TEXT_EDIT_KEYS, + ) ) { return false; } @@ -673,9 +980,11 @@ function isValidWorkspaceEditRequest(request: Record): boolean if ( typeof candidate.oldText !== 'string' || candidate.oldText.length === 0 || - Buffer.from(candidate.oldText).toString('utf8') !== candidate.oldText || + Buffer.from(candidate.oldText).toString('utf8') !== + candidate.oldText || typeof candidate.newText !== 'string' || - Buffer.from(candidate.newText).toString('utf8') !== candidate.newText + Buffer.from(candidate.newText).toString('utf8') !== + candidate.newText ) { return false; } @@ -698,7 +1007,7 @@ function hasOnlyKeys( value: Record, allowed: ReadonlySet, ): boolean { - return Object.keys(value).every((key) => allowed.has(key)); + return Object.keys(value).every(key => allowed.has(key)); } export function isWorkspaceToolRequest( @@ -723,7 +1032,8 @@ export function isWorkspaceToolRequest( (request.maxLines === undefined || (Number.isSafeInteger(request.maxLines) && Number(request.maxLines) >= 1 && - Number(request.maxLines) <= BRIDGE_WORKSPACE_READ_MAX_LINES)) + Number(request.maxLines) <= + BRIDGE_WORKSPACE_READ_MAX_LINES)) ); } if (request.operation === 'search_text') { @@ -743,7 +1053,8 @@ export function isWorkspaceToolRequest( (request.maxResults === undefined || (Number.isSafeInteger(request.maxResults) && Number(request.maxResults) >= 1 && - Number(request.maxResults) <= BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS)) + Number(request.maxResults) <= + BRIDGE_WORKSPACE_SEARCH_MAX_RESULTS)) ); } if (request.operation === 'list_files') { @@ -753,12 +1064,14 @@ export function isWorkspaceToolRequest( isSafePortableRelativePath(request.path)) && (request.afterPath === undefined || (isSafePortableRelativePath(request.afterPath) && - normalizePortableRelativePath(request.afterPath) === request.afterPath && + normalizePortableRelativePath(request.afterPath) === + request.afterPath && isWithinRequestedPath(request.afterPath, request.path))) && (request.maxResults === undefined || (Number.isSafeInteger(request.maxResults) && Number(request.maxResults) >= 1 && - Number(request.maxResults) <= BRIDGE_WORKSPACE_LIST_MAX_RESULTS)) + Number(request.maxResults) <= + BRIDGE_WORKSPACE_LIST_MAX_RESULTS)) ); } if (request.operation === 'write_file') { @@ -799,7 +1112,8 @@ export function isWorkspaceToolRequest( !request.command.includes('\0') && new TextEncoder().encode(request.command).byteLength <= BRIDGE_WORKSPACE_COMMAND_MAX_BYTES && - (request.cwd === undefined || isSafePortableRelativePath(request.cwd)) && + (request.cwd === undefined || + isSafePortableRelativePath(request.cwd)) && (request.timeoutMs === undefined || (Number.isSafeInteger(request.timeoutMs) && Number(request.timeoutMs) >= 1 && @@ -838,13 +1152,19 @@ export function isWorkspaceToolResult( if (request.operation === 'read_file') { const startLine = request.startLine ?? 1; const maxLines = request.maxLines ?? 200; - const content = typeof result.content === 'string' ? result.content : null; + const content = + typeof result.content === 'string' ? result.content : null; const reportedLineCount = - Number.isSafeInteger(result.endLine) && Number(result.endLine) >= startLine - 1 + Number.isSafeInteger(result.endLine) && + Number(result.endLine) >= startLine - 1 ? Number(result.endLine) - startLine + 1 : -1; const actualLineCount = - content === null ? -1 : content.length === 0 ? reportedLineCount : content.split('\n').length; + content === null + ? -1 + : content.length === 0 + ? reportedLineCount + : content.split('\n').length; return ( hasOnlyKeys(result, WORKSPACE_READ_RESULT_KEYS) && result.path === request.path && @@ -899,7 +1219,10 @@ export function isWorkspaceToolResult( (enforcesPaginationContract && (normalizedPath !== path || (previousPath !== undefined && - comparePortableRelativePaths(normalizedPath, previousPath) <= 0))) + comparePortableRelativePaths( + normalizedPath, + previousPath, + ) <= 0))) ) { return false; } @@ -909,7 +1232,8 @@ export function isWorkspaceToolResult( if (!enforcesPaginationContract) { return result.nextAfterPath === undefined; } - if (result.truncated !== true) return result.nextAfterPath === undefined; + if (result.truncated !== true) + return result.nextAfterPath === undefined; return ( result.paths.length > 0 && result.nextAfterPath === result.paths[result.paths.length - 1] @@ -942,7 +1266,8 @@ export function isWorkspaceToolResult( if (request.operation === 'preview_edit') { const replacements = request.edits?.length ?? 1; - const content = typeof result.content === 'string' ? result.content : null; + const content = + typeof result.content === 'string' ? result.content : null; return ( hasOnlyKeys(result, WORKSPACE_PREVIEW_EDIT_RESULT_KEYS) && result.path === request.path && @@ -964,7 +1289,8 @@ export function isWorkspaceToolResult( const stdout = typeof result.stdout === 'string' ? result.stdout : null; const stderr = typeof result.stderr === 'string' ? result.stderr : null; const outputLimit = - request.maxOutputBytes ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; + request.maxOutputBytes ?? + BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES; return ( hasOnlyKeys(result, WORKSPACE_COMMAND_RESULT_KEYS) && stdout !== null && @@ -980,7 +1306,8 @@ export function isWorkspaceToolResult( Number(result.exitCode) <= 255)) && (result.signal === undefined || (typeof result.signal === 'string' && - result.signal.length <= BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH && + result.signal.length <= + BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH && /^SIG[A-Z0-9]+$/.test(result.signal))) && typeof result.truncated === 'boolean' && typeof result.timedOut === 'boolean' && @@ -995,7 +1322,7 @@ export function isWorkspaceToolResult( return ( hasOnlyKeys(result, WORKSPACE_SEARCH_RESULT_KEYS) && result.matches.length <= maxResults && - result.matches.every((match) => { + result.matches.every(match => { if (typeof match !== 'object' || match === null) return false; const candidate = match as Record; return ( @@ -1007,7 +1334,8 @@ export function isWorkspaceToolResult( Number.isSafeInteger(candidate.column) && Number(candidate.column) >= 1 && typeof candidate.text === 'string' && - candidate.text.length <= BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH && + candidate.text.length <= + BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH && candidate.text.includes(request.query) ); }) @@ -1025,7 +1353,7 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.operations.length < 1 || capabilities.operations.length > 7 || !capabilities.operations.every( - (operation) => + operation => operation === 'read_file' || operation === 'search_text' || operation === 'list_files' || @@ -1034,7 +1362,8 @@ export function isValidBridgeWorkspaceToolCapabilities( operation === 'edit_file' || operation === 'execute_command', ) || - new Set(capabilities.operations).size !== capabilities.operations.length || + new Set(capabilities.operations).size !== + capabilities.operations.length || !Array.isArray(capabilities.workspaces) || capabilities.workspaces.length < 1 || capabilities.workspaces.length > BRIDGE_WORKSPACE_MAX_COUNT @@ -1049,7 +1378,7 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.writeFileModes.length > 2 || !capabilities.operations.includes('write_file') || !capabilities.writeFileModes.every( - (mode) => mode === 'replace' || mode === 'create', + mode => mode === 'replace' || mode === 'create', ) || new Set(capabilities.writeFileModes).size !== capabilities.writeFileModes.length) @@ -1065,7 +1394,7 @@ export function isValidBridgeWorkspaceToolCapabilities( (!capabilities.operations.includes('edit_file') && !capabilities.operations.includes('preview_edit')) || !capabilities.editFileModes.every( - (mode) => mode === 'single' || mode === 'batch', + mode => mode === 'single' || mode === 'batch', ) || new Set(capabilities.editFileModes).size !== capabilities.editFileModes.length) @@ -1093,13 +1422,23 @@ export function isValidBridgeWorkspaceToolCapabilities( return false; } + if ( + capabilities.programmaticLanguages !== undefined && + (!Array.isArray(capabilities.programmaticLanguages) || + capabilities.programmaticLanguages.length !== 1 || + !capabilities.operations.includes('execute_command') || + capabilities.programmaticLanguages[0] !== 'bash') + ) { + return false; + } + const workspaceIds = new Set(); - return capabilities.workspaces.every((workspace) => { + return capabilities.workspaces.every(workspace => { if (typeof workspace !== 'object' || workspace === null) return false; const descriptor = workspace as Record; if ( Object.keys(descriptor).some( - (key) => key !== 'id' && key !== 'name' && key !== 'operations', + key => key !== 'id' && key !== 'name' && key !== 'operations', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || @@ -1107,17 +1446,21 @@ export function isValidBridgeWorkspaceToolCapabilities( (descriptor.name !== undefined && (typeof descriptor.name !== 'string' || descriptor.name.trim().length === 0 || - descriptor.name.length > BRIDGE_WORKSPACE_NAME_MAX_LENGTH)) || + descriptor.name.length > + BRIDGE_WORKSPACE_NAME_MAX_LENGTH)) || (descriptor.operations !== undefined && (!Array.isArray(descriptor.operations) || descriptor.operations.length < 1 || descriptor.operations.length > (capabilities.operations as unknown[]).length || descriptor.operations.some( - (operation) => - !(capabilities.operations as unknown[]).includes(operation), + operation => + !(capabilities.operations as unknown[]).includes( + operation, + ), ) || - new Set(descriptor.operations).size !== descriptor.operations.length)) + new Set(descriptor.operations).size !== + descriptor.operations.length)) ) { return false; } @@ -1139,11 +1482,12 @@ export function isValidBridgeWorkerCapabilities( typeof capabilities.statefulWorkspace === 'boolean' && typeof capabilities.sandboxProfile === 'string' && capabilities.sandboxProfile.trim().length > 0 && - capabilities.sandboxProfile.length <= BRIDGE_SANDBOX_PROFILE_MAX_LENGTH && + capabilities.sandboxProfile.length <= + BRIDGE_SANDBOX_PROFILE_MAX_LENGTH && Array.isArray(capabilities.runtimes) && capabilities.runtimes.length <= BRIDGE_RUNTIME_MAX_COUNT && capabilities.runtimes.every( - (runtime) => + runtime => typeof runtime === 'string' && runtime.length > 0 && runtime.length <= BRIDGE_RUNTIME_MAX_LENGTH, diff --git a/packages/code/src/worker-slots.test.ts b/packages/code/src/worker-slots.test.ts index ba65a3cf..6e8cc8b1 100644 --- a/packages/code/src/worker-slots.test.ts +++ b/packages/code/src/worker-slots.test.ts @@ -312,3 +312,51 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b assert.equal(executed, true); assert.equal(internals.activeWorkspaceAssignments.size, 0); }); + +test('programmatic work on an independent workspace bypasses another root cleanup', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + }); + const internals = worker as unknown as { + activeWorkspaceAssignments: Map< + string, + { id: string; done: Promise } + >; + executeOwned: (assignment: BridgeAssignment) => Promise; + }; + internals.activeWorkspaceAssignments.set('a', { + id: 'previous', + done: new Promise(() => {}), + }); + let executed = false; + internals.executeOwned = async () => { + executed = true; + assert.equal(internals.activeWorkspaceAssignments.get('b')?.id, 'next'); + }; + await worker.executeAndSettle({ + assignmentId: 'next', + executionKind: 'workspace_programmatic', + workspaceId: 'b', + remainingMs: 1_000, + request: { + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }, + } as BridgeAssignment); + assert.equal(executed, true); + assert.equal(internals.activeWorkspaceAssignments.has('b'), false); + assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'previous'); +}); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 19f219da..a0517bd9 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -5,6 +5,7 @@ import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError, bridgeWorkerPath, + isBridgeWorkspaceProgrammaticRequest, isWorkspaceToolResult, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; @@ -21,6 +22,7 @@ import type { BridgeWorkerCredentialResponse, BridgeWorkerRegistrationResponse, BridgeWorkspaceToolOperation, + BridgeWorkspaceProgrammaticRequest, } from './protocol.js'; import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; import type { WorkspaceToolExecutor } from './workspace.js'; @@ -35,6 +37,13 @@ export interface BridgeWorkerOptions { runtimeSupervisor?: RuntimeSupervisor; capabilities: BridgeWorkerCapabilities; workspaceTools?: WorkspaceToolExecutor; + workspaceProgrammatic?: { + executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise; + }; workspaceMutationQuarantine?: WorkspaceMutationQuarantine; /** Required per-root durable guards when opting into concurrent workspace leases. */ workspaceQuarantines?: ReadonlyMap; @@ -164,6 +173,11 @@ function workspaceCapabilitiesMatch( (advertised.listFileFeatures?.every( (feature, index) => feature === executor.listFileFeatures?.[index], ) ?? executor.listFileFeatures == null) && + advertised.programmaticLanguages?.length === + executor.programmaticLanguages?.length && + (advertised.programmaticLanguages?.every( + (language, index) => language === executor.programmaticLanguages?.[index], + ) ?? executor.programmaticLanguages == null) && advertised.workspaces.length === executor.workspaces.length && advertised.workspaces.every( (workspace, index) => @@ -223,6 +237,7 @@ function registrationCompatibleCapabilities( editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + programmaticLanguages: _programmaticLanguages, ...compatibleWorkspaceTools } = workspaceTools; return { @@ -302,11 +317,16 @@ function supportedWorkspaceCapabilities( const listFileFeatures = desired.listFileFeatures?.filter((feature) => registration.supportedWorkspaceListFileFeatures?.includes(feature), ); + const programmaticLanguages = desired.programmaticLanguages?.filter( + (language) => + registration.supportedWorkspaceProgrammaticLanguages?.includes(language), + ); const { writeFileModes: _writeFileModes, editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + programmaticLanguages: _programmaticLanguages, ...compatibleDesired } = desired; return { @@ -327,6 +347,10 @@ function supportedWorkspaceCapabilities( ...(operations.includes('list_files') && listFileFeatures?.length ? { listFileFeatures } : {}), + ...(operations.includes('execute_command') && + programmaticLanguages?.length + ? { programmaticLanguages } + : {}), }, }; } @@ -406,6 +430,16 @@ export class BridgeWorker { 'Workspace tool capabilities require a matching executor', ); } + if ( + (options.workspaceProgrammatic != null) !== + (options.capabilities.workspaceTools?.programmaticLanguages?.includes( + 'bash', + ) === true) + ) { + throw new BridgeProtocolError( + 'Workspace programmatic capability requires a matching executor', + ); + } if ( options.capabilities.workspaceTools?.operations.some( (operation) => @@ -1130,11 +1164,7 @@ export class BridgeWorker { assignment: BridgeAssignment, signal?: AbortSignal, ): Promise { - const root = - assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) - ? assignment.request.workspaceId - : undefined; + const root = this.assignmentWorkspaceId(assignment); const waitingAt = Date.now(); while (root != null && this.activeWorkspaceAssignments.has(root)) { const active = this.activeWorkspaceAssignments.get(root)!; @@ -1212,14 +1242,31 @@ export class BridgeWorker { private workspaceGuard( assignment: BridgeAssignment, ): WorkspaceMutationQuarantine | undefined { - return assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) - ? (this.options.workspaceQuarantines?.get( - assignment.request.workspaceId, - ) ?? this.options.workspaceMutationQuarantine) + const workspaceId = this.assignmentWorkspaceId(assignment); + return workspaceId != null + ? (this.options.workspaceQuarantines?.get(workspaceId) ?? + this.options.workspaceMutationQuarantine) : this.options.workspaceMutationQuarantine; } + private assignmentWorkspaceId( + assignment: BridgeAssignment, + ): string | undefined { + if ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ) { + return assignment.request.workspaceId; + } + if ( + assignment.executionKind === 'workspace_programmatic' && + typeof assignment.workspaceId === 'string' + ) { + return assignment.workspaceId; + } + return undefined; + } + private async executeOwned( assignment: BridgeAssignment, signal?: AbortSignal, @@ -1472,6 +1519,76 @@ export class BridgeWorker { 'Bridge assignment expired during workspace execution', ); } + } else if (assignment.executionKind === 'workspace_programmatic') { + const workspaceId = assignment.workspaceId; + if ( + workspaceId == null || + this.options.workspaceProgrammatic == null || + !isBridgeWorkspaceProgrammaticRequest(assignment.request) + ) { + throw new BridgeProtocolError( + 'Worker does not provide valid selected-workspace programmatic execution', + ); + } + try { + if (this.quarantinedWorkspaces.has(workspaceId)) { + throw new Error('Workspace requires an explicit quarantine reset'); + } + if (this.options.workspaceQuarantines != null) + await guard?.assertAvailable(); + } catch (error) { + throw new BridgeWorkspaceQuarantinedError( + 'Workspace is quarantined', + error, + ); + } + const advertised = this.activeCapabilities.workspaceTools; + const workspace = advertised?.workspaces.find( + (candidate) => candidate.id === workspaceId, + ); + if ( + workspace == null || + !advertised?.operations.includes('execute_command') || + (workspace.operations != null && + !workspace.operations.includes('execute_command')) || + !advertised.programmaticLanguages?.includes('bash') + ) { + throw new BridgeProtocolError( + 'Selected-workspace programmatic execution is not advertised', + ); + } + this.mutationGuardArmed = true; + try { + this.armedWorkspaces.add(workspaceId); + await guard!.arm( + 'Workspace programmatic execution is pending settlement', + assignment.assignmentId, + ); + workspaceMutationArmed = true; + } catch (error) { + this.mutationGuardArmed = false; + throw new BridgeWorkspaceQuarantinedError( + 'Workspace mutation quarantine could not be armed before execution', + error, + ); + } + payload = await this.options.workspaceProgrammatic.executeProgrammatic( + workspaceId, + assignment.request, + executionController.signal, + ); + workspaceMutationApplied = true; + if (executionController.signal.aborted) { + throw ( + executionController.signal.reason ?? + new DOMException('aborted', 'AbortError') + ); + } + if (Date.now() >= localDeadlineAtMs) { + throw new BridgeProtocolError( + 'Bridge assignment expired during programmatic execution', + ); + } } else { runtimeLease = await this.runtimeSupervisor.acquire( assignment, @@ -1581,7 +1698,8 @@ export class BridgeWorker { leaseToken: assignment.leaseToken, incarnationId: this.incarnationId, status: 'rejected', - ...(assignment.executionKind === 'workspace_tool' && + ...((assignment.executionKind === 'workspace_tool' || + assignment.executionKind === 'workspace_programmatic') && error instanceof WorkspaceToolError ? { errorCode: error.code } : {}), @@ -1693,12 +1811,8 @@ export class BridgeWorker { clearTimeout(timer); } } - if ( - assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) - ) { - this.armedWorkspaces.delete(assignment.request.workspaceId); - } + const workspaceId = this.assignmentWorkspaceId(assignment); + if (workspaceId != null) this.armedWorkspaces.delete(workspaceId); this.mutationGuardArmed = false; } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -1957,11 +2071,12 @@ export class BridgeWorker { const fulfilledWorkspaceMutation = workspaceMutationApplied && settlement.status === 'fulfilled' && - assignment.executionKind === 'workspace_tool' && - isWorkspaceToolRequest(assignment.request) && - (assignment.request.operation === 'write_file' || - assignment.request.operation === 'edit_file' || - assignment.request.operation === 'execute_command'); + (assignment.executionKind === 'workspace_programmatic' || + (assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) && + (assignment.request.operation === 'write_file' || + assignment.request.operation === 'edit_file' || + assignment.request.operation === 'execute_command'))); if (signal?.aborted === true) { if (assignment.runtimeSessionId != null || fulfilledWorkspaceMutation) { throw await this.quarantineWorkspace( diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 8f9fcd45..fa08a84c 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -950,6 +950,88 @@ test('worker executes a workspace tool assignment locally without acquiring a sa }); }); +test('worker executes programmatic Bash in the selected workspace and preserves its fence', async () => { + const programmaticRequests: object[] = []; + const quarantineEvents: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + async executeProgrammatic(workspaceId, request) { + programmaticRequests.push({ workspaceId, request }); + return { + session_id: 'session-1', + language: 'bash', + version: '5.2', + files: [], + run: { stdout: 'ready\n', stderr: '', code: 0, signal: null }, + }; + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine( + (reason) => quarantineEvents.push(`quarantine:${reason}`), + (reason) => quarantineEvents.push(`arm:${reason}`), + () => quarantineEvents.push('clear'), + ), + ], + ]), + fetchImpl: async () => Response.json({ protocolVersion: 1, accepted: true }), + }); + const request = { + body: { + language: 'bash' as const, + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + headers: {}, + }; + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-1', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request, + }); + + assert.deepEqual(programmaticRequests, [{ workspaceId: 'primary', request }]); + assert.deepEqual(quarantineEvents, [ + 'arm:Workspace programmatic execution is pending settlement', + 'clear', + ]); +}); + test('worker stops after Code API rejects a fulfilled workspace mutation', async () => { let quarantinedReason: string | undefined; let armed = 0; diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 5bfc07fe..dfda49fc 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -112,6 +112,8 @@ export interface SandboxWorkspaceToolsOptions { commandSandbox: WorkspaceCommandSandbox; /** Workspace IDs whose sandbox is configured and may run commands. */ commandWorkspaces: string[]; + /** Optional execution-scoped languages supplied by the same command sandbox. */ + programmaticLanguages?: BridgeWorkspaceToolCapabilities['programmaticLanguages']; } const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; @@ -1657,6 +1659,9 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { ...(base.listFileFeatures != null ? { listFileFeatures: base.listFileFeatures } : {}), + ...(options.programmaticLanguages?.length + ? { programmaticLanguages: [...options.programmaticLanguages] } + : {}), workspaces: base.workspaces.map((workspace) => ({ ...workspace, operations: [ diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 25b0bdaf..369b306c 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -414,6 +414,7 @@ router.post( supportedWorkspaceEditFileModes: ['single', 'batch'], supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], supportedWorkspaceListFileFeatures: ['after_path'], + supportedWorkspaceProgrammaticLanguages: ['bash'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/bridge/selection.ts b/service/src/bridge/selection.ts index 0959279f..f3926e8a 100644 --- a/service/src/bridge/selection.ts +++ b/service/src/bridge/selection.ts @@ -1,4 +1,5 @@ export const CODEAPI_BRIDGE_WORKER_HEADER = 'X-LibreChat-Code-Worker-ID'; +export const CODEAPI_BRIDGE_WORKSPACE_HEADER = 'X-LibreChat-Code-Workspace-ID'; export const BRIDGE_WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; export class BridgeWorkerSelectionError extends Error { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index b09d3792..d6b91469 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -151,6 +151,26 @@ function supportsWorkspaceTool( return true; } +function supportsWorkspaceProgrammatic( + registration: RegisteredBridgeWorker, + workspaceId: string, + language: string, +): boolean { + const capabilities = registration.capabilities.workspaceTools; + const workspace = capabilities?.workspaces.find( + (candidate) => candidate.id === workspaceId, + ); + return ( + workspace != null && + capabilities?.operations.includes('execute_command') === true && + (workspace.operations == null || + workspace.operations.includes('execute_command')) && + capabilities.programmaticLanguages?.includes( + language as 'bash', + ) === true + ); +} + function workerKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}`; } @@ -700,6 +720,7 @@ export class RedisBridgeStore { body: t.PayloadBody; headers: Record; workspaceRequest?: WorkspaceToolRequest; + workspaceId?: string; runtimeSessionId?: string; deadlineAtMs: number; executionTimeoutMs?: number; @@ -709,6 +730,12 @@ export class RedisBridgeStore { registration: RegisteredBridgeWorker, ) => Promise; }): Promise { + if (args.workspaceRequest != null && args.workspaceId != null) { + throw new BridgeStoreError( + 'ASSIGNMENT_INVALID', + 'A bridge assignment cannot be both a workspace tool and programmatic execution', + ); + } if ( args.executionTimeoutMs !== undefined && (args.workspaceRequest == null || @@ -773,6 +800,19 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} does not advertise the requested workspace tool`, ); } + if ( + args.workspaceId != null && + !supportsWorkspaceProgrammatic( + registration, + args.workspaceId, + args.body.language, + ) + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + `Bridge worker ${args.workerId} does not advertise programmatic execution for the selected workspace`, + ); + } if ( args.runtimeSessionId !== undefined && (await this.dispatchCommand( @@ -799,14 +839,16 @@ export class RedisBridgeStore { const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; let workspaceLeaseSlot: number | undefined; + const selectedWorkspaceId = + args.workspaceRequest?.workspaceId ?? args.workspaceId; const workspaceSlots = - args.workspaceRequest != null && + selectedWorkspaceId != null && (registration.capabilities.workspaceLeaseSlots ?? 1) > 1 ? new BridgeWorkspaceSlots(this.redis) : undefined; let resultCommitted = false; const admission = - args.workspaceRequest == null + selectedWorkspaceId == null ? undefined : new BridgeAdmissionQueue(this.redis); try { @@ -820,7 +862,7 @@ export class RedisBridgeStore { args.deadlineAtMs, workspaceSlots == null ? undefined - : args.workspaceRequest?.workspaceId, + : selectedWorkspaceId, ), args, 'Bridge admission enqueue', @@ -855,7 +897,7 @@ export class RedisBridgeStore { workerId: args.workerId, incarnationId: lockIncarnationId, assignmentId, - workspaceId: args.workspaceRequest!.workspaceId, + workspaceId: selectedWorkspaceId!, capacity: registration.capabilities.workspaceLeaseSlots!, expiresAtMs: Date.now() + ttlSeconds * 1000, }), @@ -910,7 +952,14 @@ export class RedisBridgeStore { ); } if ( - !supportsWorkspaceTool(current.registration, args.workspaceRequest!) + (args.workspaceRequest != null && + !supportsWorkspaceTool(current.registration, args.workspaceRequest)) || + (args.workspaceId != null && + !supportsWorkspaceProgrammatic( + current.registration, + args.workspaceId, + args.body.language, + )) ) { throw new BridgeStoreError( 'WORKER_MISMATCH', @@ -935,11 +984,14 @@ export class RedisBridgeStore { generation, leaseToken, leaseTokenHash: tokenHash(leaseToken), + ...(selectedWorkspaceId == null ? {} : { + workspaceFence: `native-workspace:${selectedWorkspaceId}`, + }), ...(workspaceLeaseSlot === undefined ? {} : { workspaceLeaseSlot, - workspaceFence: `native-workspace:${args.workspaceRequest!.workspaceId}`, + workspaceFence: `native-workspace:${selectedWorkspaceId!}`, }), ...(registration.identityId != null ? { workerIdentityId: registration.identityId } @@ -951,6 +1003,15 @@ export class RedisBridgeStore { executionKind: 'workspace_tool' as const, request: args.workspaceRequest, } + : args.workspaceId != null + ? { + executionKind: 'workspace_programmatic' as const, + workspaceId: args.workspaceId, + request: { + body: args.body, + headers: args.headers, + }, + } : { request: { body: args.body, @@ -1011,6 +1072,19 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} no longer advertises the requested workspace tool`, ); } + if ( + args.workspaceId != null && + !supportsWorkspaceProgrammatic( + replacement.registration, + args.workspaceId, + args.body.language, + ) + ) { + throw new BridgeStoreError( + 'WORKER_MISMATCH', + `Bridge worker ${args.workerId} no longer advertises programmatic execution for the selected workspace`, + ); + } registration = replacement.registration; readyToken = replacement.readyToken; } @@ -1034,11 +1108,24 @@ export class RedisBridgeStore { resultCommitted = true; return result; } catch (error) { - if (args.runtimeSessionId !== undefined) { + if (assignment.workspaceFence != null) { + // Native roots retain their own fence through result restoration. + // Do not quarantine unrelated roots or invalidate the worker lease. + await boundedCommand(this.redis.eval( + [ + "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", + "redis.call('SET', KEYS[1], 'quarantined:' .. ARGV[1])", + 'return 1', + ].join('\n'), + 1, + workspaceQuarantineKey(args.workerId, assignment.workspaceFence), + assignment.assignmentId, + ), this.redisCommandTimeoutMs, 'Bridge native workspace finalization quarantine'); + } else if (assignmentWorkspace(assignment) !== undefined) { await this.quarantine( args.workerId, assignment.incarnationId, - args.runtimeSessionId, + assignmentWorkspace(assignment)!, ); } throw error; @@ -1904,10 +1991,11 @@ export class RedisBridgeStore { : undefined; const cancelledMutation = signal.aborted && - workspaceRequest != null && - (workspaceRequest.operation === 'write_file' || - workspaceRequest.operation === 'edit_file' || - workspaceRequest.operation === 'execute_command'); + (assignment.executionKind === 'workspace_programmatic' || + (workspaceRequest != null && + (workspaceRequest.operation === 'write_file' || + workspaceRequest.operation === 'edit_file' || + workspaceRequest.operation === 'execute_command'))); if (cancelledMutation) { try { // Keep the acknowledged assignment available long enough for the diff --git a/service/src/bridge/workspace-store.test.ts b/service/src/bridge/workspace-store.test.ts index 5b0186d0..f248aa40 100644 --- a/service/src/bridge/workspace-store.test.ts +++ b/service/src/bridge/workspace-store.test.ts @@ -77,6 +77,123 @@ test('dispatches a workspace tool only to a worker advertising its workspace and }); }); +for (const finalizationFails of [false, true]) test(`single-slot programmatic finalization retains the workspace fence (failure=${finalizationFails})`, async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['execute_command'], + programmaticLanguages: ['bash'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const body = { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }; + const completion = store.dispatch({ + workerId: 'workspace-worker', + body, + headers: {}, + workspaceId: 'primary', + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + finalize: async settlement => { + if (finalizationFails) throw new Error('artifact restoration failed'); + return settlement; + }, + }); + + const assignment = await store.lease('workspace-worker', incarnationId, 1_000); + expect(assignment).toMatchObject({ + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { body }, + }); + await store.settle('workspace-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: { + session_id: 'session-1', + language: 'bash', + version: '5.2', + files: [], + run: { + stdout: 'ready\n', + stderr: '', + code: 0, + signal: null, + output: 'ready\n', + memory: null, + message: null, + status: null, + cpu_time: null, + wall_time: 0.01, + }, + }, + }); + + if (finalizationFails) { + await expect(completion).rejects.toThrow('artifact restoration failed'); + await expect(store.dispatch({ workerId: 'workspace-worker', body, headers: {}, + workspaceId: 'primary', deadlineAtMs: Date.now() + 1000, + signal: new AbortController().signal, + })).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED' }); + return; + } + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + result: { session_id: 'session-1' }, + }); +}); + +test('rejects programmatic execution without the workspace capability', async () => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['execute_command'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatch({ + workerId: 'workspace-worker', + body: { + language: 'bash', + version: '5.2', + session_id: 'session-2', + files: [{ name: 'main.sh', content: 'echo denied' }], + }, + headers: {}, + workspaceId: 'primary', + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); +}); + test('drains an acknowledged workspace mutation cancellation before releasing it', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index f86ce656..873db173 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -45,6 +45,7 @@ import { parseBoundedContentLength } from './http-limits'; import { validateEgressGatewayHardenedConfig } from './secure-startup'; import { isOpaqueObjectContentDisposition } from './file-metadata'; import { mapObjectDetails } from './file-object-resolver'; +import { isSupportedBridgeArtifactName } from '../../packages/code/src/protocol'; export const app: Express = express(); app.disable('x-powered-by'); @@ -52,29 +53,6 @@ validateEgressGatewayHardenedConfig(); app.use(traceHttpRequest('codeapi.egress_gateway.request')); app.use(httpMetricsMiddleware); -const SUPPORTED_OUTPUT_EXTENSIONS = new Set([ - '.c', '.cs', '.cpp', '.go', '.java', '.js', '.kt', '.kts', '.lua', - '.php', '.pl', '.ps1', '.py', '.r', '.rb', '.rs', '.scala', '.sh', - '.sql', '.swift', '.ts', '.jsx', '.tsx', '.groovy', - '.css', '.htm', '.html', '.less', '.sass', '.scss', '.svg', '.svelte', '.vue', - '.adoc', '.asciidoc', '.md', '.rst', '.tex', '.txt', '.wiki', - '.csv', '.json', '.bson', '.json5', '.jsonl', '.parquet', '.tsv', - '.xml', '.yaml', '.yml', - '.ics', '.ical', '.ifb', '.icalendar', - '.conf', '.env', '.gitignore', '.ini', '.properties', '.toml', - '.doc', '.docx', '.pdf', '.ppt', '.pptx', '.xls', '.xlsx', - '.odt', '.ods', '.odp', '.rtf', - '.avif', '.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', - '.tif', '.tiff', '.webp', - '.eot', '.ttf', '.woff', '.woff2', - '.7z', '.bz2', '.gz', '.gzip', '.rar', '.tar', '.zip', - '.tf', '.tfvars', '.tfstate', '.hcl', - '.dockerfile', '.Dockerfile', '.dockerignore', - '.helmignore', '.helmfile', '.jenkinsfile', '.vagrantfile', - '.eslintrc', '.prettierrc', '.editorconfig', '.nomad', - '.bat', '.cmd', '.deb', '.log', '.rpm', '.vbs', -]); - type EgressAuditFields = { execHash?: string; requestExecHash?: string; @@ -248,18 +226,7 @@ function assertOutputFilenameAllowed(name: string): void { throw new EgressGrantError('malformed', 'Output filename must be canonical'); } if (!isDirkeepName(name)) { - const basename = path.posix.basename(name); - const ext = path.posix.extname(basename).toLowerCase(); - const dottedBasename = `.${basename}`; - const allowed = - (ext !== '' && SUPPORTED_OUTPUT_EXTENSIONS.has(ext)) || - SUPPORTED_OUTPUT_EXTENSIONS.has(basename) || - SUPPORTED_OUTPUT_EXTENSIONS.has(basename.toLowerCase()) || - (ext === '' && ( - SUPPORTED_OUTPUT_EXTENSIONS.has(dottedBasename) || - SUPPORTED_OUTPUT_EXTENSIONS.has(dottedBasename.toLowerCase()) - )); - if (!allowed) { + if (!isSupportedBridgeArtifactName(name)) { throw new EgressGrantError('scope_mismatch', 'Output filename extension is not supported'); } } diff --git a/service/src/egress-grant.test.ts b/service/src/egress-grant.test.ts index 32ef2988..cd26fcd7 100644 --- a/service/src/egress-grant.test.ts +++ b/service/src/egress-grant.test.ts @@ -5,6 +5,7 @@ import { env } from './config'; import { normalizeEgressGatewayUrl, normalizeProgrammaticTimeoutMs, + normalizeSelectedWorkspaceProgrammaticTimeoutMs, prepareSandboxJobSecurity, refreshEgressGrantClaims, timeoutMsToGrantSeconds, @@ -478,6 +479,13 @@ describe('egress encrypted grants and handles', () => { expect(() => normalizeProgrammaticTimeoutMs(0, 300000)).toThrow('timeout must be a positive number'); }); + test('budgets both selected-workspace replay passes inside the worker deadline', () => { + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(undefined, 300_000)).toBe(82_500); + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(120_000, 300_000)).toBe(82_500); + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(300_000, 300_000)).toBe(82_500); + expect(normalizeSelectedWorkspaceProgrammaticTimeoutMs(10_000, 20_000)).toBe(2_167); + }); + test('normalizes the gateway callback URL for sandbox-originated PTC', () => { expect(normalizeEgressGatewayUrl(' http://egress-gateway:3190/// ')).toBe('http://egress-gateway:3190'); expect(() => normalizeEgressGatewayUrl(' ')).toThrow('EGRESS_GATEWAY_URL is required'); diff --git a/service/src/preamble-bash.test.ts b/service/src/preamble-bash.test.ts index 815040d3..e35a77f5 100644 --- a/service/src/preamble-bash.test.ts +++ b/service/src/preamble-bash.test.ts @@ -1,10 +1,19 @@ import { execFileSync } from 'child_process'; -import { mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'fs'; import { tmpdir } from 'os'; import { join } from 'path'; import { describe, expect, test } from 'bun:test'; import { extractPendingFromStdout, type LCTool } from './preamble'; -import { generateBashReplayPostamble, generateBashReplayPreamble } from './preamble-bash'; +import { + generateBashReplayPostamble, + generateBashReplayPreamble, +} from './preamble-bash'; interface BashRunResult { stdout: string; @@ -60,9 +69,13 @@ function assemble(userCode: string, toolSet: LCTool[] = tools): string { ].join('\n'); } -function runBash(script: string, options: number | BashRunOptions = {}): BashRunResult { - const timeoutMs = typeof options === 'number' ? options : options.timeoutMs ?? 3000; - const history = typeof options === 'number' ? {} : options.history ?? {}; +function runBash( + script: string, + options: number | BashRunOptions = {}, +): BashRunResult { + const timeoutMs = + typeof options === 'number' ? options : (options.timeoutMs ?? 3000); + const history = typeof options === 'number' ? {} : (options.history ?? {}); const dir = mkdtempSync(join(tmpdir(), 'ptc-bash-unit-')); const file = join(dir, 'main.sh'); const historyPath = join(dir, 'history.json'); @@ -99,46 +112,136 @@ function pendingNames(stdout: string): string[] { return (parsed.pending ?? []).map(call => call.tool_name).sort(); } +describe('generateBashReplayPreamble - private runtime directory', () => { + test('creates every replay tempfile beneath TMPDIR', () => { + const dir = mkdtempSync(join(tmpdir(), 'ptc-bash-private-tmp-')); + const dataDir = join(dir, 'data'); + const runtimeDir = join(dir, 'runtime'); + const file = join(dir, 'main.sh'); + const historyPath = join(dataDir, 'history.json'); + mkdirSync(dataDir, { recursive: true }); + mkdirSync(runtimeDir, { recursive: true }); + writeFileSync(historyPath, '{}'); + writeFileSync( + file, + assemble(` +printf '%s\\n' "$_PTC_PENDING_FILE" "$_PTC_ERROR_FILE" "$_PTC_COUNTER_FILE" +`), + { mode: 0o755 }, + ); + + try { + const stdout = execFileSync('bash', [file], { + env: { + ...process.env, + PTC_HISTORY_PATH: historyPath, + TMPDIR: runtimeDir, + }, + encoding: 'utf8', + }); + const paths = stdout.trim().split('\n'); + expect(paths).toHaveLength(3); + expect( + paths.every(value => value.startsWith(`${runtimeDir}/`)), + ).toBe(true); + expect( + generateBashReplayPreamble({ executionId, tools }), + ).not.toContain('mktemp -t'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test('persists pending calls through the private native control path', () => { + const dir = mkdtempSync(join(tmpdir(), 'ptc-bash-control-')); + const file = join(dir, 'main.sh'); + const historyPath = join(dir, 'history.json'); + const controlPath = join(dir, 'control.json'); + writeFileSync(file, assemble(`get_weather '{"city":"Paris"}'`), { + mode: 0o755, + }); + writeFileSync(historyPath, '{}'); + try { + execFileSync('bash', [file], { + env: { + ...process.env, + PTC_HISTORY_PATH: historyPath, + LIBRECHAT_CODE_CONTROL_PATH: controlPath, + TMPDIR: dir, + }, + encoding: 'utf8', + }); + expect(JSON.parse(readFileSync(controlPath, 'utf8'))).toMatchObject( + { + pending: [ + { + call_id: 'call_001', + tool_name: 'get_weather', + input: { city: 'Paris' }, + }, + ], + }, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe('generateBashReplayPreamble - command substitution pending emission', () => { test('emits ClickHouse-style object input with SQL quotes from double-quoted JSON', () => { - const run = runBash(assemble(` + const run = runBash( + assemble( + ` SVC="45886e06-932b-4cff-bb49-3f7281d80717" result=$(run_select_query_mcp_ClickHouse "{\\"serviceId\\":\\"$SVC\\",\\"query\\":\\"SELECT name, round(avg(tempAvg)/10.0, 2) AS avg_temp_c FROM system.columns WHERE database='default' AND table='uk_prices_3' AND tempAvg != -9999\\"}") echo "AFTER: $result" -`, [clickHouseTool])); +`, + [clickHouseTool], + ), + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); expect(parsed.pending).toHaveLength(1); - expect(parsed.pending?.[0]?.tool_name).toBe('run_select_query_mcp_ClickHouse'); + expect(parsed.pending?.[0]?.tool_name).toBe( + 'run_select_query_mcp_ClickHouse', + ); expect(parsed.pending?.[0]?.input).toEqual({ serviceId: '45886e06-932b-4cff-bb49-3f7281d80717', - query: - "SELECT name, round(avg(tempAvg)/10.0, 2) AS avg_temp_c FROM system.columns WHERE database='default' AND table='uk_prices_3' AND tempAvg != -9999", + query: "SELECT name, round(avg(tempAvg)/10.0, 2) AS avg_temp_c FROM system.columns WHERE database='default' AND table='uk_prices_3' AND tempAvg != -9999", }); expect(parsed.stdout).not.toContain('AFTER'); }); test('emits ClickHouse-style object input with shell-escaped SQL quotes', () => { - const run = runBash(assemble(` + const run = runBash( + assemble( + ` result=$(run_select_query_mcp_ClickHouse '{"serviceId":"45886e06-932b-4cff-bb49-3f7281d80717","query":"SELECT name, type FROM system.columns WHERE database='"'"'default'"'"' AND table='"'"'uk_prices_3'"'"' ORDER BY position"}') echo "AFTER: $result" -`, [clickHouseTool])); +`, + [clickHouseTool], + ), + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); expect(parsed.pending).toHaveLength(1); - expect(parsed.pending?.[0]?.tool_name).toBe('run_select_query_mcp_ClickHouse'); + expect(parsed.pending?.[0]?.tool_name).toBe( + 'run_select_query_mcp_ClickHouse', + ); expect(parsed.pending?.[0]?.input).toEqual({ serviceId: '45886e06-932b-4cff-bb49-3f7281d80717', - query: - "SELECT name, type FROM system.columns WHERE database='default' AND table='uk_prices_3' ORDER BY position", + query: "SELECT name, type FROM system.columns WHERE database='default' AND table='uk_prices_3' ORDER BY position", }); expect(parsed.stdout).not.toContain('AFTER'); }); test('batches parallel ClickHouse-style command substitutions into one pending block', () => { - const run = runBash(assemble(` + const run = runBash( + assemble( + ` SVC="45886e06-932b-4cff-bb49-3f7281d80717" { @@ -158,7 +261,11 @@ SVC="45886e06-932b-4cff-bb49-3f7281d80717" wait echo "AFTER" -`, [clickHouseTool]), 3000); +`, + [clickHouseTool], + ), + 3000, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); @@ -168,7 +275,11 @@ echo "AFTER" 'run_select_query_mcp_ClickHouse', 'run_select_query_mcp_ClickHouse', ]); - expect(parsed.pending?.map(call => (call.input as { query: string }).query).sort()).toEqual([ + expect( + parsed.pending + ?.map(call => (call.input as { query: string }).query) + .sort(), + ).toEqual([ "SELECT name, engine, total_rows, formatReadableSize(total_bytes) AS size, sorting_key, partition_key FROM system.tables WHERE database='default' AND name IN ('uk_prices_3','weather_noaa_mt')", "SELECT name, type, comment FROM system.columns WHERE database='default' AND table='uk_prices_3' ORDER BY position", "SELECT name, type, comment FROM system.columns WHERE database='default' AND table='weather_noaa_mt' ORDER BY position", @@ -177,12 +288,14 @@ echo "AFTER" }); test('emits a command-substitution tool call before later user code while another job is running', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` sleep 0.2 & result=$(get_weather '{"city":"Madrid"}') echo "AFTER: $result" wait -`)); +`), + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -194,12 +307,15 @@ wait }); test('batches background and command-substitution tool calls before command-substitution side effects', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Oslo"}' & result=$(calculate '{"expression":"2+3"}') echo "SIDE_EFFECT: $result" wait -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -210,10 +326,13 @@ wait }); test('waits for background compound commands that invoke tools later', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` (sleep 0.2; get_weather '{"city":"Paris"}') & echo "AFTER LAUNCH" -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -225,10 +344,13 @@ echo "AFTER LAUNCH" }); test('does not wait for unrelated background commands with tool names as arguments', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` bash -c 'sleep 2' get_weather & echo "DONE" -`), 700); +`), + 700, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -238,14 +360,17 @@ echo "DONE" }); test('does not treat arithmetic expansion as command substitution while batching background tools', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Oslo"}' & sleep 0.1 x=$((1+1)) calculate '{"expression":"2+3"}' & wait echo "DONE $x" -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -256,12 +381,15 @@ echo "DONE $x" }); test('handles backtick command substitution without waiting for unrelated background jobs', () => { - const run = runBash(assemble(` + const run = runBash( + assemble(` sleep 5 & result=\`get_weather '{"city":"Porto"}'\` echo "AFTER: $result" wait -`), 1500); +`), + 1500, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -281,7 +409,10 @@ wait echo "DONE" `; const firstRun = runBash(assemble(userCode)); - const firstParsed = extractPendingFromStdout(firstRun.stdout, executionId); + const firstParsed = extractPendingFromStdout( + firstRun.stdout, + executionId, + ); expect(firstRun.exitCode).toBe(0); expect(firstParsed.pending).toHaveLength(2); @@ -302,7 +433,10 @@ echo "DONE" }), ); const replayRun = runBash(assemble(userCode), { history }); - const replayParsed = extractPendingFromStdout(replayRun.stdout, executionId); + const replayParsed = extractPendingFromStdout( + replayRun.stdout, + executionId, + ); expect(replayRun.exitCode).toBe(0); expect(replayParsed.pending).toBeNull(); expect(replayParsed.stdout).toContain('"slot":"first"'); @@ -326,12 +460,15 @@ echo "DONE" }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' get_weather '{"city":"Paris"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); @@ -352,12 +489,15 @@ printf '\\nDONE\\n' }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' calculate '{"expression":"2+3"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.signal).not.toBe('SIGTERM'); @@ -385,12 +525,15 @@ printf '\\nDONE\\n' }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' calculate '{"expression":"2+3"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); @@ -418,12 +561,15 @@ printf '\\nDONE\\n' }, }; - const run = runBash(assemble(` + const run = runBash( + assemble(` get_weather '{"city":"Paris"}' printf '\\n' calculate '{"expression":"2+3"}' printf '\\nDONE\\n' -`), { history }); +`), + { history }, + ); const parsed = extractPendingFromStdout(run.stdout, executionId); expect(run.exitCode).toBe(0); diff --git a/service/src/preamble-bash.ts b/service/src/preamble-bash.ts index 4ec3fc05..c442b495 100644 --- a/service/src/preamble-bash.ts +++ b/service/src/preamble-bash.ts @@ -1,8 +1,5 @@ import type { LCTool } from './preamble'; -import { - buildScopedSentinel, - PTC_HISTORY_SANDBOX_PATH, -} from './ptc-constants'; +import { buildScopedSentinel, PTC_HISTORY_SANDBOX_PATH } from './ptc-constants'; export interface BashReplayPreambleConfig { executionId: string; @@ -42,12 +39,55 @@ export class BashToolNameCollisionError extends Error { } const BASH_RESERVED = new Set([ - 'if', 'then', 'else', 'elif', 'fi', 'case', 'esac', 'for', 'select', - 'while', 'until', 'do', 'done', 'in', 'function', 'time', 'coproc', - 'return', 'exit', 'break', 'continue', 'shift', 'export', 'readonly', - 'local', 'declare', 'typeset', 'unset', 'alias', 'unalias', 'source', - 'echo', 'printf', 'read', 'cd', 'pwd', 'kill', 'trap', 'wait', 'eval', - 'exec', 'jobs', 'bg', 'fg', 'set', 'let', 'test', 'true', 'false', + 'if', + 'then', + 'else', + 'elif', + 'fi', + 'case', + 'esac', + 'for', + 'select', + 'while', + 'until', + 'do', + 'done', + 'in', + 'function', + 'time', + 'coproc', + 'return', + 'exit', + 'break', + 'continue', + 'shift', + 'export', + 'readonly', + 'local', + 'declare', + 'typeset', + 'unset', + 'alias', + 'unalias', + 'source', + 'echo', + 'printf', + 'read', + 'cd', + 'pwd', + 'kill', + 'trap', + 'wait', + 'eval', + 'exec', + 'jobs', + 'bg', + 'fg', + 'set', + 'let', + 'test', + 'true', + 'false', ]); function normalizeBashFunctionName(name: string): string { @@ -60,10 +100,7 @@ function normalizeBashFunctionName(name: string): string { * the end-of-preamble `readonly -f` lockdown runs. Compared case- * insensitively because the `_PTC_` prefix is used for variables and * `_ptc_` for functions, and both live in the same identifier space. */ - if ( - BASH_RESERVED.has(normalized) || - /^_ptc_/i.test(normalized) - ) { + if (BASH_RESERVED.has(normalized) || /^_ptc_/i.test(normalized)) { normalized = normalized + '_tool'; } if (normalized === '') normalized = 'tool'; @@ -95,9 +132,12 @@ function escapeForBashEre(s: string): string { * Users capture results via command substitution; input is passed as a single * JSON object string argument (validated by jq). */ -export function generateBashReplayPreamble(config: BashReplayPreambleConfig): string { +export function generateBashReplayPreamble( + config: BashReplayPreambleConfig, +): string { const { executionId, tools } = config; - const { start: scopedStart, end: scopedEnd } = buildScopedSentinel(executionId); + const { start: scopedStart, end: scopedEnd } = + buildScopedSentinel(executionId); let preamble = `#!/bin/bash # ============================================================================ @@ -109,20 +149,25 @@ _PTC_EXECUTION_ID="${executionId}" _PTC_SENTINEL_START="${scopedStart}" _PTC_SENTINEL_END="${scopedEnd}" _PTC_HISTORY_PATH="\${PTC_HISTORY_PATH:-${PTC_HISTORY_SANDBOX_PATH}}" -_PTC_PENDING_FILE="$(mktemp -t _ptc_pending.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_pending.XXXXXX)" -_PTC_ERROR_FILE="$(mktemp -t _ptc_error.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_error.XXXXXX)" -_PTC_CONSUMED_FILE="$(mktemp -t _ptc_consumed.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_consumed.XXXXXX)" -_PTC_SAW_BARE_TOOL_FILE="$(mktemp -t _ptc_saw_tool.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_saw_tool.XXXXXX)" -_PTC_PRE_TOOL_JOBS_FILE="$(mktemp -t _ptc_pre_tool_jobs.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_pre_tool_jobs.XXXXXX)" -_PTC_PRE_TOOL_JOBS_READY_FILE="$(mktemp -t _ptc_pre_tool_jobs_ready.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_pre_tool_jobs_ready.XXXXXX)" -_PTC_TOOL_JOBS_FILE="$(mktemp -t _ptc_tool_jobs.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_tool_jobs.XXXXXX)" -_PTC_WAIT_RAN_FILE="$(mktemp -t _ptc_wait_ran.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_wait_ran.XXXXXX)" -_PTC_SUPPRESS_SUBSHELL_TOOL_FILE="$(mktemp -t _ptc_suppress_subshell_tool.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_suppress_subshell_tool.XXXXXX)" -_PTC_SUPPRESS_SUBSHELL_TOOL_CLEAR_FILE="$(mktemp -t _ptc_suppress_subshell_tool_clear.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_suppress_subshell_tool_clear.XXXXXX)" +_PTC_CONTROL_PATH="\${LIBRECHAT_CODE_CONTROL_PATH:-}" +_PTC_RUNTIME_DIR="\${TMPDIR:-/tmp}" +_ptc_mktemp() { + mktemp "\${_PTC_RUNTIME_DIR%/}/$1.XXXXXX" +} +_PTC_PENDING_FILE="$(_ptc_mktemp _ptc_pending)" +_PTC_ERROR_FILE="$(_ptc_mktemp _ptc_error)" +_PTC_CONSUMED_FILE="$(_ptc_mktemp _ptc_consumed)" +_PTC_SAW_BARE_TOOL_FILE="$(_ptc_mktemp _ptc_saw_tool)" +_PTC_PRE_TOOL_JOBS_FILE="$(_ptc_mktemp _ptc_pre_tool_jobs)" +_PTC_PRE_TOOL_JOBS_READY_FILE="$(_ptc_mktemp _ptc_pre_tool_jobs_ready)" +_PTC_TOOL_JOBS_FILE="$(_ptc_mktemp _ptc_tool_jobs)" +_PTC_WAIT_RAN_FILE="$(_ptc_mktemp _ptc_wait_ran)" +_PTC_SUPPRESS_SUBSHELL_TOOL_FILE="$(_ptc_mktemp _ptc_suppress_subshell_tool)" +_PTC_SUPPRESS_SUBSHELL_TOOL_CLEAR_FILE="$(_ptc_mktemp _ptc_suppress_subshell_tool_clear)" # Counter must persist across subshells (command substitution) so call_ids # stay deterministic across cached/uncached calls. Bash variables set in a # subshell don't propagate back, so we use a file. -_PTC_COUNTER_FILE="$(mktemp -t _ptc_counter.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_counter.XXXXXX)" +_PTC_COUNTER_FILE="$(_ptc_mktemp _ptc_counter)" _PTC_LOCK_DIR="\${_PTC_PENDING_FILE}.lock" printf '0' > "$_PTC_COUNTER_FILE" : > "$_PTC_CONSUMED_FILE" @@ -243,7 +288,7 @@ _ptc_prune_finished_tool_jobs() { return 0 fi local _ptc_tmp_file - _ptc_tmp_file="$(mktemp -t _ptc_tool_jobs_live.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_tool_jobs_live.XXXXXX)" + _ptc_tmp_file="$(_ptc_mktemp _ptc_tool_jobs_live)" while IFS= read -r _ptc_pid; do [ -n "$_ptc_pid" ] || continue if kill -0 "$_ptc_pid" 2>/dev/null; then @@ -330,6 +375,17 @@ _ptc_maybe_emit_pending() { trap - DEBUG EXIT exit 1 fi + # Native BYOM workers use this private execution-scoped control file so a + # large stdout stream cannot truncate away the replay frame. Other + # backends continue to consume the stdout sentinel below. + if [ -n "$_PTC_CONTROL_PATH" ]; then + printf '%s' "$_ptc_payload" > "$_PTC_CONTROL_PATH" || { + printf 'failed to persist pending PTC tool calls\n' >&2 + _ptc_cleanup_tempfiles + trap - DEBUG EXIT + exit 1 + } + fi if [ "\${BASH_SUBSHELL:-0}" -eq 1 ]; then trap - DEBUG EXIT exit 0 @@ -512,7 +568,7 @@ _ptc_call_tool() { # Large input can exceed ARG_MAX via --argjson; write once, reuse path below. local _ptc_input_tmp - _ptc_input_tmp="$(mktemp -t _ptc_input.XXXXXX 2>/dev/null || mktemp /tmp/_ptc_input.XXXXXX)" + _ptc_input_tmp="$(_ptc_mktemp _ptc_input)" printf '%s' "$_ptc_input" > "$_ptc_input_tmp" local _ptc_matches @@ -668,8 +724,12 @@ exit $_ptc_user_exit_code function generateBashToolStub(tool: LCTool): string { const fnName = normalizeBashFunctionName(tool.name); - const desc = (tool.description ?? '').split('\n').map(l => `# ${l}`).join('\n'); - const nameComment = fnName !== tool.name ? `# Original tool name: ${tool.name}\n` : ''; + const desc = (tool.description ?? '') + .split('\n') + .map(l => `# ${l}`) + .join('\n'); + const nameComment = + fnName !== tool.name ? `# Original tool name: ${tool.name}\n` : ''; const escapedToolName = escapeForBashDoubleQuote(tool.name); return `${nameComment}${desc ? desc + '\n' : ''}${fnName}() { local _default_input='{}' @@ -683,7 +743,8 @@ function generateBashToolStub(tool: LCTool): string { } function generateBashPendingDeferHelper(tools: readonly LCTool[]): string { - const toolNamesPattern = tools + const toolNamesPattern = + tools .map(tool => normalizeBashFunctionName(tool.name)) .map(escapeForBashEre) .join('|') || 'a^'; diff --git a/service/src/preamble.test.ts b/service/src/preamble.test.ts index bf50e3ec..803e1839 100644 --- a/service/src/preamble.test.ts +++ b/service/src/preamble.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test'; import { buildScopedSentinel, createProgrammaticPayload, + extractPendingFromControlPayload, extractPendingFromStdout, generatePreamble, } from './preamble'; @@ -33,7 +34,9 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { expect(preamble).toMatch(/AF_UNIX/); expect(preamble).toMatch(/\.connect\(_TOOL_CALL_SOCKET\)/); /* Regression guard against reintroducing the user-spoofable check. */ - expect(preamble).not.toMatch(/if\s+os\.path\.exists\(_TOOL_CALL_SOCKET\)/); + expect(preamble).not.toMatch( + /if\s+os\.path\.exists\(_TOOL_CALL_SOCKET\)/, + ); }); test('caches the probe verdict at module load (before user code can plant a spoof)', () => { @@ -41,10 +44,14 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { /* The probe call must appear at top level of the preamble, NOT * inside _do_request. Otherwise a user could plant a regular file * at the path between calls and flip the gate per-request. */ - const probeCallIdx = preamble.indexOf('_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket()'); + const probeCallIdx = preamble.indexOf( + '_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket()', + ); expect(probeCallIdx).toBeGreaterThan(-1); /* _do_request must consult the cached verdict, not re-probe. */ - const doReqMatch = preamble.match(/def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/); + const doReqMatch = preamble.match( + /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, + ); expect(doReqMatch).not.toBeNull(); expect(doReqMatch![0]).toContain('_USE_TOOL_CALL_SOCKET'); expect(doReqMatch![0]).not.toContain('_probe_tool_call_socket('); @@ -55,7 +62,9 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { /* The fallback must still construct the URL from _CALLBACK_URL and * delegate to _tcp_request. Without this, runners without the * proxy bind-mount would have no way to reach the orchestrator. */ - const doReqMatch = preamble.match(/def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/); + const doReqMatch = preamble.match( + /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, + ); expect(doReqMatch).not.toBeNull(); expect(doReqMatch![0]).toContain('_CALLBACK_URL + path'); expect(doReqMatch![0]).toContain('_tcp_request('); @@ -67,24 +76,47 @@ describe('generatePreamble — Unix-vs-TCP transport gate', () => { * exported. The path must remain hardcoded so the preamble does * not depend on env-var injection. */ expect(preamble).toContain('_TOOL_CALL_SOCKET = "/tmp/tcs.sock"'); - expect(preamble).not.toMatch(/os\.environ\.get\(['"]TOOL_CALL_SOCKET['"]/); + expect(preamble).not.toMatch( + /os\.environ\.get\(['"]TOOL_CALL_SOCKET['"]/, + ); expect(preamble).not.toMatch(/os\.environ\[['"]TOOL_CALL_SOCKET['"]\]/); }); }); describe('extractPendingFromStdout — input hash metadata', () => { + test('normalizes native control payload hashes instead of trusting the sandbox', () => { + const forgedHash = hashToolInput({ resource: 'B' }); + const expectedHash = hashToolInput({ resource: 'A' }); + const pending = extractPendingFromControlPayload( + JSON.stringify({ + pending: [ + { + call_id: 'call_001', + tool_name: 'authorize', + input: { resource: 'A' }, + input_hash: forgedHash, + }, + ], + }), + ); + expect(pending?.[0]?.input_hash).toBe(expectedHash); + expect(pending?.[0]?.input_hash).not.toBe(forgedHash); + }); + test('ignores sandbox-supplied input_hash and uses the parsed input hash', () => { const executionId = 'exec_hash_guard'; const { start, end } = buildScopedSentinel(executionId); const forgedHash = hashToolInput({ resource: 'B' }); const expectedHash = hashToolInput({ resource: 'A' }); const payload = { - pending: [{ + pending: [ + { call_id: 'call_001', tool_name: 'authorize', input: { resource: 'A' }, input_hash: forgedHash, - }], + }, + ], }; const parsed = extractPendingFromStdout( diff --git a/service/src/preamble.ts b/service/src/preamble.ts index 9ff75dc7..94685c6c 100644 --- a/service/src/preamble.ts +++ b/service/src/preamble.ts @@ -2,7 +2,10 @@ import fs from 'fs'; import path from 'path'; import type * as t from './types'; import { planLimits } from './config'; -import { generateBashReplayPreamble, generateBashReplayPostamble } from './preamble-bash'; +import { + generateBashReplayPreamble, + generateBashReplayPostamble, +} from './preamble-bash'; import { PTC_HISTORY_FILENAME, PTC_HISTORY_SANDBOX_PATH, @@ -11,10 +14,16 @@ import { buildScopedSentinel, isReservedPtcFilename, } from './ptc-constants'; -import { hashToolInput, pendingInputHashesFromRawPayload } from './tool-input-signature'; +import { + hashToolInput, + pendingInputHashesFromRawPayload, +} from './tool-input-signature'; // Load async matplotlib template for programmatic tool calling -const templateCodeAsync = fs.readFileSync(path.join(__dirname, 'matplotlib-async.py'), 'utf8'); +const templateCodeAsync = fs.readFileSync( + path.join(__dirname, 'matplotlib-async.py'), + 'utf8', +); // ============================================================================= // Programmatic Tool Calling Types & Preamble Generation @@ -88,11 +97,41 @@ function normalizePythonFunctionName(name: string): string { // Python keywords to avoid const pythonKeywords = new Set([ - 'False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', - 'break', 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', - 'finally', 'for', 'from', 'global', 'if', 'import', 'in', 'is', - 'lambda', 'nonlocal', 'not', 'or', 'pass', 'raise', 'return', - 'try', 'while', 'with', 'yield' + 'False', + 'None', + 'True', + 'and', + 'as', + 'assert', + 'async', + 'await', + 'break', + 'class', + 'continue', + 'def', + 'del', + 'elif', + 'else', + 'except', + 'finally', + 'for', + 'from', + 'global', + 'if', + 'import', + 'in', + 'is', + 'lambda', + 'nonlocal', + 'not', + 'or', + 'pass', + 'raise', + 'return', + 'try', + 'while', + 'with', + 'yield', ]); if (pythonKeywords.has(normalized)) { @@ -137,11 +176,14 @@ function jsonSchemaToPythonType(schema: JsonSchemaProperty): string { * Sort property names so required parameters come before optional ones. * Uses a Set for O(1) lookups instead of repeated array includes() calls. */ -function getSortedPropertyNames(propertyNames: string[], required: string[]): string[] { +function getSortedPropertyNames( + propertyNames: string[], + required: string[], +): string[] { const requiredSet = new Set(required); return [ ...propertyNames.filter(name => requiredSet.has(name)), - ...propertyNames.filter(name => !requiredSet.has(name)) + ...propertyNames.filter(name => !requiredSet.has(name)), ]; } @@ -155,7 +197,10 @@ function schemaToParams(schema?: JsonSchema): string { const required = schema.required ?? []; const requiredSet = new Set(required); - const sortedNames = getSortedPropertyNames(Object.keys(schema.properties), required); + const sortedNames = getSortedPropertyNames( + Object.keys(schema.properties), + required, + ); const params: string[] = []; @@ -192,7 +237,11 @@ function inferReturnType(description?: string): string { const desc = description.toLowerCase(); - if (desc.includes('returns list') || desc.includes('returns array') || desc.includes('list of')) { + if ( + desc.includes('returns list') || + desc.includes('returns array') || + desc.includes('list of') + ) { return 'List[Dict[str, Any]]'; } if (desc.includes('returns dict') || desc.includes('returns object')) { @@ -225,7 +274,10 @@ function generateDocstring(tool: LCTool): string { doc += '\n\n Parameters:'; const required = tool.parameters.required ?? []; const requiredSet = new Set(required); - const sortedNames = getSortedPropertyNames(Object.keys(tool.parameters.properties), required); + const sortedNames = getSortedPropertyNames( + Object.keys(tool.parameters.properties), + required, + ); for (const name of sortedNames) { const propSchema = tool.parameters.properties[name]; @@ -253,7 +305,8 @@ function generateToolStub(tool: LCTool): string { const pythonFunctionName = normalizePythonFunctionName(tool.name); // If name was changed, add a comment - const nameComment = pythonFunctionName !== tool.name + const nameComment = + pythonFunctionName !== tool.name ? ` # Original tool name: ${tool.name}\n` : ''; @@ -451,7 +504,8 @@ async def _execute_tool_internal_async(tool_name: str, tool_input: Dict[str, Any */ export function generateReplayPreamble(config: ReplayPreambleConfig): string { const { executionId, tools } = config; - const { start: scopedStart, end: scopedEnd } = buildScopedSentinel(executionId); + const { start: scopedStart, end: scopedEnd } = + buildScopedSentinel(executionId); let preamble = ` # ============================================================================ @@ -564,6 +618,63 @@ export interface ExtractPendingResult { }> | null; } +export function extractPendingFromControlPayload( + rawPayload: string, +): ExtractPendingResult['pending'] { + let parsed: { pending?: unknown } | null = null; + try { + parsed = JSON.parse(rawPayload) as { pending?: unknown }; + } catch { + return null; + } + + const pendingField = parsed?.pending; + if (!Array.isArray(pendingField)) return null; + + const rawInputHashes = pendingInputHashesFromRawPayload(rawPayload); + type PendingWithIndex = { + c: { call_id: string; tool_name: string; input: unknown }; + index: number; + }; + const isPendingWithIndex = (entry: { + c: unknown; + index: number; + }): entry is PendingWithIndex => { + const { c } = entry; + return ( + c != null && + typeof c === 'object' && + typeof (c as { call_id?: unknown }).call_id === 'string' && + typeof (c as { tool_name?: unknown }).tool_name === 'string' + ); + }; + return pendingField + .map((c, index) => ({ c, index })) + .filter(isPendingWithIndex) + .map(({ c, index }) => { + const callSite = (c as { call_site?: unknown }).call_site; + const rawInputHash = rawInputHashes[index]; + const hasObjectInput = + c.input != null && typeof c.input === 'object'; + const input = (hasObjectInput ? c.input : {}) as Record< + string, + unknown + >; + return { + call_id: c.call_id, + tool_name: c.tool_name, + input, + input_hash: + hasObjectInput && typeof rawInputHash === 'string' + ? rawInputHash + : hashToolInput(input), + ...(typeof callSite === 'string' + ? { call_site: callSite } + : {}), + }; + }); +} + /** * Locate the last line whose trimmed content exactly equals `marker`. * Using full-line anchoring prevents user-provided tool payloads that happen @@ -581,7 +692,8 @@ function findSentinelLine( for (let i = lines.length - 1; i >= searchFromLine; i--) { if (lines[i].trim() === marker) { const startOffset = lineStartOffsets[i]; - const endOffset = i + 1 < lineStartOffsets.length + const endOffset = + i + 1 < lineStartOffsets.length ? lineStartOffsets[i + 1] - 1 : startOffset + lines[i].length; return { line: i, startOffset, endOffset }; @@ -619,48 +731,8 @@ export function extractPendingFromStdout( const payloadLines = lines.slice(startLine.line + 1, endLine.line); const rawPayload = payloadLines.join('\n').trim(); - let parsed: { pending?: unknown } | null = null; - try { - parsed = JSON.parse(rawPayload) as { pending?: unknown }; - } catch { - return { stdout, pending: null }; - } - - const pendingField = parsed?.pending; - if (!Array.isArray(pendingField)) return { stdout, pending: null }; - - const rawInputHashes = pendingInputHashesFromRawPayload(rawPayload); - type PendingWithIndex = { - c: { call_id: string; tool_name: string; input: unknown }; - index: number; - }; - const isPendingWithIndex = (entry: { c: unknown; index: number }): entry is PendingWithIndex => { - const { c } = entry; - return ( - c != null && - typeof c === 'object' && - typeof (c as { call_id?: unknown }).call_id === 'string' && - typeof (c as { tool_name?: unknown }).tool_name === 'string' - ); - }; - const pending = pendingField - .map((c, index) => ({ c, index })) - .filter(isPendingWithIndex) - .map(({ c, index }) => { - const callSite = (c as { call_site?: unknown }).call_site; - const rawInputHash = rawInputHashes[index]; - const hasObjectInput = c.input != null && typeof c.input === 'object'; - const input = (hasObjectInput ? c.input : {}) as Record; - return { - call_id: c.call_id, - tool_name: c.tool_name, - input, - input_hash: hasObjectInput && typeof rawInputHash === 'string' - ? rawInputHash - : hashToolInput(input), - ...(typeof callSite === 'string' ? { call_site: callSite } : {}), - }; - }); + const pending = extractPendingFromControlPayload(rawPayload); + if (pending == null) return { stdout, pending: null }; /** Strip only the sentinel block and leave every other byte of user * stdout untouched. Both the Python and bash preambles defensively @@ -674,7 +746,10 @@ export function extractPendingFromStdout( * emission, and anything else that depends on byte-accurate stdout. */ const rawHead = stdout.slice(0, startLine.startOffset); const head = rawHead.endsWith('\n') ? rawHead.slice(0, -1) : rawHead; - const tailStart = endLine.endOffset < stdout.length ? endLine.endOffset + 1 : stdout.length; + const tailStart = + endLine.endOffset < stdout.length + ? endLine.endOffset + 1 + : stdout.length; const tail = stdout.slice(tailStart); const cleaned = head + tail; @@ -688,11 +763,14 @@ export function extractPendingFromStdout( function wrapUserCodeInAsync(userCode: string): string { const lines = userCode.split('\n'); - let wrapped = '# ============================================================================\n'; + let wrapped = + '# ============================================================================\n'; wrapped += '# USER CODE BEGINS BELOW\n'; - wrapped += '# ============================================================================\n\n'; + wrapped += + '# ============================================================================\n\n'; wrapped += 'async def __user_main__():\n'; - wrapped += ' """Auto-generated wrapper for user code to support top-level await"""\n'; + wrapped += + ' """Auto-generated wrapper for user code to support top-level await"""\n'; // Indent all user code for (const line of lines) { @@ -743,10 +821,21 @@ const PROGRAMMATIC_RUN_TIMEOUT = 300000; // 5 minutes wall time * Create a payload for programmatic tool calling execution * Combines the tool preamble with user code */ -export function createProgrammaticPayload(options: CreateProgrammaticPayloadOptions): t.PayloadBody { +export function createProgrammaticPayload( + options: CreateProgrammaticPayloadOptions, +): t.PayloadBody { const { - req, session_id, execution_id, callbackUrl, callbackToken, tools, timeout, - mode = 'blocking', history, codeOverride, filesOverride, + req, + session_id, + execution_id, + callbackUrl, + callbackToken, + tools, + timeout, + mode = 'blocking', + history, + codeOverride, + filesOverride, language = 'python', } = options; const body = req.body as t.ProgrammaticRequestBody; @@ -762,7 +851,14 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti throw new Error('bash PTC is only supported in replay mode'); } return buildBashPayload({ - req, execution_id, session_id, tools, userCode, files, history, timeout, + req, + execution_id, + session_id, + tools, + userCode, + files, + history, + timeout, }); } @@ -771,7 +867,9 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti preamble = generateReplayPreamble({ executionId: execution_id, tools }); } else { if (!callbackUrl || !callbackToken) { - throw new Error('blocking PTC mode requires callbackUrl and callbackToken'); + throw new Error( + 'blocking PTC mode requires callbackUrl and callbackToken', + ); } preamble = generatePreamble({ callbackUrl, @@ -781,15 +879,21 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti }); } - const isPyPlot = userCode.includes('import matplotlib') || userCode.includes('import seaborn'); + const isPyPlot = + userCode.includes('import matplotlib') || + userCode.includes('import seaborn'); let finalCode: string; if (isPyPlot) { - const indentedUserCode = userCode.trim().split('\n').map(line => ` ${line}`).join('\n'); + const indentedUserCode = userCode + .trim() + .split('\n') + .map(line => ` ${line}`) + .join('\n'); const wrappedUserCode = templateCodeAsync.replace( /# BEGIN USER CODE\n[\s\S]*?# END USER CODE/, - `# BEGIN USER CODE\n${indentedUserCode}\n # END USER CODE` + `# BEGIN USER CODE\n${indentedUserCode}\n # END USER CODE`, ); finalCode = preamble + '\n' + wrappedUserCode; } else { @@ -797,7 +901,9 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti finalCode = preamble + wrappedUserCode; } - const run_memory_limit = planLimits[req.planId ?? '']?.run_memory_limit ?? planLimits.default.run_memory_limit; + const run_memory_limit = + planLimits[req.planId ?? '']?.run_memory_limit ?? + planLimits.default.run_memory_limit; const run_timeout = timeout ?? PROGRAMMATIC_RUN_TIMEOUT; const payload: t.PayloadBody = { @@ -809,8 +915,8 @@ export function createProgrammaticPayload(options: CreateProgrammaticPayloadOpti files: [ { name: 'main.py', - content: finalCode - } + content: finalCode, + }, ], session_id, }; @@ -851,13 +957,27 @@ function buildBashPayload(args: { history?: Record; timeout?: number; }): t.PayloadBody { - const { req, execution_id, session_id, tools, userCode, files, history, timeout } = args; - - const preamble = generateBashReplayPreamble({ executionId: execution_id, tools }); + const { + req, + execution_id, + session_id, + tools, + userCode, + files, + history, + timeout, + } = args; + + const preamble = generateBashReplayPreamble({ + executionId: execution_id, + tools, + }); const postamble = generateBashReplayPostamble(); const finalCode = preamble + userCode + '\n' + postamble; - const run_memory_limit = planLimits[req.planId ?? '']?.run_memory_limit ?? planLimits.default.run_memory_limit; + const run_memory_limit = + planLimits[req.planId ?? '']?.run_memory_limit ?? + planLimits.default.run_memory_limit; const run_timeout = timeout ?? PROGRAMMATIC_RUN_TIMEOUT; const payload: t.PayloadBody = { @@ -865,6 +985,8 @@ function buildBashPayload(args: { run_timeout, language: 'bash', version: '5.2.0', + execution_id, + replay_tool_count: tools.length, files: [ { name: 'main.sh', diff --git a/service/src/ptc-constants.test.ts b/service/src/ptc-constants.test.ts new file mode 100644 index 00000000..5028e385 --- /dev/null +++ b/service/src/ptc-constants.test.ts @@ -0,0 +1,9 @@ +import { expect, test } from 'bun:test'; +import { isReservedPtcFilename } from './ptc-constants'; + +test('reserves replay inputs and output control channels after normalization', () => { + for (const name of ['_ptc_history.json', '_ptc_pending_result.json', '_PTC_PENDING_RESULT.JSON', 'sub/../_ptc_pending_result.json', 'sub\\_ptc_pending_result.json']) { + expect(isReservedPtcFilename(name)).toBe(true); + } + expect(isReservedPtcFilename('_ptc_data.csv')).toBe(false); +}); diff --git a/service/src/ptc-constants.ts b/service/src/ptc-constants.ts index 5009c3c3..b7b3999b 100644 --- a/service/src/ptc-constants.ts +++ b/service/src/ptc-constants.ts @@ -15,8 +15,9 @@ export const PTC_HISTORY_SANDBOX_PATH = `/mnt/data/${PTC_HISTORY_FILENAME}`; * Returns `true` for any filename the submission layer must refuse. * * Two things make a name "reserved": - * 1. Its post-normalization basename is `_ptc_history.json` — the single - * runtime fixture the replay preamble injects into the submission dir. + * 1. Its post-normalization basename is `_ptc_history.json` or + * `_ptc_pending_result.json`, compared case-insensitively for macOS. + * These are the replay input and output control channels. * Any user-supplied file with that exact basename would shadow our * injected history and silently corrupt replay correctness, so we * reject it on the request path. The bash preamble's `_ptc_pending.*` @@ -61,7 +62,7 @@ export function isReservedPtcFilename(name: string): boolean { } if (escapes) return true; const basename = segments.length > 0 ? segments[segments.length - 1] : ''; - return basename === PTC_HISTORY_FILENAME; + return [PTC_HISTORY_FILENAME, '_ptc_pending_result.json'].includes(basename.toLowerCase()); } /** diff --git a/service/src/sandbox-backend/remote-bridge.test.ts b/service/src/sandbox-backend/remote-bridge.test.ts index a65bc89e..697271ee 100644 --- a/service/src/sandbox-backend/remote-bridge.test.ts +++ b/service/src/sandbox-backend/remote-bridge.test.ts @@ -62,6 +62,34 @@ describe('RemoteBridgeSandboxBackend', () => { }); }); + test('preserves an authenticated selected workspace on remote dispatch', async () => { + let dispatched: Parameters[0] | undefined; + const store = { + dispatch: async ( + args: Parameters[0], + ): ReturnType => { + dispatched = args; + return { + protocolVersion: 1 as const, + generation: 1, + leaseToken: 'a'.repeat(32), + incarnationId: 'incarnation-00000001', + status: 'fulfilled' as const, + result: { session_id: 'session-1', language: 'bash', version: '5.2', files: [] }, + }; + }, + } satisfies Pick; + const backend = new RemoteBridgeSandboxBackend(store, 'default-vm'); + + await backend.execute(request(), { ...context(), workspaceId: 'project-a' }); + + expect(dispatched).toMatchObject({ + workerId: 'user-vm', + workspaceId: 'project-a', + requireTenantBinding: true, + }); + }); + test('maps tenant authorization rejection to a bridge backend error', async () => { const store = { dispatch: async (): ReturnType => { diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index 0bee0038..6e06eda8 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -60,6 +60,7 @@ export class RemoteBridgeSandboxBackend implements SandboxBackend { (this.dynamicWorkers || ctx.bridgeWorkerId !== this.workerId), body: req.body, headers: req.headers, + ...(ctx.workspaceId != null ? { workspaceId: ctx.workspaceId } : {}), runtimeSessionId: ctx.runtimeSessionId, deadlineAtMs: ctx.deadlineAtMs ?? Date.now() + env.JOB_TIMEOUT, signal: ctx.signal, diff --git a/service/src/sandbox-backend/types.ts b/service/src/sandbox-backend/types.ts index fbaa2d20..e21982d9 100644 --- a/service/src/sandbox-backend/types.ts +++ b/service/src/sandbox-backend/types.ts @@ -39,6 +39,8 @@ export interface SandboxExecuteContext { canonicalUserId?: string; /** Trusted API-selected outbound worker. Presence requires a tenant-bound credential. */ bridgeWorkerId?: string; + /** Trusted selected workspace for native replay-mode PTC. */ + workspaceId?: string; /** Stable identifier for this queued iteration, used to derive an idempotent * stateless launch token. PTC replay reuses one executionId across every * iteration, so the executionId alone cannot separate them; the request body @@ -63,6 +65,7 @@ export type SandboxRawResponse = t.ExecuteResponse & { session_id: string; files?: t.FileRefs; run?: t.ExecuteResponse['run']; + pending_tool_calls_payload?: string; }; export interface SandboxBackend { diff --git a/service/src/sandbox-dispatch.test.ts b/service/src/sandbox-dispatch.test.ts index 435be9aa..5b1ef749 100644 --- a/service/src/sandbox-dispatch.test.ts +++ b/service/src/sandbox-dispatch.test.ts @@ -12,8 +12,10 @@ import { } from './execution-manifest'; const SECRET = 'test-secret'; -const PRIVATE_KEY = 'MC4CAQAwBQYDK2VwBCIEIBoxzSJjQ5jTVyuohHtlD+uDGqv/tZ6hQS2CmxuOg2Wn'; -const PUBLIC_KEY = 'MCowBQYDK2VwAyEAeY3PRoTS3adfU6E3gQUB5hSZdrdMSw6OrKkH4UhYh0U='; +const PRIVATE_KEY = + 'MC4CAQAwBQYDK2VwBCIEIBoxzSJjQ5jTVyuohHtlD+uDGqv/tZ6hQS2CmxuOg2Wn'; +const PUBLIC_KEY = + 'MCowBQYDK2VwAyEAeY3PRoTS3adfU6E3gQUB5hSZdrdMSw6OrKkH4UhYh0U='; function payload(overrides: Partial = {}): t.PayloadBody { return { @@ -25,14 +27,22 @@ function payload(overrides: Partial = {}): t.PayloadBody { }; } -function claims(overrides: Partial = {}): ExecutionManifestClaims { +function claims( + overrides: Partial = {}, +): ExecutionManifestClaims { return { v: EXECUTION_MANIFEST_VERSION, exec_id: 'exec_123', tenant_id: 'tenant_abc', user_id: 'user_123', session_key: 'tenant:tenant_abc:user:user_123', - input_files: [{ id: 'file_123', session_id: 'sess_input', name: 'inputs/data.csv' }], + input_files: [ + { + id: 'file_123', + session_id: 'sess_input', + name: 'inputs/data.csv', + }, + ], read_sessions: ['sess_input'], output_session_id: 'sess_output', max_upload_bytes: 1024, @@ -46,6 +56,20 @@ function claims(overrides: Partial = {}): ExecutionMani } describe('sandbox execute request dispatch', () => { + test('budgets every input and output batch before signing the request', () => { + const request = buildSandboxExecuteRequest({ + payload: payload({ files: Array.from({ length: 9 }, (_, index) => ({ name: `${index}.txt`, id: `file_${index}`, storage_session_id: 'input' })) }), + programmaticTransferReserveMs: 60_000, + executionManifestClaims: claims({ max_output_files: 10 }), + executionManifestSecret: SECRET, + executionManifestTtlSeconds: 300, + nowSeconds: 1_000, + }); + // Three download batches plus three upload batches share one reserve. + expect(request.body.transfer_timeout_ms).toBe(10_000); + const verified = verifyExecutionManifest(request.body.execution_manifest!, SECRET, { nowSeconds: 1_000 }); + expect(verified.execute_body_sha256).toBe(executionManifestBodySha256(request.body)); + }); test('keeps large egress grants out of HTTP headers', () => { const largeGrant = `ceg1.${'a'.repeat(24_000)}`; const request = buildSandboxExecuteRequest({ @@ -64,18 +88,27 @@ describe('sandbox execute request dispatch', () => { const request = buildSandboxExecuteRequest({ payload: payload(), executionManifestClaims: claims(), + maxOutputFileBytes: 1_000, executionManifestSecret: SECRET, executionManifestTtlSeconds: 300, nowSeconds: 1_000, }); expect(request.headers[EXECUTION_MANIFEST_HEADER]).toBeUndefined(); + expect(request.body.max_output_files).toBe(10); + expect(request.body.max_output_file_bytes).toBe(1_000); expect(request.body.execution_manifest).toEqual(expect.any(String)); - expect(verifyExecutionManifest(request.body.execution_manifest!, SECRET, { nowSeconds: 1_100 })).toEqual(claims({ + expect( + verifyExecutionManifest(request.body.execution_manifest!, SECRET, { + nowSeconds: 1_100, + }), + ).toEqual( + claims({ execute_body_sha256: executionManifestBodySha256(request.body), iat: 1_000, exp: 1_300, - })); + }), + ); }); test('signs execution manifests with a private key when configured', () => { @@ -88,11 +121,19 @@ describe('sandbox execute request dispatch', () => { nowSeconds: 1_000, }); - expect(verifyExecutionManifestWithPublicKey(request.body.execution_manifest!, PUBLIC_KEY, { nowSeconds: 1_100 })).toEqual(claims({ + expect( + verifyExecutionManifestWithPublicKey( + request.body.execution_manifest!, + PUBLIC_KEY, + { nowSeconds: 1_100 }, + ), + ).toEqual( + claims({ execute_body_sha256: executionManifestBodySha256(request.body), iat: 1_000, exp: 1_300, - })); + }), + ); }); test('binds body-carried egress grants into signed execution manifests', () => { @@ -106,10 +147,16 @@ describe('sandbox execute request dispatch', () => { }); expect(request.body.egress_grant).toBe('ceg1.sealed-grant'); - expect(verifyExecutionManifest(request.body.execution_manifest!, SECRET, { nowSeconds: 1_100 })).toEqual(claims({ + expect( + verifyExecutionManifest(request.body.execution_manifest!, SECRET, { + nowSeconds: 1_100, + }), + ).toEqual( + claims({ execute_body_sha256: executionManifestBodySha256(request.body), iat: 1_000, exp: 1_300, - })); + }), + ); }); }); diff --git a/service/src/sandbox-dispatch.ts b/service/src/sandbox-dispatch.ts index e3066905..340834b5 100644 --- a/service/src/sandbox-dispatch.ts +++ b/service/src/sandbox-dispatch.ts @@ -1,5 +1,14 @@ import type * as t from './types'; -import { executionManifestBodySha256, signExecutionManifestWithKey, type ExecutionManifestClaims } from './execution-manifest'; +import { + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY, + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS, +} from '../../packages/code/src/protocol'; +import { + executionManifestBodySha256, + signExecutionManifestWithKey, + type ExecutionManifestClaims, +} from './execution-manifest'; interface BuildSandboxExecuteRequestArgs { payload: t.PayloadBody; @@ -9,6 +18,8 @@ interface BuildSandboxExecuteRequestArgs { executionManifestSecret: string; executionManifestTtlSeconds: number; nowSeconds?: number; + maxOutputFileBytes?: number; + programmaticTransferReserveMs?: number; } interface SandboxExecuteRequest { @@ -21,15 +32,31 @@ interface SandboxExecuteRequest { * ride in the JSON body instead of HTTP headers. Otherwise skill-heavy jobs can * fail with 431 before sandbox-runner reaches capability validation. */ -export function buildSandboxExecuteRequest(args: BuildSandboxExecuteRequestArgs): SandboxExecuteRequest { +export function buildSandboxExecuteRequest( + args: BuildSandboxExecuteRequestArgs, +): SandboxExecuteRequest { const body: t.PayloadBody = { ...args.payload }; - const headers: Record = { 'Content-Type': 'application/json' }; + const headers: Record = { + 'Content-Type': 'application/json', + }; if (args.egressGrantToken) { body.egress_grant = args.egressGrantToken; } + if (args.maxOutputFileBytes != null) { + body.max_output_file_bytes = args.maxOutputFileBytes; + } + if (args.programmaticTransferReserveMs != null) { + const batches = Math.max(1, + Math.ceil(body.files.filter(file => 'id' in file).length / BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY) + + Math.ceil((args.executionManifestClaims?.max_output_files ?? BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY), + ); + body.transfer_timeout_ms = Math.min(BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS, + Math.max(1, Math.floor(args.programmaticTransferReserveMs / batches))); + } if (args.executionManifestClaims) { + body.max_output_files = args.executionManifestClaims.max_output_files; const nowSeconds = args.nowSeconds ?? Math.floor(Date.now() / 1000); body.execution_manifest = signExecutionManifestWithKey( { diff --git a/service/src/sandbox-egress.ts b/service/src/sandbox-egress.ts index f6549c09..e47806a2 100644 --- a/service/src/sandbox-egress.ts +++ b/service/src/sandbox-egress.ts @@ -3,6 +3,7 @@ import { env } from './config'; import { createGatewayPtcCallbackToken } from './egress-gateway-client'; import type { ExecutionManifestClaims } from './execution-manifest'; import type * as t from './types'; +import { programmaticTransferReserveMs } from '../../packages/code/src/protocol'; export type SandboxJobSecurity = { payload: t.PayloadBody; @@ -65,6 +66,9 @@ export function timeoutMsToGrantSeconds(timeoutMs: number): number { } const DEFAULT_PROGRAMMATIC_TIMEOUT_MS = 300000; +const SELECTED_WORKSPACE_REPLAY_PASSES = 2; +const SELECTED_WORKSPACE_SETTLEMENT_RESERVE_MS = 5_000; +const SELECTED_WORKSPACE_MAX_QUEUE_RESERVE_MS = 30_000; export function normalizeProgrammaticTimeoutMs( rawTimeout: unknown, @@ -80,6 +84,31 @@ export function normalizeProgrammaticTimeoutMs( return Math.min(Math.ceil(rawTimeout), maxTimeout); } +/** + * Selected-workspace Bash replay may run one read-only probe and one commit + * pass in its final iteration. Bound each pass so both plus settlement reserve + * fit inside the worker-owned JOB_TIMEOUT instead of advertising a duration + * the assignment cannot complete. + */ +export function normalizeSelectedWorkspaceProgrammaticTimeoutMs( + rawTimeout: unknown, + jobTimeoutMs = env.JOB_TIMEOUT, +): number { + const totalBudget = Math.max(1, Math.floor(jobTimeoutMs)); + const queueReserve = Math.min( + SELECTED_WORKSPACE_MAX_QUEUE_RESERVE_MS, + Math.floor(totalBudget / 5), + ); + const executionBudget = Math.max( + 1, + totalBudget - queueReserve - SELECTED_WORKSPACE_SETTLEMENT_RESERVE_MS - programmaticTransferReserveMs(totalBudget), + ); + return normalizeProgrammaticTimeoutMs( + rawTimeout, + Math.max(1, Math.floor(executionBudget / SELECTED_WORKSPACE_REPLAY_PASSES)), + ); +} + export async function sealPtcCallbackTokenForGateway(args: { executionId: string; sessionId: string; diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 9896518c..4de4a4d8 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -11,12 +11,19 @@ import { connection, getExecutionQueueBinding, } from '../queue'; -import { createProgrammaticPayload, extractPendingFromStdout } from '../preamble'; +import { + createProgrammaticPayload, + extractPendingFromControlPayload, + extractPendingFromStdout, +} from '../preamble'; import { findBashToolNameCollision } from '../preamble-bash'; import type { LCTool } from '../preamble'; import { isReservedPtcFilename } from '../ptc-constants'; import { internalServiceHeaders } from '../internal-service-auth'; -import { resolveOutputBucketSessionKey, SessionKeyResolutionError } from '../session-key'; +import { + resolveOutputBucketSessionKey, + SessionKeyResolutionError, +} from '../session-key'; import { getCredentialId, getPrincipalOrReject } from '../auth/principal'; import { getExecutionIdentity } from '../execution-identity'; import { PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION } from '../runtime-session/job-policy'; @@ -34,15 +41,25 @@ import { publicExecutionFailure } from '../utils'; import { normalizeEgressGatewayUrl, normalizeProgrammaticTimeoutMs, + normalizeSelectedWorkspaceProgrammaticTimeoutMs, prepareSandboxJobSecurity, sealPtcCallbackTokenForGateway, timeoutMsToGrantSeconds, } from '../sandbox-egress'; import { findUnregisteredToolCall } from '../tool-scope'; import { summarizeRequestedFiles } from '../execution-log'; -import { pollBlockingExecution, type BlockingPendingState } from './blocking-poll'; -import { clearSessionOwnership, recordSessionOwnership } from '../session-ownership'; -import { FileRefAuthorizationError, authorizeRequestedFiles } from './file-authorization'; +import { + pollBlockingExecution, + type BlockingPendingState, +} from './blocking-poll'; +import { + clearSessionOwnership, + recordSessionOwnership, +} from '../session-ownership'; +import { + FileRefAuthorizationError, + authorizeRequestedFiles, +} from './file-authorization'; import { buildReplayExecutionState, resolveReplayStateSandboxBackend, @@ -50,8 +67,10 @@ import { import { BridgeWorkerSelectionError, CODEAPI_BRIDGE_WORKER_HEADER, + CODEAPI_BRIDGE_WORKSPACE_HEADER, resolveBridgeWorkerSelection, } from '../bridge/selection'; +import { isValidBridgeWorkerId, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES } from '../../../packages/code/src/protocol'; import logger from '../logger'; import { type ExecutionState, @@ -145,26 +164,41 @@ async function retryToolCallServerRequest( ): Promise { let lastError: Error | undefined; - for (let attempt = 1; attempt <= TOOL_CALL_SERVER_RETRY_ATTEMPTS; attempt++) { + for ( + let attempt = 1; + attempt <= TOOL_CALL_SERVER_RETRY_ATTEMPTS; + attempt++ + ) { try { return await requestFn(); } catch (error) { lastError = error as Error; if (axios.isAxiosError(error)) { - if (error.response && error.response.status >= 400 && error.response.status < 500) { + if ( + error.response && + error.response.status >= 400 && + error.response.status < 500 + ) { throw error; } } if (attempt < TOOL_CALL_SERVER_RETRY_ATTEMPTS) { - logger.warn(`${context} failed (attempt ${attempt}/${TOOL_CALL_SERVER_RETRY_ATTEMPTS}), retrying...`, { + logger.warn( + `${context} failed (attempt ${attempt}/${TOOL_CALL_SERVER_RETRY_ATTEMPTS}), retrying...`, + { error: lastError.message, - }); - await new Promise(resolve => setTimeout(resolve, TOOL_CALL_SERVER_RETRY_DELAY * attempt)); + }, + ); + await new Promise(resolve => + setTimeout(resolve, TOOL_CALL_SERVER_RETRY_DELAY * attempt), + ); } } } - logger.error(`${context} failed after ${TOOL_CALL_SERVER_RETRY_ATTEMPTS} attempts`); + logger.error( + `${context} failed after ${TOOL_CALL_SERVER_RETRY_ATTEMPTS} attempts`, + ); throw lastError; } @@ -179,7 +213,9 @@ setInterval(() => { }, STALE_CLEANUP_INTERVAL_MS); function generateContinuationToken(execution_id: string): string { - return Buffer.from(JSON.stringify({ execution_id, ts: Date.now() })).toString('base64'); + return Buffer.from( + JSON.stringify({ execution_id, ts: Date.now() }), + ).toString('base64'); } /** Map a replay-continuation HTTP status to its operational outcome @@ -200,14 +236,21 @@ function classifyContinuationOutcome(statusCode: number): string { * timestamp is older than the execution-state TTL — without this, the * `ts` field was dead data and a client could replay an ancient token * against a freshly-reused-execution-id window. */ -function decodeContinuationToken(token: string): { execution_id: string } | null { +function decodeContinuationToken( + token: string, +): { execution_id: string } | null { try { - const parsed: unknown = JSON.parse(Buffer.from(token, 'base64').toString('utf-8')); + const parsed: unknown = JSON.parse( + Buffer.from(token, 'base64').toString('utf-8'), + ); if (parsed === null || typeof parsed !== 'object') { return null; } const candidate = parsed as { execution_id?: unknown; ts?: unknown }; - if (typeof candidate.execution_id !== 'string' || candidate.execution_id.length === 0) { + if ( + typeof candidate.execution_id !== 'string' || + candidate.execution_id.length === 0 + ) { return null; } if (typeof candidate.ts === 'number' && Number.isFinite(candidate.ts)) { @@ -226,13 +269,17 @@ function decodeContinuationToken(token: string): { execution_id: string } | null // Blocking mode (legacy path) // --------------------------------------------------------------------------- -function waitForExecutionState(execution_id: string, timeout: number): ReturnType { +function waitForExecutionState( + execution_id: string, + timeout: number, +): ReturnType { return pollBlockingExecution(execution_id, timeout, { getExecutionState, getBlockingResult, - getPending: async (id) => { + getPending: async id => { const response = await retryToolCallServerRequest( - () => axios.get( + () => + axios.get( `${env.TOOL_CALL_SERVER_URL}/sessions/${id}/pending`, { headers: internalServiceHeaders() }, ), @@ -240,7 +287,8 @@ function waitForExecutionState(execution_id: string, timeout: number): ReturnTyp ); return response.data; }, - isNotFound: (error) => axios.isAxiosError(error) && error.response?.status === 404, + isNotFound: error => + axios.isAxiosError(error) && error.response?.status === 404, sleep: () => new Promise(resolve => setTimeout(resolve, POLL_INTERVAL)), now: Date.now, }); @@ -295,7 +343,8 @@ async function runReplayIteration( }); if (DEBUG_MODE) { - const firstFile = rawPayload.files[0] as { content?: string } | undefined; + const firstFile = rawPayload.files[0] as + { content?: string } | undefined; logger.debug('Replay enqueue details', { execution_id: state.execution_id, historySize: Object.keys(history).length, @@ -320,7 +369,9 @@ async function runReplayIteration( state.executionProfile ?? env.EXECUTION_PROFILE, state.executionProfileSource ?? env.EXECUTION_PROFILE_SOURCE, ); - const job = await queue.add(Jobs.execute, { + const job = await queue.add( + Jobs.execute, + { code: state.userCode ?? '', userId, payload: sandboxSecurity.payload, @@ -332,17 +383,24 @@ async function runReplayIteration( canonicalUserId: state.canonicalUserId, executionProfile: state.executionProfile ?? env.EXECUTION_PROFILE, sandboxBackend: replayBackend, - ...(state.bridgeWorkerId != null ? { bridgeWorkerId: state.bridgeWorkerId } : {}), + ...(state.bridgeWorkerId != null + ? { bridgeWorkerId: state.bridgeWorkerId } + : {}), + ...(state.workspaceId != null + ? { workspaceId: state.workspaceId } + : {}), runtimeSessionMode: 'stateless', runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, executionManifestClaims: sandboxSecurity.executionManifestClaims, egressGrantClaims: sandboxSecurity.egressGrantClaims, egressGrantToken: sandboxSecurity.egressGrantToken, - }, { + }, + { removeOnComplete: { age: 60, count: 1 }, removeOnFail: { age: 180, count: 1 }, attempts: 1, - }); + }, + ); jobsSubmitted.inc({ language }); return job.waitUntilFinished(events, JOB_COMPLETION_WAIT_TIMEOUT_MS); @@ -365,18 +423,24 @@ async function handleReplayInitial( apiKeyId: string; userId: string; bridgeWorkerId?: string; + workspaceId?: string; }, ): Promise { - const { apiKeyId, userId, bridgeWorkerId } = params; - const { - code, - tools, - user_id, - files, - } = req.body as t.ProgrammaticRequestBody; + const { apiKeyId, userId, bridgeWorkerId, workspaceId } = params; + const { code, tools, user_id, files } = + req.body as t.ProgrammaticRequestBody; let timeout: number; try { - timeout = normalizeProgrammaticTimeoutMs((req.body as t.ProgrammaticRequestBody).timeout); + if (workspaceId != null && Array.isArray(files) && files.length > BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES) { + throw new Error(`Selected-workspace execution allows at most ${BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES} input files; main and replay history occupy two reserved slots`); + } + timeout = workspaceId != null + ? normalizeSelectedWorkspaceProgrammaticTimeoutMs( + (req.body as t.ProgrammaticRequestBody).timeout, + ) + : normalizeProgrammaticTimeoutMs( + (req.body as t.ProgrammaticRequestBody).timeout, + ); } catch (error) { res.status(400).json({ error: (error as Error).message }); return; @@ -400,14 +464,23 @@ async function handleReplayInitial( }); return; } - const language: 'python' | 'bash' = requestedLanguage === 'bash' ? 'bash' : 'python'; + const language: 'python' | 'bash' = + requestedLanguage === 'bash' ? 'bash' : 'python'; + if (workspaceId != null && language !== 'bash') { + res.status(400).json({ + error: 'Selected-workspace programmatic execution supports bash only', + }); + return; + } if (!code) { res.status(400).json({ error: 'Missing required field: code' }); return; } - if (!tools || !Array.isArray(tools) || tools.length === 0) { - res.status(400).json({ error: 'Missing required field: tools (must be a non-empty array)' }); + if (!Array.isArray(tools) || (tools.length === 0 && workspaceId == null)) { + res.status(400).json({ + error: 'Missing required field: tools (must be non-empty unless a selected workspace executes bash)', + }); return; } if (tools.length > MAX_TOOLS_PER_REQUEST) { @@ -442,7 +515,8 @@ async function handleReplayInitial( files, store: connection, }); - (req.body as t.ProgrammaticRequestBody).files = authorizedFiles.length > 0 ? authorizedFiles : undefined; + (req.body as t.ProgrammaticRequestBody).files = + authorizedFiles.length > 0 ? authorizedFiles : undefined; } catch (error) { if (sendFileRefAuthorizationError(error, res, req)) return; logger.error('Error authorizing replay file refs:', error); @@ -456,7 +530,14 @@ async function handleReplayInitial( try { sessionKey = resolveOutputBucketSessionKey(req); } catch (error) { - if (sendSessionKeyResolutionError(error, res, req, 'programmatic /exec: resolveOutputBucketSessionKey')) { + if ( + sendSessionKeyResolutionError( + error, + res, + req, + 'programmatic /exec: resolveOutputBucketSessionKey', + ) + ) { return; } throw error; @@ -466,9 +547,9 @@ async function handleReplayInitial( const execution_id = nanoid(); const authContext = req.codeApiAuthContext; const identity = getExecutionIdentity(req, userId); - const isPyPlot = language === 'python' && ( - code.includes('import matplotlib') || code.includes('import seaborn') - ); + const isPyPlot = + language === 'python' && + (code.includes('import matplotlib') || code.includes('import seaborn')); await recordSessionOwnership(connection, session_id, sessionKey); @@ -487,6 +568,7 @@ async function handleReplayInitial( timeout, language, bridgeWorkerId, + workspaceId, executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: resolveReplayStateSandboxBackend({ @@ -506,13 +588,16 @@ async function handleReplayInitial( await setExecutionState(state); } catch (err) { if (err instanceof ExecutionStateTooLargeError) { - logger.warn('Rejecting replay request: ExecutionState exceeds Redis cap', { + logger.warn( + 'Rejecting replay request: ExecutionState exceeds Redis cap', + { execution_id, userId, apiKeyId, bytes: err.bytes, cap: err.cap, - }); + }, + ); await clearSessionOwnership(connection, session_id).catch(() => {}); ptcReplayStateOversize.inc(); res.status(413).json({ @@ -568,9 +653,15 @@ async function handleReplayContinuation( * adds outcome plumbing through `runAndRespond`. */ const startMs = performance.now(); res.once('finish', () => { - const labels = { mode: 'replay' as const, outcome: classifyContinuationOutcome(res.statusCode) }; + const labels = { + mode: 'replay' as const, + outcome: classifyContinuationOutcome(res.statusCode), + }; ptcReplayContinuations.inc(labels); - ptcReplayContinuationDuration.observe(labels, (performance.now() - startMs) / 1000); + ptcReplayContinuationDuration.observe( + labels, + (performance.now() - startMs) / 1000, + ); }); /** Reject oversized batches before we spend any CPU on per-entry @@ -621,9 +712,14 @@ async function handleReplayContinuation( call_site: emitted.call_site, }; }); - const deltaOrError = await computeToolHistoryDelta(state.execution_id, enrichedResults); + const deltaOrError = await computeToolHistoryDelta( + state.execution_id, + enrichedResults, + ); if ('error' in deltaOrError) { - res.status(deltaOrError.status ?? 400).json({ error: deltaOrError.error }); + res.status(deltaOrError.status ?? 400).json({ + error: deltaOrError.error, + }); return; } const delta = deltaOrError; @@ -642,7 +738,9 @@ async function handleReplayContinuation( }); if (!pre.ok) { if (pre.status === 403) { - logger.warn('Unauthorized replay continuation request rejected', { + logger.warn( + 'Unauthorized replay continuation request rejected', + { execution_id: state.execution_id, requestUserId: userId, requestApiKeyId: apiKeyId, @@ -650,7 +748,8 @@ async function handleReplayContinuation( executionUserId: state.userId, executionApiKeyId: state.apiKeyId, executionTenantId: state.tenantId, - }); + }, + ); } if (pre.cleanupOnReject === true) { await cleanupExecution(state.execution_id, 'replay'); @@ -672,7 +771,10 @@ async function handleReplayContinuation( * Redis MULTI/EXEC so counters and the hash can't drift out of sync * on a partial failure. */ state.callCount = (state.callCount ?? 0) + delta.newCallIds.length; - state.historyBytes = Math.max(0, (state.historyBytes ?? 0) + delta.bytesDelta); + state.historyBytes = Math.max( + 0, + (state.historyBytes ?? 0) + delta.bytesDelta, + ); state.lastActivity = Date.now(); try { await commitToolHistoryAndState(state, delta); @@ -687,14 +789,19 @@ async function handleReplayContinuation( * forward is a fresh execution with smaller inputs. Reap the * old execution to free the lock and Redis keys, then return * an actionable 413 instead of a generic 500. */ - logger.warn('Replay continuation rejected: ExecutionState exceeds Redis cap', { + logger.warn( + 'Replay continuation rejected: ExecutionState exceeds Redis cap', + { execution_id: state.execution_id, bytes: err.bytes, cap: err.cap, callCount: state.callCount, historyBytes: state.historyBytes, - }); - await cleanupExecution(state.execution_id, 'replay').catch(() => {}); + }, + ); + await cleanupExecution(state.execution_id, 'replay').catch( + () => {}, + ); ptcReplayStateOversize.inc(); res.status(413).json({ status: 'error', @@ -715,10 +822,13 @@ async function handleReplayContinuation( * the throw bubble to the top-level catch and become an opaque * 500 — clients (and load balancers) treat 5xx classes very * differently for retry policy. */ - logger.error('Failed to commit replay continuation; returning retryable 503', { + logger.error( + 'Failed to commit replay continuation; returning retryable 503', + { execution_id: state.execution_id, err: (err as Error).message, - }); + }, + ); res.status(503).json({ status: 'error', error: 'Failed to persist replay continuation; please retry the same request', @@ -764,11 +874,15 @@ async function runAndRespond( try { result = await runReplayIteration(req, state, apiKeyId, userId); } catch (err) { - logger.error('Replay iteration failed', { execution_id: state.execution_id, err }); + logger.error('Replay iteration failed', { + execution_id: state.execution_id, + err, + }); await cleanupExecution(state.execution_id, 'replay'); if (!isDisconnected()) { const publicFailure = publicExecutionFailure(err); - const message = publicFailure?.body.message ?? (err as Error).message; + const message = + publicFailure?.body.message ?? (err as Error).message; res.status(200).json({ status: 'error', error: message !== '' ? message : 'Sandbox execution failed', @@ -786,10 +900,19 @@ async function runAndRespond( return; } - const { stdout: cleanStdout, pending } = extractPendingFromStdout( + const extracted = extractPendingFromStdout( result.stdout, state.execution_id, ); + const cleanStdout = extracted.stdout; + const controlPayload = result.pending_tool_calls_payload; + const hasControlPayload = typeof controlPayload === 'string'; + const controlPending = hasControlPayload + ? extractPendingFromControlPayload(controlPayload) + : null; + const pending = hasControlPayload + ? (controlPending ?? []) + : extracted.pending; if (pending != null) { if (pending.length === 0) { @@ -806,7 +929,10 @@ async function runAndRespond( }); return; } - const unregisteredToolCall = findUnregisteredToolCall(pending, state.tools); + const unregisteredToolCall = findUnregisteredToolCall( + pending, + state.tools, + ); if (unregisteredToolCall != null) { logger.warn('Sandbox requested unregistered replay tool call', { execution_id: state.execution_id, @@ -866,11 +992,16 @@ async function runAndRespond( await setExecutionState(state); await refreshExecutionTtl(state.execution_id); } catch (err) { - logger.error('Failed to persist execution state before continuation; aborting', { + logger.error( + 'Failed to persist execution state before continuation; aborting', + { execution_id: state.execution_id, err: (err as Error).message, - }); - await cleanupExecution(state.execution_id, 'replay').catch(() => {}); + }, + ); + await cleanupExecution(state.execution_id, 'replay').catch( + () => {}, + ); if (!isDisconnected()) { if (err instanceof ExecutionStateTooLargeError) { /** A continuation that pushes `emittedCallIds` past the @@ -887,8 +1018,7 @@ async function runAndRespond( } else { res.status(503).json({ status: 'error', - error: - 'Failed to persist replay state; please retry the request from scratch', + error: 'Failed to persist replay state; please retry the request from scratch', session_id: state.session_id, }); } @@ -912,7 +1042,8 @@ async function runAndRespond( if (!isSandboxRunSuccess(result)) { await cleanupExecution(state.execution_id, 'replay'); - const errorMessage = result.message != null && result.message !== '' + const errorMessage = + result.message != null && result.message !== '' ? result.message : `Sandbox exited with code ${result.code ?? 'unknown'}`; res.status(200).json({ @@ -941,7 +1072,10 @@ async function runAndRespond( // Request entrypoint // --------------------------------------------------------------------------- -router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedRequest, res) => { +router.post( + '/exec/programmatic', + executionLimiter, + async (req: t.AuthenticatedRequest, res) => { const principal = getPrincipalOrReject(req, res); if (!principal) return; const apiKeyId = getCredentialId(req); @@ -954,13 +1088,12 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR return res.status(503).json({ error: 'Service is starting up' }); } - const { - continuation_token, - tool_results, - } = req.body as t.ProgrammaticRequestBody; + const { continuation_token, tool_results } = + req.body as t.ProgrammaticRequestBody; const rawBody = req.body as Record; const requestedLanguage: unknown = rawBody.language ?? rawBody.lang; let bridgeWorkerId: string | undefined; + let workspaceId: string | undefined; if (continuation_token == null || continuation_token === '') { try { const bridgeSelection = resolveBridgeWorkerSelection({ @@ -970,12 +1103,35 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), trustedWorkerId: principal.codeWorkerId, }); - bridgeWorkerId = bridgeSelection?.explicit === true - ? bridgeSelection.workerId - : undefined; + bridgeWorkerId = + bridgeSelection?.explicit === true || + (bridgeSelection != null && !env.BRIDGE_DYNAMIC_WORKERS) + ? bridgeSelection.workerId + : undefined; + const requestedWorkspaceId = req + .header(CODEAPI_BRIDGE_WORKSPACE_HEADER) + ?.trim(); + if ( + requestedWorkspaceId != null && + requestedWorkspaceId !== '' + ) { + if (bridgeWorkerId == null) { + return res.status(400).json({ + error: 'Workspace selection requires an authenticated bridge worker', + }); + } + if (!isValidBridgeWorkerId(requestedWorkspaceId)) { + return res + .status(400) + .json({ error: 'Invalid code workspace ID' }); + } + workspaceId = requestedWorkspaceId; + } } catch (error) { if (error instanceof BridgeWorkerSelectionError) { - return res.status(error.status).json({ error: error.message }); + return res + .status(error.status) + .json({ error: error.message }); } throw error; } @@ -1013,7 +1169,9 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR } const decoded = decodeContinuationToken(continuation_token); if (!decoded) { - return res.status(400).json({ error: 'Invalid continuation token' }); + return res + .status(400) + .json({ error: 'Invalid continuation token' }); } const existing = await getExecutionState(decoded.execution_id); if (existing?.mode === 'replay') { @@ -1036,9 +1194,23 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR }); } if (env.PTC_MODE === 'replay') { - return await handleReplayInitial(req, res, { apiKeyId, userId, bridgeWorkerId }); + return await handleReplayInitial(req, res, { + apiKeyId, + userId, + bridgeWorkerId, + workspaceId, + }); } - return await handleBlocking(req, res, { apiKeyId, userId, bridgeWorkerId }); + if (workspaceId != null) { + return res.status(400).json({ + error: 'Selected-workspace programmatic execution requires replay mode', + }); + } + return await handleBlocking(req, res, { + apiKeyId, + userId, + bridgeWorkerId, + }); } catch (err) { logger.error(`[${INSTANCE_ID}] Programmatic routing error:`, err); if (!res.headersSent) { @@ -1046,7 +1218,8 @@ router.post('/exec/programmatic', executionLimiter, async (req: t.AuthenticatedR } return; } -}); + }, +); // --------------------------------------------------------------------------- // Blocking-mode handler (extracted from the original implementation). @@ -1059,46 +1232,47 @@ async function handleBlocking( params: { apiKeyId: string; userId: string; bridgeWorkerId?: string }, ): Promise> { const { apiKeyId, userId, bridgeWorkerId } = params; - const { - code, - tools, - user_id, - files, - continuation_token, - tool_results, - } = req.body as t.ProgrammaticRequestBody; + const { code, tools, user_id, files, continuation_token, tool_results } = + req.body as t.ProgrammaticRequestBody; let timeout: number; try { - timeout = normalizeProgrammaticTimeoutMs((req.body as t.ProgrammaticRequestBody).timeout); + timeout = normalizeProgrammaticTimeoutMs( + (req.body as t.ProgrammaticRequestBody).timeout, + ); } catch (error) { return res.status(400).json({ error: (error as Error).message }); } // CASE 1: Continuation - if (continuation_token != null && continuation_token !== '' && tool_results) { + if ( + continuation_token != null && + continuation_token !== '' && + tool_results + ) { const decoded = decodeContinuationToken(continuation_token); if (!decoded) { - return res.status(400).json({ error: 'Invalid continuation token' }); + return res + .status(400) + .json({ error: 'Invalid continuation token' }); } const { execution_id } = decoded; const execution = await getExecutionState(execution_id); if (!execution) { - return res.status(404).json({ error: 'Execution not found or expired' }); + return res + .status(404) + .json({ error: 'Execution not found or expired' }); } const identity = getExecutionIdentity(req, userId); if ( execution.userId !== userId || (execution.apiKeyId != null && execution.apiKeyId !== apiKeyId) || - ( - execution.tenantId != null && - execution.tenantId !== identity.storageNamespace - ) || - ( - execution.authContextHash != null && - execution.authContextHash !== req.codeApiAuthContext?.authContextHash - ) + (execution.tenantId != null && + execution.tenantId !== identity.storageNamespace) || + (execution.authContextHash != null && + execution.authContextHash !== + req.codeApiAuthContext?.authContextHash) ) { logger.warn('Unauthorized blocking continuation request rejected', { execution_id, @@ -1122,14 +1296,19 @@ async function handleBlocking( try { await retryToolCallServerRequest( - () => axios.post(`${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}/results`, { + () => + axios.post( + `${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}/results`, + { results: tool_results.map(r => ({ call_id: r.call_id, result: r.result, is_error: r.is_error ?? false, error_message: r.error_message, })), - }, { headers: internalServiceHeaders() }), + }, + { headers: internalServiceHeaders() }, + ), 'Submit tool results', ); @@ -1174,14 +1353,21 @@ async function handleBlocking( return res.status(400).json({ error: 'Missing required field: code' }); } if (!tools || !Array.isArray(tools) || tools.length === 0) { - return res.status(400).json({ error: 'Missing required field: tools (must be a non-empty array)' }); + return res + .status(400) + .json({ + error: 'Missing required field: tools (must be a non-empty array)', + }); } if (tools.length > MAX_TOOLS_PER_REQUEST) { - logger.warn(`Too many tools provided: ${tools.length}, limit is ${MAX_TOOLS_PER_REQUEST}`, { + logger.warn( + `Too many tools provided: ${tools.length}, limit is ${MAX_TOOLS_PER_REQUEST}`, + { execution_id: 'pre-creation', userId, toolCount: tools.length, - }); + }, + ); return res.status(400).json({ error: `Too many tools provided (${tools.length}). Maximum is ${MAX_TOOLS_PER_REQUEST}.`, }); @@ -1202,7 +1388,8 @@ async function handleBlocking( files, store: connection, }); - (req.body as t.ProgrammaticRequestBody).files = authorizedFiles.length > 0 ? authorizedFiles : undefined; + (req.body as t.ProgrammaticRequestBody).files = + authorizedFiles.length > 0 ? authorizedFiles : undefined; } catch (error) { if (sendFileRefAuthorizationError(error, res, req)) return; logger.error('Error authorizing programmatic file refs:', error); @@ -1215,7 +1402,14 @@ async function handleBlocking( try { sessionKey = resolveOutputBucketSessionKey(req); } catch (error) { - if (sendSessionKeyResolutionError(error, res, req, 'programmatic /exec-blocking: resolveOutputBucketSessionKey')) { + if ( + sendSessionKeyResolutionError( + error, + res, + req, + 'programmatic /exec-blocking: resolveOutputBucketSessionKey', + ) + ) { return; } throw error; @@ -1270,24 +1464,34 @@ async function handleBlocking( try { callbackUrl = normalizeEgressGatewayUrl(env.EGRESS_GATEWAY_URL); } catch (error) { - logger.error('Blocking PTC requires egress gateway callback URL:', error); + logger.error( + 'Blocking PTC requires egress gateway callback URL:', + error, + ); await cleanupExecution(execution_id, 'blocking'); - return res.status(503).json({ error: 'Egress gateway unavailable' }); + return res + .status(503) + .json({ error: 'Egress gateway unavailable' }); } let callbackToken: string; try { const toolCallResponse = await retryToolCallServerRequest( - () => axios.post<{ + () => + axios.post<{ success: boolean; callback_token: string; - }>(`${env.TOOL_CALL_SERVER_URL}/sessions`, { + }>( + `${env.TOOL_CALL_SERVER_URL}/sessions`, + { execution_id, session_id, timeout, tools, - }, { headers: internalServiceHeaders() }), + }, + { headers: internalServiceHeaders() }, + ), 'Create Tool Call Server session', ); @@ -1299,9 +1503,14 @@ async function handleBlocking( allowedToolNames: tools.map(tool => tool.name), }); } catch (error) { - logger.error('Failed to create Tool Call Server session or callback token:', error); + logger.error( + 'Failed to create Tool Call Server session or callback token:', + error, + ); await cleanupExecution(execution_id, 'blocking'); - return res.status(503).json({ error: 'Tool Call Server unavailable' }); + return res + .status(503) + .json({ error: 'Tool Call Server unavailable' }); } let rawPayload: t.PayloadBody; @@ -1316,10 +1525,15 @@ async function handleBlocking( timeout, }); } catch (error) { - logger.error('Failed to create payload', { execution_id, error: (error as Error).message }); + logger.error('Failed to create payload', { + execution_id, + error: (error as Error).message, + }); await cleanupExecution(execution_id, 'blocking'); return res.status(400).json({ - error: (error as Error).message || 'Failed to generate code payload', + error: + (error as Error).message || + 'Failed to generate code payload', }); } const sandboxSecurity = prepareSandboxJobSecurity({ @@ -1331,7 +1545,9 @@ async function handleBlocking( payload: rawPayload, }); - const job = await pyQueue.add(Jobs.execute, { + const job = await pyQueue.add( + Jobs.execute, + { code, userId, payload: sandboxSecurity.payload, @@ -1350,18 +1566,24 @@ async function handleBlocking( ...(bridgeWorkerId != null ? { bridgeWorkerId } : {}), runtimeSessionMode: 'stateless', runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, - executionManifestClaims: sandboxSecurity.executionManifestClaims, + executionManifestClaims: + sandboxSecurity.executionManifestClaims, egressGrantClaims: sandboxSecurity.egressGrantClaims, egressGrantToken: sandboxSecurity.egressGrantToken, - }, { + }, + { removeOnComplete: { age: 60, count: 1 }, removeOnFail: { age: 180, count: 1 }, attempts: 1, jobId: session_id, - }); + }, + ); jobsSubmitted.inc({ language: 'python' }); - logger.info('Job queued, polling for tool calls', { execution_id, session_id }); + logger.info('Job queued, polling for tool calls', { + execution_id, + session_id, + }); let clientDisconnected = false; req.on('close', async () => { @@ -1372,21 +1594,27 @@ async function handleBlocking( await job.remove(); await cleanupExecution(execution_id, 'blocking'); } catch (error) { - logger.error('Error cleaning up after client disconnect:', error); + logger.error( + 'Error cleaning up after client disconnect:', + error, + ); } }); job.waitUntilFinished(pyQueueEvents, JOB_COMPLETION_WAIT_TIMEOUT_MS) - .then(async (result) => { + .then(async result => { if (clientDisconnected) return; await setExecutionResult(execution_id, result); }) - .catch(async (error) => { + .catch(async error => { if (clientDisconnected) return; await setExecutionError(execution_id, error); }); - const state = await waitForExecutionState(execution_id, Math.min(timeout, MAX_POLL_TIME)); + const state = await waitForExecutionState( + execution_id, + Math.min(timeout, MAX_POLL_TIME), + ); if (state.status === 'waiting' && state.pending_calls) { return res.status(200).json({ @@ -1416,7 +1644,10 @@ async function handleBlocking( session_id, }); } catch (error) { - logger.error(`[${INSTANCE_ID}] Session ID: ${session_id} | Execution ID: ${execution_id} | Error:`, error); + logger.error( + `[${INSTANCE_ID}] Session ID: ${session_id} | Execution ID: ${execution_id} | Error:`, + error, + ); await cleanupExecution(execution_id, 'blocking'); return res.status(500).json({ error: 'Internal server error' }); } diff --git a/service/src/service/programmatic-state.test.ts b/service/src/service/programmatic-state.test.ts index fc84d8f8..81405021 100644 --- a/service/src/service/programmatic-state.test.ts +++ b/service/src/service/programmatic-state.test.ts @@ -83,6 +83,7 @@ describe('buildReplayExecutionState', () => { const state = build({ authContext, bridgeWorkerId: 'code-user_123', + workspaceId: 'project-a', sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', @@ -102,6 +103,7 @@ describe('buildReplayExecutionState', () => { authContextHash: 'hash_123', apiKeyId: 'key_legacy', bridgeWorkerId: 'code-user_123', + workspaceId: 'project-a', sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', diff --git a/service/src/service/programmatic-state.ts b/service/src/service/programmatic-state.ts index 25571fed..e6bda59a 100644 --- a/service/src/service/programmatic-state.ts +++ b/service/src/service/programmatic-state.ts @@ -38,6 +38,7 @@ export interface BuildReplayExecutionStateParams { timeout: number; language: 'python' | 'bash'; bridgeWorkerId?: string; + workspaceId?: string; sandboxBackend?: SandboxBackendName; executionProfile: ExecutionProfile; executionProfileSource: ExecutionProfileSource; @@ -66,6 +67,7 @@ export function buildReplayExecutionState( authContextHash: identity.authContextHash, apiKeyId: params.apiKeyId, bridgeWorkerId: params.bridgeWorkerId, + workspaceId: params.workspaceId, sandboxBackend: params.sandboxBackend, executionProfile: params.executionProfile, executionProfileSource: params.executionProfileSource, diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 2254ee21..3b65cedf 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -116,6 +116,8 @@ export interface ExecutionState { apiKeyId?: string; /** Authenticated worker selection retained across every replay iteration. */ bridgeWorkerId?: string; + /** Selected workspace retained and bound across every replay iteration. */ + workspaceId?: string; /** Original queue/backend target retained across replay continuations. */ sandboxBackend?: SandboxBackendName; /** Original producer profile retained so continuations use the same queue. */ diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 2a90eac7..f0a6da3b 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -3,7 +3,10 @@ import type { Request } from 'express'; import type { ExecutionManifestClaims } from '../execution-manifest'; import type { ExecutionIdentity } from '../execution-identity'; import type { CodeApiPrincipal } from '../auth/principal'; -import type { ExecutionProfile, SandboxBackendName } from '../execution-profile'; +import type { + ExecutionProfile, + SandboxBackendName, +} from '../execution-profile'; import { Jobs } from '@/enum/service'; /** @@ -149,7 +152,11 @@ export interface RequestBody { runtime_session_hint?: string; } -export type CreatePayload = { req: AuthenticatedRequest, session_id: string; isPyPlot?: boolean }; +export type CreatePayload = { + req: AuthenticatedRequest; + session_id: string; + isPyPlot?: boolean; +}; export interface FileObject { name: string; id: string; @@ -160,10 +167,12 @@ export interface FileObject { size?: number; lastModified?: string; etag?: string; - metadata?: { + metadata?: + | { 'content-type': string; 'original-filename': string; - } | undefined; + } + | undefined; versionId?: string | null; contentType?: string; } @@ -184,6 +193,14 @@ export type PayloadFileRef = { export interface PayloadBody { language: string; version: string; + /** Stable identity shared by all replay iterations of one execution. */ + execution_id?: string; + replay_tool_count?: number; + /** Manifest-bound upload ceiling exposed to remote workers. */ + max_output_files?: number; + /** Effective per-file ceiling after manifest and gateway policy intersect. */ + max_output_file_bytes?: number; + transfer_timeout_ms?: number; run_memory_limit?: number; run_timeout?: number; run_cpu_time?: number; @@ -238,6 +255,8 @@ export type ExecuteResult = { message?: string | null; status?: string | null; wall_time?: number | null; + /** Trusted worker control channel; avoids losing replay calls to stdout truncation. */ + pending_tool_calls_payload?: string; }; export interface LanguageConfig { @@ -265,6 +284,8 @@ export type JobData = { canonicalUserId?: string; /** Trusted dynamic outbound worker selection. */ bridgeWorkerId?: string; + /** Trusted selected workspace for native replay-mode PTC. */ + workspaceId?: string; /** Producer deployment identity. Optional only for pre-profile queued jobs. */ executionProfile?: ExecutionProfile; /** Required sandbox transport. Optional only for jobs queued before fencing. */ diff --git a/service/src/workers.ts b/service/src/workers.ts index ad20fd0f..dc2f491b 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -21,6 +21,7 @@ import { validateQueuedExecutionProfile, validateQueuedSandboxBackend, } from './execution-profile'; +import { BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, programmaticTransferReserveMs } from '../../packages/code/src/protocol'; const { INSTANCE_ID } = env; const WORKER_ID = `${INSTANCE_ID}-${process.pid}`; @@ -91,9 +92,15 @@ async function processJobInner(job: t.ExecuteJob): Promise { const delivery = prepareInputDelivery(payload, sandboxPayload); const sandboxRequest = buildSandboxExecuteRequest({ + ...(job.data.workspaceId == null ? {} : { programmaticTransferReserveMs: programmaticTransferReserveMs(env.JOB_TIMEOUT) }), payload: delivery.payload, egressGrantToken, executionManifestClaims, + maxOutputFileBytes: Math.min( + executionManifestClaims?.max_upload_bytes ?? env.EGRESS_GATEWAY_MAX_FILE_BYTES, + env.EGRESS_GATEWAY_MAX_FILE_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + ), executionManifestPrivateKey: env.EXECUTION_MANIFEST_PRIVATE_KEY, executionManifestSecret: env.EXECUTION_MANIFEST_SECRET, executionManifestTtlSeconds: env.EXECUTION_MANIFEST_TTL_SECONDS, @@ -149,6 +156,7 @@ async function processJobInner(job: t.ExecuteJob): Promise { tenantId: job.data.tenantId, canonicalUserId: job.data.canonicalUserId, bridgeWorkerId: job.data.bridgeWorkerId, + workspaceId: job.data.workspaceId, runtimeSessionId: runtimeSession.runtimeSessionId, runtimeSessionMode: runtimeSession.runtimeSessionMode, /* Stateful backends run this as a commit barrier after user code but @@ -184,6 +192,9 @@ async function processJobInner(job: t.ExecuteJob): Promise { : {}), stdout, stderr, + ...(responseData.pending_tool_calls_payload != null + ? { pending_tool_calls_payload: responseData.pending_tool_calls_payload } + : {}), }; if (run) { From e4815fa6572a4380bd82678b5d500ce2d4c49fb2 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 13:59:24 -0400 Subject: [PATCH 17/42] feat: Report Truncated Output Artifacts (#199) * feat: report truncated output artifacts * fix: classify omitted artifacts precisely * fix: preserve artifact scan invariants * fix: bound depth truncation probes * fix: bound capped directory enumeration * fix: constrain truncation probes across the job * fix: stop exhausted artifact probes --- api/README.md | 11 + api/src/job.ts | 279 ++++++++++++++- api/src/walker.test.ts | 375 +++++++++++++++++++++ service/src/execution-log.test.ts | 13 + service/src/execution-log.ts | 18 + service/src/service/blocking-poll.test.ts | 4 + service/src/service/blocking-poll.ts | 2 + service/src/service/programmatic-router.ts | 6 + service/src/types/service.ts | 12 + service/src/workers.ts | 3 + 10 files changed, 711 insertions(+), 12 deletions(-) diff --git a/api/README.md b/api/README.md index bbbe36ec..665a2931 100644 --- a/api/README.md +++ b/api/README.md @@ -94,6 +94,17 @@ Other package-format-compatible runtimes (Go, Rust, Java, GCC) can be installed Execute code in a sandboxed environment. +When supported output files are omitted because the response reaches its file +count limit, nesting or path limits, file-size limit, or a filesystem entry +cannot be read, the response includes `artifact_truncation`. Its `reasons` +object counts detected omissions by cause, `skipped_count` reports the total +detected omissions, and `skipped` contains up to 20 relative paths so callers +can match an expected output. Intentional filters such as unsupported file +extensions, hidden runtime directories, and unchanged session files do not +produce this marker when they can be classified within the bounded scan. A +depth-capped subtree that exceeds the metadata probe budget is reported +conservatively rather than allowing post-execution traversal to run unbounded. + ### `GET /api/v2/runtimes` List available language runtimes. diff --git a/api/src/job.ts b/api/src/job.ts index b7b82148..27604d29 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -36,9 +36,9 @@ import { SANDBOX_DIR_MODE, SANDBOX_FILE_MODE, ValidationError, + checkPathShape, hasRunnableSource, isDirkeep, - isValidPathShape, validateFilePath, isValidFilePath, } from './validation'; @@ -60,6 +60,15 @@ export { const AUTO_LOAD_DIRKEEP_TIMEOUT_MS = 10000; const AUTO_LOAD_DIRKEEP_RETRIES = 2; +const PTC_HISTORY_FILENAME = '_ptc_history.json'; +const TRUNCATION_PROBE_MAX_ENTRIES = 1000; +const TRUNCATION_PROBE_MAX_LEVELS = 10; +const TRUNCATION_PROBE_MAX_HASH_BYTES = 50_000_000; + +interface TruncationProbeState { + remainingEntries: number; + remainingHashBytes: number; +} /** Replaying the same sealed grant cannot repair an authorization denial. */ class InputAuthorizationError extends Error { @@ -647,8 +656,20 @@ interface ExecuteResult { session_id: string; files: FileRef[]; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; } +export type ArtifactTruncationReason = 'max_files' | 'depth' | 'size' | 'path' | 'unreadable'; + +export interface ArtifactTruncation { + code: 'artifact_truncated'; + reasons: Partial>; + skipped: string[]; + skipped_count: number; +} + +const MAX_REPORTED_TRUNCATED_PATHS = 20; + const jobQueue: Array<() => void> = []; async function acquireJobIdentity(log: Logger): Promise { @@ -693,6 +714,11 @@ export class Job { private pendingSurfaced = new Map(); private sessionFiles: FileRef[] = []; private inheritedRefs: FileRef[] = []; + private artifactTruncation: ArtifactTruncation | undefined; + private truncationProbeState: TruncationProbeState = { + remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, + remainingHashBytes: TRUNCATION_PROBE_MAX_HASH_BYTES, + }; private inputFileHashes = new Map(); private inputManifest = new Map(); private inputDestinations = new Map(); @@ -1673,6 +1699,7 @@ export class Job { version: this.runtime.version.raw, session_id: this.outputSessionId, files: this.sessionFiles, + ...(this.artifactTruncation ? { artifact_truncation: this.artifactTruncation } : {}), }; } @@ -1680,6 +1707,11 @@ export class Job { this.generatedFiles = []; this.sessionFiles = []; this.inheritedRefs = []; + this.artifactTruncation = undefined; + this.truncationProbeState = { + remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, + remainingHashBytes: TRUNCATION_PROBE_MAX_HASH_BYTES, + }; const inputByName = new Map(); for (const f of this.files) inputByName.set(f.name, f); @@ -1699,6 +1731,23 @@ export class Job { if (remaining > 0 && this.inheritedRefs.length > 0) { this.sessionFiles.push(...this.inheritedRefs.slice(0, remaining)); } + for (const ref of this.inheritedRefs.slice(remaining)) { + this.recordArtifactTruncation('max_files', ref.name); + } + } + + private recordArtifactTruncation(reason: ArtifactTruncationReason, relativePath: string): void { + this.artifactTruncation ??= { + code: 'artifact_truncated', + reasons: {}, + skipped: [], + skipped_count: 0, + }; + this.artifactTruncation.reasons[reason] = (this.artifactTruncation.reasons[reason] ?? 0) + 1; + this.artifactTruncation.skipped_count++; + if (this.artifactTruncation.skipped.length < MAX_REPORTED_TRUNCATED_PATHS) { + this.artifactTruncation.skipped.push(relativePath); + } } /** @@ -1722,6 +1771,7 @@ export class Job { isRegularFile = st.isFile(); } catch (err) { this.log.debug({ path: relativePath, err }, 'walkDir: failed to lstat entry'); + this.recordArtifactTruncation('unreadable', relativePath); return 'skip'; } } @@ -1742,7 +1792,14 @@ export class Job { inputByName: Map, ): Promise<{ collected: boolean; truncated: boolean }> { const keepPath = path.join(relativePath, DIRKEEP); - if (!isValidPathShape(keepPath)) return { collected: false, truncated: false }; + const pathShapeError = checkPathShape(keepPath); + if (pathShapeError) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + keepPath, + ); + return { collected: false, truncated: true }; + } const keepFullPath = path.join(fullPath, DIRKEEP); const inheritedKeep = inputByName.get(keepPath); @@ -1770,6 +1827,7 @@ export class Job { return this.createDirkeepMarker(keepPath, keepFullPath); } if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } const id = nanoid(); @@ -1819,6 +1877,7 @@ export class Job { if (!keepModified || keepInfo?.readOnly === true) return this.echoInheritedKeep(keepPath, inheritedKeep); if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } const refreshedId = nanoid(); @@ -1860,6 +1919,7 @@ export class Job { inheritedKeep: TFile, ): { collected: boolean; truncated: boolean } { if (this.inheritedRefs.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } this.inheritedRefs.push({ @@ -1886,6 +1946,7 @@ export class Job { keepFullPath: string, ): Promise<{ collected: boolean; truncated: boolean }> { if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', keepPath); return { collected: false, truncated: true }; } try { @@ -1944,6 +2005,7 @@ export class Job { if (existingFile.id && existingFile.storage_session_id) { if (this.inheritedRefs.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', relativePath); return { collected: false, truncated: true }; } this.inheritedRefs.push({ @@ -2003,9 +2065,29 @@ export class Job { size = st.size; } catch (err) { this.log.debug({ path: relativePath, err }, 'walkDir: unable to stat file'); + this.recordArtifactTruncation('unreadable', relativePath); return { collected: false, truncated: false, stopLoop: false }; } + + const inputFileInfo = this.inputFileHashes.get(relativePath); + const existingFile = inputByName.get(relativePath); if (size > this.runtime.max_file_size) { + /* Only an inline entrypoint needs hashing to decide whether this is + * intentional request-input suppression. Every other oversized file + * is rejected immediately, preserving the scan's bounded I/O cost. */ + if (!inputFileInfo || existingFile?.id != null || relativePath !== this.entryPointName) { + this.recordArtifactTruncation('size', relativePath); + return { collected: false, truncated: false, stopLoop: false }; + } + try { + const currentHash = await this.computeFileHash(fullPath, true); + if (currentHash === inputFileInfo.hash) { + return { collected: true, truncated: false, stopLoop: false }; + } + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed to hash oversized entrypoint'); + } + this.recordArtifactTruncation('size', relativePath); return { collected: false, truncated: false, stopLoop: false }; } @@ -2015,8 +2097,6 @@ export class Job { * stat-only signature would wrongly suppress. Compute once per session/input * file and reuse for the suppression check, wasModified, and the surfaced * mark; non-session jobs still only hash their inputs. */ - const inputFileInfo = this.inputFileHashes.get(relativePath); - const existingFile = inputByName.get(relativePath); let contentHash: string | undefined; if (inputFileInfo != null || this.session != null) { try { @@ -2058,6 +2138,15 @@ export class Job { if (wasModified) this.log.info({ file: relativePath }, 'Input file was modified'); } + /* The unchanged inline entrypoint is executable request input, not an + * output artifact. Suppress it before applying output-size reporting; + * downloaded inputs still flow through the size limit below, preserving + * the existing response-cap behavior for inherited refs. */ + if (!wasModified && inputFileInfo && existingFile?.id == null + && relativePath === this.entryPointName) { + return { collected: true, truncated: false, stopLoop: false }; + } + const echoed = this.tryEchoUnchangedInput({ wasModified, inputFileInfo, @@ -2067,6 +2156,7 @@ export class Job { if (echoed) return { ...echoed, stopLoop: false }; if (this.generatedFiles.length >= config.max_output_files) { + this.recordArtifactTruncation('max_files', relativePath); return { collected: false, truncated: true, stopLoop: true }; } @@ -2105,10 +2195,130 @@ export class Job { const childStatus = await this.walkDir(fullPath, parentDepth + 1, inputByName); if (childStatus === 'collected') return { collected: true, truncated: false }; if (childStatus === 'skipped') return { collected: false, truncated: true }; - if (this.isOutputCapFull()) return { collected: false, truncated: true }; return this.handleEmptyDirectory(relativePath, fullPath, inputByName); } + /** Finds the first artifact that a scan cap would hide without reading file + * contents. Files below a depth boundary cannot be valid primed inputs, and + * symlinks/unsupported files/hidden runtime directories remain intentional + * exclusions. An empty directory represents a reportable `.dirkeep`. */ + private async findTruncatedArtifact( + dir: string, + inputByName: Map, + state = this.truncationProbeState, + probeDepth = 0, + rootPath = path.relative(this.submissionDir, dir) || '.', + respectSessionSuppression = false, + ): Promise { + /* The state is shared by every probe in this job. Once exhausted, return + * conservatively before opening yet another capped sibling directory. */ + if (state.remainingEntries <= 0) return rootPath; + let directory: fs.Dir; + try { + directory = await fsp.opendir(dir); + } catch (err) { + const relativeDir = path.relative(this.submissionDir, dir) || '.'; + this.log.debug({ dir, err }, 'walkDir: unable to inspect depth-capped directory'); + this.recordArtifactTruncation('unreadable', relativeDir); + return undefined; + } + + let sawVisibleEntry = false; + let sawVisibleNonHiddenEntry = false; + try { + for await (const entry of directory) { + if (entry.name === PTC_HISTORY_FILENAME) continue; + sawVisibleEntry = true; + state.remainingEntries--; + if (state.remainingEntries < 0) return rootPath; + const fullPath = path.join(dir, entry.name); + const relativePath = path.relative(this.submissionDir, fullPath); + const kind = await this.classifyDirent(entry, fullPath, relativePath); + if (kind === 'skip') { + /* Ordinary walking counts symlinks/special entries as non-empty even + * though it does not surface them, so the probe must not invent a + * parent .dirkeep for that shape. */ + sawVisibleNonHiddenEntry = true; + continue; + } + if (kind === 'file') { + sawVisibleNonHiddenEntry = true; + if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; + const existingFile = inputByName.get(relativePath); + const inputFileInfo = this.inputFileHashes.get(relativePath); + if ( + respectSessionSuppression + && relativePath === this.entryPointName + && existingFile?.id == null + && inputFileInfo + ) { + try { + const st = await fsp.lstat(fullPath); + if (!st.isFile()) continue; + if (st.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= st.size; + if (await this.computeFileHash(fullPath, true) === inputFileInfo.hash) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during entrypoint cap probe'); + this.recordArtifactTruncation('unreadable', relativePath); + continue; + } + } + /* Once generated outputs fill the response cap, a persistent + * workspace may still contain unchanged artifacts from earlier + * turns. Ordinary walking suppresses those via their content hash, + * so the bounded cap probe must do the same or it reports a false + * max_files warning. Current-request inputs remain reportable: they + * would otherwise have been echoed into this response. */ + if (respectSessionSuppression && this.session && !existingFile) { + if (this.session.isPrimedReadOnly(relativePath)) continue; + try { + const st = await fsp.lstat(fullPath); + if (!st.isFile()) continue; + if (st.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= st.size; + const hash = await this.computeFileHash(fullPath, true); + if (this.session.isSurfaced(relativePath, hash)) continue; + if ( + this.session.isPrimedInput(relativePath) + && this.session.primedHash(relativePath) === hash + ) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during cap-probe hashing'); + this.recordArtifactTruncation('unreadable', relativePath); + continue; + } + } + return relativePath; + } + if (isHiddenDirectory(entry.name) && !inputsLiveUnder(inputByName, relativePath)) continue; + sawVisibleNonHiddenEntry = true; + /* The probe exists only to avoid false warnings for small, obviously + * unsupported-only subtrees. Once either budget is exhausted, report + * the capped root conservatively instead of defeating the scan bound. */ + if (probeDepth >= TRUNCATION_PROBE_MAX_LEVELS) return rootPath; + const nested = await this.findTruncatedArtifact( + fullPath, + inputByName, + state, + probeDepth + 1, + rootPath, + respectSessionSuppression, + ); + if (nested) return nested; + } + } catch (err) { + const relativeDir = path.relative(this.submissionDir, dir) || '.'; + this.log.debug({ dir, err }, 'walkDir: failed during bounded directory inspection'); + this.recordArtifactTruncation('unreadable', relativeDir); + return undefined; + } + + return sawVisibleEntry && sawVisibleNonHiddenEntry + ? undefined + : path.join(path.relative(this.submissionDir, dir), DIRKEEP); + } + /** * Recursively scans the submission directory for output files. Returns a * status distinguishing truly empty directories from scans truncated by @@ -2120,14 +2330,30 @@ export class Job { depth: number, inputByName: Map, ): Promise<'collected' | 'empty' | 'skipped'> { - if (depth >= config.max_nesting_depth) return 'skipped'; - if (this.isOutputCapFull()) return 'skipped'; - + const relativeDir = path.relative(this.submissionDir, dir) || '.'; + if (depth >= config.max_nesting_depth) { + const skippedPath = await this.findTruncatedArtifact(dir, inputByName); + if (skippedPath) this.recordArtifactTruncation('depth', skippedPath); + return 'skipped'; + } + if (this.isOutputCapFull()) { + const skippedPath = await this.findTruncatedArtifact( + dir, + inputByName, + this.truncationProbeState, + 0, + relativeDir, + true, + ); + if (skippedPath) this.recordArtifactTruncation('max_files', skippedPath); + return 'skipped'; + } let entries: fs.Dirent[]; try { entries = await fsp.readdir(dir, { withFileTypes: true }); } catch (err) { this.log.debug({ dir, err }, 'walkDir: unable to read directory'); + this.recordArtifactTruncation('unreadable', relativeDir); return 'skipped'; } @@ -2146,7 +2372,6 @@ export class Job { * separate npm packages so we can't import directly; the filename literal * is asserted-equal in `service/scripts/test-ptc-sentinel.ts` to catch * accidental drift in CI. */ - const PTC_HISTORY_FILENAME = '_ptc_history.json'; const isPtcReserved = (name: string): boolean => name === PTC_HISTORY_FILENAME; const nonDirkeepCount = entries.reduce( @@ -2166,13 +2391,10 @@ export class Job { let skippedHiddenDirs = 0; for (const entry of entries) { - if (this.isOutputCapFull()) { truncated = true; break; } if (isPtcReserved(entry.name)) continue; const fullPath = path.join(dir, entry.name); const relativePath = path.relative(this.submissionDir, fullPath); - if (!isValidPathShape(relativePath)) continue; - const kind = await this.classifyDirent(entry, fullPath, relativePath); if (kind === 'skip') continue; @@ -2189,9 +2411,42 @@ export class Job { skippedHiddenDirs++; continue; } + const pathShapeError = checkPathShape(relativePath); + if (pathShapeError) { + const skippedPath = await this.findTruncatedArtifact( + fullPath, + inputByName, + this.truncationProbeState, + 0, + relativePath, + ); + if (skippedPath) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + skippedPath, + ); + truncated = true; + } + continue; + } const res = await this.walkSubdirectory(relativePath, fullPath, depth, inputByName); if (res.collected) hasCollectedChild = true; if (res.truncated) truncated = true; + if (this.isOutputCapFull() && this.artifactTruncation?.reasons.max_files) break; + continue; + } + + /* Check intentional filename filtering before path limits. Unsupported + * files never belong in files[], regardless of how long their path is. */ + if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; + + const pathShapeError = checkPathShape(relativePath); + if (pathShapeError) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + relativePath, + ); + truncated = true; continue; } diff --git a/api/src/walker.test.ts b/api/src/walker.test.ts index 91382468..543505a9 100644 --- a/api/src/walker.test.ts +++ b/api/src/walker.test.ts @@ -26,11 +26,19 @@ interface WalkerInternals { generatedFiles: Array<{ id: string; name: string; path: string }>; sessionFiles: Array<{ id: string; name: string; storage_session_id: string; modified_from?: { id: string; storage_session_id: string }; inherited?: true; entity_id?: string }>; inheritedRefs: Array<{ id: string; name: string; storage_session_id: string; inherited?: true; entity_id?: string }>; + artifactTruncation?: { + code: 'artifact_truncated'; + reasons: Partial>; + skipped: string[]; + skipped_count: number; + }; + truncationProbeState: { remainingEntries: number; remainingHashBytes: number }; pendingSurfaced: Map; inputFileHashes: Map; files: TFile[]; reusePrimedInput: (file: TFile) => Promise; writeFile: (file: TFile) => Promise; + computeFileHash: (filePath: string, noFollow?: boolean) => Promise; walkDir: (dir: string, depth: number, inputByName: Map) => Promise<'collected' | 'empty' | 'skipped'>; handleSessionFiles: () => Promise; } @@ -743,6 +751,10 @@ describe('walkDir / output caps', () => { await internals.walkDir(tmpDir, 0, new Map()); expect(internals.generatedFiles.length).toBeLessThanOrEqual(cap); + expect(internals.artifactTruncation).toMatchObject({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + }); }); it('respects max_output_files cap on inherited refs', async () => { @@ -771,6 +783,11 @@ describe('walkDir / output caps', () => { expect(internals.inheritedRefs.length).toBeLessThanOrEqual(cap); expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toMatchObject({ + code: 'artifact_truncated', + reasons: { max_files: 5 }, + skipped_count: 5, + }); }); }); @@ -792,6 +809,364 @@ describe('walkDir / depth cap', () => { const deepName = path.relative(tmpDir, path.join(cursor, 'deep.py')); expect(internals.generatedFiles.map(f => f.name)).not.toContain(deepName); + expect(internals.artifactTruncation).toMatchObject({ + code: 'artifact_truncated', + reasons: { depth: 1 }, + skipped_count: 1, + }); + expect(internals.artifactTruncation?.skipped[0]).toBe(deepName); + }); + + it('does not report a depth cap when the skipped subtree has only unsupported files', async () => { + let cursor = tmpDir; + for (let i = 0; i < config.max_nesting_depth; i++) { + cursor = path.join(cursor, `d${i}`); + await fsp.mkdir(cursor); + } + await fsp.writeFile(path.join(cursor, 'cache.bin'), 'ignored'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('bounds depth-cap eligibility probes and reports the capped subtree conservatively', async () => { + let cursor = tmpDir; + const totalDepth = config.max_nesting_depth + 12; + for (let i = 0; i < totalDepth; i++) { + cursor = path.join(cursor, `d${i}`); + await fsp.mkdir(cursor); + } + await fsp.writeFile(path.join(cursor, 'cache.bin'), 'ignored'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + const cappedRoot = Array.from( + { length: config.max_nesting_depth }, + (_, i) => `d${i}`, + ).join(path.sep); + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { depth: 1 }, + skipped: [cappedRoot], + skipped_count: 1, + }); + }); +}); + +describe('walkDir / artifact truncation details', () => { + it('reports oversized supported outputs while leaving them out of files', async () => { + await fsp.writeFile(path.join(tmpDir, 'large.txt'), 'too large'); + const job = makeJob({ maxFileSize: 3 }); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { size: 1 }, + skipped: ['large.txt'], + skipped_count: 1, + }); + }); + + it('reports overlong output paths', async () => { + const directory = 'a'.repeat(200); + await fsp.mkdir(path.join(tmpDir, directory)); + const name = path.join(directory, `${'b'.repeat(60)}.txt`); + await fsp.writeFile(path.join(tmpDir, name), 'content'); + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [name], + skipped_count: 1, + }); + }); + + it('does not report an overlong path for an unsupported output', async () => { + const directory = 'a'.repeat(200); + await fsp.mkdir(path.join(tmpDir, directory)); + await fsp.writeFile(path.join(tmpDir, directory, `${'b'.repeat(60)}.bin`), 'ignored'); + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('reports an empty-directory marker whose appended path is too long', async () => { + const directory = 'a'.repeat(config.max_path_length - 6); + await fsp.mkdir(path.join(tmpDir, directory)); + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [path.join(directory, DIRKEEP)], + skipped_count: 1, + }); + }); + + it('reports an explicit overlong .dirkeep exactly once', async () => { + const directory = 'a'.repeat(config.max_path_length - 6); + await fsp.mkdir(path.join(tmpDir, directory)); + const keepName = path.join(directory, DIRKEEP); + await fsp.writeFile(path.join(tmpDir, keepName), ''); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [keepName], + skipped_count: 1, + }); + }); + + it('uses a bounded probe instead of recursively walking an overlong directory', async () => { + const first = 'a'.repeat(200); + const second = 'b'.repeat(60); + const overlongDir = path.join(first, second); + await fsp.mkdir(path.join(tmpDir, overlongDir), { recursive: true }); + for (let i = 0; i < 1001; i++) { + await fsp.writeFile(path.join(tmpDir, overlongDir, `ignored-${i}.bin`), 'ignored'); + } + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [overlongDir], + skipped_count: 1, + }); + }); + + it('does not report an unchanged oversized inline entrypoint', async () => { + const name = 'main.py'; + const content = 'print(1)'; + const full = path.join(tmpDir, name); + await fsp.writeFile(full, content); + const inline: TFile = { name, content }; + const job = makeJob({ files: [inline], maxFileSize: 3 }); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.entryPointName = name; + internals.inputFileHashes.set(name, { hash: sha256(content), path: full }); + + await internals.walkDir(tmpDir, 0, buildInputByName([inline])); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('inspects a capped directory before deciding whether an artifact was omitted', async () => { + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + await fsp.mkdir(path.join(tmpDir, 'ignored')); + await fsp.writeFile(path.join(tmpDir, 'ignored', 'cache.bin'), 'ignored'); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('reports a capped empty-directory marker', async () => { + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + await fsp.mkdir(path.join(tmpDir, 'empty')); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: [path.join('empty', DIRKEEP)], + skipped_count: 1, + }); + }); + + it('does not hash ordinary oversized files in session mode', async () => { + await fsp.writeFile(path.join(tmpDir, 'large.txt'), 'too large'); + const session = new SessionWorkspace({ runtimeSessionId: 'rt_large' }); + const internals = asInternals(makeJob({ maxFileSize: 3, session })); + internals.submissionDir = tmpDir; + let hashCalls = 0; + internals.computeFileHash = async () => { + hashCalls++; + return sha256('too large'); + }; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(hashCalls).toBe(0); + expect(internals.artifactTruncation?.reasons).toEqual({ size: 1 }); + }); + + it('keeps scanning for generated outputs when only inherited refs are capped', async () => { + await fsp.mkdir(path.join(tmpDir, 'a-ignored')); + await fsp.writeFile(path.join(tmpDir, 'a-ignored', 'cache.bin'), 'ignored'); + await fsp.writeFile(path.join(tmpDir, 'z-generated.txt'), 'new'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.inheritedRefs = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `inherited-${i}.txt`, + storage_session_id: 'previous', + inherited: true, + })); + internals.artifactTruncation = { + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: ['another-inherited.txt'], + skipped_count: 1, + }; + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.generatedFiles.map(file => file.name)).toContain('z-generated.txt'); + }); + + it('bounds output-cap eligibility probes for wide unsupported-only directories', async () => { + const job = makeJob(); + const internals = asInternals(job); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + for (let i = 0; i < 1001; i++) { + await fsp.writeFile(path.join(tmpDir, `ignored-${i}.bin`), 'ignored'); + } + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: ['.'], + skipped_count: 1, + }); + }); + + it('shares the output-cap probe budget across sibling subtrees', async () => { + for (const dirname of ['b-ignored', 'c-ignored']) { + await fsp.mkdir(path.join(tmpDir, dirname)); + for (let i = 0; i < 600; i++) { + await fsp.writeFile(path.join(tmpDir, dirname, `ignored-${i}.bin`), 'ignored'); + } + } + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(path.join(tmpDir, 'b-ignored'), 1, new Map()); + await internals.walkDir(path.join(tmpDir, 'c-ignored'), 1, new Map()); + + expect(internals.artifactTruncation?.reasons).toEqual({ max_files: 1 }); + expect(internals.artifactTruncation?.skipped).toEqual(['c-ignored']); + }); + + it('does not report surfaced session artifacts during output-cap probing', async () => { + const name = 'old-output.txt'; + const content = 'already returned'; + await fsp.writeFile(path.join(tmpDir, name), content); + const session = new SessionWorkspace({ runtimeSessionId: 'rt_capped' }); + session.markSurfaced(name, sha256(content)); + const internals = asInternals(makeJob({ session })); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toBeUndefined(); + }); + + it('does not reopen capped directories after the shared probe budget is exhausted', async () => { + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.truncationProbeState.remainingEntries = 0; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + const absentDir = path.join(tmpDir, 'not-opened'); + + await internals.walkDir(absentDir, 1, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { max_files: 1 }, + skipped: ['not-opened'], + skipped_count: 1, + }); + }); + + it('does not report an unchanged inline entrypoint during output-cap probing', async () => { + const directory = path.join(tmpDir, 'src'); + const name = path.join('src', 'main.py'); + const content = 'print(1)'; + await fsp.mkdir(directory); + await fsp.writeFile(path.join(tmpDir, name), content); + const inline: TFile = { name, content }; + const internals = asInternals(makeJob({ files: [inline] })); + internals.submissionDir = tmpDir; + internals.entryPointName = name; + internals.inputFileHashes.set(name, { hash: sha256(content), path: path.join(tmpDir, name) }); + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(directory, 1, buildInputByName([inline])); + + expect(internals.artifactTruncation).toBeUndefined(); }); }); diff --git a/service/src/execution-log.test.ts b/service/src/execution-log.test.ts index f04bad90..0f538125 100644 --- a/service/src/execution-log.test.ts +++ b/service/src/execution-log.test.ts @@ -28,6 +28,12 @@ describe('execution log summaries', () => { failed: 1, detail: 'private storage failure', }, + artifact_truncation: { + code: 'artifact_truncated', + reasons: { max_files: 2 }, + skipped: ['secret-one.txt', 'secret-two.txt'], + skipped_count: 2, + }, run: { code: 0, stdout: 'top secret stdout', @@ -42,6 +48,7 @@ describe('execution log summaries', () => { expect(JSON.stringify(summary)).not.toContain('sensitive stderr'); expect(JSON.stringify(summary)).not.toContain('combined output'); expect(JSON.stringify(summary)).not.toContain('private storage failure'); + expect(JSON.stringify(summary)).not.toContain('secret-one.txt'); expect(summary).toMatchObject({ session_id: 'sess_123', files: { count: 2, inheritedCount: 1, modifiedCount: 1 }, @@ -52,6 +59,12 @@ describe('execution log summaries', () => { delivered: 2, failed: 1, }, + artifact_truncation: { + code: 'artifact_truncated', + reasons: { max_files: 2 }, + skipped_count: 2, + reported_paths: 2, + }, run: { stdout: { length: 17, present: true }, stderr: { length: 16, present: true }, diff --git a/service/src/execution-log.ts b/service/src/execution-log.ts index 48a93cce..143e5541 100644 --- a/service/src/execution-log.ts +++ b/service/src/execution-log.ts @@ -19,6 +19,7 @@ type SandboxResponseLike = { version?: unknown; files?: unknown; artifact_delivery?: unknown; + artifact_truncation?: unknown; run?: RunLike; }; @@ -40,6 +41,22 @@ function summarizeArtifactDelivery(value: unknown): Record | un }; } +function summarizeArtifactTruncation(value: unknown): Record | undefined { + if (value == null || typeof value !== 'object' || Array.isArray(value)) return undefined; + const truncation = value as { + code?: unknown; + reasons?: unknown; + skipped?: unknown; + skipped_count?: unknown; + }; + return { + code: truncation.code, + reasons: truncation.reasons, + skipped_count: truncation.skipped_count, + reported_paths: Array.isArray(truncation.skipped) ? truncation.skipped.length : undefined, + }; +} + export function summarizeText(value: unknown): { length: number; present: boolean } { if (typeof value !== 'string') { return { length: 0, present: false }; @@ -87,6 +104,7 @@ export function summarizeSandboxResponse(data: SandboxResponseLike): Record { expect(await pollBlockingExecution('exec', 5, deps)).toEqual({ status: 'completed', stdout: result.stdout, stderr: '', files: [], artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, }); expect(deps.now()).toBe(2); }); diff --git a/service/src/service/blocking-poll.ts b/service/src/service/blocking-poll.ts index 8ce07fa8..03e386d4 100644 --- a/service/src/service/blocking-poll.ts +++ b/service/src/service/blocking-poll.ts @@ -34,6 +34,7 @@ export async function pollBlockingExecution( stderr?: string; files?: t.FileRefs; artifact_delivery?: t.ArtifactDeliveryFailure; + artifact_truncation?: t.ArtifactTruncation; }> { const start = deps.now(); while (deps.now() - start < timeout) { @@ -48,6 +49,7 @@ export async function pollBlockingExecution( stderr: result.stderr, files: result.files, artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, }; } } diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 4de4a4d8..93bbc0c5 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -1051,6 +1051,9 @@ async function runAndRespond( error: errorMessage, stdout: cleanStdout, stderr: result.stderr, + files: result.files, + artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, session_id: state.session_id, }); return; @@ -1064,6 +1067,7 @@ async function runAndRespond( stderr: result.stderr, files: result.files, artifact_delivery: result.artifact_delivery, + artifact_truncation: result.artifact_truncation, session_id: state.session_id, }); } @@ -1331,6 +1335,7 @@ async function handleBlocking( stderr: state.stderr ?? '', files: state.files ?? [], artifact_delivery: state.artifact_delivery, + artifact_truncation: state.artifact_truncation, session_id: execution.session_id, }); } @@ -1633,6 +1638,7 @@ async function handleBlocking( stderr: state.stderr ?? '', files: state.files ?? [], artifact_delivery: state.artifact_delivery, + artifact_truncation: state.artifact_truncation, session_id, }); } diff --git a/service/src/types/service.ts b/service/src/types/service.ts index f0a6da3b..d17b5a99 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -114,6 +114,15 @@ export interface ArtifactDeliveryFailure { failed: number; } +export type ArtifactTruncationReason = 'max_files' | 'depth' | 'size' | 'path' | 'unreadable'; + +export interface ArtifactTruncation { + code: 'artifact_truncated'; + reasons: Partial>; + skipped: string[]; + skipped_count: number; +} + export type ExecuteResponse = { run?: { stdout: string; @@ -133,6 +142,7 @@ export type ExecuteResponse = { session_id: string; files: FileRefs; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; }; export interface RequestBody { @@ -250,6 +260,7 @@ export type ExecuteResult = { stderr: string; files: FileRefs; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; code?: number | null; signal?: string | null; message?: string | null; @@ -392,6 +403,7 @@ export interface ProgrammaticResponse { stderr?: string; files?: FileRefs; artifact_delivery?: ArtifactDeliveryFailure; + artifact_truncation?: ArtifactTruncation; /** Top-level execution session id (one sandbox PTC invocation). */ session_id?: string; tool_calls_made?: number; diff --git a/service/src/workers.ts b/service/src/workers.ts index dc2f491b..f11a5420 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -190,6 +190,9 @@ async function processJobInner(job: t.ExecuteJob): Promise { ...(responseData.artifact_delivery != null ? { artifact_delivery: responseData.artifact_delivery } : {}), + ...(responseData.artifact_truncation != null + ? { artifact_truncation: responseData.artifact_truncation } + : {}), stdout, stderr, ...(responseData.pending_tool_calls_payload != null From f181b4deaa4ddf37a214875d9256932cb812a656 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 14:12:25 -0400 Subject: [PATCH 18/42] fix: cancel selected-workspace PTC across processes (#196) * fix: cancel replay jobs across API and worker processes * fix: drain worker cancellation watches promptly * fix: close programmatic cancellation races * fix: preserve cancellation response ordering * fix: close distributed cancellation races * fix: harden cancellation under concurrent load * fix: make cancellation ownership durable through completion * fix: recover durable replay outcomes across lost replies * fix: return atomic cancellation outcomes with aligned retention * fix: commit native results inside the workspace mutation fence * fix: claim programmatic execution before stalled-job redelivery --- docs/remote-bridge/README.md | 17 + packages/code/src/worker.test.ts | 643 ++++++++++++++--- packages/code/src/worker.ts | 203 ++++-- packages/code/src/workspace-worker.test.ts | 244 +++++++ service/src/bridge/concurrent-worker.test.ts | 20 +- service/src/config.spec.ts | 5 + service/src/config.ts | 9 +- service/src/job-cancellation-commit.test.ts | 431 ++++++++++++ service/src/job-cancellation.test.ts | 599 ++++++++++++++++ service/src/job-cancellation.ts | 664 ++++++++++++++++++ service/src/metrics.ts | 6 + service/src/middleware/limits.ts | 13 + service/src/programmatic-cancellation.test.ts | 234 ++++++ service/src/programmatic-cancellation.ts | 182 +++++ service/src/queue.ts | 39 +- service/src/redis-options.test.ts | 12 +- service/src/redis-options.ts | 6 + service/src/request-disconnect.test.ts | 60 ++ service/src/request-disconnect.ts | 49 ++ service/src/service/programmatic-router.ts | 351 +++++++-- service/src/types/service.ts | 6 + service/src/workers.ts | 399 ++++++++--- 22 files changed, 3872 insertions(+), 320 deletions(-) create mode 100644 service/src/job-cancellation-commit.test.ts create mode 100644 service/src/job-cancellation.test.ts create mode 100644 service/src/job-cancellation.ts create mode 100644 service/src/programmatic-cancellation.test.ts create mode 100644 service/src/programmatic-cancellation.ts create mode 100644 service/src/request-disconnect.test.ts create mode 100644 service/src/request-disconnect.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 99a0dece..b9de3ac2 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -256,6 +256,23 @@ execution. the currently registered incarnation. - Request cancellation is polled by the worker and aborts the local sandbox request. +- Replay PTC clients may attach a fresh `X-LibreChat-Code-Request-ID` to each + `/exec/programmatic` request and send that same opaque ID to + `POST /v1/exec/programmatic/cancel`. Code API binds the short-lived request + record to the authenticated principal, durably marks cancellation in Redis, + and publishes it to the worker process holding the BullMQ job. This explicit + path avoids relying on HTTP connection teardown, frees waiting jobs + immediately, and interrupts active remote-bridge assignments without polling + once per active job. + Cancellation and completed-result publication use an atomic Redis decision: + a late cancel returns `already_completed` instead of acknowledging Stop after + completion won. Ambiguous enqueue/cancellation errors retain replay ownership + until a durable fence or the original job deadline. Completed results are + retained temporarily (bounded to 16 MiB) so a lost BullMQ completion reply + does not cause sandbox effects to be repeated. Reconnect reconciliation reads + only small status markers, using one subscriber per process. + Roll out the matching Code API queue-worker processes before enabling this + endpoint on API replicas; pre-cancellation workers do not observe its markers. - A leased assignment remains in a Redis-backed delivery claim until the worker explicitly acknowledges it; reconnecting before acknowledgement redelivers the same fenced assignment instead of losing it after an HTTP disconnect. diff --git a/packages/code/src/worker.test.ts b/packages/code/src/worker.test.ts index 68d95b28..e26f212a 100644 --- a/packages/code/src/worker.test.ts +++ b/packages/code/src/worker.test.ts @@ -252,7 +252,10 @@ test('worker asks its supervisor to quarantine an ambiguous stateful runtime', a const quarantined: Array<{ sessionId: string; reason: string }> = []; const supervisor: RuntimeSupervisor = { async acquire() { - return { endpoint: 'http://127.0.0.1:3000/runtime', sessionId: 'rt-user-1' }; + return { + endpoint: 'http://127.0.0.1:3000/runtime', + sessionId: 'rt-user-1', + }; }, async reset() {}, async quarantine(sessionId, reason) { @@ -388,7 +391,10 @@ test('worker continues after an assignment-scoped settlement conflict', async () registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (init?.signal?.aborted === true) { @@ -397,15 +403,25 @@ test('worker continues after an assignment-scoped settlement conflict', async () if (url.endsWith('/lease')) { leases += 1; return new Response( - JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0, assignment }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 0, + assignment, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/ack')) { leaseAcknowledged = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -511,7 +527,10 @@ test('worker refreshes its registration during a long assignment', async () => { registeredAt: new Date().toISOString(), leaseTtlMs: 100, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -576,7 +595,10 @@ test('worker schedules registration freshness from request start', async () => { registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -589,7 +611,10 @@ test('worker schedules registration freshness from request start', async () => { if (url.endsWith('/cancelled')) { return new Response( JSON.stringify({ protocolVersion: 1, cancelled: false }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } return new Response( @@ -658,10 +683,13 @@ test('worker continues cancellation polling after a stalled response', async () }); } settlementAttempted = true; - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', @@ -694,6 +722,175 @@ test('worker continues cancellation polling after a stalled response', async () assert.equal(settlementAttempted, true); }); +test('worker stops an outstanding cancellation request before settling completed work', async () => { + let startCancellation!: () => void; + const cancellationStarted = new Promise((resolve) => { + startCancellation = resolve; + }); + let cancellationAborted = false; + let settlementAttempted = false; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/execute')) { + await cancellationStarted; + return Response.json({ session_id: 'run-1', files: [] }); + } + if (url.endsWith('/cancellation')) { + startCancellation(); + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => { + cancellationAborted = true; + reject(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + }); + } + settlementAttempted = true; + return Response.json({ protocolVersion: 1, accepted: true }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + cancellationPollIntervalMs: 1, + cancellationTransportTimeoutMs: 10_000, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'complete-while-cancellation-polling', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(cancellationAborted, true); + assert.equal(settlementAttempted, true); +}); + +test('worker aborts a retryable cancellation error body before settling completed work', async () => { + let cancellationBodyStarted!: () => void; + const cancellationStarted = new Promise((resolve) => { + cancellationBodyStarted = resolve; + }); + let cancellationBodyAborted = false; + let settlementAttempted = false; + const fetchImpl: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith('/execute')) { + await cancellationStarted; + return Response.json({ session_id: 'run-1', files: [] }); + } + if (url.endsWith('/cancellation')) { + return new Response( + new ReadableStream({ + start(controller) { + cancellationBodyStarted(); + init?.signal?.addEventListener( + 'abort', + () => { + cancellationBodyAborted = true; + controller.error(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ); + }, + }), + { status: 500 }, + ); + } + settlementAttempted = true; + return Response.json({ protocolVersion: 1, accepted: true }); + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + cancellationPollIntervalMs: 1, + cancellationTransportTimeoutMs: 10_000, + fetchImpl, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'complete-during-retryable-cancellation-response', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(cancellationBodyAborted, true); + assert.equal(settlementAttempted, true); +}); + +test('worker stops its cancellation delay before settling immediately completed work', async () => { + let settlementAttempted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + cancellationPollIntervalMs: 10_000, + fetchImpl: async (input) => { + const url = String(input); + if (url.endsWith('/execute')) { + return Response.json({ session_id: 'run-1', files: [] }); + } + if (url.endsWith('/cancellation')) { + throw new Error('cancellation transport should not start'); + } + settlementAttempted = true; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'complete-before-cancellation-polling', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 1_000).toISOString(), + remainingMs: 1_000, + request: { body: { language: 'bash' }, headers: {} }, + }); + + assert.equal(settlementAttempted, true); +}); + test('worker routes a hintless assignment to an ephemeral template session', async () => { let executeUrl = ''; let runtimeSessionHeader = ''; @@ -852,10 +1049,13 @@ test('worker preserves status for a non-JSON settlement rejection', async () => }, fetchImpl: async (input) => { if (String(input).endsWith('/execute')) { - return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ session_id: 'run-1', files: [] }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); } settlementAttempts += 1; return new Response('assignment fenced', { @@ -1032,7 +1232,10 @@ test('worker retries a known-clean rejection after shutdown until acknowledged', registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/execute')) { @@ -1046,18 +1249,20 @@ test('worker retries a known-clean rejection after shutdown until acknowledged', controller.abort(); throw new TypeError('connection reset'); } - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1097,8 +1302,7 @@ test('worker preserves a definite rejection when its heartbeat fails', async () token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1118,7 +1322,10 @@ test('worker preserves a definite rejection when its heartbeat fails', async () registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/execute')) { @@ -1139,7 +1346,10 @@ test('worker preserves a definite rejection when its heartbeat fails', async () } return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1170,8 +1380,7 @@ test('worker quarantines a stateful workspace after a sandbox 5xx response', asy token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1187,7 +1396,10 @@ test('worker quarantines a stateful workspace after a sandbox 5xx response', asy settlementAttempted = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1217,8 +1429,7 @@ test('worker treats a non-JSON sandbox 4xx as a definite rejection', async () => token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1235,7 +1446,10 @@ test('worker treats a non-JSON sandbox 4xx as a definite rejection', async () => JSON.parse(String(init?.body) || '{}').status === 'rejected'; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1268,18 +1482,20 @@ test('worker quarantines a stateful workspace after the sandbox request aborts', }); } settlementAttempted = true; - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1331,13 +1547,23 @@ test('worker surfaces quarantine when shutdown aborts stateful execution', async registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/lease')) { return new Response( - JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0, assignment }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 0, + assignment, + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (url.endsWith('/execute')) { @@ -1351,18 +1577,20 @@ test('worker surfaces quarantine when shutdown aborts stateful execution', async ); }); } - return new Response(JSON.stringify({ protocolVersion: 1, accepted: true }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ protocolVersion: 1, accepted: true }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); }; const worker = new BridgeWorker({ codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1510,7 +1738,10 @@ test('worker subtracts lease response transit from the server budget', async () if (String(input).endsWith('/ack')) { return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } now += 50; @@ -1527,10 +1758,16 @@ test('worker subtracts lease response transit from the server budget', async () leaseToken: 'lease-token-that-is-long-enough-for-testing', expiresAt: new Date(0).toISOString(), remainingMs: 1_000, - request: { body: { language: 'bash' }, headers: {} }, + request: { + body: { language: 'bash' }, + headers: {}, + }, }, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1572,21 +1809,28 @@ test('worker rejects a lease whose acknowledgement exhausts its budget', async ( registeredAt: new Date().toISOString(), leaseTtlMs: 50, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/ack')) { now += 10; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/settle')) { settlementAttempts += 1; - abandonedSettlement = JSON.parse( - String(init?.body), - ) as Record; + abandonedSettlement = JSON.parse(String(init?.body)) as Record< + string, + unknown + >; if (settlementAttempts === 1) { return new Response(JSON.stringify({ error: 'unavailable' }), { status: 503, @@ -1595,7 +1839,10 @@ test('worker rejects a lease whose acknowledgement exhausts its budget', async ( } return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } return new Response( @@ -1611,15 +1858,24 @@ test('worker rejects a lease whose acknowledgement exhausts its budget', async ( leaseToken: 'lease-token-that-is-long-enough-for-testing', expiresAt: new Date(0).toISOString(), remainingMs: 10, - request: { body: { language: 'bash' }, headers: {} }, + request: { + body: { language: 'bash' }, + headers: {}, + }, }, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); - await assert.rejects(worker.lease(), /expired during lease acknowledgement/); + await assert.rejects( + worker.lease(), + /expired during lease acknowledgement/, + ); assert.equal(abandonedSettlement?.status, 'rejected'); assert.ok(registrations > 0); assert.equal(settlementAttempts, 2); @@ -1653,7 +1909,10 @@ test('worker rejects an assignment after ambiguous acknowledgement delivery', as JSON.parse(String(init?.body) || '{}').status === 'rejected'; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } if (String(input).endsWith('/workers/register')) { @@ -1665,7 +1924,10 @@ test('worker rejects an assignment after ambiguous acknowledgement delivery', as registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); } return new Response( @@ -1685,7 +1947,10 @@ test('worker rejects an assignment after ambiguous acknowledgement delivery', as request: { body: { language: 'bash' }, headers: {} }, }, }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1702,8 +1967,7 @@ test('worker clamps rejected settlement errors to the protocol limit', async () token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1716,11 +1980,16 @@ test('worker clamps rejected settlement errors to the protocol limit', async () headers: { 'Content-Type': 'application/json' }, }); } - const settlement = JSON.parse(String(init?.body)) as { error: string }; + const settlement = JSON.parse(String(init?.body)) as { + error: string; + }; rejection = settlement.error; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1747,8 +2016,7 @@ test('worker quarantines an explicitly dirty stateful sandbox response', async ( token: 'worker-secret', workerId: 'vm-1', incarnationId: 'incarnation-00000001', - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', @@ -1761,13 +2029,19 @@ test('worker quarantines an explicitly dirty stateful sandbox response', async ( error: 'session_workspace_dirty', message: 'restore required', }), - { status: 409, headers: { 'Content-Type': 'application/json' } }, + { + status: 409, + headers: { 'Content-Type': 'application/json' }, + }, ); } settlementAttempted = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1859,15 +2133,21 @@ test('worker uses the server-relative lease budget despite VM clock skew', async }, fetchImpl: async (input) => { if (String(input).endsWith('/execute')) { - return new Response(JSON.stringify({ session_id: 'run-1', files: [] }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }); + return new Response( + JSON.stringify({ session_id: 'run-1', files: [] }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, + ); } settlementAttempted = true; return new Response( JSON.stringify({ protocolVersion: 1, accepted: true }), - { status: 200, headers: { 'Content-Type': 'application/json' } }, + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }, ); }, }); @@ -1886,7 +2166,6 @@ test('worker uses the server-relative lease budget despite VM clock skew', async assert.equal(settlementAttempted, true); }); - test('worker continues after an expired assignment settlement conflict', async () => { const controller = new AbortController(); let registrations = 0; @@ -1921,18 +2200,22 @@ test('worker continues after an expired assignment settlement conflict', async ( leases += 1; return Response.json({ protocolVersion: 1, - assignment: leases === 1 - ? { - protocolVersion: 1, - assignmentId: 'assignment-expired', - workerId: 'vm-1', - incarnationId, - generation: 1, - leaseToken: 'lease-token-that-is-long-enough-for-testing', - expiresAt: new Date(Date.now() + 10_000).toISOString(), - request: { body: { language: 'bash' }, headers: {} }, - } - : undefined, + assignment: + leases === 1 + ? { + protocolVersion: 1, + assignmentId: 'assignment-expired', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 10_000).toISOString(), + request: { + body: { language: 'bash' }, + headers: {}, + }, + } + : undefined, }); } if (url.endsWith('/execute')) { @@ -1940,7 +2223,10 @@ test('worker continues after an expired assignment settlement conflict', async ( } if (url.endsWith('/settle')) { return Response.json( - { error: 'Bridge assignment has expired', code: 'ASSIGNMENT_EXPIRED' }, + { + error: 'Bridge assignment has expired', + code: 'ASSIGNMENT_EXPIRED', + }, { status: 409 }, ); } @@ -2172,7 +2458,10 @@ test('paired worker rotates credentials throughout a long assignment', async () } if (url.endsWith('/execute')) { await new Promise((resolve) => setTimeout(resolve, 55)); - return Response.json({ session_id: 'run-long-rotation', files: [] }); + return Response.json({ + session_id: 'run-long-rotation', + files: [], + }); } return Response.json({ protocolVersion: 1, accepted: true }); }; @@ -2276,6 +2565,117 @@ test('paired worker cancels a stalled credential refresh after execution', async assert.equal(refreshAborted, true); }); +test('one concurrent caller cannot abort a credential refresh another caller still needs', async () => { + const key = createBridgeIdentity(); + const first = new AbortController(); + const second = new AbortController(); + let releaseRefresh!: () => void; + let refreshStarted!: () => void; + const started = new Promise((resolve) => { + refreshStarted = resolve; + }); + const released = new Promise((resolve) => { + releaseRefresh = resolve; + }); + let transportAborted = false; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-before-shared-refresh', + expiresAt: new Date(Date.now() + 5).toISOString(), + }, + credentialRefreshWindowMs: 10, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (input, init) => { + assert.match(String(input), /credentials\/refresh$/); + refreshStarted(); + init?.signal?.addEventListener('abort', () => { + transportAborted = true; + }); + await released; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-after-shared-refresh-value', + expiresAt: new Date(Date.now() + 120_000).toISOString(), + }); + }, + }); + + const firstRefresh = worker.refreshCredential(first.signal); + await started; + const secondRefresh = worker.refreshCredential(second.signal); + first.abort(); + + await assert.rejects(firstRefresh, { name: 'AbortError' }); + assert.equal(transportAborted, false); + releaseRefresh(); + await secondRefresh; + assert.equal(transportAborted, false); +}); + +test('a new caller starts a fresh credential refresh after the last waiter aborts', async () => { + const key = createBridgeIdentity(); + const first = new AbortController(); + let refreshCount = 0; + let firstRefreshStarted!: () => void; + const started = new Promise((resolve) => { + firstRefreshStarted = resolve; + }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: { + privateKey: key.privateKey, + credential: 'credential-before-replacement-refresh', + expiresAt: new Date(Date.now() + 5).toISOString(), + }, + credentialRefreshWindowMs: 10, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + }, + fetchImpl: async (_input, init) => { + refreshCount += 1; + if (refreshCount === 1) { + firstRefreshStarted(); + return await new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ); + }); + } + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'credential-after-replacement-refresh', + expiresAt: new Date(Date.now() + 120_000).toISOString(), + }); + }, + }); + + const abandoned = worker.refreshCredential(first.signal, Date.now() + 1_000); + await started; + first.abort(); + await assert.rejects(abandoned, { name: 'AbortError' }); + await worker.refreshCredential(undefined, Date.now() + 1_000); + + assert.equal(refreshCount, 2); +}); + test('paired worker refreshes conservatively before server clock calibration', async () => { const key = createBridgeIdentity(); let refreshCount = 0; @@ -2347,8 +2747,7 @@ test('paired worker charges initial credential refresh against the assignment de sandboxStarted = true; } if (url.endsWith('/settle')) { - rejected = - JSON.parse(String(init?.body)).status === 'rejected'; + rejected = JSON.parse(String(init?.body)).status === 'rejected'; } return Response.json({ protocolVersion: 1, @@ -2389,8 +2788,7 @@ test('paired worker rechecks the deadline after request serialization', async () codeApiUrl: 'https://code.example/v1', workerId: 'vm-1', incarnationId, - sandboxEndpoint: - 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', + sandboxEndpoint: 'http://127.0.0.1:2000/sessions/{runtimeSessionId}/api/v2', identity: { privateKey: key.privateKey, credential: 'credential-valid-during-serialization', @@ -2408,8 +2806,7 @@ test('paired worker rechecks the deadline after request serialization', async () sandboxStarted = true; } if (url.endsWith('/settle')) { - rejected = - JSON.parse(String(init?.body)).status === 'rejected'; + rejected = JSON.parse(String(init?.body)).status === 'rejected'; } return Response.json({ protocolVersion: 1, @@ -2471,8 +2868,7 @@ test('paired worker keeps endpoint validation failures known-clean', async () => const url = String(input); if (url.endsWith('/execute')) sandboxStarted = true; if (url.endsWith('/settle')) { - rejected = - JSON.parse(String(init?.body)).status === 'rejected'; + rejected = JSON.parse(String(init?.body)).status === 'rejected'; } return Response.json({ protocolVersion: 1, @@ -2758,11 +3154,13 @@ test('sandbox completion does not cancel an in-flight credential rotation', asyn } if (url.endsWith('/execute')) { await refreshStartedPromise; - return Response.json({ session_id: 'run-rotation-race', files: [] }); + return Response.json({ + session_id: 'run-rotation-race', + files: [], + }); } - settleAuthorization = ( - init?.headers as Record - ).Authorization; + settleAuthorization = (init?.headers as Record) + .Authorization; return Response.json({ protocolVersion: 1, accepted: true }); }; const worker = new BridgeWorker({ @@ -2799,8 +3197,41 @@ test('sandbox completion does not cancel an in-flight credential rotation', asyn ); }); +test('settlement does not drain another lane credential renewal', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', workerId: 'vm-1', token: 'fixture', + incarnationId, sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'] }, + fetchImpl: async (input) => String(input).endsWith('/execute') + ? Response.json({ session_id: 'independent-lane', files: [] }) + : Response.json({ protocolVersion: 1, accepted: true }), + }); + // A different lane owns this pending renewal. The settling lane has no + // maintenance waiter and must not consume its own lease on that promise. + Object.assign(worker, { credentialInFlight: { + promise: new Promise(() => {}), controller: new AbortController(), waiters: 1, + }, refreshCredential: async () => {} }); + const startedAt = Date.now(); + await worker.executeAndSettle({ + protocolVersion: 1, assignmentId: 'independent-lane', workerId: 'vm-1', + incarnationId, generation: 5, leaseToken: 'independent-lane-lease-token', + expiresAt: new Date(Date.now() + 600_000).toISOString(), + request: { body: { language: 'bash' }, headers: {} }, + }); + assert.ok(Date.now() - startedAt < 500, 'unrelated renewal must not add a one-second drain'); +}); + test('reconnect delay uses bounded exponential jitter', () => { - assert.equal(reconnectDelayMs(0, 1_000, 30_000, () => 0), 500); - assert.equal(reconnectDelayMs(0, 1_000, 30_000, () => 1), 1_000); - assert.equal(reconnectDelayMs(10, 1_000, 30_000, () => 1), 30_000); + assert.equal( + reconnectDelayMs(0, 1_000, 30_000, () => 0), + 500, + ); + assert.equal( + reconnectDelayMs(0, 1_000, 30_000, () => 1), + 1_000, + ); + assert.equal( + reconnectDelayMs(10, 1_000, 30_000, () => 1), + 30_000, + ); }); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index a0517bd9..ffe1b350 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -38,6 +38,11 @@ export interface BridgeWorkerOptions { capabilities: BridgeWorkerCapabilities; workspaceTools?: WorkspaceToolExecutor; workspaceProgrammatic?: { + /** + * True when a WorkspaceToolError without mutation uncertainty proves the + * selected workspace was not changed. + */ + mutationFailuresAreAtomic?: true; executeProgrammatic( workspaceId: string, request: BridgeWorkspaceProgrammaticRequest, @@ -99,6 +104,7 @@ const DEFAULT_REGISTRATION_TRANSPORT_TIMEOUT_MS = 10_000; const DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS = 10_000; const DEFAULT_CANCELLATION_POLL_INTERVAL_MS = 500; const DEFAULT_CANCELLATION_TRANSPORT_TIMEOUT_MS = 2_000; +const CREDENTIAL_REFRESH_SETTLEMENT_GRACE_MS = 1_000; const MIN_REGISTRATION_HEARTBEAT_MS = 25; const REGISTRATION_RETRY_DELAY_MS = 100; const CREDENTIAL_REFRESH_RETRY_DELAY_MS = 100; @@ -158,26 +164,29 @@ function workspaceCapabilitiesMatch( advertised.writeFileModes?.length === executor.writeFileModes?.length && (advertised.writeFileModes?.every( (mode, index) => mode === executor.writeFileModes?.[index], - ) ?? executor.writeFileModes == null) && + ) ?? + executor.writeFileModes == null) && advertised.editFileModes?.length === executor.editFileModes?.length && (advertised.editFileModes?.every( (mode, index) => mode === executor.editFileModes?.[index], - ) ?? executor.editFileModes == null) && - advertised.editFileFeatures?.length === - executor.editFileFeatures?.length && + ) ?? + executor.editFileModes == null) && + advertised.editFileFeatures?.length === executor.editFileFeatures?.length && (advertised.editFileFeatures?.every( (feature, index) => feature === executor.editFileFeatures?.[index], - ) ?? executor.editFileFeatures == null) && - advertised.listFileFeatures?.length === - executor.listFileFeatures?.length && + ) ?? + executor.editFileFeatures == null) && + advertised.listFileFeatures?.length === executor.listFileFeatures?.length && (advertised.listFileFeatures?.every( (feature, index) => feature === executor.listFileFeatures?.[index], - ) ?? executor.listFileFeatures == null) && + ) ?? + executor.listFileFeatures == null) && advertised.programmaticLanguages?.length === executor.programmaticLanguages?.length && (advertised.programmaticLanguages?.every( (language, index) => language === executor.programmaticLanguages?.[index], - ) ?? executor.programmaticLanguages == null) && + ) ?? + executor.programmaticLanguages == null) && advertised.workspaces.length === executor.workspaces.length && advertised.workspaces.every( (workspace, index) => @@ -189,7 +198,8 @@ function workspaceCapabilitiesMatch( (operation, operationIndex) => operation === executor.workspaces[index]?.operations?.[operationIndex], - ) ?? executor.workspaces[index]?.operations == null), + ) ?? + executor.workspaces[index]?.operations == null), ) ); } @@ -201,8 +211,7 @@ function registrationCompatibleCapabilities( if ( workspaceTools == null || (workspaceTools.operations.every( - (operation) => - operation === 'read_file' || operation === 'search_text', + (operation) => operation === 'read_file' || operation === 'search_text', ) && workspaceTools.workspaces.every( (workspace) => workspace.operations == null, @@ -211,8 +220,7 @@ function registrationCompatibleCapabilities( return capabilities; } const operations = workspaceTools.operations.filter( - (operation) => - operation === 'read_file' || operation === 'search_text', + (operation) => operation === 'read_file' || operation === 'search_text', ); if (operations.length === 0) { const { workspaceTools: _workspaceTools, ...compatible } = capabilities; @@ -221,7 +229,9 @@ function registrationCompatibleCapabilities( const workspaces = workspaceTools.workspaces.flatMap((workspace) => { if ( workspace.operations != null && - !operations.every((operation) => workspace.operations?.includes(operation)) + !operations.every((operation) => + workspace.operations?.includes(operation), + ) ) { return []; } @@ -386,7 +396,11 @@ export class BridgeWorker { private negotiatedWorkspaceSlots = 1; private concurrentRunning = false; private registrationInFlight?: Promise; - private credentialInFlight?: Promise; + private credentialInFlight?: { + promise: Promise; + controller: AbortController; + waiters: number; + }; private serverClockOffsetMs = MAX_PROOF_CLOCK_SKEW_MS; constructor(private readonly options: BridgeWorkerOptions) { @@ -434,7 +448,8 @@ export class BridgeWorker { (options.workspaceProgrammatic != null) !== (options.capabilities.workspaceTools?.programmaticLanguages?.includes( 'bash', - ) === true) + ) === + true) ) { throw new BridgeProtocolError( 'Workspace programmatic capability requires a matching executor', @@ -553,7 +568,9 @@ export class BridgeWorker { if (signal?.aborted) { abortRegistration(); } else { - signal?.addEventListener('abort', abortRegistration, { once: true }); + signal?.addEventListener('abort', abortRegistration, { + once: true, + }); } const timeoutMs = Math.min( Math.max(1, this.registrationTtlMs - 1), @@ -575,7 +592,10 @@ export class BridgeWorker { workerId: this.options.workerId, incarnationId: this.incarnationId, capabilities: this.maintenanceOnly - ? { ...capabilities, requiresReadyConfirmation: true } + ? { + ...capabilities, + requiresReadyConfirmation: true, + } : capabilities, }, registrationController.signal, @@ -709,7 +729,9 @@ export class BridgeWorker { } await this.runtimeSupervisor.reset(runtimeSessionId, signal); await this.timedRequest( - `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/workspaces/reset`, + `${this.codeApiUrl}${bridgeWorkerPath( + this.options.workerId, + )}/workspaces/reset`, { protocolVersion: BRIDGE_PROTOCOL_VERSION, incarnationId: this.incarnationId, @@ -745,7 +767,9 @@ export class BridgeWorker { // machine-local guard before the remote fence can be removed. await guard.assertAvailable(); await this.timedRequest( - `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/workspaces/reset`, + `${this.codeApiUrl}${bridgeWorkerPath( + this.options.workerId, + )}/workspaces/reset`, { protocolVersion: BRIDGE_PROTOCOL_VERSION, incarnationId: this.incarnationId, @@ -1051,20 +1075,58 @@ export class BridgeWorker { transportTimeoutMs = Number.POSITIVE_INFINITY, ): Promise { while (this.credentialInFlight) { - await this.credentialInFlight; + await this.waitForCredentialRefresh(this.credentialInFlight, signal); // A longer-lived caller may still need another refresh after this one. } + const controller = new AbortController(); const pending = this.refreshCredentialOwned( - signal, + controller.signal, validThroughMs, transportTimeoutMs, ); - this.credentialInFlight = pending; + const entry = { promise: pending, controller, waiters: 0 }; + this.credentialInFlight = entry; + void pending.then( + () => { + if (this.credentialInFlight === entry) + this.credentialInFlight = undefined; + }, + () => { + if (this.credentialInFlight === entry) + this.credentialInFlight = undefined; + }, + ); + await this.waitForCredentialRefresh(entry, signal); + } + + private async waitForCredentialRefresh( + entry: NonNullable, + signal?: AbortSignal, + ): Promise { + entry.waiters += 1; + let removeAbortListener = (): void => {}; + const aborted = new Promise((_, reject) => { + if (signal == null) return; + const abort = (): void => + reject( + signal.reason instanceof Error + ? signal.reason + : new DOMException('aborted', 'AbortError'), + ); + removeAbortListener = (): void => + signal.removeEventListener('abort', abort); + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) abort(); + }); try { - await pending; + await Promise.race([entry.promise, aborted]); } finally { - if (this.credentialInFlight === pending) + removeAbortListener(); + entry.waiters -= 1; + if (entry.waiters === 0 && this.credentialInFlight === entry) { this.credentialInFlight = undefined; + entry.controller.abort(); + } } } @@ -1118,7 +1180,7 @@ export class BridgeWorker { assignment: BridgeAssignment, stopSignal: AbortSignal, serverClockOffsetMs: number, - requestSignal?: AbortSignal, + maintenance: { refresh?: Promise }, ): Promise { const identity = this.options.identity; if (identity == null) return; @@ -1136,18 +1198,18 @@ export class BridgeWorker { await abortableDelay(waitMs, stopSignal); if (stopSignal.aborted || Date.now() >= assignmentDeadlineMs) return; try { - await this.refreshCredential( - requestSignal, + maintenance.refresh = this.refreshCredential( + stopSignal, Date.now() + serverClockOffsetMs + refreshWindowMs, ); + await maintenance.refresh; } catch (error) { if (stopSignal.aborted) return; const terminal = error instanceof BridgeProtocolError && (error.status === 401 || error.status === 403); const credentialRemainingMs = - Date.parse(identity.expiresAt) - - (Date.now() + serverClockOffsetMs); + Date.parse(identity.expiresAt) - (Date.now() + serverClockOffsetMs); if (terminal || credentialRemainingMs <= 0) throw error; await abortableDelay( Math.min( @@ -1156,6 +1218,8 @@ export class BridgeWorker { ), stopSignal, ); + } finally { + maintenance.refresh = undefined; } } } @@ -1352,6 +1416,7 @@ export class BridgeWorker { ); let credentialMaintenanceError: unknown; let credentialMaintenance: Promise | undefined; + const ownCredentialMaintenance: { refresh?: Promise } = {}; let settlement: BridgeSettlement; let ambiguousSandboxError: unknown; let ambiguousWorkspaceMutationError: unknown; @@ -1368,7 +1433,7 @@ export class BridgeWorker { assignment, credentialController.signal, serverClockOffsetMs, - signal, + ownCredentialMaintenance, ).catch((error) => { credentialMaintenanceError = error; executionController.abort(); @@ -1674,14 +1739,22 @@ export class BridgeWorker { ) { workspaceMutationGuardError = error; } + const knownAtomicWorkspaceToolFailure = + assignment.executionKind === 'workspace_tool' && + error instanceof WorkspaceToolError && + this.options.workspaceTools?.mutationFailuresAreAtomic === true && + !error.requiresQuarantine; + const knownAtomicProgrammaticFailure = + assignment.executionKind === 'workspace_programmatic' && + error instanceof WorkspaceToolError && + this.options.workspaceProgrammatic?.mutationFailuresAreAtomic === + true && + !error.requiresQuarantine; if ( workspaceMutationApplied || (workspaceMutationArmed && - !( - error instanceof WorkspaceToolError && - this.options.workspaceTools?.mutationFailuresAreAtomic === true && - !error.requiresQuarantine - )) + !knownAtomicWorkspaceToolFailure && + !knownAtomicProgrammaticFailure) ) { ambiguousWorkspaceMutationError = error; } @@ -1713,12 +1786,30 @@ export class BridgeWorker { clearTimeout(deadlineTimer); cancellationController.abort(); await cancellationWatcher; + // Only drain renewal joined by this assignment, never an unrelated lane's + // refresh. Leave settlement time inside the original assignment budget. + const credentialInFlight = ownCredentialMaintenance.refresh; + if (credentialInFlight != null && !credentialController.signal.aborted) { + let drainTimer: ReturnType | undefined; + await Promise.race([ + credentialInFlight.catch(() => undefined), + new Promise((resolve) => { + drainTimer = setTimeout( + resolve, + Math.min(CREDENTIAL_REFRESH_SETTLEMENT_GRACE_MS, + Math.max(0, Date.parse(assignment.expiresAt) - serverClockOffsetMs - Date.now() - 5_000)), + ); + }), + ]); + if (drainTimer != null) clearTimeout(drainTimer); + } credentialController.abort(); await credentialMaintenance; try { if (workspaceMutationGuardError != null) throw workspaceMutationGuardError; if (ambiguousWorkspaceMutationError != null) { + this.options.onError?.(ambiguousWorkspaceMutationError); throw await this.quarantineWorkspace( undefined, 'Worker stopped after a workspace mutation completed without a fulfilled settlement', @@ -1913,7 +2004,9 @@ export class BridgeWorker { return await lease.execute({ body, headers, signal }); } if (lease.endpoint == null) { - throw new BridgeProtocolError('Runtime lease does not provide an execution transport'); + throw new BridgeProtocolError( + 'Runtime lease does not provide an execution transport', + ); } const endpoint = lease.endpoint.replace(/\/+$/, ''); const response = await this.fetchImpl(`${endpoint}/execute`, { @@ -2170,17 +2263,23 @@ export class BridgeWorker { signal: AbortSignal, ): Promise { while (!signal.aborted && !executionController.signal.aborted) { - await this.delay( - Math.max( - 1, - this.options.cancellationPollIntervalMs ?? - DEFAULT_CANCELLATION_POLL_INTERVAL_MS, - ), - signal, - ); + try { + await this.delay( + Math.max( + 1, + this.options.cancellationPollIntervalMs ?? + DEFAULT_CANCELLATION_POLL_INTERVAL_MS, + ), + signal, + ); + } catch (error) { + if (signal.aborted || executionController.signal.aborted) return; + throw error; + } if (signal.aborted || executionController.signal.aborted) return; const pollController = new AbortController(); const abortPoll = (): void => pollController.abort(); + signal.addEventListener('abort', abortPoll, { once: true }); executionController.signal.addEventListener('abort', abortPoll, { once: true, }); @@ -2200,6 +2299,15 @@ export class BridgeWorker { incarnationId: this.incarnationId, }, pollController.signal, + (response) => { + // Once response headers arrive, drain the bounded body before a + // successful execution can settle. Otherwise a cancellation=true + // response racing command completion can be discarded. The + // transport timer and execution signal still cap the drain. + if (response.ok || response.status === 404) { + signal.removeEventListener('abort', abortPoll); + } + }, ); if (response.cancelled) { executionController.abort(); @@ -2213,6 +2321,7 @@ export class BridgeWorker { if (signal.aborted) return; } finally { clearTimeout(timeout); + signal.removeEventListener('abort', abortPoll); executionController.signal.removeEventListener('abort', abortPoll); } } @@ -2222,6 +2331,7 @@ export class BridgeWorker { url: string, body: object, signal?: AbortSignal, + onResponseHeaders?: (response: Response) => void, ): Promise { const requestBody = JSON.stringify(body); const response = await this.fetchImpl(url, { @@ -2233,6 +2343,7 @@ export class BridgeWorker { body: requestBody, signal, }); + onResponseHeaders?.(response); let payload: unknown; try { payload = await response.json(); diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index fa08a84c..f6205cd4 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -1032,6 +1032,250 @@ test('worker executes programmatic Bash in the selected workspace and preserves ]); }); +test('worker keeps a selected workspace usable after an atomic programmatic setup failure', async () => { + const lifecycle: string[] = []; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + mutationFailuresAreAtomic: true, + async executeProgrammatic() { + throw new WorkspaceToolError( + 'Programmatic input download failed', + 'COMMAND_UNAVAILABLE', + ); + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + ], + ]), + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-setup-failure', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + headers: {}, + }, + }); + + assert.deepEqual(lifecycle, ['arm', 'clear']); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'COMMAND_UNAVAILABLE'); +}); + +test('worker keeps a selected workspace usable after confirmed programmatic cancellation cleanup', async () => { + const lifecycle: string[] = []; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + mutationFailuresAreAtomic: true, + async executeProgrammatic() { + throw new WorkspaceToolError( + 'Workspace command execution aborted', + 'EXECUTION_ABORTED', + true, + false, + ); + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine( + () => lifecycle.push('quarantine'), + () => lifecycle.push('arm'), + () => lifecycle.push('clear'), + ), + ], + ]), + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-cancelled-cleanly', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'sleep 30' }], + }, + headers: {}, + }, + }); + + assert.deepEqual(lifecycle, ['arm', 'clear']); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'EXECUTION_ABORTED'); +}); + +test('worker reports the underlying cause before quarantining an uncertain programmatic mutation', async () => { + const rootCause = new WorkspaceToolError( + 'Programmatic output upload failed', + 'COMMAND_UNAVAILABLE', + true, + true, + ); + let reported: unknown; + let quarantined = false; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const], + programmaticLanguages: ['bash' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute() { + throw new Error('workspace tool executor must not run'); + }, + }, + workspaceProgrammatic: { + mutationFailuresAreAtomic: true, + async executeProgrammatic() { + throw rootCause; + }, + }, + workspaceQuarantines: new Map([ + [ + 'primary', + mutationQuarantine(() => { + quarantined = true; + }), + ], + ]), + onError(error) { + reported = error; + }, + fetchImpl: async () => { + throw new Error('settlement must not run'); + }, + }); + + await assert.rejects( + worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-programmatic-uncertain-failure', + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_programmatic', + workspaceId: 'primary', + request: { + body: { + language: 'bash', + version: '5.2', + session_id: 'session-1', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + headers: {}, + }, + }), + BridgeWorkspaceQuarantinedError, + ); + + assert.equal(reported, rootCause); + assert.equal(quarantined, true); +}); + test('worker stops after Code API rejects a fulfilled workspace mutation', async () => { let quarantinedReason: string | undefined; let armed = 0; diff --git a/service/src/bridge/concurrent-worker.test.ts b/service/src/bridge/concurrent-worker.test.ts index 5aec49cf..ea5ac601 100644 --- a/service/src/bridge/concurrent-worker.test.ts +++ b/service/src/bridge/concurrent-worker.test.ts @@ -243,11 +243,25 @@ for (const failure of [ status: 'fulfilled', value: { status: 'fulfilled' }, }); - for (let i = 0; i < 300 && errors.length === 0; i++) + const diagnosticCount = cleanupFailure ? 1 : 2; + const quarantineAttemptCount = + failure === 'lost-response' + ? 2 + : failure === 'all-responses-lost' || failure === 'delivery-outage' + ? 3 + : 1; + for ( + let i = 0; + i < 300 && + (errors.length < diagnosticCount || + (!cleanupFailure && quarantineAttempts < quarantineAttemptCount)); + i++ + ) { await new Promise((resolve) => setTimeout(resolve, 5)); + } if (failure === 'delivery-outage') - expect(errors.length).toBeGreaterThanOrEqual(1); - else expect(errors.length).toBe(1); + expect(errors.length).toBeGreaterThanOrEqual(2); + else expect(errors.length).toBe(diagnosticCount); if (failure === 'lost-response') expect(quarantineAttempts).toBe(2); if (failure === 'delivery-outage') expect(quarantineAttempts).toBeGreaterThanOrEqual(3); diff --git a/service/src/config.spec.ts b/service/src/config.spec.ts index 44b64b8b..69b416d8 100644 --- a/service/src/config.spec.ts +++ b/service/src/config.spec.ts @@ -99,6 +99,11 @@ describe('egress grant TTL configuration', () => { }); describe('job deadline accounting', () => { + it('never extends the producer deadline when worker configuration differs', () => { + expect(jobDeadlineAtMs(1_000, 300_000, 50_000, 91_000)).toBe(91_000); + expect(jobDeadlineAtMs(1_000, 30_000, 50_000, 91_000)).toBe(31_000); + expect(jobDeadlineAtMs(1_000, 300_000, 50_000, Number.NaN)).toBe(0); + }); it('counts time spent waiting in BullMQ against JOB_TIMEOUT', () => { expect(jobDeadlineAtMs(1_000, 300_000, 50_000)).toBe(301_000); }); diff --git a/service/src/config.ts b/service/src/config.ts index d025831e..55570dfe 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -95,10 +95,17 @@ export function jobDeadlineAtMs( enqueuedAtMs: number | undefined, timeoutMs: number, nowMs: number = Date.now(), + producerDeadlineAtMs?: number, ): number { - return Number.isFinite(enqueuedAtMs) && (enqueuedAtMs as number) > 0 + const localDeadline = Number.isFinite(enqueuedAtMs) && (enqueuedAtMs as number) > 0 ? (enqueuedAtMs as number) + timeoutMs : nowMs + timeoutMs; + if (producerDeadlineAtMs === undefined) return localDeadline; + // A worker with a larger JOB_TIMEOUT must not outlive the admission fence + // retained by its API producer. Malformed explicit deadlines fail closed. + return Number.isFinite(producerDeadlineAtMs) + ? Math.min(localDeadline, producerDeadlineAtMs) + : 0; } /** The worker stops user work at JOB_TIMEOUT, then may still need to terminate diff --git a/service/src/job-cancellation-commit.test.ts b/service/src/job-cancellation-commit.test.ts new file mode 100644 index 00000000..8ded29f2 --- /dev/null +++ b/service/src/job-cancellation-commit.test.ts @@ -0,0 +1,431 @@ +import { afterEach, beforeEach, expect, test } from 'bun:test'; +import { startTestRedis } from './test/redis'; +import { RedisBridgeStore } from './bridge/store'; +import { + commitJobResult, + readCommittedJobResult, + requestJobCancellation, + JobCancellationRegistry, + jobCancellationInternals, + fenceJobCancellation, + waitForJobWithCancellation, + jobCancellationRetentionSeconds, + claimJobExecution, +} from './job-cancellation'; + +let redis: Awaited>; +beforeEach(async () => { + redis = await startTestRedis(); +}); +afterEach(async () => { + await redis.closeTestServer(); +}); +const target = { queueName: 'other', jobId: 'commit-race' }; + +for (const outcome of ['commit', 'stop', 'duplicate']) + test(`native mutation handoff commits or quarantines before root release (${outcome})`, async () => { + const store = new RedisBridgeStore(redis); + const workerId = 'handoff-worker'; + const incarnationId = 'incarnation-handoff-01'; + await store.register({ + protocolVersion: 1, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: 1, + operations: ['execute_command'], + programmaticLanguages: ['bash'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const controller = new AbortController(); + const dispatchArgs = { + workerId, + workspaceId: 'primary', + headers: {}, + body: { + language: 'bash', + version: '5.2', + session_id: 'handoff-session', + files: [{ name: 'main.sh', content: 'echo mutation' }], + }, + deadlineAtMs: Date.now() + 5_000, + signal: controller.signal, + }; + const completion = store.dispatch({ + ...dispatchArgs, + finalize: async settlement => { + if (outcome === 'stop') await requestJobCancellation(redis, target, 60); + if (outcome === 'duplicate') + await commitJobResult( + redis, + target, + { stdout: 'first mutation' }, + 60, + ); + if ( + (await commitJobResult( + redis, + target, + { stdout: 'mutation settled' }, + 60, + )) !== 'committed' + ) + throw new Error('handoff did not win'); + // This represents Stop during post-handoff egress cleanup. It must no + // longer turn the applied mutation into an acknowledged cancellation. + expect(await requestJobCancellation(redis, target, 60)).toBe(false); + return settlement; + }, + }); + void completion.catch(() => undefined); + const assignment = await store.lease(workerId, incarnationId, 1_000); + if (assignment == null) throw new Error('Missing assignment'); + await store.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'fulfilled', + result: { + session_id: 'handoff-session', + language: 'bash', + version: '5.2', + files: [], + }, + }); + if (outcome !== 'commit') { + await expect(completion).rejects.toThrow('handoff did not win'); + await expect(store.dispatch(dispatchArgs)).rejects.toMatchObject({ + code: 'WORKSPACE_QUARANTINED', + }); + } else { + await expect(completion).resolves.toMatchObject({ + status: 'fulfilled', + }); + expect(await readCommittedJobResult(redis, target)).toEqual({ + result: { stdout: 'mutation settled' }, + }); + } + }); + +test('concurrent stalled-job redelivery claims at most one sandbox execution', async () => { + let executions = 0; + const attempt = async () => { + const claim = await claimJobExecution(redis, target, 60); + if (claim.status === 'claimed') executions += 1; + return claim; + }; + const results = await Promise.allSettled([attempt(), attempt()]); + expect(executions).toBe(1); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength( + 1, + ); + expect(results.filter(result => result.status === 'rejected')).toHaveLength( + 1, + ); + await expect(attempt()).rejects.toThrow('already claimed'); + expect(executions).toBe(1); + await commitJobResult(redis, target, { stdout: 'first result' }, 60); + expect(await attempt()).toEqual({ + status: 'completed', + result: { stdout: 'first result' }, + }); + expect(executions).toBe(1); + expect( + await commitJobResult(redis, target, { stdout: 'different result' }, 60), + ).toBe('already_completed'); + expect(await readCommittedJobResult(redis, target)).toEqual({ + result: { stdout: 'first result' }, + }); +}); + +test('a lost execution-claim reply never authorizes a second attempt', async () => { + const lostReply = { + eval: async (...args: Parameters) => { + await redis.eval(...args); + throw new Error('claim reply lost'); + }, + } as unknown as typeof redis; + await expect(claimJobExecution(lostReply, target, 60)).rejects.toThrow( + 'claim reply lost', + ); + await expect(claimJobExecution(redis, target, 60)).rejects.toThrow( + 'already claimed', + ); +}); + +test('cancel-before-claim and missing completion payload fail closed', async () => { + await requestJobCancellation(redis, target, 60); + await expect(claimJobExecution(redis, target, 60)).rejects.toThrow( + 'cancelled', + ); + const completedTarget = { ...target, jobId: 'missing-payload-claim' }; + await commitJobResult(redis, completedTarget, { stdout: 'done' }, 60); + await redis.del( + `${jobCancellationInternals.cancellationKey(completedTarget)}:result`, + ); + await expect(claimJobExecution(redis, completedTarget, 60)).rejects.toThrow( + 'refusing re-execution', + ); +}); + +for (const corrupt of [false, true]) + test(`invalid committed result fails immediately without Redis retries (corrupt=${corrupt})`, async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 60); + const key = jobCancellationInternals.cancellationKey(target); + if (corrupt) await redis.set(`${key}:result`, '{invalid'); + else await redis.del(`${key}:result`); + let calls = 0; + const commands = { + eval: (...args: Parameters) => { + calls += 1; + return redis.eval(...args); + }, + } as unknown as typeof redis; + await expect( + fenceJobCancellation({ + commands, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 30_000, + }), + ).rejects.toThrow(); + expect(calls).toBe(1); + }); + +test('completion retention includes the API producer across timeout configuration drift', async () => { + const ttl = jobCancellationRetentionSeconds(30_000, 430); + expect(ttl).toBe(430); + expect(jobCancellationRetentionSeconds(300_000, 430)).toBe(780); + await commitJobResult(redis, target, { stdout: 'done' }, ttl); + const key = jobCancellationInternals.cancellationKey(target); + expect(await redis.ttl(key)).toBeGreaterThanOrEqual(429); + expect(await redis.ttl(`${key}:result`)).toBeGreaterThanOrEqual(429); +}); + +test('a late Stop renews completion evidence along with its request tombstone', async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 1); + expect(await requestJobCancellation(redis, target, 60)).toBe(false); + const key = jobCancellationInternals.cancellationKey(target); + expect(await redis.ttl(key)).toBeGreaterThanOrEqual(59); + expect(await redis.ttl(`${key}:result`)).toBeGreaterThanOrEqual(59); +}); + +test('retention renewal does not lose subsecond time to rounded TTL readings', async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 60); + const key = jobCancellationInternals.cancellationKey(target); + await redis.pexpire(key, 59_900); + const expiration = async () => + Number( + await redis.eval( + ` + local now = redis.call('TIME') + return tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) + redis.call('PTTL', KEYS[1]) + `, + 1, + key, + ), + ); + const before = await expiration(); + await requestJobCancellation(redis, target, 60); + expect(await expiration()).toBeGreaterThan(before); +}); + +test('fencing returns the committed result without a vulnerable second Redis read', async () => { + const result = { stdout: 'one committed effect' }; + await commitJobResult(redis, target, result, 60); + let calls = 0; + const connectionDropsAfterDecision = { + eval: async (...args: Parameters) => { + calls += 1; + return redis.eval(...args); + }, + get: async () => { + throw new Error('connection lost after decision'); + }, + mget: async () => { + throw new Error('connection lost after decision'); + }, + } as unknown as typeof redis; + expect( + await fenceJobCancellation({ + commands: connectionDropsAfterDecision, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 1_000, + }), + ).toEqual({ status: 'completed', result }); + expect(calls).toBe(1); +}); + +test('disconnect returns a known completed result without waiting for a lost queue event', async () => { + const result = { stdout: 'done' }; + await commitJobResult(redis, target, result, 60); + const registry = new JobCancellationRegistry(redis); + const controller = new AbortController(); + controller.abort(); + const job = { + id: target.jobId, + queueName: target.queueName, + waitUntilFinished: () => new Promise(() => {}), + } as unknown as Parameters[0]['job']; + try { + expect( + await waitForJobWithCancellation({ + commands: redis, + registry, + job, + events: {} as Parameters< + typeof waitForJobWithCancellation + >[0]['events'], + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + signal: controller.signal, + }), + ).toEqual(result); + } finally { + await registry.close(); + } +}); + +test('durable cancellation wins even before its subscriber notification arrives', async () => { + expect(await requestJobCancellation(redis, target, 60)).toBe(true); + expect(await commitJobResult(redis, target, { stdout: 'late' }, 60)).toBe( + 'cancelled', + ); + expect(await readCommittedJobResult(redis, target)).toBeUndefined(); +}); + +test('committed results reject late Stop and survive a lost BullMQ completion reply', async () => { + const result = { stdout: 'one mutation', files: [] }; + expect(await commitJobResult(redis, target, result, 60)).toBe('committed'); + expect(await requestJobCancellation(redis, target, 60)).toBe(false); + expect(await readCommittedJobResult(redis, target)).toEqual({ result }); + expect( + await redis.get(jobCancellationInternals.cancellationKey(target)), + ).toBe('completed'); + const registry = new JobCancellationRegistry(redis); + const controller = new AbortController(); + try { + await registry.register(target, controller); + expect(controller.signal.aborted).toBe(false); + } finally { + await registry.close(); + } +}); + +test('concurrent cancellation and completion have exactly one winner', async () => { + const [cancelled, committed] = await Promise.all([ + requestJobCancellation(redis, target, 60), + commitJobResult(redis, target, { stdout: 'result' }, 60), + ]); + expect(Number(cancelled) + Number(committed === 'committed')).toBe(1); +}); + +test('a missing committed result fails closed instead of re-executing', async () => { + await commitJobResult(redis, target, { stdout: 'already applied' }, 60); + await redis.del(`${jobCancellationInternals.cancellationKey(target)}:result`); + await expect(readCommittedJobResult(redis, target)).rejects.toThrow( + 'refusing re-execution', + ); +}); + +test('an enqueue failure can recover a result that won cancellation fencing', async () => { + const result = { stdout: 'effect already applied' }; + await commitJobResult(redis, target, result, 60); + expect( + await fenceJobCancellation({ + commands: redis, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 5_000, + }), + ).toEqual({ status: 'completed', result }); + expect(await readCommittedJobResult(redis, target)).toEqual({ result }); +}); + +test('enqueue fencing still recovers completion after the original deadline', async () => { + await commitJobResult(redis, target, { stdout: 'done' }, 60); + expect( + await fenceJobCancellation({ + commands: redis, + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() - 1_000, + }), + ).toEqual({ status: 'completed', result: { stdout: 'done' } }); +}); + +test('Redis rejects commitment when recovery happens after the producer deadline', async () => { + const delayed = { + eval: async (...args: Parameters) => { + await new Promise(resolve => setTimeout(resolve, 150)); + return redis.eval(...args); + }, + } as unknown as typeof redis; + await expect( + commitJobResult(delayed, target, { stdout: 'late' }, 60, Date.now() + 100), + ).rejects.toThrow('exceeded its deadline'); + expect(await readCommittedJobResult(redis, target)).toBeUndefined(); +}); + +test('a timely durable commit remains successful when only its acknowledgement is late', async () => { + const delayedReply = { + eval: async (...args: Parameters) => { + const value = await redis.eval(...args); + await new Promise(resolve => setTimeout(resolve, 150)); + return value; + }, + } as unknown as typeof redis; + expect( + await commitJobResult( + delayedReply, + target, + { stdout: 'committed' }, + 60, + Date.now() + 100, + ), + ).toBe('committed'); + expect(await readCommittedJobResult(redis, target)).toEqual({ + result: { stdout: 'committed' }, + }); +}); + +for (const failedStage of ['subscription', 'completion'] as const) { + test(`a lost ${failedStage} reply recovers the committed result instead of reporting failure`, async () => { + const result = { stdout: 'already applied once' }; + await commitJobResult(redis, target, result, 60); + const registry = new JobCancellationRegistry(redis); + if (failedStage === 'subscription') + registry.register = async () => { + throw new Error('lost reply'); + }; + const job = { + id: target.jobId, + queueName: target.queueName, + waitUntilFinished: () => Promise.reject(new Error('lost result event')), + } as unknown as Parameters[0]['job']; + try { + expect( + await waitForJobWithCancellation({ + commands: redis, + registry, + job, + events: {} as Parameters< + typeof waitForJobWithCancellation + >[0]['events'], + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + }), + ).toEqual(result); + } finally { + await registry.close(); + } + }); +} diff --git a/service/src/job-cancellation.test.ts b/service/src/job-cancellation.test.ts new file mode 100644 index 00000000..e23186e4 --- /dev/null +++ b/service/src/job-cancellation.test.ts @@ -0,0 +1,599 @@ +import { expect, test } from 'bun:test'; +import { EventEmitter } from 'node:events'; +import type IORedis from 'ioredis'; +import type { Job, QueueEvents } from 'bullmq'; +import { + CLIENT_DISCONNECT_REASON, + JobCancellationRegistry, + jobResultCommitFailure, + jobCancellationInternals, + removeJobIfWaiting, + requestJobCancellation, + throwIfJobAborted, + waitForJobWithCancellation, + fenceJobCancellation, +} from './job-cancellation'; + +class FakeSubscriber extends EventEmitter { + subscribed?: string; + closed = false; + subscribeFailures = 0; + + async subscribe(channel: string): Promise { + if (this.subscribeFailures > 0) { + this.subscribeFailures -= 1; + throw new Error('subscriber unavailable'); + } + this.subscribed = channel; + return 1; + } + + async quit(): Promise<'OK'> { + this.closed = true; + return 'OK'; + } + + disconnect(): void { + this.closed = true; + } +} + +class FakeTransaction { + readonly operations: unknown[][] = []; + + set(...args: unknown[]): this { + this.operations.push(['set', ...args]); + return this; + } + + publish(...args: unknown[]): this { + this.operations.push(['publish', ...args]); + return this; + } + + async exec(): Promise> { + return this.operations.map(() => [null, 'OK']); + } +} + +class FakeRedis { + readonly subscriber = new FakeSubscriber(); + duplicateCalls = 0; + readonly existing = new Set(); + readonly deleted: string[] = []; + readonly transactions: FakeTransaction[] = []; + mgetFailures = 0; + cancellationFailures = 0; + cancellationAttempts = 0; + + duplicate(): FakeSubscriber { + this.duplicateCalls += 1; + return this.subscriber; + } + + async get(key: string): Promise { + return this.existing.has(key) ? '1' : null; + } + + async eval( + _script: string, + _keys: number, + key: string, + _resultKey: string, + ttl: number, + channel: string, + payload: string, + ): Promise { + this.cancellationAttempts += 1; + if (this.cancellationFailures-- > 0) throw new Error('Redis unavailable'); + const transaction = this.multi(); + transaction.set(key, '1', 'EX', ttl); + transaction.publish(channel, payload); + await transaction.exec(); + return [1]; + } + + async mget(...keys: string[]): Promise> { + if (this.mgetFailures > 0) { + this.mgetFailures -= 1; + throw new Error('command connection unavailable'); + } + return keys.map(key => (this.existing.has(key) ? '1' : null)); + } + + async del(key: string): Promise { + this.deleted.push(key); + this.existing.delete(key); + return 1; + } + + multi(): FakeTransaction { + const transaction = new FakeTransaction(); + this.transactions.push(transaction); + return transaction; + } +} + +function redis(fake: FakeRedis): IORedis { + return fake as unknown as IORedis; +} + +test('idle registries allocate no subscriber connection', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + + await registry.close(); + + expect(fake.duplicateCalls).toBe(0); +}); + +test('shutdown disconnects a subscriber whose startup is still waiting for Redis', async () => { + const fake = new FakeRedis(); + fake.subscriber.subscribe = async () => new Promise(() => {}); + const registry = new JobCancellationRegistry(redis(fake)); + void registry + .register( + { queueName: 'other', jobId: 'shutdown-startup' }, + new AbortController(), + ) + .catch(() => undefined); + await registry.close(); + expect(fake.subscriber.closed).toBe(true); + expect(fake.subscriber.listenerCount('message')).toBe(0); +}, 1_000); + +test('failed subscription startup removes handlers before a bounded retry', async () => { + const fake = new FakeRedis(); + fake.subscriber.subscribeFailures = 1; + const registry = new JobCancellationRegistry(redis(fake)); + const first = new AbortController(); + + await expect( + registry.register({ queueName: 'other', jobId: 'job-failed-start' }, first), + ).rejects.toThrow('subscriber unavailable'); + expect(fake.subscriber.listenerCount('message')).toBe(0); + expect(fake.subscriber.listenerCount('ready')).toBe(0); + expect(fake.subscriber.listenerCount('error')).toBe(0); + + const second = new AbortController(); + await registry.register({ queueName: 'other', jobId: 'job-retry' }, second); + expect(fake.duplicateCalls).toBe(2); + expect(fake.subscriber.listenerCount('message')).toBe(1); + await registry.close(); +}); + +test('registry catches durable cancellation before subscriber registration', async () => { + const fake = new FakeRedis(); + const target = { queueName: 'other', jobId: 'job-1' }; + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + const registry = new JobCancellationRegistry(redis(fake)); + const controller = new AbortController(); + + await registry.register(target, controller); + + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + expect(fake.subscriber.subscribed).toBe(jobCancellationInternals.channel); + await registry.unregister(target); + await registry.close(); + expect(fake.subscriber.closed).toBe(true); +}); + +test('one pubsub listener cancels only the matching active job', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const first = new AbortController(); + const second = new AbortController(); + await registry.register({ queueName: 'other', jobId: 'job-1' }, first); + await registry.register({ queueName: 'other', jobId: 'job-2' }, second); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify({ queueName: 'other', jobId: 'job-2' }), + ); + + expect(first.signal.aborted).toBe(false); + expect(second.signal.aborted).toBe(true); + await registry.close(); +}); + +test('one pubsub listener wakes every local waiter for the same job', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-shared' }; + const first = new AbortController(); + const second = new AbortController(); + await registry.register(target, first); + await registry.register(target, second); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify(target), + ); + + expect(first.signal.aborted).toBe(true); + expect(second.signal.aborted).toBe(true); + expect(fake.duplicateCalls).toBe(1); + await registry.close(); +}); + +test('unregistering one local waiter preserves other waiters for the job', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-shared-unregister' }; + const first = new AbortController(); + const second = new AbortController(); + await registry.register(target, first); + await registry.register(target, second); + await registry.unregister(target, first); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify(target), + ); + + expect(first.signal.aborted).toBe(false); + expect(second.signal.aborted).toBe(true); + await registry.close(); +}); + +test('subscriber reconnect reconciles active jobs against durable markers', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-reconnect' }; + const controller = new AbortController(); + await registry.register(target, controller); + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + + fake.subscriber.emit('ready'); + await new Promise(resolve => setTimeout(resolve, 10)); + + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + await registry.close(); +}); + +test('subscriber reconnect retries durable-marker reconciliation', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-retry-reconcile' }; + const controller = new AbortController(); + await registry.register(target, controller); + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + fake.mgetFailures = 1; + + fake.subscriber.emit('ready'); + await new Promise(resolve => setTimeout(resolve, 150)); + + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + await registry.close(); +}); + +test('terminal subscriber disconnect rebuilds the subscription and reconciles markers', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const target = { queueName: 'other', jobId: 'job-terminal-reconnect' }; + const controller = new AbortController(); + await registry.register(target, controller); + fake.existing.add(jobCancellationInternals.cancellationKey(target)); + + fake.subscriber.emit('end'); + await new Promise(resolve => setTimeout(resolve, 10)); + + expect(fake.duplicateCalls).toBe(2); + expect(controller.signal.aborted).toBe(true); + expect(controller.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + await registry.close(); +}); + +test('cancellation writes a durable marker before publishing', async () => { + const fake = new FakeRedis(); + const target = { queueName: 'other', jobId: 'job-3' }; + + await requestJobCancellation(redis(fake), target, 42); + + expect(fake.transactions).toHaveLength(1); + expect(fake.transactions[0]?.operations).toEqual([ + ['set', jobCancellationInternals.cancellationKey(target), '1', 'EX', 42], + ['publish', jobCancellationInternals.channel, JSON.stringify(target)], + ]); +}); + +test('result commit barrier rejects cancellation observed after execution', () => { + const controller = new AbortController(); + expect(() => throwIfJobAborted(controller.signal)).not.toThrow(); + controller.abort(CLIENT_DISCONNECT_REASON); + expect(() => throwIfJobAborted(controller.signal)).toThrow( + CLIENT_DISCONNECT_REASON, + ); +}); + +test('result cleanup maps late cancellation to stable worker failures', () => { + const disconnected = new AbortController(); + disconnected.abort(CLIENT_DISCONNECT_REASON); + expect(jobResultCommitFailure(disconnected.signal, 30_000)?.message).toBe( + 'Job cancelled after client disconnected', + ); + + const deadline = new AbortController(); + deadline.abort('deadline'); + expect(jobResultCommitFailure(deadline.signal, 30_000)?.message).toBe( + 'Job timed out after 30000ms', + ); + expect( + jobResultCommitFailure(new AbortController().signal, 30_000), + ).toBeUndefined(); +}); + +test('disconnect frees a waiting job and rejects promptly', async () => { + const fake = new FakeRedis(); + const controller = new AbortController(); + let removed = false; + const never = new Promise(() => {}); + const job = { + id: 'job-4', + queueName: 'other', + waitUntilFinished: () => never, + getState: async () => 'waiting', + remove: async () => { + removed = true; + }, + } as unknown as Job; + + const registry = new JobCancellationRegistry(redis(fake)); + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + signal: controller.signal, + }); + controller.abort(CLIENT_DISCONNECT_REASON); + + await expect(waiting).rejects.toMatchObject({ name: 'AbortError' }); + expect(removed).toBe(true); + expect(fake.cancellationAttempts).toBe(1); + expect(fake.transactions[0]?.operations[0]).toEqual([ + 'set', + jobCancellationInternals.cancellationKey({ + queueName: 'other', + jobId: 'job-4', + }), + '1', + 'EX', + 120, + ]); + await registry.close(); +}); + +test('registration failure fences and removes the already-enqueued job', async () => { + const fake = new FakeRedis(); + fake.subscriber.subscribeFailures = 1; + let removed = false; + const job = { + id: 'job-register-failure', + queueName: 'other', + waitUntilFinished: () => new Promise(() => {}), + getState: async () => 'waiting', + remove: async () => { + removed = true; + }, + } as unknown as Job; + const registry = new JobCancellationRegistry(redis(fake)); + + await expect( + waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + }), + ).rejects.toThrow('subscriber unavailable'); + + expect(removed).toBe(true); + expect(fake.transactions[0]?.operations[0]).toEqual([ + 'set', + jobCancellationInternals.cancellationKey({ + queueName: 'other', + jobId: 'job-register-failure', + }), + '1', + 'EX', + 120, + ]); + await registry.close(); +}); + +test('a result rejection is owned while subscription registration is pending', async () => { + const fake = new FakeRedis(); + let release!: () => void; + fake.subscriber.subscribe = async () => { + await new Promise(resolve => { + release = resolve; + }); + return 1; + }; + const registry = new JobCancellationRegistry(redis(fake)); + const job = { + id: 'pending-registration', + queueName: 'other', + waitUntilFinished: () => Promise.reject(new Error('completion timeout')), + getState: async () => 'active', + remove: async () => {}, + } as unknown as Job; + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 1_000, + cancellationTtlSeconds: 60, + }); + const rejection = waiting.catch((error: Error) => error); + // An unowned rejection fails the test runner on this event-loop turn. + await new Promise(resolve => setImmediate(resolve)); + release(); + expect(await rejection).toMatchObject({ message: 'completion timeout' }); + await registry.close(); +}); + +test('external cancellation frees a waiting job before rejecting its waiter', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + let removed = false; + const job = { + id: 'job-external-waiting', + queueName: 'other', + waitUntilFinished: () => new Promise(() => {}), + getState: async () => 'waiting', + remove: async () => { + removed = true; + }, + } as unknown as Job; + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + }); + await new Promise(resolve => setImmediate(resolve)); + + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify({ queueName: 'other', jobId: 'job-external-waiting' }), + ); + + await expect(waiting).rejects.toMatchObject({ name: 'AbortError' }); + expect(removed).toBe(true); + await registry.close(); +}); + +test('a separate cancellation request wakes the original job waiter', async () => { + const fake = new FakeRedis(); + const registry = new JobCancellationRegistry(redis(fake)); + const never = new Promise(() => {}); + const job = { + id: 'job-external-cancel', + queueName: 'other', + waitUntilFinished: () => never, + getState: async () => 'active', + remove: async () => undefined, + } as unknown as Job; + + const waiting = waitForJobWithCancellation({ + commands: redis(fake), + registry, + job, + events: {} as QueueEvents, + timeoutMs: 60_000, + cancellationTtlSeconds: 120, + }); + await new Promise(resolve => setImmediate(resolve)); + fake.subscriber.emit( + 'message', + jobCancellationInternals.channel, + JSON.stringify({ queueName: 'other', jobId: 'job-external-cancel' }), + ); + + await expect(waiting).rejects.toMatchObject({ name: 'AbortError' }); + expect(fake.deleted).toEqual([]); + await registry.close(); +}); + +test('queued removal never removes an active job', async () => { + let removed = false; + const job = { + getState: async () => 'active' as const, + remove: async () => { + removed = true; + }, + }; + + expect(await removeJobIfWaiting(job)).toBe(false); + expect(removed).toBe(false); +}); + +test('a failed cancellation write retains ownership until a durable retry succeeds', async () => { + const fake = new FakeRedis(); + fake.cancellationFailures = 2; + const target = { queueName: 'other', jobId: 'ambiguous-enqueue' }; + let released = false; + const fencing = fenceJobCancellation({ + commands: redis(fake), + target, + ttlSeconds: 60, + deadlineAtMs: Date.now() + 500, + }).then(() => { + released = true; + }); + await new Promise(resolve => setTimeout(resolve, 10)); + expect(released).toBe(false); + await fencing; + expect(fake.cancellationAttempts).toBe(3); + expect(released).toBe(true); +}); + +test('an unavailable Redis cannot release ownership before the fixed job deadline', async () => { + const fake = new FakeRedis(); + fake.cancellationFailures = 1_000; + const startedAt = Date.now(); + await fenceJobCancellation({ + commands: redis(fake), + target: { queueName: 'other', jobId: 'offline' }, + ttlSeconds: 60, + deadlineAtMs: startedAt + 80, + }); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(80); + expect(fake.cancellationAttempts).toBeLessThanOrEqual(4); +}); + +test('a pending Redis write allocates no retry backlog and waits until the deadline', async () => { + const fake = new FakeRedis(); + let calls = 0; + fake.eval = async () => { + calls += 1; + return new Promise(() => {}); + }; + const startedAt = Date.now(); + await fenceJobCancellation({ + commands: redis(fake), + target: { queueName: 'other', jobId: 'pending-write' }, + ttlSeconds: 60, + deadlineAtMs: startedAt + 40, + }); + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(39); + expect(calls).toBe(1); +}); + +test('queued removal frees waiting capacity and tolerates an activation race', async () => { + let removals = 0; + expect( + await removeJobIfWaiting({ + getState: async () => 'waiting', + remove: async () => { + removals += 1; + }, + }), + ).toBe(true); + expect( + await removeJobIfWaiting({ + getState: async () => 'waiting', + remove: async () => { + removals += 1; + throw new Error('job is active'); + }, + }), + ).toBe(false); + expect(removals).toBe(2); +}); diff --git a/service/src/job-cancellation.ts b/service/src/job-cancellation.ts new file mode 100644 index 00000000..4da3253d --- /dev/null +++ b/service/src/job-cancellation.ts @@ -0,0 +1,664 @@ +import type IORedis from 'ioredis'; +import type { Job, QueueEvents } from 'bullmq'; + +const JOB_CANCELLATION_PREFIX = 'codeapi:job-cancellation:v1'; +const JOB_CANCELLATION_CHANNEL = `${JOB_CANCELLATION_PREFIX}:events`; +export const CLIENT_DISCONNECT_REASON = 'client_disconnected'; +export const JOB_CANCELLED_MESSAGE = 'Job cancelled after client disconnected'; + +interface JobTarget { + queueName: string; + jobId: string; +} + +function targetKey(target: JobTarget): string { + return `${target.queueName}:${target.jobId}`; +} + +function cancellationKey(target: JobTarget): string { + return `${JOB_CANCELLATION_PREFIX}:${encodeURIComponent( + target.queueName, + )}:${encodeURIComponent(target.jobId)}`; +} + +function parseTarget(raw: string): JobTarget | undefined { + try { + const parsed = JSON.parse(raw) as Partial; + if ( + typeof parsed.queueName !== 'string' || + parsed.queueName.length === 0 || + parsed.queueName.length > 256 || + typeof parsed.jobId !== 'string' || + parsed.jobId.length === 0 || + parsed.jobId.length > 256 + ) { + return undefined; + } + return { queueName: parsed.queueName, jobId: parsed.jobId }; + } catch { + return undefined; + } +} + +/** + * Cross-process cancellation for BullMQ work. + * + * The durable marker closes the publish-before-subscribe race while one + * process-wide pub/sub connection makes active cancellation O(events), not + * O(active jobs) Redis polling. Only explicitly cancellable replay jobs use + * this path, so ordinary queue traffic pays no extra Redis round trips. + */ +export class JobCancellationRegistry { + private subscriber?: IORedis; + private readonly controllers = new Map< + string, + { target: JobTarget; controllers: Set } + >(); + private startPromise?: Promise; + private readonly subscriberEndHandlers = new WeakMap void>(); + private reconcileTimer?: ReturnType; + private subscriberRestartTimer?: ReturnType; + private reconcileRetryMs = 100; + private closed = false; + + constructor(private readonly commands: IORedis) {} + + private readonly onSubscriberError = (): void => { + // ioredis reconnects using the shared policy. The listener prevents a + // transient subscriber outage from becoming an uncaught process error. + }; + + private readonly onSubscriberReady = (): void => { + this.scheduleReconcile(0); + }; + + private readonly onSubscriberMessage = ( + channel: string, + raw: string, + ): void => { + if (channel !== JOB_CANCELLATION_CHANNEL) return; + const target = parseTarget(raw); + if (target == null) return; + for (const controller of this.controllers.get(targetKey(target)) + ?.controllers ?? []) { + controller.abort(CLIENT_DISCONNECT_REASON); + } + }; + + private detachSubscriber(subscriber: IORedis): void { + subscriber.removeListener('error', this.onSubscriberError); + subscriber.removeListener('ready', this.onSubscriberReady); + subscriber.removeListener('message', this.onSubscriberMessage); + const onEnd = this.subscriberEndHandlers.get(subscriber); + if (onEnd != null) subscriber.removeListener('end', onEnd); + this.subscriberEndHandlers.delete(subscriber); + } + + private restartAfterTerminalDisconnect(subscriber: IORedis): void { + if (this.closed || this.subscriber !== subscriber) return; + this.detachSubscriber(subscriber); + this.subscriber = undefined; + this.startPromise = undefined; + if (this.controllers.size === 0) return; + void this.start().then( + () => this.scheduleReconcile(0), + () => this.scheduleSubscriberRestart(), + ); + } + + private scheduleSubscriberRestart(): void { + if ( + this.closed || + this.controllers.size === 0 || + this.startPromise != null || + this.subscriberRestartTimer != null + ) + return; + const retryMs = this.reconcileRetryMs; + this.reconcileRetryMs = Math.min(2_000, retryMs * 2); + this.subscriberRestartTimer = setTimeout(() => { + this.subscriberRestartTimer = undefined; + if ( + this.closed || + this.controllers.size === 0 || + this.startPromise != null + ) + return; + void this.start().then( + () => { + this.reconcileRetryMs = 100; + this.scheduleReconcile(0); + }, + () => this.scheduleSubscriberRestart(), + ); + }, retryMs); + } + + private async reconcile(): Promise { + const entries = [...this.controllers.values()]; + if (entries.length === 0) return; + const cancelled = await this.commands.mget( + ...entries.map(({ target }) => cancellationKey(target)), + ); + cancelled.forEach((value, index) => { + if (value === '1') { + for (const controller of entries[index]?.controllers ?? []) { + controller.abort(CLIENT_DISCONNECT_REASON); + } + } + }); + } + + private scheduleReconcile(delayMs: number): void { + if ( + this.closed || + this.controllers.size === 0 || + this.reconcileTimer != null + ) { + return; + } + this.reconcileTimer = setTimeout(() => { + this.reconcileTimer = undefined; + void this.reconcile().then( + () => { + this.reconcileRetryMs = 100; + }, + () => { + const retryMs = this.reconcileRetryMs; + this.reconcileRetryMs = Math.min(2_000, retryMs * 2); + this.scheduleReconcile(retryMs); + }, + ); + }, delayMs); + } + + private start(): Promise { + if (this.closed) { + return Promise.reject(new Error('Job cancellation registry is closed')); + } + if (this.startPromise != null) return this.startPromise; + const starting = (async (): Promise => { + const subscriber = this.commands.duplicate(); + this.subscriber = subscriber; + subscriber.on('error', this.onSubscriberError); + subscriber.on('ready', this.onSubscriberReady); + subscriber.on('message', this.onSubscriberMessage); + const onEnd = (): void => this.restartAfterTerminalDisconnect(subscriber); + this.subscriberEndHandlers.set(subscriber, onEnd); + subscriber.on('end', onEnd); + try { + await subscriber.subscribe(JOB_CANCELLATION_CHANNEL); + } catch (error) { + this.detachSubscriber(subscriber); + if (this.subscriber === subscriber) this.subscriber = undefined; + subscriber.disconnect(false); + throw error; + } + })(); + this.startPromise = starting; + void starting.catch(() => { + if (this.startPromise === starting) this.startPromise = undefined; + }); + return starting; + } + + async register( + target: JobTarget, + controller: AbortController, + ): Promise { + const key = targetKey(target); + const entry = this.controllers.get(key) ?? { + target, + controllers: new Set(), + }; + entry.controllers.add(controller); + this.controllers.set(key, entry); + try { + await this.start(); + if ((await this.commands.get(cancellationKey(target))) === '1') { + controller.abort(CLIENT_DISCONNECT_REASON); + } + } catch (error) { + entry.controllers.delete(controller); + if (entry.controllers.size === 0) this.controllers.delete(key); + throw error; + } + } + + async unregister( + target: JobTarget, + controller?: AbortController, + ): Promise { + const key = targetKey(target); + const entry = this.controllers.get(key); + if (controller == null) { + this.controllers.delete(key); + } else if (entry != null) { + entry.controllers.delete(controller); + if (entry.controllers.size === 0) this.controllers.delete(key); + } + // Markers expire by TTL. Deleting one here can erase the only evidence + // needed by another replica whose subscriber was reconnecting. + } + + async close(): Promise { + this.closed = true; + this.controllers.clear(); + if (this.reconcileTimer != null) clearTimeout(this.reconcileTimer); + this.reconcileTimer = undefined; + if (this.subscriberRestartTimer != null) + clearTimeout(this.subscriberRestartTimer); + this.subscriberRestartTimer = undefined; + const subscriber = this.subscriber; + this.subscriber = undefined; + this.startPromise = undefined; + if (subscriber == null) return; + this.detachSubscriber(subscriber); + // This socket only carries notifications. Disconnect it before awaiting + // anything: subscribe() may be queued through an indefinite Redis outage. + subscriber.disconnect(false); + } +} + +async function cancelJobInRedis( + commands: IORedis, + target: JobTarget, + ttlSeconds: number, + includeResult: boolean, +): Promise { + // Cancellation and result publication have ONE durable winner. Pub/sub is + // only a notification; it must not decide whether Stop was accepted. + return commands.eval( + ` + local state = redis.call('GET', KEYS[1]) + if state == 'completed' then + -- Keep completion evidence at least as long as the requesting process + -- requires, even across API/worker config differences. Attached request + -- tombstones never renew independently of this decision. + local requestedTtlMs = tonumber(ARGV[1]) * 1000 + for i = 1, 2 do + if redis.call('PTTL', KEYS[i]) < requestedTtlMs then + redis.call('PEXPIRE', KEYS[i], requestedTtlMs) + end + end + if ARGV[4] == '1' then return {0, redis.call('GET', KEYS[2])} end + return {0} + end + if state and state ~= '1' then return {-1} end + redis.call('SET', KEYS[1], '1', 'EX', ARGV[1]) + redis.call('PUBLISH', ARGV[2], ARGV[3]) + return {1} + `, + 2, + cancellationKey(target), + `${cancellationKey(target)}:result`, + Math.max(1, ttlSeconds), + JOB_CANCELLATION_CHANNEL, + JSON.stringify(target), + includeResult ? '1' : '0', + ); +} + +export async function requestJobCancellation( + commands: IORedis, + target: JobTarget, + ttlSeconds: number, +): Promise { + const decision = await cancelJobInRedis(commands, target, ttlSeconds, false); + if (!Array.isArray(decision) || ![0, 1].includes(decision[0])) { + throw new Error('Invalid durable cancellation decision'); + } + return decision[0] === 1; +} + +export type JobFenceOutcome = + | { status: 'cancelled' | 'expired' } + | { status: 'completed'; result: T }; + +function decodeCommittedResult(value: unknown): { result: T } { + if (typeof value !== 'string') { + throw new Error( + 'Committed programmatic result expired; refusing re-execution', + ); + } + const decoded: unknown = JSON.parse(value); + if ( + decoded == null || + typeof decoded !== 'object' || + !Object.prototype.hasOwnProperty.call(decoded, 'result') + ) { + throw new Error( + 'Invalid committed programmatic result; refusing re-execution', + ); + } + return decoded as { result: T }; +} + +export function jobCancellationRetentionSeconds( + localTimeoutMs: number, + producerTtlSeconds = 0, +): number { + return Math.max( + Math.ceil(localTimeoutMs / 1_000) * 2 + 180, + Number.isFinite(producerTtlSeconds) ? producerTtlSeconds : 0, + ); +} + +/** Retain the actual result so a BullMQ retry after a lost completion reply + * cannot repeat sandbox mutations. Keep the status small: reconnect MGETs must + * never load every active job's output into each API/worker replica. */ +export async function commitJobResult( + commands: IORedis, + target: JobTarget, + result: T, + ttlSeconds: number, + deadlineAtMs = Number.MAX_SAFE_INTEGER, +): Promise<'committed' | 'cancelled' | 'already_completed'> { + const serialized = JSON.stringify({ result }); + if (Buffer.byteLength(serialized) > 16 * 1024 * 1024) { + throw new Error('Programmatic completion exceeds the 16 MiB result limit'); + } + const decision = await commands.eval( + ` + local state = redis.call('GET', KEYS[1]) + if state == '1' then return 0 end + if state == 'completed' then return 2 end + if state then return -2 end + if not state then + local now = redis.call('TIME') + if tonumber(now[1]) * 1000 + math.floor(tonumber(now[2]) / 1000) >= tonumber(ARGV[3]) then + return -1 + end + -- One write command, so an OOM cannot publish just half the decision. + redis.call('MSET', KEYS[1], 'completed', KEYS[2], ARGV[1]) + redis.call('EXPIRE', KEYS[1], ARGV[2]) + redis.call('EXPIRE', KEYS[2], ARGV[2]) + end + return 1 + `, + 2, + cancellationKey(target), + `${cancellationKey(target)}:result`, + serialized, + Math.max(1, ttlSeconds), + deadlineAtMs, + ); + if (decision === -1) + throw new Error('Job result commitment exceeded its deadline'); + if (decision === 1) return 'committed'; + if (decision === 0) return 'cancelled'; + if (decision === 2) return 'already_completed'; + throw new Error('Invalid durable result commitment'); +} + +/** BullMQ lock loss can redeliver a job while its first processor still runs. + * Claim once before any sandbox work, retaining the claim through the job's + * recovery horizon. An ambiguous/stalled attempt is never permission to rerun. + * Completion lookup and claim are atomic, so there is no read-then-start gap. */ +export async function claimJobExecution( + commands: IORedis, + target: JobTarget, + ttlSeconds: number, +): Promise<{ status: 'claimed' } | { status: 'completed'; result: T }> { + const key = cancellationKey(target); + const decision = await commands.eval( + ` + local state = redis.call('GET', KEYS[1]) + if state == 'completed' then return {0, redis.call('GET', KEYS[2])} end + if state == '1' then return {-1} end + if state then return {-3} end + if redis.call('SET', KEYS[3], '1', 'NX', 'EX', ARGV[1]) then return {1} end + return {-2} + `, + 3, + key, + `${key}:result`, + `${key}:execution`, + Math.max(1, ttlSeconds), + ); + if (!Array.isArray(decision)) throw new Error('Invalid execution claim'); + if (decision[0] === 1) return { status: 'claimed' }; + if (decision[0] === 0) + return { + status: 'completed', + result: decodeCommittedResult(decision[1]).result, + }; + if (decision[0] === -1) throw new Error(JOB_CANCELLED_MESSAGE); + if (decision[0] === -2) + throw new Error( + 'Programmatic job already claimed; refusing duplicate execution', + ); + throw new Error('Invalid durable execution claim'); +} + +export async function readCommittedJobResult( + commands: IORedis, + target: JobTarget, +): Promise<{ result: T } | undefined> { + const [state, value] = await commands.mget( + cancellationKey(target), + `${cancellationKey(target)}:result`, + ); + if (state !== 'completed') return undefined; + return decodeCommittedResult(value); +} + +/** Do not release replay ownership on an ambiguous Redis failure. Keep one + * outstanding marker write, retry rejected writes with bounded backoff, and + * retain ownership until it succeeds or the job's ORIGINAL deadline expires. + * A delayed queue.add must carry that same timestamp into the worker. */ +export async function fenceJobCancellation(args: { + commands: IORedis; + target: JobTarget; + ttlSeconds: number; + deadlineAtMs: number; +}): Promise> { + let retryMs = 25; + let firstAttempt = true; + while (firstAttempt || Date.now() < args.deadlineAtMs) { + firstAttempt = false; + // If a lost enqueue reply arrives after the execution deadline, still + // give a healthy Redis one bounded opportunity to return completion's + // winning decision. Never translate a known committed effect to failure. + const remainingMs = args.deadlineAtMs - Date.now(); + let timer: ReturnType | undefined; + let decision: unknown; + try { + decision = await Promise.race([ + cancelJobInRedis(args.commands, args.target, args.ttlSeconds, true), + new Promise(resolve => { + timer = setTimeout( + () => resolve(undefined), + remainingMs > 0 ? remainingMs : 1_000, + ); + }), + ]); + } catch { + await new Promise(resolve => + setTimeout( + resolve, + Math.min(retryMs, Math.max(0, args.deadlineAtMs - Date.now())), + ), + ); + retryMs = Math.min(1_000, retryMs * 2); + continue; + } finally { + if (timer != null) clearTimeout(timer); + } + // Only transport failures retry. Corrupt/missing durable results are + // deterministic invariant failures, not an invitation to extend their TTL. + if (decision === undefined) return { status: 'expired' }; + if (!Array.isArray(decision)) + throw new Error('Invalid durable cancellation decision'); + if (decision[0] === 1) return { status: 'cancelled' }; + if (decision[0] === 0) + return { + status: 'completed', + result: decodeCommittedResult(decision[1]).result, + }; + throw new Error('Invalid durable cancellation decision'); + } + return { status: 'expired' }; +} + +const REMOVABLE_JOB_STATES = new Set([ + 'waiting', + 'delayed', + 'prioritized', + 'waiting-children', +]); + +/** Frees queued capacity without ever removing an active or settled job. */ +export async function removeJobIfWaiting( + job: Pick, +): Promise { + if (!REMOVABLE_JOB_STATES.has(await job.getState())) return false; + try { + await job.remove(); + return true; + } catch { + // A worker may have activated the job between getState() and remove(). + // The durable marker remains authoritative for that race. + return false; + } +} + +export function programmaticCancellationError(): Error { + return new DOMException( + 'Programmatic execution request disconnected', + 'AbortError', + ); +} + +/** Commit barrier for result-processing stages that may yield after execution. */ +export function throwIfJobAborted(signal: AbortSignal): void { + if (!signal.aborted) return; + if (signal.reason instanceof Error) throw signal.reason; + throw new DOMException( + typeof signal.reason === 'string' ? signal.reason : 'Job aborted', + 'AbortError', + ); +} + +/** Maps cancellation observed during asynchronous result cleanup to the same + * stable worker failure used by the main execution catch path. */ +export function jobResultCommitFailure( + signal: AbortSignal, + jobTimeoutMs: number, +): Error | undefined { + if (!signal.aborted) return undefined; + return new Error( + signal.reason === CLIENT_DISCONNECT_REASON + ? JOB_CANCELLED_MESSAGE + : `Job timed out after ${jobTimeoutMs}ms`, + ); +} + +export async function waitForJobWithCancellation(args: { + commands: IORedis; + registry: JobCancellationRegistry; + job: Job; + events: QueueEvents; + timeoutMs: number; + cancellationTtlSeconds: number; + deadlineAtMs?: number; + signal?: AbortSignal; +}): Promise { + const { + commands, + registry, + job, + events, + timeoutMs, + cancellationTtlSeconds, + signal, + } = args; + const completion = job.waitUntilFinished(events, timeoutMs); + // Subscription startup can itself wait for Redis recovery. Own the losing + // promise immediately, before any await, rather than after registration. + void completion.catch(() => undefined); + const target = { queueName: job.queueName, jobId: String(job.id) }; + const deadlineAtMs = args.deadlineAtMs ?? Date.now() + timeoutMs; + let fencing: Promise> | undefined; + const fence = (): Promise> => + (fencing ??= fenceJobCancellation({ + commands, + target, + ttlSeconds: cancellationTtlSeconds, + deadlineAtMs, + })); + const externalController = new AbortController(); + try { + await registry.register(target, externalController); + } catch (error) { + void completion.catch(() => undefined); + const outcome = await fence(); + if (outcome.status === 'completed') return outcome.result; + await removeJobIfWaiting(job).catch(() => false); + throw error; + } + + let removeAbortListener = (): void => {}; + const disconnected = new Promise((resolve, reject) => { + let cancelling = false; + const cancel = (): void => { + if (cancelling) return; + cancelling = true; + void fence() + .then(async outcome => { + if (outcome.status === 'completed') { + resolve(outcome.result); + return; + } + // Removing a waiting job immediately frees queue capacity. An active + // job cannot be removed; its worker observes the durable marker or + // pub/sub event and aborts the sandbox transport instead. + await removeJobIfWaiting(job).catch(() => false); + reject(programmaticCancellationError()); + }) + .catch(reject); + }; + if (signal != null) { + removeAbortListener = (): void => + signal.removeEventListener('abort', cancel); + signal.addEventListener('abort', cancel, { once: true }); + if (signal.aborted) cancel(); + } + }); + const cancelled = new Promise((_, reject) => { + const cancel = (): void => { + void removeJobIfWaiting(job).then( + () => reject(programmaticCancellationError()), + () => reject(programmaticCancellationError()), + ); + }; + externalController.signal.addEventListener('abort', cancel, { + once: true, + }); + if (externalController.signal.aborted) cancel(); + }); + + // A cancelled request stops awaiting the BullMQ result, so attach a sink to + // the losing promise before racing it to avoid an unhandled late rejection. + void completion.catch(() => undefined); + try { + return await Promise.race([completion, disconnected, cancelled]); + } catch (error) { + // Includes waitUntilFinished timeouts and registration/transport errors, + // not only explicit Stop. Replay cleanup is unsafe until this barrier. + const outcome = await fence(); + if (outcome.status === 'completed') return outcome.result; + throw error; + } finally { + removeAbortListener(); + await registry + .unregister(target, externalController) + .catch(() => undefined); + } +} + +export const jobCancellationInternals = { + channel: JOB_CANCELLATION_CHANNEL, + cancellationKey, + parseTarget, +}; diff --git a/service/src/metrics.ts b/service/src/metrics.ts index adfd9872..42c5536f 100644 --- a/service/src/metrics.ts +++ b/service/src/metrics.ts @@ -117,6 +117,12 @@ export const jobsFailed = new Counter({ labelNames: ['language'] as const, }); +export const jobsCancelled = new Counter({ + name: 'codeapi_jobs_cancelled_total', + help: 'Total number of jobs cancelled after the calling client disconnected', + labelNames: ['language'] as const, +}); + export const activeJobs = new Gauge({ name: 'codeapi_active_jobs', help: 'Number of jobs currently being processed', diff --git a/service/src/middleware/limits.ts b/service/src/middleware/limits.ts index 099261a1..9ffaeb9d 100644 --- a/service/src/middleware/limits.ts +++ b/service/src/middleware/limits.ts @@ -182,6 +182,19 @@ export const executionLimiter = createRateLimiter( } ); +/** Keep Stop available when execution admission is full, while independently + * bounding request-id churn in the cancellation registry. */ +export const cancellationLimiter = createRateLimiter( + 'exec-cancel', + env.EXEC_LIMIT_WINDOW, + Math.max(80, env.EXEC_MAX_REQUESTS * 4), + { + message: 'Too many CodeAPI cancellation requests.', + structuredBody: true, + logRejections: true, + } +); + export const uploadLimiter = createRateLimiter( 'upload', env.UPLOAD_LIMIT_WINDOW, diff --git a/service/src/programmatic-cancellation.test.ts b/service/src/programmatic-cancellation.test.ts new file mode 100644 index 00000000..53a4340e --- /dev/null +++ b/service/src/programmatic-cancellation.test.ts @@ -0,0 +1,234 @@ +import { afterEach, beforeEach, expect, test } from 'bun:test'; +import type IORedis from 'ioredis'; +import { startTestRedis } from './test/redis'; +import { commitJobResult, requestJobCancellation } from './job-cancellation'; +import { + attachProgrammaticCancellationTarget, + cancelProgrammaticRequest, + normalizeProgrammaticRequestId, + programmaticCancellationInternals, + releaseProgrammaticCancellation, + reserveProgrammaticCancellation, +} from './programmatic-cancellation'; + +let redis: IORedis & { closeTestServer(): Promise }; + +beforeEach(async () => { + redis = await startTestRedis(); +}); + +afterEach(async () => { + await redis.closeTestServer(); +}); + +test('normalizes only bounded opaque request IDs', () => { + expect(normalizeProgrammaticRequestId('request_123456789')).toBe( + 'request_123456789', + ); + expect(normalizeProgrammaticRequestId(' short ')).toBeUndefined(); + expect( + normalizeProgrammaticRequestId('../request_123456789'), + ).toBeUndefined(); + expect(normalizeProgrammaticRequestId('a'.repeat(129))).toBeUndefined(); +}); + +test('Stop cannot extend an attached tombstone before the outcome command succeeds', async () => { + const requestId = 'request_no_split_renewal'; + const owner = 'owner-a'; + const target = { queueName: 'other', jobId: 'split-renewal' }; + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }); + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner, + target, + ttlSeconds: 60, + }); + await commitJobResult(redis, target, { stdout: 'done' }, 60); + const key = programmaticCancellationInternals.requestKey(requestId); + await redis.pexpire(key, 5_000); + const before = await redis.pttl(key); + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner, + ttlSeconds: 600, + }), + ).toEqual({ status: 'accepted', target }); + // Simulate losing Redis before requestJobCancellation: no second command. + expect(await redis.pttl(key)).toBeLessThanOrEqual(before); + expect(await requestJobCancellation(redis, target, 600)).toBe(false); +}); + +test('cancellation before queue attachment is retained atomically', async () => { + const requestId = 'request_early_cancel_123'; + const owner = 'owner-a'; + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toEqual({ status: 'accepted' }); + + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('cancelled'); + expect( + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner, + target: { queueName: 'other', jobId: '42' }, + ttlSeconds: 60, + }), + ).toBe('cancelled'); +}); + +test('cancellation after attachment returns the exact queue target', async () => { + const requestId = 'request_attached_cancel_1'; + const owner = 'owner-a'; + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('active'); + expect( + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner, + target: { queueName: 'other', jobId: '43' }, + ttlSeconds: 60, + }), + ).toBe('active'); + + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toEqual({ + status: 'accepted', + target: { queueName: 'other', jobId: '43' }, + }); +}); + +test('overlapping requests from the same owner cannot share cancellation state', async () => { + const requestId = 'request_duplicate_owner_1'; + const owner = 'owner-a'; + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('active'); + + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner, + ttlSeconds: 60, + }), + ).toBe('duplicate'); +}); + +test('a different principal cannot reserve, attach, cancel, or release a request', async () => { + const requestId = 'request_owned_cancel_123'; + await reserveProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-a', + ttlSeconds: 60, + }); + + expect( + await reserveProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-b', + ttlSeconds: 60, + }), + ).toBe('forbidden'); + expect( + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner: 'owner-b', + target: { queueName: 'other', jobId: '44' }, + ttlSeconds: 60, + }), + ).toBe('forbidden'); + expect( + await cancelProgrammaticRequest({ + redis, + requestId, + owner: 'owner-b', + ttlSeconds: 60, + }), + ).toEqual({ status: 'forbidden' }); + await releaseProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-b', + }); + expect( + await redis.exists(programmaticCancellationInternals.requestKey(requestId)), + ).toBe(1); +}); + +test('settlement retains a bounded target tombstone for late Stop classification', async () => { + const requestId = 'request_release_cancel_1'; + await reserveProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-a', + ttlSeconds: 60, + }); + const target = { queueName: 'other', jobId: 'settled-job' }; + await attachProgrammaticCancellationTarget({ + redis, + requestId, + owner: 'owner-a', + target, + ttlSeconds: 60, + }); + await commitJobResult(redis, target, { stdout: 'done' }, 60); + await releaseProgrammaticCancellation({ + redis, + requestId, + owner: 'owner-a', + }); + const key = programmaticCancellationInternals.requestKey(requestId); + expect(await redis.exists(key)).toBe(1); + expect(await redis.ttl(key)).toBeGreaterThan(0); + expect(await redis.ttl(key)).toBeLessThanOrEqual(60); + const cancelled = await cancelProgrammaticRequest({ + redis, + requestId, + owner: 'owner-a', + ttlSeconds: 60, + }); + expect(cancelled).toEqual({ status: 'accepted', target }); + expect(await requestJobCancellation(redis, target, 60)).toBe(false); +}); diff --git a/service/src/programmatic-cancellation.ts b/service/src/programmatic-cancellation.ts new file mode 100644 index 00000000..12a6694d --- /dev/null +++ b/service/src/programmatic-cancellation.ts @@ -0,0 +1,182 @@ +import { createHash } from 'node:crypto'; +import type IORedis from 'ioredis'; +import type { AuthenticatedRequest } from './types'; +import { getCredentialId } from './auth/principal'; +import { getExecutionIdentity } from './execution-identity'; + +export const CODEAPI_PROGRAMMATIC_REQUEST_HEADER = + 'X-LibreChat-Code-Request-ID'; +const REQUEST_PREFIX = 'codeapi:programmatic-cancellation:v1'; +const REQUEST_ID_PATTERN = /^[A-Za-z0-9_-]{16,128}$/; + +interface CancellationTarget { + queueName: string; + jobId: string; +} + +export type CancellationRequestResult = + | { status: 'accepted'; target?: CancellationTarget } + | { status: 'forbidden' }; + +function requestKey(requestId: string): string { + return `${REQUEST_PREFIX}:${requestId}`; +} + +export function normalizeProgrammaticRequestId( + value: unknown, +): string | undefined { + if (typeof value !== 'string') return undefined; + const trimmed = value.trim(); + return REQUEST_ID_PATTERN.test(trimmed) ? trimmed : undefined; +} + +export function programmaticCancellationOwner( + req: AuthenticatedRequest, + userId: string, +): string { + const identity = getExecutionIdentity(req, userId); + return createHash('sha256') + .update( + JSON.stringify([ + identity.storageNamespace, + identity.canonicalUserId, + getCredentialId(req), + identity.authContextHash ?? '', + ]), + ) + .digest('hex'); +} + +const RESERVE_SCRIPT = ` +local key = KEYS[1] +local owner = ARGV[1] +local ttl = tonumber(ARGV[2]) +local existing = redis.call('HGET', key, 'owner') +if existing and existing ~= owner then return -1 end +if redis.call('HGET', key, 'reserved') == '1' then return -2 end +if not existing then + redis.call('HSET', key, 'owner', owner, 'cancelled', '0') +end +redis.call('HSET', key, 'reserved', '1') +redis.call('EXPIRE', key, ttl) +return tonumber(redis.call('HGET', key, 'cancelled') or '0') +`; + +const ATTACH_SCRIPT = ` +local key = KEYS[1] +local owner = ARGV[1] +local queueName = ARGV[2] +local jobId = ARGV[3] +local ttl = tonumber(ARGV[4]) +if redis.call('HGET', key, 'owner') ~= owner then return -1 end +redis.call('HSET', key, 'queueName', queueName, 'jobId', jobId) +redis.call('EXPIRE', key, ttl) +return tonumber(redis.call('HGET', key, 'cancelled') or '0') +`; + +const CANCEL_SCRIPT = ` +local key = KEYS[1] +local owner = ARGV[1] +local ttl = tonumber(ARGV[2]) +local existing = redis.call('HGET', key, 'owner') +if existing and existing ~= owner then return {-1} end +if not existing then redis.call('HSET', key, 'owner', owner) end +redis.call('HSET', key, 'cancelled', '1') +local queueName = redis.call('HGET', key, 'queueName') +local jobId = redis.call('HGET', key, 'jobId') +-- Once attached, never extend this mapping independently of the job decision. +-- Its original admission TTL already covers execution and late cancellation. +if queueName and jobId then return {1, queueName, jobId} end +redis.call('EXPIRE', key, ttl) +return {1} +`; + +const RELEASE_SCRIPT = ` +if redis.call('HGET', KEYS[1], 'owner') == ARGV[1] then + -- Keep the owner/target tombstone through its existing bounded TTL. A Stop + -- racing response delivery must still reach the job's completion decision. + return redis.call('HSET', KEYS[1], 'finished', '1') +end +return 0 +`; + +export async function reserveProgrammaticCancellation(args: { + redis: IORedis; + requestId: string; + owner: string; + ttlSeconds: number; +}): Promise<'active' | 'cancelled' | 'duplicate' | 'forbidden'> { + const result = Number( + await args.redis.eval( + RESERVE_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + Math.max(1, args.ttlSeconds), + ), + ); + if (result === -1) return 'forbidden'; + if (result === -2) return 'duplicate'; + return result === 1 ? 'cancelled' : 'active'; +} + +export async function attachProgrammaticCancellationTarget(args: { + redis: IORedis; + requestId: string; + owner: string; + target: CancellationTarget; + ttlSeconds: number; +}): Promise<'active' | 'cancelled' | 'forbidden'> { + const result = Number( + await args.redis.eval( + ATTACH_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + args.target.queueName, + args.target.jobId, + Math.max(1, args.ttlSeconds), + ), + ); + if (result < 0) return 'forbidden'; + return result === 1 ? 'cancelled' : 'active'; +} + +export async function cancelProgrammaticRequest(args: { + redis: IORedis; + requestId: string; + owner: string; + ttlSeconds: number; +}): Promise { + const raw = await args.redis.eval( + CANCEL_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + Math.max(1, args.ttlSeconds), + ); + const result = Array.isArray(raw) ? raw.map(String) : []; + if (result[0] === '-1') return { status: 'forbidden' }; + if (result.length >= 3) { + return { + status: 'accepted', + target: { queueName: result[1]!, jobId: result[2]! }, + }; + } + return { status: 'accepted' }; +} + +export async function releaseProgrammaticCancellation(args: { + redis: IORedis; + requestId: string; + owner: string; +}): Promise { + await args.redis.eval( + RELEASE_SCRIPT, + 1, + requestKey(args.requestId), + args.owner, + ); +} + +export const programmaticCancellationInternals = { requestKey }; diff --git a/service/src/queue.ts b/service/src/queue.ts index 54fea308..72c2fbb1 100644 --- a/service/src/queue.ts +++ b/service/src/queue.ts @@ -1,6 +1,7 @@ // src/queue.ts import IORedis from 'ioredis'; import { Queue, QueueEvents } from 'bullmq'; +import type { Job } from 'bullmq'; import { setMaxListeners } from 'events'; import type { CommonRedisOptions } from 'ioredis'; import type * as tls from 'tls'; @@ -17,19 +18,13 @@ import type { SandboxBackendName, } from './execution-profile'; import logger from './logger'; -import { redisKeepAliveOptions } from './redis-options'; +import { redisKeepAliveOptions, redisReconnectDelay } from './redis-options'; import { bullmqQueueJobs, registerBullmqQueueMetricsCollector } from './metrics'; - -const MAX_RECONNECT_ATTEMPTS = 5; -const RECONNECT_DELAY = 2000; +import { JobCancellationRegistry } from './job-cancellation'; const retryStrategy: CommonRedisOptions['retryStrategy'] = (times) => { - if (times > MAX_RECONNECT_ATTEMPTS) { - logger.error(`Failed to connect to Redis after ${times} attempts`); - return null; - } logger.warn(`Retrying Redis connection attempt ${times}`); - return RECONNECT_DELAY; + return redisReconnectDelay(times); }; const reconnectOnError: CommonRedisOptions['reconnectOnError'] = (err) => { @@ -60,6 +55,7 @@ const connection = new IORedis({ ? { dnsLookup: (address: string, callback: (err: Error | null, addr: string) => void): void => callback(null, address) } : {}) }); +const jobCancellationRegistry = new JobCancellationRegistry(connection); // Global queues - no INSTANCE_ID prefix // This enables horizontal scaling where any worker can process any job @@ -110,6 +106,19 @@ export function getExecutionQueueBinding( return { ...getQueueResources(name), language }; } +/** + * Resolve a job only from this deployment's already-open queue set. Every + * homogeneous API replica opens both execution queues at startup, so this + * supports cross-replica cancellation without allocating attacker-shaped + * QueueEvents connections for arbitrary names recovered from Redis. + */ +export async function getExistingExecutionJob( + queueName: string, + jobId: string, +): Promise | undefined> { + return queueResources.get(queueName)?.queue.getJob(jobId); +} + const { queue: pyQueue, events: pyQueueEvents } = getQueueResources(queueNames.python); const { queue: otherQueue, events: otherQueueEvents } = getQueueResources(queueNames.other); @@ -163,8 +172,16 @@ export async function closeQueueConnections(): Promise { [...queueResources.values()].flatMap(({ queue, events }) => [ queue.close(), events.close(), - ]), + ]).concat(jobCancellationRegistry.close()), ); } -export { pyQueue, otherQueue, pyQueueEvents, otherQueueEvents, queueNames, connection }; +export { + pyQueue, + otherQueue, + pyQueueEvents, + otherQueueEvents, + queueNames, + connection, + jobCancellationRegistry, +}; diff --git a/service/src/redis-options.test.ts b/service/src/redis-options.test.ts index 1bb942e6..9795e279 100644 --- a/service/src/redis-options.test.ts +++ b/service/src/redis-options.test.ts @@ -1,5 +1,15 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { redisKeepAliveMs, redisKeepAliveOptions } from './redis-options'; +import { + redisKeepAliveMs, + redisKeepAliveOptions, + redisReconnectDelay, +} from './redis-options'; + +test('long-lived command connections keep recovering with a bounded retry delay', () => { + expect(redisReconnectDelay(1)).toBe(100); + expect(redisReconnectDelay(6)).toBe(600); + expect(redisReconnectDelay(1_000)).toBe(2_000); +}); describe('Redis keepalive options', () => { afterEach(() => { diff --git a/service/src/redis-options.ts b/service/src/redis-options.ts index 98d3f5b1..1f2c8e42 100644 --- a/service/src/redis-options.ts +++ b/service/src/redis-options.ts @@ -1,5 +1,11 @@ import type { CommonRedisOptions } from 'ioredis'; +/** Long-lived queue/cancellation command and subscriber connections must both + * recover after an outage. Never leave a live process with a terminal client. */ +export function redisReconnectDelay(attempt: number): number { + return Math.min(2_000, 100 * Math.max(1, attempt)); +} + export function redisKeepAliveMs(): number { const raw = process.env.REDIS_KEEP_ALIVE_MS; const trimmed = raw?.trim(); diff --git a/service/src/request-disconnect.test.ts b/service/src/request-disconnect.test.ts new file mode 100644 index 00000000..15660aca --- /dev/null +++ b/service/src/request-disconnect.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from 'bun:test'; +import { EventEmitter } from 'node:events'; +import type { Response } from 'express'; +import type { AuthenticatedRequest } from './types'; +import { CLIENT_DISCONNECT_REASON } from './job-cancellation'; +import { observeRequestDisconnect } from './request-disconnect'; + +function requestAndResponse(options: { + requestAborted?: boolean; + requestDestroyed?: boolean; + responseDestroyed?: boolean; +} = {}): { + req: AuthenticatedRequest & EventEmitter; + res: Response & EventEmitter; +} { + const req = Object.assign(new EventEmitter(), { + aborted: options.requestAborted ?? false, + destroyed: options.requestDestroyed ?? false, + }) as AuthenticatedRequest & EventEmitter; + const res = Object.assign(new EventEmitter(), { + destroyed: options.responseDestroyed ?? false, + writableFinished: false, + }) as Response & EventEmitter; + return { req, res }; +} + +test('a consumed Bun request stream is not mistaken for a disconnect', () => { + const { req, res } = requestAndResponse({ requestDestroyed: true }); + const observer = observeRequestDisconnect(req, res); + + expect(observer.isDisconnected()).toBe(false); + expect(observer.signal.aborted).toBe(false); + observer.dispose(); +}); + +test('current and future transport abandonment abort exactly once', () => { + const current = requestAndResponse({ requestAborted: true }); + const currentObserver = observeRequestDisconnect(current.req, current.res); + expect(currentObserver.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + + const future = requestAndResponse(); + const futureObserver = observeRequestDisconnect(future.req, future.res); + future.res.emit('close'); + future.req.emit('aborted'); + expect(futureObserver.signal.reason).toBe(CLIENT_DISCONNECT_REASON); + expect(future.req.listenerCount('aborted')).toBe(0); + expect(future.res.listenerCount('close')).toBe(0); +}); + +test('a completed response disposes listeners without aborting', () => { + const { req, res } = requestAndResponse(); + const observer = observeRequestDisconnect(req, res); + (res as unknown as { writableFinished: boolean }).writableFinished = true; + res.emit('finish'); + res.emit('close'); + + expect(observer.isDisconnected()).toBe(false); + expect(req.listenerCount('aborted')).toBe(0); + expect(res.listenerCount('close')).toBe(0); +}); diff --git a/service/src/request-disconnect.ts b/service/src/request-disconnect.ts new file mode 100644 index 00000000..2a4d7a9d --- /dev/null +++ b/service/src/request-disconnect.ts @@ -0,0 +1,49 @@ +import type { Response } from 'express'; +import type { AuthenticatedRequest } from './types'; +import { CLIENT_DISCONNECT_REASON } from './job-cancellation'; + +export interface RequestDisconnectObserver { + signal: AbortSignal; + isDisconnected(): boolean; + dispose(): void; +} + +/** + * Observe a genuinely abandoned HTTP response across Node and Bun. + * + * Bun may mark the consumed IncomingMessage stream as `destroyed` while the + * response remains healthy, so request stream destruction is deliberately not + * treated as a disconnect. Express' `aborted` event and ServerResponse's + * pre-finish `close` event are the portable abandonment signals. + */ +export function observeRequestDisconnect( + req: AuthenticatedRequest, + res: Response, +): RequestDisconnectObserver { + const controller = new AbortController(); + let disposed = false; + const dispose = (): void => { + if (disposed) return; + disposed = true; + req.removeListener('aborted', disconnect); + res.removeListener('close', disconnect); + res.removeListener('finish', dispose); + }; + const disconnect = (): void => { + if (!res.writableFinished && !controller.signal.aborted) { + controller.abort(CLIENT_DISCONNECT_REASON); + } + dispose(); + }; + + req.once('aborted', disconnect); + res.once('close', disconnect); + res.once('finish', dispose); + if (req.aborted || res.destroyed) disconnect(); + + return { + signal: controller.signal, + isDisconnected: () => controller.signal.aborted, + dispose, + }; +} diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 93bbc0c5..bb140013 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -4,13 +4,32 @@ import { Router } from 'express'; import type { Response } from 'express'; import type * as t from '../types'; import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; -import { executionLimiter } from '../middleware/limits'; +import { cancellationLimiter, executionLimiter } from '../middleware/limits'; import { pyQueue, pyQueueEvents, connection, + jobCancellationRegistry, getExecutionQueueBinding, + getExistingExecutionJob, } from '../queue'; +import { + JOB_CANCELLED_MESSAGE, + programmaticCancellationError, + removeJobIfWaiting, + requestJobCancellation, + fenceJobCancellation, + waitForJobWithCancellation, +} from '../job-cancellation'; +import { + CODEAPI_PROGRAMMATIC_REQUEST_HEADER, + attachProgrammaticCancellationTarget, + cancelProgrammaticRequest, + normalizeProgrammaticRequestId, + programmaticCancellationOwner, + releaseProgrammaticCancellation, + reserveProgrammaticCancellation, +} from '../programmatic-cancellation'; import { createProgrammaticPayload, extractPendingFromControlPayload, @@ -38,6 +57,7 @@ import { Jobs } from '../enum'; import { env, jobCompletionWaitTimeoutMs } from '../config'; import { resolveQueuedSandboxBackend } from '../execution-profile'; import { publicExecutionFailure } from '../utils'; +import { observeRequestDisconnect } from '../request-disconnect'; import { normalizeEgressGatewayUrl, normalizeProgrammaticTimeoutMs, @@ -107,6 +127,18 @@ const JOB_COMPLETION_WAIT_TIMEOUT_MS = jobCompletionWaitTimeoutMs( env.LAMBDA_MICROVM_LAUNCH_TIMEOUT_MS, env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS, ); +const PROGRAMMATIC_CANCELLATION_TTL_SECONDS = + Math.ceil(JOB_COMPLETION_WAIT_TIMEOUT_MS / 1000) + 60; + +interface ReplayRequestCancellation { + signal: AbortSignal; + isDisconnected(): boolean; + request?: { + requestId: string; + owner: string; + cancelledBeforeStart: boolean; + }; +} const router = Router(); @@ -322,7 +354,10 @@ async function runReplayIteration( state: ExecutionState, apiKeyId: string, userId: string, + signal?: AbortSignal, + cancellation?: { requestId: string; owner: string }, ): Promise { + if (signal?.aborted) throw programmaticCancellationError(); const history = await loadToolHistory(state.execution_id); const rawPayload = buildReplayPayload(req, state, history); const sessionKey = state.sessionKey ?? state.userId; @@ -369,41 +404,83 @@ async function runReplayIteration( state.executionProfile ?? env.EXECUTION_PROFILE, state.executionProfileSource ?? env.EXECUTION_PROFILE_SOURCE, ); - const job = await queue.add( - Jobs.execute, - { - code: state.userCode ?? '', - userId, - payload: sandboxSecurity.payload, - apiKeyId, - isPyPlot: state.isPyPlot ?? false, - principalSource: state.principalSource, - executionId: state.execution_id, - tenantId: state.tenantId, - canonicalUserId: state.canonicalUserId, - executionProfile: state.executionProfile ?? env.EXECUTION_PROFILE, - sandboxBackend: replayBackend, - ...(state.bridgeWorkerId != null - ? { bridgeWorkerId: state.bridgeWorkerId } - : {}), - ...(state.workspaceId != null - ? { workspaceId: state.workspaceId } - : {}), - runtimeSessionMode: 'stateless', - runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, - executionManifestClaims: sandboxSecurity.executionManifestClaims, - egressGrantClaims: sandboxSecurity.egressGrantClaims, - egressGrantToken: sandboxSecurity.egressGrantToken, - }, - { - removeOnComplete: { age: 60, count: 1 }, - removeOnFail: { age: 180, count: 1 }, - attempts: 1, - }, + if (signal?.aborted) throw programmaticCancellationError(); + const cancellationTarget = { queueName: queue.name, jobId: nanoid() }; + if (cancellation != null) { + const attachment = await attachProgrammaticCancellationTarget({ + redis: connection, + requestId: cancellation.requestId, + owner: cancellation.owner, + target: cancellationTarget, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + }); + if (attachment === 'forbidden') { + throw new Error('Programmatic cancellation request ownership changed'); + } + if (attachment === 'cancelled') { + throw programmaticCancellationError(); + } + } + const submittedAtMs = Date.now(); + const deadlineAtMs = submittedAtMs + env.JOB_TIMEOUT; + let job: Awaited>; + try { + job = await queue.add( + Jobs.execute, + { + code: state.userCode ?? '', + userId, + payload: sandboxSecurity.payload, + apiKeyId, + isPyPlot: state.isPyPlot ?? false, + principalSource: state.principalSource, + executionId: state.execution_id, + tenantId: state.tenantId, + canonicalUserId: state.canonicalUserId, + executionProfile: state.executionProfile ?? env.EXECUTION_PROFILE, + sandboxBackend: replayBackend, + ...(state.bridgeWorkerId != null ? { bridgeWorkerId: state.bridgeWorkerId } : {}), + ...(state.workspaceId != null ? { workspaceId: state.workspaceId } : {}), + cancellable: true, + deadlineAtMs, + cancellationTtlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + runtimeSessionMode: 'stateless', + runtimeSessionExemption: PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION, + executionManifestClaims: sandboxSecurity.executionManifestClaims, + egressGrantClaims: sandboxSecurity.egressGrantClaims, + egressGrantToken: sandboxSecurity.egressGrantToken, + }, + { + removeOnComplete: { age: 60, count: 1 }, + removeOnFail: { age: 180, count: 1 }, + attempts: 1, + jobId: cancellationTarget.jobId, + timestamp: submittedAtMs, + }, ); + } catch (error) { + // Redis may have enqueued the job even though its reply was lost. + // Preserve replay ownership until cancellation is durable or the job's + // fixed worker deadline prevents a late admission from executing. + const outcome = await fenceJobCancellation({ + commands: connection, target: cancellationTarget, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, deadlineAtMs, + }); + if (outcome.status === 'completed') return outcome.result; + throw error; + } jobsSubmitted.inc({ language }); - return job.waitUntilFinished(events, JOB_COMPLETION_WAIT_TIMEOUT_MS); + return waitForJobWithCancellation({ + commands: connection, + registry: jobCancellationRegistry, + job, + events, + timeoutMs: JOB_COMPLETION_WAIT_TIMEOUT_MS, + cancellationTtlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + deadlineAtMs, + signal, + }); } function isSandboxRunSuccess(result: t.ExecuteResult): boolean { @@ -425,6 +502,7 @@ async function handleReplayInitial( bridgeWorkerId?: string; workspaceId?: string; }, + cancellation: ReplayRequestCancellation, ): Promise { const { apiKeyId, userId, bridgeWorkerId, workspaceId } = params; const { code, tools, user_id, files } = @@ -543,6 +621,19 @@ async function handleReplayInitial( throw error; } + if ( + cancellation.signal.aborted || + cancellation.request?.cancelledBeforeStart === true + ) { + if (!cancellation.isDisconnected()) { + res.status(200).json({ + status: 'error', + error: 'Programmatic execution request cancelled', + }); + } + return; + } + const session_id = nanoid(); const execution_id = nanoid(); const authContext = req.codeApiAuthContext; @@ -622,7 +713,7 @@ async function handleReplayInitial( timeout, }); - await runAndRespond(req, res, state, apiKeyId, userId); + await runAndRespond(req, res, state, apiKeyId, userId, cancellation); } async function handleReplayContinuation( @@ -634,6 +725,7 @@ async function handleReplayContinuation( decoded: { execution_id: string }; tool_results: NonNullable; }, + cancellation: ReplayRequestCancellation, ): Promise { const { apiKeyId, userId, decoded, tool_results } = params; @@ -692,6 +784,20 @@ async function handleReplayContinuation( res.status(404).json({ error: 'Execution not found or expired' }); return; } + if ( + cancellation.signal.aborted || + cancellation.request?.cancelledBeforeStart === true + ) { + await cleanupExecution(state.execution_id, 'replay'); + if (!cancellation.isDisconnected()) { + res.status(200).json({ + status: 'error', + error: 'Programmatic execution request cancelled', + session_id: state.session_id, + }); + } + return; + } /** Compute the delta against already-persisted history first so the * cap checks see the real impact of this batch (new call_ids only * advance `callCount`; overwrites may shrink or grow `historyBytes` @@ -845,7 +951,14 @@ async function handleReplayContinuation( }); } - await runAndRespond(req, res, state, apiKeyId, userId); + await runAndRespond( + req, + res, + state, + apiKeyId, + userId, + cancellation, + ); } finally { await releaseExecutionLock(decoded.execution_id, lockToken); } @@ -857,29 +970,29 @@ async function runAndRespond( state: ExecutionState, apiKeyId: string, userId: string, + cancellation: ReplayRequestCancellation, ): Promise { - /** Read disconnect state through `isDisconnected()` rather than a - * direct boolean. The `req.on('close', ...)` handler flips the flag - * during awaits, but `@typescript-eslint/no-unnecessary-condition` - * (correctly per TS semantics) narrows a directly-mutated `let`/object - * member to its literal value after an early-return `if (...) return`, - * even across awaits. A function call is opaque to that narrowing. */ - let disconnected = false; - const isDisconnected = (): boolean => disconnected; - req.on('close', () => { - if (!res.writableEnded) disconnected = true; - }); - let result: t.ExecuteResult; try { - result = await runReplayIteration(req, state, apiKeyId, userId); + result = await runReplayIteration( + req, + state, + apiKeyId, + userId, + cancellation.signal, + cancellation.request, + ); } catch (err) { - logger.error('Replay iteration failed', { - execution_id: state.execution_id, - err, - }); + const cancelled = + (err as Error).name === 'AbortError' || + (err as Error).message === JOB_CANCELLED_MESSAGE; + logger.log(cancelled ? 'info' : 'error', 'Replay iteration failed', { + execution_id: state.execution_id, + cancelled, + err, + }); await cleanupExecution(state.execution_id, 'replay'); - if (!isDisconnected()) { + if (!cancellation.isDisconnected()) { const publicFailure = publicExecutionFailure(err); const message = publicFailure?.body.message ?? (err as Error).message; @@ -892,7 +1005,7 @@ async function runAndRespond( return; } - if (isDisconnected()) { + if (cancellation.isDisconnected()) { logger.info('Client disconnected during replay; cleaning up', { execution_id: state.execution_id, }); @@ -1002,7 +1115,7 @@ async function runAndRespond( await cleanupExecution(state.execution_id, 'replay').catch( () => {}, ); - if (!isDisconnected()) { + if (!cancellation.isDisconnected()) { if (err instanceof ExecutionStateTooLargeError) { /** A continuation that pushes `emittedCallIds` past the * `MAX_EXECUTION_STATE_BYTES` cap is a client-input sizing @@ -1076,6 +1189,67 @@ async function runAndRespond( // Request entrypoint // --------------------------------------------------------------------------- +router.post( + '/exec/programmatic/cancel', + cancellationLimiter, + async (req: t.AuthenticatedRequest, res) => { + const principal = getPrincipalOrReject(req, res); + if (!principal) return; + const requestId = normalizeProgrammaticRequestId( + (req.body as Record)?.request_id, + ); + if (requestId == null) { + res.status(400).json({ error: 'Invalid or missing request_id' }); + return; + } + try { + const owner = programmaticCancellationOwner(req, principal.userId); + const cancellation = await cancelProgrammaticRequest({ + redis: connection, + requestId, + owner, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + }); + if (cancellation.status === 'forbidden') { + res.status(403).json({ error: 'Programmatic request belongs to another principal' }); + return; + } + if (cancellation.target != null) { + const accepted = await requestJobCancellation( + connection, + cancellation.target, + PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + ); + if (!accepted) { + res.status(200).json({ status: 'already_completed' }); + return; + } + try { + const queuedJob = await getExistingExecutionJob( + cancellation.target.queueName, + cancellation.target.jobId, + ); + if (queuedJob != null) await removeJobIfWaiting(queuedJob); + } catch (error) { + logger.warn('Failed to remove cancelled waiting execution', { + requestId, + queueName: cancellation.target?.queueName, + jobId: cancellation.target?.jobId, + error: (error as Error).message, + }); + } + } + res.status(202).json({ status: 'cancellation_requested' }); + } catch (error) { + logger.error('Failed to request programmatic execution cancellation', { + requestId, + error: (error as Error).message, + }); + res.status(503).json({ error: 'Cancellation service unavailable' }); + } + }, +); + router.post( '/exec/programmatic', executionLimiter, @@ -1095,6 +1269,11 @@ router.post( const { continuation_token, tool_results } = req.body as t.ProgrammaticRequestBody; const rawBody = req.body as Record; + const rawRequestId = req.header(CODEAPI_PROGRAMMATIC_REQUEST_HEADER); + const requestId = normalizeProgrammaticRequestId(rawRequestId); + if (rawRequestId != null && requestId == null) { + return res.status(400).json({ error: 'Invalid programmatic request ID' }); + } const requestedLanguage: unknown = rawBody.language ?? rawBody.lang; let bridgeWorkerId: string | undefined; let workspaceId: string | undefined; @@ -1151,7 +1330,51 @@ router.post( }); } + const disconnectObserver = observeRequestDisconnect(req, res); + + const cancellation: ReplayRequestCancellation = { + signal: disconnectObserver.signal, + isDisconnected: disconnectObserver.isDisconnected, + }; + let reservedCancellation: { requestId: string; owner: string } | undefined; + try { + if (requestId != null) { + const owner = programmaticCancellationOwner(req, userId); + let reservation: Awaited>; + try { + reservation = await reserveProgrammaticCancellation({ + redis: connection, + requestId, + owner, + ttlSeconds: PROGRAMMATIC_CANCELLATION_TTL_SECONDS, + }); + } catch (error) { + logger.error('Failed to reserve programmatic cancellation request', { + requestId, + error: (error as Error).message, + }); + if (!cancellation.isDisconnected()) { + return res.status(503).json({ error: 'Cancellation service unavailable' }); + } + return; + } + if (reservation === 'forbidden' || reservation === 'duplicate') { + if (!cancellation.isDisconnected()) { + return res.status(409).json({ + error: 'Programmatic request ID is already in use', + }); + } + return; + } + reservedCancellation = { requestId, owner }; + cancellation.request = { + requestId, + owner, + cancelledBeforeStart: reservation === 'cancelled', + }; + } + /** For continuations, peek at the stored execution to route by the * mode it was started in rather than the current process default. * Without this, a replay-mode execution resumed via an instance @@ -1184,7 +1407,7 @@ router.post( userId, decoded, tool_results, - }); + }, cancellation); } return await handleBlocking(req, res, { apiKeyId, userId }); } @@ -1203,7 +1426,7 @@ router.post( userId, bridgeWorkerId, workspaceId, - }); + }, cancellation); } if (workspaceId != null) { return res.status(400).json({ @@ -1221,6 +1444,20 @@ router.post( return res.status(500).json({ error: 'Internal server error' }); } return; + } finally { + disconnectObserver.dispose(); + if (reservedCancellation != null) { + await releaseProgrammaticCancellation({ + redis: connection, + requestId: reservedCancellation.requestId, + owner: reservedCancellation.owner, + }).catch(error => { + logger.warn('Failed to release programmatic cancellation request', { + requestId: reservedCancellation?.requestId, + error: (error as Error).message, + }); + }); + } } }, ); diff --git a/service/src/types/service.ts b/service/src/types/service.ts index d17b5a99..a0c78486 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -297,6 +297,12 @@ export type JobData = { bridgeWorkerId?: string; /** Trusted selected workspace for native replay-mode PTC. */ workspaceId?: string; + /** Opts replay jobs into durable client-disconnect cancellation. */ + cancellable?: boolean; + /** Absolute producer budget; queue-worker configuration may only tighten it. */ + deadlineAtMs?: number; + /** Producer request tombstones must never outlive the completion decision. */ + cancellationTtlSeconds?: number; /** Producer deployment identity. Optional only for pre-profile queued jobs. */ executionProfile?: ExecutionProfile; /** Required sandbox transport. Optional only for jobs queued before fencing. */ diff --git a/service/src/workers.ts b/service/src/workers.ts index f11a5420..a7153b53 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -1,69 +1,148 @@ import axios from 'axios'; import { Worker } from 'bullmq'; import type * as t from './types'; -import { filterSystemLogs, applySystemReplacements, getAxiosErrorDetails, sandboxErrorMessageFromAxios } from './utils'; -import { jobProcessingDuration, jobsCompleted, jobsFailed, activeJobs, workerRunning } from './metrics'; -import { connection, queueNames } from './queue'; +import { + filterSystemLogs, + applySystemReplacements, + getAxiosErrorDetails, + sandboxErrorMessageFromAxios, +} from './utils'; +import { + jobProcessingDuration, + jobsCancelled, + jobsCompleted, + jobsFailed, + activeJobs, + workerRunning, +} from './metrics'; +import { connection, jobCancellationRegistry, queueNames } from './queue'; import { env, jobDeadlineAtMs } from './config'; import { summarizeSandboxResponse, summarizeText } from './execution-log'; -import { createGatewayEgressGrant, restoreGatewaySandboxResult, revokeGatewayEgressGrant } from './egress-gateway-client'; +import { + createGatewayEgressGrant, + restoreGatewaySandboxResult, + revokeGatewayEgressGrant, +} from './egress-gateway-client'; import { refreshEgressGrantClaims } from './sandbox-egress'; import { buildSandboxExecuteRequest } from './sandbox-dispatch'; import { prepareInputDelivery } from './runtime-session/input-delivery'; import { SessionFilesError } from './runtime-session/files'; import { resolveRuntimeSessionForJob } from './runtime-session/job-policy'; -import { getSandboxBackend, SandboxBackendError, type SandboxRawResponse } from './sandbox-backend'; +import { + getSandboxBackend, + SandboxBackendError, + type SandboxRawResponse, +} from './sandbox-backend'; import { isSyntheticPrincipalSource } from './auth/synthetic'; import { withSpan, withTraceContext } from './telemetry'; import { workerDeadlineFailure } from './worker-error'; +import { + CLIENT_DISCONNECT_REASON, + JOB_CANCELLED_MESSAGE, + jobResultCommitFailure, + commitJobResult, + claimJobExecution, + jobCancellationRetentionSeconds, + throwIfJobAborted, +} from './job-cancellation'; import logger from './logger'; import { validateQueuedExecutionProfile, validateQueuedSandboxBackend, } from './execution-profile'; -import { BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, programmaticTransferReserveMs } from '../../packages/code/src/protocol'; +import { + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, + programmaticTransferReserveMs, +} from '../../packages/code/src/protocol'; const { INSTANCE_ID } = env; const WORKER_ID = `${INSTANCE_ID}-${process.pid}`; function isAbortError(error: unknown): boolean { - return axios.isAxiosError(error) && (error.name === 'AbortError' || error.code === 'ERR_CANCELED'); + return ( + axios.isAxiosError(error) && + (error.name === 'AbortError' || error.code === 'ERR_CANCELED') + ); } async function processJob(job: t.ExecuteJob): Promise { - return withTraceContext(job.data._otel, () => withSpan('codeapi.job.process', { - 'messaging.system': 'bullmq', - 'messaging.operation.name': 'process', - 'messaging.message.id': typeof job.id === 'string' ? job.id : String(job.id ?? ''), - 'codeapi.language': job.data.payload?.language ?? 'unknown', - 'codeapi.execution_profile': job.data.executionProfile ?? 'legacy', - 'codeapi.worker_execution_profile': env.EXECUTION_PROFILE, - }, () => processJobInner(job), 'CONSUMER')); + return withTraceContext(job.data._otel, () => + withSpan( + 'codeapi.job.process', + { + 'messaging.system': 'bullmq', + 'messaging.operation.name': 'process', + 'messaging.message.id': + typeof job.id === 'string' ? job.id : String(job.id ?? ''), + 'codeapi.language': job.data.payload?.language ?? 'unknown', + 'codeapi.execution_profile': job.data.executionProfile ?? 'legacy', + 'codeapi.worker_execution_profile': env.EXECUTION_PROFILE, + }, + () => processJobInner(job), + 'CONSUMER', + ), + ); } async function processJobInner(job: t.ExecuteJob): Promise { const { payload, isPyPlot } = job.data; - const isSyntheticJob = job.data.isSynthetic === true || isSyntheticPrincipalSource(job.data.principalSource); + const isSyntheticJob = + job.data.isSynthetic === true || + isSyntheticPrincipalSource(job.data.principalSource); const language = payload?.language ?? 'unknown'; const endTimer = jobProcessingDuration.startTimer({ language }); activeJobs.inc({ language }); const controller = new AbortController(); - const deadlineAtMs = jobDeadlineAtMs(job.timestamp, env.JOB_TIMEOUT); + const cancellationTarget = + job.data.cancellable === true && job.id != null + ? { queueName: job.queueName, jobId: String(job.id) } + : undefined; + let cancellationRegistered = false; + const deadlineAtMs = jobDeadlineAtMs( + job.timestamp, + env.JOB_TIMEOUT, + Date.now(), + job.data.deadlineAtMs, + ); const remainingBudgetMs = Math.max(0, deadlineAtMs - Date.now()); - const timer = remainingBudgetMs > 0 - ? setTimeout(() => controller.abort(), remainingBudgetMs) - : undefined; - if (remainingBudgetMs === 0) controller.abort(); + const timer = + remainingBudgetMs > 0 + ? setTimeout(() => controller.abort('deadline'), remainingBudgetMs) + : undefined; + if (remainingBudgetMs === 0) controller.abort('deadline'); let egressGrantId: string | undefined; let egressGrantTokenForRestore: string | undefined; let revokeReason = 'completed'; + let completedResult = false; + let resultToCommit: t.ExecuteResult | undefined; + let resultCommittedAtHandoff = false; + const commitAtHandoff = + cancellationTarget != null && + job.data.workspaceId != null && + env.SANDBOX_BACKEND === 'remote-bridge'; try { + if (cancellationTarget != null) { + await jobCancellationRegistry.register(cancellationTarget, controller); + cancellationRegistered = true; + const claim = await claimJobExecution( + connection, + cancellationTarget, + jobCancellationRetentionSeconds( + env.JOB_TIMEOUT, + job.data.cancellationTtlSeconds, + ), + ); + if (claim.status === 'completed') return claim.result; + } if (controller.signal.aborted) { throw new Error(`Job timed out after ${env.JOB_TIMEOUT}ms`); } - validateQueuedExecutionProfile(job.data.executionProfile, env.EXECUTION_PROFILE); + validateQueuedExecutionProfile( + job.data.executionProfile, + env.EXECUTION_PROFILE, + ); validateQueuedSandboxBackend( job.data.sandboxBackend, env.SANDBOX_BACKEND, @@ -77,7 +156,10 @@ async function processJobInner(job: t.ExecuteJob): Promise { const nowSeconds = Math.floor(Date.now() / 1000); const prepared = await createGatewayEgressGrant({ payload, - claims: refreshEgressGrantClaims(job.data.egressGrantClaims, nowSeconds), + claims: refreshEgressGrantClaims( + job.data.egressGrantClaims, + nowSeconds, + ), isSynthetic: isSyntheticJob, signal: controller.signal, }); @@ -85,19 +167,27 @@ async function processJobInner(job: t.ExecuteJob): Promise { sandboxPayload = prepared.payload; egressGrantToken = prepared.egressGrantToken; egressGrantTokenForRestore = prepared.egressGrantToken; - executionManifestClaims = (env.EXECUTION_MANIFEST_PRIVATE_KEY || env.EXECUTION_MANIFEST_SECRET) - ? prepared.executionManifestClaims - : undefined; + executionManifestClaims = + env.EXECUTION_MANIFEST_PRIVATE_KEY || env.EXECUTION_MANIFEST_SECRET + ? prepared.executionManifestClaims + : undefined; } const delivery = prepareInputDelivery(payload, sandboxPayload); const sandboxRequest = buildSandboxExecuteRequest({ - ...(job.data.workspaceId == null ? {} : { programmaticTransferReserveMs: programmaticTransferReserveMs(env.JOB_TIMEOUT) }), + ...(job.data.workspaceId == null + ? {} + : { + programmaticTransferReserveMs: programmaticTransferReserveMs( + env.JOB_TIMEOUT, + ), + }), payload: delivery.payload, egressGrantToken, executionManifestClaims, maxOutputFileBytes: Math.min( - executionManifestClaims?.max_upload_bytes ?? env.EGRESS_GATEWAY_MAX_FILE_BYTES, + executionManifestClaims?.max_upload_bytes ?? + env.EGRESS_GATEWAY_MAX_FILE_BYTES, env.EGRESS_GATEWAY_MAX_FILE_BYTES, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, ), @@ -121,21 +211,41 @@ async function processJobInner(job: t.ExecuteJob): Promise { * the transformed object makes that second call an idempotent no-op. */ const resultRestoreToken = egressGrantTokenForRestore; const finalizedSandboxResults = new WeakSet(); - const finalizeSandboxResult = async (result: SandboxRawResponse): Promise => { - if ( - resultRestoreToken === undefined || - resultRestoreToken.length === 0 || - finalizedSandboxResults.has(result) - ) { - return result; + const finalizeSandboxResult = async ( + result: SandboxRawResponse, + ): Promise => { + if (finalizedSandboxResults.has(result)) return result; + const restored = + resultRestoreToken == null || resultRestoreToken.length === 0 + ? result + : await restoreGatewaySandboxResult({ + grantId: egressGrantId, + egressGrantToken: resultRestoreToken, + result, + isSynthetic: isSyntheticJob, + signal: controller.signal, + }); + if (commitAtHandoff && cancellationTarget != null) { + // The bridge still owns its mutation fence here. A failed/ambiguous + // commit quarantines that root before it can serve a caller retry. + throwIfJobAborted(controller.signal); + const mapped = mapSandboxResult(restored); + const committed = await commitJobResult( + connection, + cancellationTarget, + mapped, + jobCancellationRetentionSeconds( + env.JOB_TIMEOUT, + job.data.cancellationTtlSeconds, + ), + deadlineAtMs, + ); + if (committed === 'cancelled') throw new Error(JOB_CANCELLED_MESSAGE); + if (committed === 'already_completed') + throw new Error('Duplicate mutation handoff; quarantining workspace'); + resultToCommit = mapped; + resultCommittedAtHandoff = true; } - const restored = await restoreGatewaySandboxResult({ - grantId: egressGrantId, - egressGrantToken: resultRestoreToken, - result, - isSynthetic: isSyntheticJob, - signal: controller.signal, - }); finalizedSandboxResults.add(restored); return restored; }; @@ -162,76 +272,115 @@ async function processJobInner(job: t.ExecuteJob): Promise { /* Stateful backends run this as a commit barrier after user code but * before checkpointing/reusing the mutated workspace. Stateless/HTTP * paths retain the worker-owned fallback immediately below. */ - sessionResultFinalizer: resultRestoreToken !== undefined && resultRestoreToken.length > 0 - ? finalizeSandboxResult - : undefined, + sessionResultFinalizer: + commitAtHandoff || + (resultRestoreToken !== undefined && resultRestoreToken.length > 0) + ? finalizeSandboxResult + : undefined, }, ); const responseData = await finalizeSandboxResult(responseRaw); + // Cancellation can arrive after sandbox exit while artifact restoration + // yields. Do not let BullMQ commit a success after Stop was acknowledged. + if (!resultCommittedAtHandoff) throwIfJobAborted(controller.signal); - if (!isSyntheticJob) { - logger.info('Sandbox response', summarizeSandboxResponse(responseData)); - } + function mapSandboxResult( + responseData: SandboxRawResponse, + ): t.ExecuteResult { + if (!isSyntheticJob) { + logger.info('Sandbox response', summarizeSandboxResponse(responseData)); + } - const { files } = responseData; - const run = responseData.run; - const stdout = applySystemReplacements(run?.stdout ?? ''); - const stderr = filterSystemLogs(run?.stderr ?? '', isPyPlot); + const { files } = responseData; + const run = responseData.run; + const stdout = applySystemReplacements(run?.stdout ?? ''); + const stderr = filterSystemLogs(run?.stderr ?? '', isPyPlot); - const result: t.ExecuteResult = { - session_id: responseData.session_id, - /* `files` is optional on the sandbox response (e.g. dry-run - * execute with no outputs); the public `ExecuteResult.files` is - * required and downstream callers always iterate it. Default to - * `[]` so the strictened response type from Phase B doesn't - * surface a regression that wasn't there before. */ - files: files ?? [], - ...(responseData.artifact_delivery != null - ? { artifact_delivery: responseData.artifact_delivery } - : {}), - ...(responseData.artifact_truncation != null - ? { artifact_truncation: responseData.artifact_truncation } - : {}), - stdout, - stderr, - ...(responseData.pending_tool_calls_payload != null - ? { pending_tool_calls_payload: responseData.pending_tool_calls_payload } - : {}), - }; + const result: t.ExecuteResult = { + session_id: responseData.session_id, + /* `files` is optional on the sandbox response (e.g. dry-run + * execute with no outputs); the public `ExecuteResult.files` is + * required and downstream callers always iterate it. Default to + * `[]` so the strictened response type from Phase B doesn't + * surface a regression that wasn't there before. */ + files: files ?? [], + ...(responseData.artifact_delivery != null + ? { artifact_delivery: responseData.artifact_delivery } + : {}), + ...(responseData.artifact_truncation != null + ? { artifact_truncation: responseData.artifact_truncation } + : {}), + stdout, + stderr, + ...(responseData.pending_tool_calls_payload != null + ? { + pending_tool_calls_payload: + responseData.pending_tool_calls_payload, + } + : {}), + }; - if (run) { - result.code = run.code ?? null; - result.signal = run.signal != null ? String(run.signal) : null; - result.message = run.message ?? null; - result.status = run.status ?? null; - result.wall_time = (run as Record).wall_time as number | null ?? null; - } + if (run) { + result.code = run.code ?? null; + result.signal = run.signal != null ? String(run.signal) : null; + result.message = run.message ?? null; + result.status = run.status ?? null; + result.wall_time = + ((run as Record).wall_time as number | null) ?? null; + } - if (result.message || result.signal) { - logger.warn('Sandbox execution error metadata', { - session_id: responseData.session_id, - code: result.code, - signal: result.signal, - message: summarizeText(result.message), - status: result.status, - wall_time: result.wall_time, - }); + if (result.message || result.signal) { + logger.warn('Sandbox execution error metadata', { + session_id: responseData.session_id, + code: result.code, + signal: result.signal, + message: summarizeText(result.message), + status: result.status, + wall_time: result.wall_time, + }); + } + + return result; } + const result = resultToCommit ?? mapSandboxResult(responseData); + completedResult = true; + resultToCommit = result; return result; } catch (error) { - revokeReason = controller.signal.aborted || isAbortError(error) ? 'timeout' : 'failed'; + // Bridge fence cleanup can fail after the outcome was durably committed. + // Preserve the winning result; the bridge retains/quarantines its fence. + if (resultCommittedAtHandoff && resultToCommit != null) + return resultToCommit; + const clientDisconnected = + controller.signal.aborted && + controller.signal.reason === CLIENT_DISCONNECT_REASON; + revokeReason = clientDisconnected + ? 'cancelled' + : controller.signal.aborted || isAbortError(error) + ? 'timeout' + : 'failed'; const errorDetails = getAxiosErrorDetails(error); - logger.error('Error processing job', errorDetails); + if (clientDisconnected) { + logger.info('Job cancelled after client disconnected', { + queueName: job.queueName, + jobId: job.id, + executionId: job.data.executionId, + }); + } else { + logger.error('Error processing job', errorDetails); + } const deadlineFailure = workerDeadlineFailure( error, - controller.signal.aborted, + controller.signal.aborted && !clientDisconnected, env.JOB_TIMEOUT, ); if (deadlineFailure) { throw deadlineFailure; + } else if (clientDisconnected) { + throw new Error(JOB_CANCELLED_MESSAGE); } else if (error instanceof SandboxBackendError) { throw new Error(`${error.code}: ${error.message}`); } else if (error instanceof SessionFilesError) { @@ -251,17 +400,67 @@ async function processJobInner(job: t.ExecuteJob): Promise { if (egressGrantId || egressGrantTokenForRestore) { await revokeGatewayEgressGrant({ grantId: egressGrantId, - egressGrantToken: egressGrantId ? undefined : egressGrantTokenForRestore, + egressGrantToken: egressGrantId + ? undefined + : egressGrantTokenForRestore, isSynthetic: isSyntheticJob, reason: revokeReason, timeoutMs: env.EGRESS_GATEWAY_REVOKE_TIMEOUT_MS, }).catch(error => { - logger.error('Failed to revoke egress grant', { grantId: egressGrantId, error: getAxiosErrorDetails(error) }); + logger.error('Failed to revoke egress grant', { + grantId: egressGrantId, + error: getAxiosErrorDetails(error), + }); }); } + let lateCommitFailure = + completedResult && !resultCommittedAtHandoff + ? jobResultCommitFailure(controller.signal, env.JOB_TIMEOUT) + : undefined; + if ( + completedResult && + !resultCommittedAtHandoff && + cancellationTarget != null && + lateCommitFailure == null + ) { + try { + const committed = await commitJobResult( + connection, + cancellationTarget, + resultToCommit, + jobCancellationRetentionSeconds( + env.JOB_TIMEOUT, + job.data.cancellationTtlSeconds, + ), + deadlineAtMs, + ); + if (committed === 'cancelled') { + lateCommitFailure = new Error(JOB_CANCELLED_MESSAGE); + } else if (committed === 'already_completed') { + lateCommitFailure = new Error( + 'Duplicate result handoff; refusing replacement', + ); + } + } catch (error) { + lateCommitFailure = + error instanceof Error ? error : new Error('Result commit failed'); + } + } if (timer) clearTimeout(timer); + if (cancellationTarget != null && cancellationRegistered) { + await jobCancellationRegistry + .unregister(cancellationTarget, controller) + .catch(error => { + logger.warn('Failed to clear queued execution cancellation state', { + queueName: cancellationTarget.queueName, + jobId: cancellationTarget.jobId, + error: getAxiosErrorDetails(error), + }); + }); + } endTimer(); activeJobs.dec({ language }); + if (lateCommitFailure != null) throw lateCommitFailure; } } @@ -304,21 +503,31 @@ otherWorker.on('completed', job => { }); pyWorker.on('failed', (job, err) => { + if (err.message === JOB_CANCELLED_MESSAGE) { + logger.info(`[${WORKER_ID}] Python job ${job?.id} cancelled`); + jobsCancelled.inc({ language: 'python' }); + return; + } logger.error(`[${WORKER_ID}] Python job ${job?.id} failed`, err); jobsFailed.inc({ language: 'python' }); }); otherWorker.on('failed', (job, err) => { + if (err.message === JOB_CANCELLED_MESSAGE) { + logger.info(`[${WORKER_ID}] Other job ${job?.id} cancelled`); + jobsCancelled.inc({ language: 'other' }); + return; + } logger.error(`[${WORKER_ID}] Other job ${job?.id} failed`, err); jobsFailed.inc({ language: 'other' }); }); -pyWorker.on('error', (err) => { +pyWorker.on('error', err => { logger.error(`[${WORKER_ID}] Python worker error`, err); workerRunning.set({ worker_type: 'python' }, 0); }); -otherWorker.on('error', (err) => { +otherWorker.on('error', err => { logger.error(`[${WORKER_ID}] Other worker error`, err); workerRunning.set({ worker_type: 'other' }, 0); }); From 3e7b107c74960ab42197fe01d2b05a1709e4b762 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:43:49 -0400 Subject: [PATCH 19/42] fix: classify capped artifact probe candidates (#206) --- api/src/job.ts | 56 +++++++++++++++++++++++++++++++++--------- api/src/walker.test.ts | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 12 deletions(-) diff --git a/api/src/job.ts b/api/src/job.ts index 27604d29..a5b7ded2 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -2208,7 +2208,7 @@ export class Job { state = this.truncationProbeState, probeDepth = 0, rootPath = path.relative(this.submissionDir, dir) || '.', - respectSessionSuppression = false, + isOutputCapProbe = false, ): Promise { /* The state is shared by every probe in this job. Once exhausted, return * conservatively before opening yet another capped sibling directory. */ @@ -2246,17 +2246,51 @@ export class Job { if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; const existingFile = inputByName.get(relativePath); const inputFileInfo = this.inputFileHashes.get(relativePath); + let capProbeStat: fs.Stats | undefined; + if (isOutputCapProbe) { + const pathShapeError = checkPathShape(relativePath); + if (pathShapeError) { + this.recordArtifactTruncation( + pathShapeError.includes('nesting depth') ? 'depth' : 'path', + relativePath, + ); + continue; + } + try { + capProbeStat = await fsp.lstat(fullPath); + if (!capProbeStat.isFile()) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during cap-probe stat'); + this.recordArtifactTruncation('unreadable', relativePath); + continue; + } + if (capProbeStat.size > this.runtime.max_file_size) { + /* Match handleRegularFile's one exception: an unchanged inline + * entrypoint is request input rather than an oversized output. */ + if (!inputFileInfo || existingFile?.id != null || relativePath !== this.entryPointName) { + this.recordArtifactTruncation('size', relativePath); + continue; + } + if (capProbeStat.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= capProbeStat.size; + try { + if (await this.computeFileHash(fullPath, true) === inputFileInfo.hash) continue; + } catch (err) { + this.log.debug({ path: relativePath, err }, 'walkDir: failed during oversized entrypoint cap probe'); + } + this.recordArtifactTruncation('size', relativePath); + continue; + } + } if ( - respectSessionSuppression + isOutputCapProbe && relativePath === this.entryPointName && existingFile?.id == null && inputFileInfo ) { try { - const st = await fsp.lstat(fullPath); - if (!st.isFile()) continue; - if (st.size > state.remainingHashBytes) return rootPath; - state.remainingHashBytes -= st.size; + if (capProbeStat!.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= capProbeStat!.size; if (await this.computeFileHash(fullPath, true) === inputFileInfo.hash) continue; } catch (err) { this.log.debug({ path: relativePath, err }, 'walkDir: failed during entrypoint cap probe'); @@ -2270,13 +2304,11 @@ export class Job { * so the bounded cap probe must do the same or it reports a false * max_files warning. Current-request inputs remain reportable: they * would otherwise have been echoed into this response. */ - if (respectSessionSuppression && this.session && !existingFile) { + if (isOutputCapProbe && this.session && !existingFile) { if (this.session.isPrimedReadOnly(relativePath)) continue; try { - const st = await fsp.lstat(fullPath); - if (!st.isFile()) continue; - if (st.size > state.remainingHashBytes) return rootPath; - state.remainingHashBytes -= st.size; + if (capProbeStat!.size > state.remainingHashBytes) return rootPath; + state.remainingHashBytes -= capProbeStat!.size; const hash = await this.computeFileHash(fullPath, true); if (this.session.isSurfaced(relativePath, hash)) continue; if ( @@ -2303,7 +2335,7 @@ export class Job { state, probeDepth + 1, rootPath, - respectSessionSuppression, + isOutputCapProbe, ); if (nested) return nested; } diff --git a/api/src/walker.test.ts b/api/src/walker.test.ts index 543505a9..ba6f4af3 100644 --- a/api/src/walker.test.ts +++ b/api/src/walker.test.ts @@ -1021,6 +1021,49 @@ describe('walkDir / artifact truncation details', () => { }); }); + it('classifies an oversized supported file by size when the output cap is full', async () => { + await fsp.writeFile(path.join(tmpDir, 'oversized.txt'), 'too large'); + const internals = asInternals(makeJob({ maxFileSize: 3 })); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { size: 1 }, + skipped: ['oversized.txt'], + skipped_count: 1, + }); + }); + + it('classifies an overlong supported path by path when the output cap is full', async () => { + const directory = 'a'.repeat(200); + const filename = path.join(directory, `${'b'.repeat(60)}.txt`); + await fsp.mkdir(path.join(tmpDir, directory)); + await fsp.writeFile(path.join(tmpDir, filename), 'output'); + const internals = asInternals(makeJob()); + internals.submissionDir = tmpDir; + internals.generatedFiles = Array.from({ length: config.max_output_files }, (_, i) => ({ + id: `id-${i}`, + name: `file-${i}.txt`, + path: path.join(tmpDir, `file-${i}.txt`), + })); + + await internals.walkDir(tmpDir, 0, new Map()); + + expect(internals.artifactTruncation).toEqual({ + code: 'artifact_truncated', + reasons: { path: 1 }, + skipped: [filename], + skipped_count: 1, + }); + }); + it('does not hash ordinary oversized files in session mode', async () => { await fsp.writeFile(path.join(tmpDir, 'large.txt'), 'too large'); const session = new SessionWorkspace({ runtimeSessionId: 'rt_large' }); From 6ca38b23fd0b40eab66d82b0d8783e9168a092f9 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:44:01 -0400 Subject: [PATCH 20/42] feat: Report Deleted Code Session Files (#200) * fix: report deleted persisted files * fix: reconcile deletions across code runtimes * fix: preserve protected session inputs * fix: classify reserved runtime paths --- api/README.md | 7 + api/src/job.ts | 57 ++++- api/src/session-workspace.test.ts | 5 + api/src/session-workspace.ts | 6 + api/src/walker.test.ts | 196 ++++++++++++++++++ packages/code/src/native-programmatic.test.ts | 112 ++++++++++ packages/code/src/native-programmatic.ts | 44 +++- service/src/service/blocking-poll.test.ts | 2 + service/src/service/blocking-poll.ts | 2 + service/src/service/programmatic-router.ts | 4 + service/src/types/service.ts | 3 + service/src/workers.ts | 3 + 12 files changed, 424 insertions(+), 17 deletions(-) diff --git a/api/README.md b/api/README.md index 665a2931..38e4cd98 100644 --- a/api/README.md +++ b/api/README.md @@ -94,6 +94,13 @@ Other package-format-compatible runtimes (Go, Rust, Java, GCC) can be installed Execute code in a sandboxed environment. +When a persisted input file is removed during execution, a complete artifact +scan reports its relative path in `deleted_files`. Callers can use this +explicit list to remove stale file references from their next session request. +The field is omitted when no persisted inputs were removed or when artifact +scanning is incomplete, so truncation or unreadable paths cannot be mistaken +for deletions. + When supported output files are omitted because the response reaches its file count limit, nesting or path limits, file-size limit, or a filesystem entry cannot be read, the response includes `artifact_truncation`. Its `reasons` diff --git a/api/src/job.ts b/api/src/job.ts index a5b7ded2..0939de08 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -655,6 +655,8 @@ interface ExecuteResult { /** Top-level execution session id (one sandbox `/exec` invocation). */ session_id: string; files: FileRef[]; + /** Persisted input paths that no longer exist after this execution. */ + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; } @@ -714,6 +716,8 @@ export class Job { private pendingSurfaced = new Map(); private sessionFiles: FileRef[] = []; private inheritedRefs: FileRef[] = []; + private presentInputFiles = new Set(); + private deletedFiles: string[] = []; private artifactTruncation: ArtifactTruncation | undefined; private truncationProbeState: TruncationProbeState = { remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, @@ -1699,6 +1703,9 @@ export class Job { version: this.runtime.version.raw, session_id: this.outputSessionId, files: this.sessionFiles, + ...(this.deletedFiles.length > 0 + ? { deleted_files: this.deletedFiles } + : {}), ...(this.artifactTruncation ? { artifact_truncation: this.artifactTruncation } : {}), }; } @@ -1707,6 +1714,8 @@ export class Job { this.generatedFiles = []; this.sessionFiles = []; this.inheritedRefs = []; + this.presentInputFiles.clear(); + this.deletedFiles = []; this.artifactTruncation = undefined; this.truncationProbeState = { remainingEntries: TRUNCATION_PROBE_MAX_ENTRIES, @@ -1720,6 +1729,26 @@ export class Job { await this.walkDir(this.submissionDir, 0, inputByName); } catch (error) { this.log.error({ err: error }, 'Error scanning submission directory'); + this.recordArtifactTruncation('unreadable', '.'); + } + + if (this.artifactTruncation == null) { + const returnedNames = new Set([ + ...this.sessionFiles.map(file => file.name), + ...this.inheritedRefs.map(file => file.name), + ]); + for (const file of this.files) { + if ( + file.id != null && + file.storage_session_id != null && + this.inputFileHashes.get(file.name)?.readOnly !== true && + !this.presentInputFiles.has(file.name) && + !returnedNames.has(file.name) + ) { + this.deletedFiles.push(file.name); + this.session?.forgetPrimed(file.name); + } + } } /* Generated files get priority in sessionFiles; fill remaining slots up @@ -2227,13 +2256,18 @@ export class Job { let sawVisibleNonHiddenEntry = false; try { for await (const entry of directory) { - if (entry.name === PTC_HISTORY_FILENAME) continue; - sawVisibleEntry = true; - state.remainingEntries--; - if (state.remainingEntries < 0) return rootPath; const fullPath = path.join(dir, entry.name); const relativePath = path.relative(this.submissionDir, fullPath); const kind = await this.classifyDirent(entry, fullPath, relativePath); + if (kind === 'file' && entry.name === PTC_HISTORY_FILENAME) { + if (inputByName.has(relativePath)) { + this.presentInputFiles.add(relativePath); + } + continue; + } + sawVisibleEntry = true; + state.remainingEntries--; + if (state.remainingEntries < 0) return rootPath; if (kind === 'skip') { /* Ordinary walking counts symlinks/special entries as non-empty even * though it does not surface them, so the probe must not invent a @@ -2243,6 +2277,9 @@ export class Job { } if (kind === 'file') { sawVisibleNonHiddenEntry = true; + if (inputByName.has(relativePath)) { + this.presentInputFiles.add(relativePath); + } if (entry.name !== DIRKEEP && !isSupportedOutputFilename(entry.name)) continue; const existingFile = inputByName.get(relativePath); const inputFileInfo = this.inputFileHashes.get(relativePath); @@ -2423,13 +2460,21 @@ export class Job { let skippedHiddenDirs = 0; for (const entry of entries) { - if (isPtcReserved(entry.name)) continue; - const fullPath = path.join(dir, entry.name); const relativePath = path.relative(this.submissionDir, fullPath); const kind = await this.classifyDirent(entry, fullPath, relativePath); if (kind === 'skip') continue; + if (kind === 'file' && inputByName.has(relativePath)) { + this.presentInputFiles.add(relativePath); + } + + /* A by-reference input may legitimately use the reserved replay-history + * basename on the ordinary execution endpoint. It remains hidden from + * output collection, but must be observed before the runtime fixture is + * skipped so an untouched input is not reported as deleted. */ + if (kind === 'file' && isPtcReserved(entry.name)) continue; + if (kind === 'dir') { /* Skip hidden directories (basename starts with `.`) unless the user * explicitly primed something under them. Matplotlib, pip, and other diff --git a/api/src/session-workspace.test.ts b/api/src/session-workspace.test.ts index 87ca71df..1a5320c6 100644 --- a/api/src/session-workspace.test.ts +++ b/api/src/session-workspace.test.ts @@ -113,6 +113,11 @@ describe('SessionWorkspace state', () => { expect(ws.isPrimedInput('in.csv')).toBe(false); ws.markPrimed('in.csv', 'file_abc'); expect(ws.primedInputId('in.csv')).toBe('file_abc'); + ws.markSurfaced('in.csv', 'old-output'); + ws.forgetPrimed('in.csv'); + expect(ws.primedInputId('in.csv')).toBeUndefined(); + expect(ws.isSurfaced('in.csv', 'old-output')).toBe(false); + ws.markPrimed('in.csv', 'file_abc'); /* read-only primes report as not-primed so the caller re-downloads them * (a reused on-disk copy could have been tampered via the writable dir). */ diff --git a/api/src/session-workspace.ts b/api/src/session-workspace.ts index 3f2105bc..82ed6c89 100644 --- a/api/src/session-workspace.ts +++ b/api/src/session-workspace.ts @@ -202,6 +202,12 @@ export class SessionWorkspace { this.primed.set(relPath, { id: storageFileId, readOnly, hash }); } + /** Clears input lineage after execution proves that the path was deleted. */ + forgetPrimed(relPath: string): void { + this.primed.delete(relPath); + this.forget(relPath); + } + markDirty(reason: string): void { this.dirty = reason; logger.error( diff --git a/api/src/walker.test.ts b/api/src/walker.test.ts index ba6f4af3..44e252a6 100644 --- a/api/src/walker.test.ts +++ b/api/src/walker.test.ts @@ -26,6 +26,8 @@ interface WalkerInternals { generatedFiles: Array<{ id: string; name: string; path: string }>; sessionFiles: Array<{ id: string; name: string; storage_session_id: string; modified_from?: { id: string; storage_session_id: string }; inherited?: true; entity_id?: string }>; inheritedRefs: Array<{ id: string; name: string; storage_session_id: string; inherited?: true; entity_id?: string }>; + presentInputFiles: Set; + deletedFiles: string[]; artifactTruncation?: { code: 'artifact_truncated'; reasons: Partial>; @@ -39,6 +41,14 @@ interface WalkerInternals { reusePrimedInput: (file: TFile) => Promise; writeFile: (file: TFile) => Promise; computeFileHash: (filePath: string, noFollow?: boolean) => Promise; + findTruncatedArtifact: ( + dir: string, + inputByName: Map, + state?: { remainingEntries: number; remainingHashBytes: number }, + probeDepth?: number, + rootPath?: string, + respectSessionSuppression?: boolean, + ) => Promise; walkDir: (dir: string, depth: number, inputByName: Map) => Promise<'collected' | 'empty' | 'skipped'>; handleSessionFiles: () => Promise; } @@ -1300,6 +1310,192 @@ describe('handleSessionFiles / priority-fill composition', () => { }); }); +describe('handleSessionFiles / persisted input deletion', () => { + it('reports a persisted input that no longer exists', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'removed.txt', + }; + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual(['removed.txt']); + }); + + it('retains a read-only persisted input when sandbox code removes its local copy', async () => { + const inherited: TFile = { + id: 'skill-id', + storage_session_id: 'skill-session', + name: path.join('skills', 'review', 'SKILL.md'), + }; + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + internals.inputFileHashes.set(inherited.name, { + hash: sha256('trusted-skill'), + path: path.join(tmpDir, inherited.name), + originalId: inherited.id, + originalSessionId: inherited.storage_session_id, + readOnly: true, + }); + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + }); + + it('tracks a persisted input using the reserved PTC history basename', async () => { + const inherited: TFile = { + id: 'history-id', + storage_session_id: 'prior-session', + name: path.join('fixtures', '_ptc_history.json'), + }; + await fsp.mkdir(path.join(tmpDir, 'fixtures')); + await fsp.writeFile(path.join(tmpDir, inherited.name), '{}'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + expect(internals.generatedFiles.map(file => file.name)).not.toContain(inherited.name); + }); + + it('traverses a directory that uses the reserved PTC history basename', async () => { + const inherited: TFile = { + id: 'nested-id', + storage_session_id: 'prior-session', + name: path.join('_ptc_history.json', 'data.csv'), + }; + await fsp.mkdir(path.join(tmpDir, '_ptc_history.json')); + await fsp.writeFile(path.join(tmpDir, inherited.name), 'persisted'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + expect(internals.presentInputFiles.has(inherited.name)).toBe(true); + }); + + it('tracks a reserved persisted input during a capped subtree probe', async () => { + const inherited: TFile = { + id: 'history-id', + storage_session_id: 'prior-session', + name: path.join('fixtures', '_ptc_history.json'), + }; + const fixtures = path.join(tmpDir, 'fixtures'); + await fsp.mkdir(fixtures); + await fsp.writeFile(path.join(tmpDir, inherited.name), '{}'); + await fsp.writeFile(path.join(fixtures, 'unsupported.bin'), 'binary'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + const skipped = await internals.findTruncatedArtifact( + fixtures, + new Map([[inherited.name, inherited]]) + ); + + expect(skipped).toBeUndefined(); + expect(internals.presentInputFiles.has(inherited.name)).toBe(true); + }); + + it('does not report a surviving input that is unsupported as an output artifact', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'archive.bin', + }; + await fsp.writeFile(path.join(tmpDir, inherited.name), 'binary-placeholder'); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.generatedFiles).toHaveLength(0); + expect(internals.deletedFiles).toEqual([]); + }); + + it('suppresses deletion reporting when the artifact scan is incomplete', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'removed.txt', + }; + await fsp.writeFile(path.join(tmpDir, 'too-large.txt'), 'too large'); + const internals = asInternals(makeJob({ files: [inherited], maxFileSize: 3 })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.artifactTruncation?.reasons).toEqual({ size: 1 }); + expect(internals.deletedFiles).toEqual([]); + }); + + it('does not report an inherited marker that is returned for an empty directory', async () => { + const name = path.join('empty', DIRKEEP); + const inherited: TFile = { + id: 'marker-id', + storage_session_id: 'prior-session', + name, + }; + await fsp.mkdir(path.join(tmpDir, 'empty')); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([]); + expect([ + ...internals.sessionFiles, + ...internals.inheritedRefs, + ].map(file => file.name)).toContain(name); + }); + + it('clears stateful priming lineage when a persisted input is deleted', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: 'removed.txt', + }; + const session = new SessionWorkspace({ runtimeSessionId: 'rt_deleted' }); + session.markPrimed(inherited.name, inherited.id!, true, 'old-hash'); + session.markSurfaced(inherited.name, 'old-output-hash'); + const internals = asInternals(makeJob({ files: [inherited], session })); + internals.submissionDir = tmpDir; + + await internals.handleSessionFiles(); + + expect(internals.deletedFiles).toEqual([inherited.name]); + expect(session.isPrimedInput(inherited.name)).toBe(false); + expect(session.isSurfaced(inherited.name, 'old-output-hash')).toBe(false); + }); + + it('tracks surviving persisted inputs during capped subtree probes', async () => { + const inherited: TFile = { + id: 'prior-id', + storage_session_id: 'prior-session', + name: path.join('assets', 'model.bin'), + }; + await fsp.mkdir(path.join(tmpDir, 'assets')); + await fsp.writeFile( + path.join(tmpDir, inherited.name), + 'unsupported-but-persisted', + ); + const internals = asInternals(makeJob({ files: [inherited] })); + internals.submissionDir = tmpDir; + + await internals.findTruncatedArtifact( + tmpDir, + new Map([[inherited.name, inherited]]), + ); + + expect(internals.presentInputFiles.has(inherited.name)).toBe(true); + }); +}); + describe('walkDir / dirent classification', () => { it('ignores symlinks (never classifies them as file or dir)', async () => { await fsp.writeFile(path.join(tmpDir, 'real.py'), 'print(1)'); diff --git a/packages/code/src/native-programmatic.test.ts b/packages/code/src/native-programmatic.test.ts index 82650f95..902c707a 100644 --- a/packages/code/src/native-programmatic.test.ts +++ b/packages/code/src/native-programmatic.test.ts @@ -121,6 +121,118 @@ test('stages skill files privately and returns generated artifacts', async () => } }); +test('reports persisted inputs deleted by selected-workspace execution', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-delete-test-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const server = createServer((_req, res) => res.end('persisted input')); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(() => resolve()))); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + await rm(join(dataDirectory, 'input.txt')); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + + const result = await executor.execute({ + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'execution-session', + egress_grant: 'grant', + files: [ + { name: 'main.sh', content: 'rm input.txt' }, + { + name: 'input.txt', + id: 'input-id', + storage_session_id: 'input-session', + }, + ], + }, + }, 'primary'); + + assert.deepEqual(result.files, []); + assert.deepEqual(result.deleted_files, ['input.txt']); +}); + +test('retains read-only persisted inputs removed by selected-workspace execution', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-readonly-delete-test-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + let downloads = 0; + const server = createServer((_req, res) => { + downloads++; + res.setHeader('X-Read-Only', 'true'); + res.end('trusted skill'); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(() => resolve()))); + const address = server.address() as AddressInfo; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: `http://127.0.0.1:${address.port}`, + sandbox: { + async createExecutionDirectory() { + return await mkdtemp(join(scratch, 'execution-')); + }, + async executeProgrammatic(request, dataDirectory) { + await rm(join(dataDirectory, 'skills', 'review', 'SKILL.md')); + return { + protocolVersion: 1, + operation: 'execute_command' as const, + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + }, + }); + const request = { + headers: {}, + body: { + language: 'bash' as const, + version: '5.2.0', + execution_id: 'readonly-execution', + session_id: 'execution-session', + egress_grant: 'grant', + files: [ + { name: 'main.sh', content: 'rm skills/review/SKILL.md' }, + { + name: 'skills/review/SKILL.md', + id: 'skill-id', + storage_session_id: 'skill-session', + input_cache_key: 'a'.repeat(64), + }, + ], + }, + }; + + const result = await executor.execute(request, 'primary'); + const replay = await executor.execute(request, 'primary'); + + assert.equal(downloads, 1); + assert.equal(result.deleted_files, undefined); + assert.equal(replay.deleted_files, undefined); +}); + test('reports unsupported and rejected artifacts without invalidating a completed command', async () => { const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-artifact-test-')); const uploads = new Map(); diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts index 42974b79..bd2700f2 100644 --- a/packages/code/src/native-programmatic.ts +++ b/packages/code/src/native-programmatic.ts @@ -50,6 +50,7 @@ type ProgrammaticResult = { version: string; session_id: string; files: ProgrammaticFileResult[]; + deleted_files?: string[]; artifact_delivery?: { code: 'artifact_delivery_failed'; status: 'partial' | 'failed'; @@ -75,8 +76,11 @@ type ProgrammaticResult = { type InputBaseline = { sha256: string; source?: { id: string; storage_session_id: string }; + readOnly?: boolean; }; +type CachedInput = { bytes: Buffer; readOnly: boolean }; + function sha256(value: Uint8Array): string { return createHash('sha256').update(value).digest('hex'); } @@ -227,7 +231,7 @@ export class NativeWorkspaceProgrammaticExecutor { /** Parent-process cache: sandboxed children cannot inspect this memory. */ private readonly inputCache = new Map< string, - { bytes: Buffer; lastUsed: number } + CachedInput & { lastUsed: number } >(); private inputCacheBytes = 0; @@ -272,14 +276,15 @@ export class NativeWorkspaceProgrammaticExecutor { : undefined; } - private cachedInput(key: string): Buffer | undefined { + private cachedInput(key: string): CachedInput | undefined { const cached = this.inputCache.get(key); if (!cached) return undefined; cached.lastUsed = Date.now(); - return cached.bytes; + return { bytes: cached.bytes, readOnly: cached.readOnly }; } - private cacheInput(key: string, bytes: Buffer): void { + private cacheInput(key: string, input: CachedInput): void { + const { bytes } = input; if (bytes.byteLength > INPUT_CACHE_MAX_BYTES) return; const existing = this.inputCache.get(key); if (existing) this.inputCacheBytes -= existing.bytes.byteLength; @@ -300,7 +305,7 @@ export class NativeWorkspaceProgrammaticExecutor { this.inputCache.get(oldestKey)!.bytes.byteLength; this.inputCache.delete(oldestKey); } - this.inputCache.set(key, { bytes, lastUsed: Date.now() }); + this.inputCache.set(key, { ...input, lastUsed: Date.now() }); this.inputCacheBytes += bytes.byteLength; } @@ -310,7 +315,7 @@ export class NativeWorkspaceProgrammaticExecutor { executionId: string | undefined, signal?: AbortSignal, transferTimeoutMs = TRANSFER_TIMEOUT_MS, - ): Promise { + ): Promise { const key = this.cacheKey(executionId, file); const cached = key ? this.cachedInput(key) : undefined; if (cached) return cached; @@ -341,8 +346,12 @@ export class NativeWorkspaceProgrammaticExecutor { response, controller.signal, ); - if (key) this.cacheInput(key, bytes); - return bytes; + const input = { + bytes, + readOnly: response.headers.get('x-read-only')?.toLowerCase() === 'true', + }; + if (key) this.cacheInput(key, input); + return input; } finally { clearTimeout(timer); signal?.removeEventListener('abort', abort); @@ -393,9 +402,9 @@ export class NativeWorkspaceProgrammaticExecutor { request.body.files, TRANSFER_CONCURRENCY, async (file): Promise => { - const bytes = + const input = 'content' in file - ? Buffer.from(file.content) + ? { bytes: Buffer.from(file.content), readOnly: false } : await this.downloadInput( file, grant!, @@ -403,6 +412,7 @@ export class NativeWorkspaceProgrammaticExecutor { signal, request.body.transfer_timeout_ms, ); + const { bytes } = input; totalInputBytes += bytes.byteLength; if ( totalInputBytes > @@ -421,6 +431,7 @@ export class NativeWorkspaceProgrammaticExecutor { await writeFile(path, bytes, { flag: 'wx', mode: 0o600 }); baselines.set(file.name, { sha256: sha256(bytes), + ...(input.readOnly ? { readOnly: true } : {}), ...('id' in file ? { source: { @@ -578,7 +589,15 @@ export class NativeWorkspaceProgrammaticExecutor { } const outputSessionId = request.body.output_session_id; - const outputNames = (await listRegularFiles(dataDirectory)).filter( + const survivingNames = new Set(await listRegularFiles(dataDirectory)); + const deletedFiles = refFiles + .filter( + file => + baselines.get(file.name)?.readOnly !== true && + !survivingNames.has(file.name), + ) + .map(file => file.name); + const outputNames = [...survivingNames].filter( name => name !== EXECUTION_MAIN_FILE && name !== EXECUTION_HISTORY_FILE && @@ -731,6 +750,7 @@ export class NativeWorkspaceProgrammaticExecutor { performance.now() - startedAt, undefined, artifactDelivery, + deletedFiles, ); } catch (error) { if (!commandDispatched) { @@ -785,6 +805,7 @@ export class NativeWorkspaceProgrammaticExecutor { elapsedMs: number, pendingToolCallsPayload?: string, artifactDelivery?: ProgrammaticResult['artifact_delivery'], + deletedFiles: string[] = [], ): ProgrammaticResult { return { language: 'bash', @@ -795,6 +816,7 @@ export class NativeWorkspaceProgrammaticExecutor { session_id: request.body.output_session_id ?? request.body.session_id, files, + ...(deletedFiles.length > 0 ? { deleted_files: deletedFiles } : {}), ...(artifactDelivery ? { artifact_delivery: artifactDelivery } : {}), ...(pendingToolCallsPayload ? { pending_tool_calls_payload: pendingToolCallsPayload } diff --git a/service/src/service/blocking-poll.test.ts b/service/src/service/blocking-poll.test.ts index 35190157..96903d1f 100644 --- a/service/src/service/blocking-poll.test.ts +++ b/service/src/service/blocking-poll.test.ts @@ -4,6 +4,7 @@ import type * as t from '../types'; const result: t.ExecuteResult = { session_id: 'session', stdout: 'successful code', stderr: '', files: [], + deleted_files: ['removed.txt'], artifact_delivery: { code: 'artifact_delivery_failed', status: 'failed', attempted: 1, delivered: 0, failed: 1, }, @@ -29,6 +30,7 @@ describe('blocking worker settlement', () => { const deps = fixture(); expect(await pollBlockingExecution('exec', 5, deps)).toEqual({ status: 'completed', stdout: result.stdout, stderr: '', files: [], + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, }); diff --git a/service/src/service/blocking-poll.ts b/service/src/service/blocking-poll.ts index 03e386d4..1505f818 100644 --- a/service/src/service/blocking-poll.ts +++ b/service/src/service/blocking-poll.ts @@ -33,6 +33,7 @@ export async function pollBlockingExecution( stdout?: string; stderr?: string; files?: t.FileRefs; + deleted_files?: string[]; artifact_delivery?: t.ArtifactDeliveryFailure; artifact_truncation?: t.ArtifactTruncation; }> { @@ -48,6 +49,7 @@ export async function pollBlockingExecution( stdout: result.stdout, stderr: result.stderr, files: result.files, + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, }; diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index bb140013..1063fbfe 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -1165,6 +1165,7 @@ async function runAndRespond( stdout: cleanStdout, stderr: result.stderr, files: result.files, + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, session_id: state.session_id, @@ -1179,6 +1180,7 @@ async function runAndRespond( stdout: cleanStdout, stderr: result.stderr, files: result.files, + deleted_files: result.deleted_files, artifact_delivery: result.artifact_delivery, artifact_truncation: result.artifact_truncation, session_id: state.session_id, @@ -1571,6 +1573,7 @@ async function handleBlocking( stdout: state.stdout ?? '', stderr: state.stderr ?? '', files: state.files ?? [], + deleted_files: state.deleted_files, artifact_delivery: state.artifact_delivery, artifact_truncation: state.artifact_truncation, session_id: execution.session_id, @@ -1874,6 +1877,7 @@ async function handleBlocking( stdout: state.stdout ?? '', stderr: state.stderr ?? '', files: state.files ?? [], + deleted_files: state.deleted_files, artifact_delivery: state.artifact_delivery, artifact_truncation: state.artifact_truncation, session_id, diff --git a/service/src/types/service.ts b/service/src/types/service.ts index a0c78486..0404ad16 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -141,6 +141,7 @@ export type ExecuteResponse = { /** Top-level execution session id (one sandbox `/exec` invocation). */ session_id: string; files: FileRefs; + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; }; @@ -259,6 +260,7 @@ export type ExecuteResult = { stdout: string; stderr: string; files: FileRefs; + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; code?: number | null; @@ -408,6 +410,7 @@ export interface ProgrammaticResponse { stdout?: string; stderr?: string; files?: FileRefs; + deleted_files?: string[]; artifact_delivery?: ArtifactDeliveryFailure; artifact_truncation?: ArtifactTruncation; /** Top-level execution session id (one sandbox PTC invocation). */ diff --git a/service/src/workers.ts b/service/src/workers.ts index a7153b53..dbfd544b 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -305,6 +305,9 @@ async function processJobInner(job: t.ExecuteJob): Promise { * `[]` so the strictened response type from Phase B doesn't * surface a regression that wasn't there before. */ files: files ?? [], + ...(responseData.deleted_files != null + ? { deleted_files: responseData.deleted_files } + : {}), ...(responseData.artifact_delivery != null ? { artifact_delivery: responseData.artifact_delivery } : {}), From 926569e38975acc39ff202ef5800c83e8e0bc464 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:44:10 -0400 Subject: [PATCH 21/42] fix: preserve trusted jq path for PTC (#207) --- packages/code/src/native-process.test.ts | 14 ++++++++++-- packages/code/src/native-process.ts | 11 +++++----- packages/code/src/native-sandbox.ts | 5 +++++ service/src/preamble-bash.test.ts | 8 +++++++ service/src/preamble-bash.ts | 27 ++++++++++++------------ 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index e96f1e65..2f38d642 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -3,7 +3,7 @@ import { EventEmitter } from 'node:events'; import test from 'node:test'; import { mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import type { ChildProcess, ForkOptions } from 'node:child_process'; import { NativeProcessWorkspaceCommandSandbox, @@ -199,6 +199,7 @@ test('programmatic executor resolves and scopes credentials to its command', asy { workspaceRoot: tmpdir(), programmaticFileUpstream: 'http://127.0.0.1:3190', + environment: { PATH: '/sandbox-only' }, maskedEnvironment: { variables: [{ name: 'TOKEN', injectHosts: ['github.com'] }], async resolve() { @@ -225,9 +226,18 @@ test('programmatic executor resolves and scopes credentials to its command', asy JSON.stringify(fake.options).includes('per-programmatic-secret'), false, ); - const message = fake.messages.find( + const message = fake.messages.find( candidate => candidate.type === 'programmatic', )!; + const prepareMessage = fake.messages.find( + candidate => candidate.type === 'prepare', + )!; + assert.equal(typeof prepareMessage.options.jqPath, 'string'); + assert.equal(prepareMessage.options.jqPath.startsWith('/'), true); + assert.equal( + '/sandbox-only'.split(':').includes(dirname(prepareMessage.options.jqPath)), + false, + ); assert.deepEqual(message.credentials, { TOKEN: 'per-programmatic-secret' }); assert.equal( message.wrappedCommand, diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index d2ea0d72..4ed3ffec 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -61,7 +61,7 @@ export async function trustedProgrammaticExecutable(candidate: string, workspace async function resolveProgrammaticShell( options: NativeProcessSandboxOptions, -): Promise { +): Promise<{ shellPath: string; jqPath: string }> { const environment = options.environment ?? process.env; const shellPath = options.shellPath != null @@ -99,7 +99,7 @@ async function resolveProgrammaticShell( 'COMMAND_UNAVAILABLE', ); } - return shellPath; + return { shellPath, jqPath }; } /** Only OS discovery and conventional proxy settings cross into the executor. @@ -211,9 +211,9 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan } private async start(): Promise { - const programmaticShellPath = this.options.programmaticFileUpstream + const programmaticExecutables = this.options.programmaticFileUpstream ? await resolveProgrammaticShell(this.options) - : this.options.shellPath; + : undefined; const child = this.forkExecutor( new URL('./native-process-child.js', import.meta.url), [], @@ -299,7 +299,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan protectedPaths, allowedDomains, homeDirectory, - shellPath: programmaticShellPath ?? shellPath, + shellPath: programmaticExecutables?.shellPath ?? shellPath, + jqPath: programmaticExecutables?.jqPath, programmaticFileUpstream, variables: this.options.maskedEnvironment?.variables, }, diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 8d150d9f..91e9832d 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -169,6 +169,8 @@ export interface NativeSrtWorkspaceCommandSandboxOptions { platform?: NodeJS.Platform; /** Trusted shell path used by SRT on POSIX hosts. */ shellPath?: string; + /** Trusted jq path used by generated programmatic scripts. */ + jqPath?: string; /** Host-owned credentials exposed only as SRT sentinels inside the sandbox. */ maskedEnvironment?: { variables: Array<{ @@ -702,6 +704,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox '_ptc_pending_result.json', ), LIBRECHAT_CODE_BASH_PATH: this.options.shellPath ?? '/bin/bash', + ...(this.options.jqPath + ? { LIBRECHAT_CODE_JQ_PATH: this.options.jqPath } + : {}), PTC_HISTORY_PATH: join( canonicalDataDirectory, '_ptc_history.json', diff --git a/service/src/preamble-bash.test.ts b/service/src/preamble-bash.test.ts index e35a77f5..e110abdc 100644 --- a/service/src/preamble-bash.test.ts +++ b/service/src/preamble-bash.test.ts @@ -186,6 +186,14 @@ printf '%s\\n' "$_PTC_PENDING_FILE" "$_PTC_ERROR_FILE" "$_PTC_COUNTER_FILE" rmSync(dir, { recursive: true, force: true }); } }); + + test('uses the trusted jq path instead of resolving jq through PATH', () => { + const preamble = generateBashReplayPreamble({ executionId, tools }); + expect(preamble).toContain( + '_PTC_JQ_PATH="${LIBRECHAT_CODE_JQ_PATH:-jq}"', + ); + expect(preamble).not.toMatch(/(^|[|;(]\s*)jq\s/m); + }); }); describe('generateBashReplayPreamble - command substitution pending emission', () => { diff --git a/service/src/preamble-bash.ts b/service/src/preamble-bash.ts index c442b495..31ed4c6b 100644 --- a/service/src/preamble-bash.ts +++ b/service/src/preamble-bash.ts @@ -150,6 +150,7 @@ _PTC_SENTINEL_START="${scopedStart}" _PTC_SENTINEL_END="${scopedEnd}" _PTC_HISTORY_PATH="\${PTC_HISTORY_PATH:-${PTC_HISTORY_SANDBOX_PATH}}" _PTC_CONTROL_PATH="\${LIBRECHAT_CODE_CONTROL_PATH:-}" +_PTC_JQ_PATH="\${LIBRECHAT_CODE_JQ_PATH:-jq}" _PTC_RUNTIME_DIR="\${TMPDIR:-/tmp}" _ptc_mktemp() { mktemp "\${_PTC_RUNTIME_DIR%/}/$1.XXXXXX" @@ -222,7 +223,7 @@ _ptc_sha256() { _ptc_hash_input() { local _ptc_canonical - _ptc_canonical=$(printf '%s' "$1" | jq -cS . 2>/dev/null) || return 1 + _ptc_canonical=$(printf '%s' "$1" | "$_PTC_JQ_PATH" -cS . 2>/dev/null) || return 1 printf '%s' "$_ptc_canonical" | _ptc_sha256 } @@ -369,7 +370,7 @@ _ptc_maybe_emit_pending() { return 0 fi local _ptc_payload - if ! _ptc_payload=$(jq -c -s '{pending:.}' "$_PTC_PENDING_FILE" 2>/dev/null); then + if ! _ptc_payload=$("$_PTC_JQ_PATH" -c -s '{pending:.}' "$_PTC_PENDING_FILE" 2>/dev/null); then printf 'failed to serialize pending PTC tool calls\\n' >&2 _ptc_cleanup_tempfiles trap - DEBUG EXIT @@ -480,7 +481,7 @@ _ptc_history_matches_by_signature() { return 0 fi # Path, not inline: large input can exceed ARG_MAX via --argjson. - jq -c \\ + "$_PTC_JQ_PATH" -c \\ --arg nm "$_ptc_name" \\ --arg site "$_ptc_call_site" \\ --arg hash "$_ptc_input_hash" \\ @@ -504,7 +505,7 @@ _ptc_first_unconsumed_history_match() { local _ptc_key while IFS= read -r _ptc_match; do [ -n "$_ptc_match" ] || continue - _ptc_key=$(printf '%s' "$_ptc_match" | jq -r '.key // empty' 2>/dev/null) + _ptc_key=$(printf '%s' "$_ptc_match" | "$_PTC_JQ_PATH" -r '.key // empty' 2>/dev/null) if [ -n "$_ptc_key" ] && ! grep -Fxq "$_ptc_key" "$_PTC_CONSUMED_FILE" 2>/dev/null; then printf '%s' "$_ptc_match" return 0 @@ -516,15 +517,15 @@ _ptc_first_unconsumed_history_match() { _ptc_print_history_entry() { local _ptc_entry="$1" local _ptc_is_err - _ptc_is_err=$(printf '%s' "$_ptc_entry" | jq -r 'if type == "object" then (.is_error // false) else false end' 2>/dev/null) + _ptc_is_err=$(printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -r 'if type == "object" then (.is_error // false) else false end' 2>/dev/null) if [ "$_ptc_is_err" = "true" ]; then local _ptc_msg - _ptc_msg=$(printf '%s' "$_ptc_entry" | jq -r '.error_message // "tool execution failed"' 2>/dev/null) + _ptc_msg=$(printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -r '.error_message // "tool execution failed"' 2>/dev/null) _ptc_write_error "$_ptc_msg" exit 1 fi local _ptc_result - _ptc_result=$(printf '%s' "$_ptc_entry" | jq -c 'if type == "object" and has("result") then .result else . end' 2>/dev/null || printf 'null') + _ptc_result=$(printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -c 'if type == "object" and has("result") then .result else . end' 2>/dev/null || printf 'null') printf '%s' "$_ptc_result" return 0 } @@ -535,7 +536,7 @@ _ptc_history_entry_matches_current_call() { local _ptc_input_file="$3" local _ptc_input_hash="$4" # Path, same ARG_MAX reason as above. - printf '%s' "$_ptc_entry" | jq -e \\ + printf '%s' "$_ptc_entry" | "$_PTC_JQ_PATH" -e \\ --arg nm "$_ptc_name" \\ --arg hash "$_ptc_input_hash" \\ --slurpfile inp_arr "$_ptc_input_file" \\ @@ -555,7 +556,7 @@ _ptc_call_tool() { local _ptc_call_site="\${BASH_LINENO[1]:-\${BASH_LINENO[0]:-0}}" # Reject extra trailing JSON values instead of silently dropping them. - if ! printf '%s' "$_ptc_input" | jq -e -n '[inputs] as $docs | ($docs | length) == 1 and ($docs[0] | type) == "object"' >/dev/null 2>&1; then + if ! printf '%s' "$_ptc_input" | "$_PTC_JQ_PATH" -e -n '[inputs] as $docs | ($docs | length) == 1 and ($docs[0] | type) == "object"' >/dev/null 2>&1; then _ptc_write_error "tool input for $_ptc_name must be a single JSON object, got: $_ptc_input" exit 1 fi @@ -583,8 +584,8 @@ _ptc_call_tool() { if [ -n "$_ptc_match" ] && [ "$_ptc_match" != "null" ]; then local _ptc_matched_call_id local _ptc_matched_entry - _ptc_matched_call_id=$(printf '%s' "$_ptc_match" | jq -r '.key' 2>/dev/null) - _ptc_matched_entry=$(printf '%s' "$_ptc_match" | jq -c '.value' 2>/dev/null) + _ptc_matched_call_id=$(printf '%s' "$_ptc_match" | "$_PTC_JQ_PATH" -r '.key' 2>/dev/null) + _ptc_matched_entry=$(printf '%s' "$_ptc_match" | "$_PTC_JQ_PATH" -c '.value' 2>/dev/null) printf '%s\\n' "$_ptc_matched_call_id" >> "$_PTC_CONSUMED_FILE" _ptc_mark_counter_at_least "$_ptc_matched_call_id" _ptc_release_lock @@ -598,7 +599,7 @@ _ptc_call_tool() { while :; do _ptc_call_id=$(_ptc_next_call_id) if [ -r "$_PTC_HISTORY_PATH" ]; then - _ptc_entry=$(jq -c --arg id "$_ptc_call_id" '.[$id] // empty' "$_PTC_HISTORY_PATH" 2>/dev/null || printf '') + _ptc_entry=$("$_PTC_JQ_PATH" -c --arg id "$_ptc_call_id" '.[$id] // empty' "$_PTC_HISTORY_PATH" 2>/dev/null || printf '') else _ptc_entry="" fi @@ -614,7 +615,7 @@ _ptc_call_tool() { fi done - if ! printf '%s' "$_ptc_input" | jq -c -n \\ + if ! printf '%s' "$_ptc_input" | "$_PTC_JQ_PATH" -c -n \\ --arg cid "$_ptc_call_id" \\ --arg nm "$_ptc_name" \\ --arg hash "$_ptc_input_hash" \\ From 10ac19b74f87ec2ff4c9ff7e81198cc1f6b5e830 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 15:51:48 -0400 Subject: [PATCH 22/42] fix: isolate native PTC readiness and watchdog phases (#208) --- packages/code/src/native-process-child.ts | 89 ++++++-- packages/code/src/native-process.test.ts | 234 +++++++++++++++++++++- packages/code/src/native-process.ts | 140 ++++++++++--- packages/code/src/native-programmatic.ts | 15 +- packages/code/src/native-sandbox.ts | 17 +- 5 files changed, 444 insertions(+), 51 deletions(-) diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index 2e8beb38..c3b9614b 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -11,7 +11,12 @@ import type { // argv credentials, bridge token, or persisted pairing material is required. let sandbox: NativeSrtWorkspaceCommandSandbox | undefined; let programmaticExecutor: NativeWorkspaceProgrammaticExecutor | undefined; +let programmaticReady: Promise | undefined; +let programmaticFileUpstream: string | undefined; let active: { id: string; controller: AbortController } | undefined; +let commitAcknowledgement: + | { id: string; acknowledge(): void } + | undefined; let busy = false; let credentials: Record = {}; let wrappedCommand: string | undefined; @@ -25,6 +30,33 @@ function reply(message: object): void { /* Parent was lost. */ } } +async function awaitCommitAcknowledgement( + id: string, + signal: AbortSignal, +): Promise { + await new Promise((resolve, reject) => { + const abort = () => { + commitAcknowledgement = undefined; + reject( + new WorkspaceToolError( + 'Programmatic execution aborted before commit', + 'EXECUTION_ABORTED', + ), + ); + }; + commitAcknowledgement = { + id, + acknowledge() { + signal.removeEventListener('abort', abort); + commitAcknowledgement = undefined; + resolve(); + }, + }; + signal.addEventListener('abort', abort, { once: true }); + reply({ id, phase: 'commit' }); + if (signal.aborted) abort(); + }); +} let shuttingDown = false; const shutdown = () => { if (shuttingDown) return; @@ -59,19 +91,29 @@ process.on('message', async (raw: unknown) => { workspaceId?: string; credentials?: Record; wrappedCommand?: string; + programmaticShellPath?: string; + programmaticJqPath?: string; }; if (!message || typeof message.id !== 'string') return; if (message.type === 'cancel') { if (active?.id === message.id) active.controller.abort(); return; } + if (message.type === 'commit-ack') { + if (commitAcknowledgement?.id === message.id) { + commitAcknowledgement.acknowledge(); + } + return; + } if (busy) return; busy = true; + let mutationStarted = false; try { let result: unknown; if (message.type === 'prepare' && !sandbox) { - const { variables, programmaticFileUpstream, ...options } = + const { variables, programmaticFileUpstream: upstream, ...options } = message.options; + programmaticFileUpstream = upstream; sandbox = new NativeSrtWorkspaceCommandSandbox({ ...options, ...(variables @@ -89,32 +131,55 @@ process.on('message', async (raw: unknown) => { : {}), }); await sandbox.prepare(); - programmaticExecutor = programmaticFileUpstream - ? new NativeWorkspaceProgrammaticExecutor({ - sandbox, - upstreamUrl: programmaticFileUpstream, - }) - : undefined; - await programmaticExecutor?.prepare(); } else if (message.type === 'execute' && sandbox) { active = { id: message.id, controller: new AbortController() }; credentials = message.credentials ?? {}; wrappedCommand = message.wrappedCommand; + mutationStarted = true; result = await sandbox.execute(message.request, active.controller.signal); } else if ( message.type === 'programmatic' && sandbox && - programmaticExecutor && + programmaticFileUpstream && message.programmaticRequest && - typeof message.workspaceId === 'string' + typeof message.workspaceId === 'string' && + typeof message.programmaticShellPath === 'string' && + typeof message.programmaticJqPath === 'string' ) { active = { id: message.id, controller: new AbortController() }; credentials = message.credentials ?? {}; wrappedCommand = message.wrappedCommand; + if (!programmaticExecutor) { + programmaticExecutor = new NativeWorkspaceProgrammaticExecutor({ + sandbox, + upstreamUrl: programmaticFileUpstream, + shellPath: message.programmaticShellPath, + jqPath: message.programmaticJqPath, + }); + programmaticReady = programmaticExecutor.prepare( + active.controller.signal, + ); + } + try { + await programmaticReady; + } catch (error) { + programmaticExecutor = undefined; + programmaticReady = undefined; + throw error; + } result = await programmaticExecutor.execute( message.programmaticRequest, message.workspaceId, active.controller.signal, + { + async beforeCommit() { + await awaitCommitAcknowledgement( + message.id, + active!.controller.signal, + ); + mutationStarted = true; + }, + }, ); } else if (message.type === 'close' && sandbox) { await sandbox.close(); @@ -134,11 +199,11 @@ process.on('message', async (raw: unknown) => { mutation: error instanceof WorkspaceToolError ? error.mutationMayHaveCommitted - : true, + : mutationStarted, requiresQuarantine: error instanceof WorkspaceToolError ? error.requiresQuarantine - : true, + : mutationStarted, }); } finally { active = undefined; diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 2f38d642..89651845 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -11,6 +11,7 @@ import { trustedProgrammaticExecutable, } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; +import { BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS } from './protocol.js'; test('preflight rejects relative and workspace-controlled executables including symlinks', async t => { const root = await mkdtemp(join(tmpdir(), 'native-ptc-path-')); @@ -99,6 +100,24 @@ function fixture( }; } +class ObservedWatchdogSandbox extends NativeProcessWorkspaceCommandSandbox { + readonly watchdogTimeouts: number[] = []; + readonly watchdogCallbacks: Array<() => void> = []; + + protected override scheduleRpcTimeout( + callback: () => void, + timeoutMs: number, + ): ReturnType { + this.watchdogTimeouts.push(timeoutMs); + this.watchdogCallbacks.push(callback); + return super.scheduleRpcTimeout(callback, timeoutMs); + } + + fireLatestWatchdog(): void { + this.watchdogCallbacks.at(-1)?.(); + } +} + test('executor bootstrap excludes bridge credentials and Node injection variables', async () => { assert.deepEqual( nativeExecutorEnvironment({ @@ -229,13 +248,12 @@ test('programmatic executor resolves and scopes credentials to its command', asy const message = fake.messages.find( candidate => candidate.type === 'programmatic', )!; - const prepareMessage = fake.messages.find( - candidate => candidate.type === 'prepare', - )!; - assert.equal(typeof prepareMessage.options.jqPath, 'string'); - assert.equal(prepareMessage.options.jqPath.startsWith('/'), true); + assert.equal(typeof message.programmaticShellPath, 'string'); + assert.equal(message.programmaticShellPath.startsWith('/'), true); + assert.equal(typeof message.programmaticJqPath, 'string'); + assert.equal(message.programmaticJqPath.startsWith('/'), true); assert.equal( - '/sandbox-only'.split(':').includes(dirname(prepareMessage.options.jqPath)), + '/sandbox-only'.split(':').includes(dirname(message.programmaticJqPath)), false, ); assert.deepEqual(message.credentials, { TOKEN: 'per-programmatic-secret' }); @@ -246,8 +264,109 @@ test('programmatic executor resolves and scopes credentials to its command', asy await sandbox.close(); }); +test('omitted PTC timeout gives the commit watchdog the protocol execution default', async () => { + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') return; + child.emit('message', { id: message.id, phase: 'commit' }); + child.emit('message', { id: message.id, ok: true, result: {} }); + }); + const sandbox = new ObservedWatchdogSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + replay_tool_count: 0, + max_output_files: 0, + files: [{ name: 'main.sh', content: 'sleep 45' }], + }, + }); + assert.ok( + sandbox.watchdogTimeouts.at(-1)! > + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + ); + await sandbox.close(); +}); + +test('PTC watchdog budgets staging separately and resets when commit begins', async () => { + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') return; + child.emit('message', { id: message.id, phase: 'commit' }); + child.emit('message', { id: message.id, ok: true, result: {} }); + }); + const sandbox = new ObservedWatchdogSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + run_timeout: 1_000, + replay_tool_count: 0, + max_output_files: 0, + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }); + assert.deepEqual(sandbox.watchdogTimeouts.slice(-2), [65_000, 6_000]); + assert.ok( + fake.messages.some(message => message.type === 'commit-ack'), + 'the child must not enter the mutating phase before the parent arms it', + ); + await sandbox.close(); +}); + +test('PTC-only preflight failures do not disable ordinary native commands', async () => { + const fake = fixture(); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: '/workspace', + shellPath: '/definitely/missing/bash', + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + assert.deepEqual(await sandbox.execute(request), result); + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'COMMAND_UNAVAILABLE' && + !error.mutationMayHaveCommitted, + ); + assert.deepEqual(await sandbox.execute(request), result); + await sandbox.close(); +}); + test('programmatic executor preserves a child-reported pre-dispatch failure', async () => { - const fake = fixture((child, message) => + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') { + child.emit('message', { id: message.id, ok: true, result }); + return; + } child.emit('message', { id: message.id, ok: false, @@ -255,8 +374,39 @@ test('programmatic executor preserves a child-reported pre-dispatch failure', as errorMessage: 'Programmatic input download failed', mutation: false, requiresQuarantine: false, + }); + }); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, }), + (error: unknown) => + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted && + !error.requiresQuarantine, ); + assert.deepEqual(await sandbox.execute(request), result); + await sandbox.close(); +}); + +test('executor loss during programmatic staging is not an uncertain workspace mutation', async () => { + const fake = fixture((child, message) => { + if (message.type === 'programmatic') child.emit('exit', 1); + }); const sandbox = new NativeProcessWorkspaceCommandSandbox( { workspaceRoot: tmpdir(), @@ -283,6 +433,76 @@ test('programmatic executor preserves a child-reported pre-dispatch failure', as await sandbox.close(); }); +test('programmatic staging watchdog expires without claiming a workspace mutation', async () => { + let staged!: () => void; + const staging = new Promise(resolve => { + staged = resolve; + }); + const fake = fixture((_child, message) => { + if (message.type === 'programmatic') staged(); + }); + const sandbox = new ObservedWatchdogSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + const execution = sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }); + await staging; + sandbox.fireLatestWatchdog(); + + await assert.rejects( + execution, + (error: unknown) => + error instanceof WorkspaceToolError && + !error.mutationMayHaveCommitted && + !error.requiresQuarantine, + ); + assert.equal(fake.killCalls, 1); + await sandbox.close(); +}); + +test('executor loss after programmatic commit starts remains an uncertain mutation', async () => { + const fake = fixture((child, message) => { + if (message.type !== 'programmatic') return; + child.emit('message', { id: message.id, phase: 'commit' }); + child.emit('exit', 1); + }); + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { + workspaceRoot: tmpdir(), + programmaticFileUpstream: 'http://127.0.0.1:3190', + }, + fake.fork, + ); + + await assert.rejects( + sandbox.executeProgrammatic('primary', { + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'echo ready' }], + }, + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.mutationMayHaveCommitted && + error.requiresQuarantine, + ); + await sandbox.close(); +}); + test('executor loss after dispatch is an uncertain mutation and is never replayed', async () => { const fake = fixture(child => child.emit('exit', 1)); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 4ed3ffec..81bbd9c4 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -7,6 +7,7 @@ import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; import { + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES, isWorkspaceToolRequest, isWorkspaceToolResult, @@ -29,6 +30,13 @@ export type NativeProcessSandboxOptions = Omit< }; const execFileAsync = promisify(execFile); +const PROGRAMMATIC_STAGING_TIMEOUT_MS = 60_000; +const PROGRAMMATIC_TRANSFER_TIMEOUT_MS = 30_000; +const RPC_SETTLEMENT_SLACK_MS = 5_000; + +type RpcTimeoutBudget = + | number + | { stagingMs: number; commitMs: number }; async function systemProgrammaticExecutable( name: string, @@ -183,11 +191,16 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan private closing?: Promise; private failed = false; private terminationTimer?: ReturnType; + private programmaticExecutables?: Promise<{ + shellPath: string; + jqPath: string; + }>; private pending?: { id: string; resolve(value: unknown): void; reject(error: Error): void; mutation: boolean; + commit?(): void; }; constructor( @@ -199,6 +212,14 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan ) => ChildProcess = fork, ) {} + /** Overridable only for deterministic watchdog tests. */ + protected scheduleRpcTimeout( + callback: () => void, + timeoutMs: number, + ): ReturnType { + return setTimeout(callback, timeoutMs); + } + async prepare(): Promise { if (this.failed || this.closing) throw this.unavailable(false); if (this.ready) return this.ready; @@ -210,10 +231,22 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return new NativeExecutorUnavailableError(mutation); } + private async resolveProgrammaticExecutables(): Promise<{ + shellPath: string; + jqPath: string; + }> { + this.programmaticExecutables ??= resolveProgrammaticShell(this.options); + try { + return await this.programmaticExecutables; + } catch (error) { + // An operator may install or repair this optional dependency while the + // worker stays online. Keep ordinary execution live and let PTC retry. + this.programmaticExecutables = undefined; + throw error; + } + } + private async start(): Promise { - const programmaticExecutables = this.options.programmaticFileUpstream - ? await resolveProgrammaticShell(this.options) - : undefined; const child = this.forkExecutor( new URL('./native-process-child.js', import.meta.url), [], @@ -237,6 +270,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan code?: unknown; errorMessage?: unknown; fatal?: unknown; + phase?: unknown; }; if ( !message || @@ -246,6 +280,25 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return; const pending = this.pending; if (!pending) return; + if (message.phase === 'commit') { + pending.mutation = true; + const commit = pending.commit; + pending.commit = undefined; + commit?.(); + try { + child.send({ type: 'commit-ack', id: pending.id }, error => { + if (!error) return; + this.failed = true; + this.terminate(); + pending.reject(this.unavailable(true)); + }); + } catch { + this.failed = true; + this.terminate(); + pending.reject(this.unavailable(true)); + } + return; + } if (message.fatal === true) this.failed = true; if (message.ok === true) pending.resolve(message.result); else { @@ -299,8 +352,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan protectedPaths, allowedDomains, homeDirectory, - shellPath: programmaticExecutables?.shellPath ?? shellPath, - jqPath: programmaticExecutables?.jqPath, + shellPath, programmaticFileUpstream, variables: this.options.maskedEnvironment?.variables, }, @@ -376,9 +428,12 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan ); let credentials: Record | undefined; let wrappedCommand: string | undefined; + let programmaticExecutables: { shellPath: string; jqPath: string }; try { await this.prepare(); if (signal?.aborted) throw new Error('aborted'); + programmaticExecutables = await this.resolveProgrammaticExecutables(); + if (signal?.aborted) throw new Error('aborted'); credentials = await this.options.maskedEnvironment?.resolve(signal); if (signal?.aborted) throw new Error('aborted'); wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( @@ -407,19 +462,11 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan workspaceId, credentials, wrappedCommand, + programmaticShellPath: programmaticExecutables.shellPath, + programmaticJqPath: programmaticExecutables.jqPath, }, - (request.body.run_timeout ?? 30_000) * - ((request.body.replay_tool_count ?? 0) > 0 ? 2 : 1) + - (Math.ceil( - request.body.files.filter(file => 'id' in file).length / 4, - ) + - Math.ceil( - (request.body.max_output_files ?? - BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / 4, - )) * - (request.body.transfer_timeout_ms ?? 30_000) + - 5_000, - true, + this.programmaticWatchdogBudget(request), + false, signal, ); if (signal?.aborted) { @@ -437,6 +484,35 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return result; } + private programmaticWatchdogBudget( + request: BridgeWorkspaceProgrammaticRequest, + ): Exclude { + const runTimeoutMs = Math.min( + request.body.run_timeout ?? BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + ); + const transferTimeoutMs = + request.body.transfer_timeout_ms ?? PROGRAMMATIC_TRANSFER_TIMEOUT_MS; + const inputBatches = Math.ceil( + request.body.files.filter(file => 'id' in file).length / 4, + ); + const outputBatches = Math.ceil( + (request.body.max_output_files ?? + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES) / 4, + ); + return { + stagingMs: + PROGRAMMATIC_STAGING_TIMEOUT_MS + + inputBatches * transferTimeoutMs + + ((request.body.replay_tool_count ?? 0) > 0 ? runTimeoutMs : 0) + + RPC_SETTLEMENT_SLACK_MS, + commitMs: + runTimeoutMs + + outputBatches * transferTimeoutMs + + RPC_SETTLEMENT_SLACK_MS, + }; + } + private async executeOnce( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, @@ -498,7 +574,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan private async rpc( type: string, payload: object, - timeoutMs: number, + timeout: RpcTimeoutBudget, mutation: boolean, signal?: AbortSignal, ): Promise { @@ -506,7 +582,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan throw this.unavailable(false); const id = randomUUID(); const child = this.child; - let timer: ReturnType; + let timer: ReturnType | undefined; const abort = () => { try { if (child.connected) @@ -518,12 +594,24 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan }; try { return await new Promise((resolve, reject) => { - this.pending = { id, resolve, reject, mutation }; - timer = setTimeout(() => { - this.failed = true; - this.terminate(); - reject(this.unavailable(mutation)); - }, timeoutMs); + const schedule = (timeoutMs: number): void => { + if (timer) clearTimeout(timer); + timer = this.scheduleRpcTimeout(() => { + this.failed = true; + this.terminate(); + reject(this.unavailable(this.pending?.mutation ?? mutation)); + }, timeoutMs); + }; + this.pending = { + id, + resolve, + reject, + mutation, + ...(typeof timeout === 'number' + ? {} + : { commit: () => schedule(timeout.commitMs) }), + }; + schedule(typeof timeout === 'number' ? timeout : timeout.stagingMs); signal?.addEventListener('abort', abort, { once: true }); const sendFailed = () => { this.failed = true; @@ -540,7 +628,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan if (signal?.aborted) abort(); }); } finally { - clearTimeout(timer!); + if (timer) clearTimeout(timer); signal?.removeEventListener('abort', abort); this.pending = undefined; } diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts index bd2700f2..9b6eea9e 100644 --- a/packages/code/src/native-programmatic.ts +++ b/packages/code/src/native-programmatic.ts @@ -216,6 +216,8 @@ export interface NativeWorkspaceProgrammaticOptions { Pick >; upstreamUrl: string; + shellPath?: string; + jqPath?: string; fetchImpl?: typeof fetch; } @@ -362,6 +364,7 @@ export class NativeWorkspaceProgrammaticExecutor { request: BridgeWorkspaceProgrammaticRequest, workspaceId: string, signal?: AbortSignal, + lifecycle?: { beforeCommit?(): Promise | void }, ): Promise { if (!isBridgeWorkspaceProgrammaticRequest(request)) { throw new WorkspaceToolError( @@ -456,7 +459,10 @@ export class NativeWorkspaceProgrammaticExecutor { errorOnExist: true, mode: constants.COPYFILE_FICLONE, }); - if (!probe) commandDispatched = true; + if (!probe) { + await lifecycle?.beforeCommit?.(); + commandDispatched = true; + } return await this.options.sandbox.executeProgrammatic( { protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -473,7 +479,12 @@ export class NativeWorkspaceProgrammaticExecutor { }, directory, signal, - { probe, workspaceRoot }, + { + probe, + workspaceRoot, + shellPath: this.options.shellPath, + jqPath: this.options.jqPath, + }, ); }; diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 91e9832d..550d0d52 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -661,7 +661,12 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox request: WorkspaceExecuteCommandRequest, dataDirectory: string, signal?: AbortSignal, - options?: { probe?: boolean; workspaceRoot?: string }, + options?: { + probe?: boolean; + workspaceRoot?: string; + shellPath?: string; + jqPath?: string; + }, ): Promise { if (this.execution || this.closing) { throw new WorkspaceToolError( @@ -703,9 +708,13 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox canonicalDataDirectory, '_ptc_pending_result.json', ), - LIBRECHAT_CODE_BASH_PATH: this.options.shellPath ?? '/bin/bash', - ...(this.options.jqPath - ? { LIBRECHAT_CODE_JQ_PATH: this.options.jqPath } + LIBRECHAT_CODE_BASH_PATH: + options?.shellPath ?? this.options.shellPath ?? '/bin/bash', + ...((options?.jqPath ?? this.options.jqPath) + ? { + LIBRECHAT_CODE_JQ_PATH: + options?.jqPath ?? this.options.jqPath, + } : {}), PTC_HISTORY_PATH: join( canonicalDataDirectory, From 3a2c2a0a974b3b01c1c509e40cc414f75b23faab Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 18:30:33 -0400 Subject: [PATCH 23/42] feat: Declare Named Worker Project Environments (#209) * feat: declare named worker project environments * fix: preserve environment trust and negotiated action boundaries * Harden environment loading and executor identity * Protect environment root traversal and exact config bytes * Reject self-controlled environment root aliases * Check filesystem identities at environment trust boundaries * Validate environment containment across Linux mount aliases * Handle stacked mounts conservatively without blocking unrelated paths --- packages/code/README.md | 52 +++ packages/code/package-lock.json | 18 +- packages/code/package.json | 3 +- packages/code/src/cli.ts | 249 +++++++++--- packages/code/src/environment-live.test.ts | 117 ++++++ packages/code/src/environment-mount.test.ts | 73 ++++ packages/code/src/environment-mount.ts | 123 ++++++ packages/code/src/environment.test.ts | 348 +++++++++++++++++ packages/code/src/environment.ts | 399 ++++++++++++++++++++ packages/code/src/private-storage.ts | 10 +- packages/code/src/protocol.ts | 239 ++++++++++-- packages/code/src/worker.ts | 21 +- packages/code/src/workspace-worker.test.ts | 50 +++ packages/code/src/workspace.ts | 3 + 14 files changed, 1617 insertions(+), 88 deletions(-) create mode 100644 packages/code/src/environment-live.test.ts create mode 100644 packages/code/src/environment-mount.test.ts create mode 100644 packages/code/src/environment-mount.ts create mode 100644 packages/code/src/environment.test.ts create mode 100644 packages/code/src/environment.ts diff --git a/packages/code/README.md b/packages/code/README.md index 87d9dded..c3bb16d3 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -652,3 +652,55 @@ To recover a quarantined native root: The workspace selector in LibreChat must preserve these registered IDs. Adding roots here does not grant a principal access or change an agent's selected root. +# Named project environments + +An operator can keep a project definition outside the coding workspace and start +the worker with `librechat-code run --environment /operator/app.yaml +--allow-workspace-commands --allow-workspace-writes`. Existing pairing settings +still identify the machine and its principal. Repeat `--environment` for independent, +non-overlapping roots (up to 32). Do not combine definitions with workspace directory, +ID, or name flags or environment variables. + +```yaml +name: app-dev +root: /projects/app +repo: example/app +ref: main +setup: + command: npm ci + timeoutMs: 300000 +actions: + - name: typecheck + command: npm run typecheck + timeoutMs: 120000 +``` + +The root must already exist; relative roots resolve from the YAML file's directory. +Repository and ref are descriptive metadata, not a clone or checkout instruction. +No Git repository is required. Definitions are loaded once at startup, hashed into +the worker's policy identity, and protected from sandbox writes. All definition +files must be outside every registered root. Unknown fields are rejected. +On Linux, startup also verifies the mount namespace so bind mounts cannot expose +definitions or their controlling paths through a workspace. The mount table is +bounded to 4 MiB, with at most 256 exposed mount boundaries; stacked and hidden +mount mappings are considered conservatively. Operators must keep mount topology stable while the +worker runs. This inspection happens at startup, not on the command hot path. + +Setup is an operator-authorized startup command under the configured native sandbox +policy. It requires commands to be enabled, runs once per worker startup before +registration, and must be idempotent for restarts. Its timeout is bounded to five +minutes and captured output to 8 KiB. Setup failure prevents registration. A crash +or uncertain termination retains the existing workspace quarantine marker; inspect +the workspace before clearing quarantine. No setup output is sent to the model. + +Named actions are fixed commands without model-supplied substitution. The bridge +advertises only their names and the definition fingerprint, never their shell source +or host root. A command request can select `environmentAction: { name, fingerprint }`; +the worker resolves the command from its loaded definition and rejects stale revisions, +unknown names, other roots, or a changed working directory. Actions use ordinary +command authorization, queueing, cancellation and quarantine. They never override +deployment approval rules or expand the pairing's principal scope. + +Rollout: update Code API and the LibreChat environment-descriptor consumer before +enabling this opt-in flag on a worker. Older validators reject the additional metadata. +Existing workers without `--environment` continue to use their existing registration. diff --git a/packages/code/package-lock.json b/packages/code/package-lock.json index ac9affc8..426b950f 100644 --- a/packages/code/package-lock.json +++ b/packages/code/package-lock.json @@ -10,7 +10,8 @@ "license": "Apache-2.0", "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.75", - "koffi": "3.2.1" + "koffi": "3.2.1", + "yaml": "2.9.1" }, "bin": { "librechat-code": "dist/cli.js" @@ -414,6 +415,21 @@ "dev": true, "license": "MIT" }, + "node_modules/yaml": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz", + "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, "node_modules/zod": { "version": "3.25.76", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", diff --git a/packages/code/package.json b/packages/code/package.json index 452a8be7..b00ca807 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -65,6 +65,7 @@ }, "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.75", - "koffi": "3.2.1" + "koffi": "3.2.1", + "yaml": "2.9.1" } } diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 289b0dc4..5498af63 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -5,6 +5,11 @@ import { realpath, stat } from 'node:fs/promises'; import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; +import { + loadCodeEnvironment, + assertEnvironmentDefinitionsOutsideRoots, + EnvironmentWorkspaceTools, +} from './environment.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { @@ -61,7 +66,8 @@ function workspaceSecurityIdentity( configuredToken: string | undefined, ): string { return ( - pairedPublicKey ?? required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken) + pairedPublicKey ?? + required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken) ); } @@ -70,7 +76,8 @@ function workspaceQuarantinePath(options: { workerId: string; workspaceRoot?: string; }): string { - const override = process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim(); + const override = + process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim(); if (override) return override; return defaultWorkspaceQuarantinePath({ ...options, @@ -88,7 +95,7 @@ function list(value: string | undefined): string[] { return ( value ?.split(',') - .map((item) => item.trim()) + .map(item => item.trim()) .filter(Boolean) ?? [] ); } @@ -127,7 +134,7 @@ function option(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index >= 0) return args[index + 1]; return args - .find((value) => value.startsWith(`${name}=`)) + .find(value => value.startsWith(`${name}=`)) ?.slice(name.length + 1); } @@ -175,7 +182,9 @@ function githubCredentials(): { try { parsedApiUrl = new URL(apiUrl); } catch { - throw new Error('LIBRECHAT_CODE_GITHUB_API_URL must be a valid URL'); + throw new Error( + 'LIBRECHAT_CODE_GITHUB_API_URL must be a valid URL', + ); } apiHost = parsedApiUrl.hostname.toLowerCase() === 'api.github.com' @@ -288,7 +297,7 @@ async function relay(): Promise { process.stdout.write( `librechat-code: file relay listening at ${handle.url}\n`, ); - await new Promise((resolve) => { + await new Promise(resolve => { process.once('SIGINT', resolve); process.once('SIGTERM', resolve); }); @@ -299,6 +308,47 @@ async function run( runtimeSessionId?: string, args: string[] = [], ): Promise { + const environmentPaths: string[] = []; + for (let i = 0; i < args.length; i++) { + if (args[i] === '--environment') { + const path = args[++i]; + if (!path || path.startsWith('--')) + throw new Error('--environment requires a YAML file'); + environmentPaths.push(path); + } else if (args[i].startsWith('--environment=')) { + const path = args[i].slice('--environment='.length); + if (!path) throw new Error('--environment requires a YAML file'); + environmentPaths.push(path); + } + } + if (environmentPaths.length > 32) + throw new Error('At most 32 environments may be registered'); + const environments = await Promise.all( + environmentPaths.map(loadCodeEnvironment), + ); + if ( + environments.length && + (runtimeSessionId != null || + args.some(arg => + [ + '--worker-dir', + '--default-workspace', + '--workspace', + '--workspace-id', + '--workspace-name', + ].some(flag => arg === flag || arg.startsWith(`${flag}=`)), + ) || + [ + process.env.LIBRECHAT_CODE_WORKER_DIR, + process.env.LIBRECHAT_CODE_WORKSPACE_ID, + process.env.LIBRECHAT_CODE_WORKSPACE_NAME, + ].some(value => value?.trim()) || + process.env.LIBRECHAT_CODE_DEFAULT_WORKSPACE?.trim().toLowerCase() === 'true') + ) { + throw new Error( + '--environment cannot be combined with workspace directory, ID, or name settings', + ); + } const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); const configuredIdentityPath = process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); @@ -347,7 +397,8 @@ async function run( ); } const nsjailDockerMode = - runtimeMode === 'docker-nsjail' || runtimeMode === 'docker-macos-nsjail'; + runtimeMode === 'docker-nsjail' || + runtimeMode === 'docker-macos-nsjail'; const sandboxEndpoint = process.env.LIBRECHAT_CODE_SANDBOX_ENDPOINT ?? 'http://127.0.0.1:2000/api/v2'; @@ -374,16 +425,18 @@ async function run( runtimeSessionId == null && (fileRelayUpstream?.length ?? 0) > 0; const workspaceId = + environments[0]?.definition.name ?? option(args, '--workspace-id') ?? process.env.LIBRECHAT_CODE_WORKSPACE_ID?.trim() ?? 'primary'; const explicitWorkerDirectory = - runtimeSessionId == null + environments[0]?.definition.root ?? + (runtimeSessionId == null ? nonEmpty( option(args, '--worker-dir') ?? process.env.LIBRECHAT_CODE_WORKER_DIR?.trim(), ) - : undefined; + : undefined); const useDefaultWorkspace = runtimeSessionId == null && (args.includes('--default-workspace') || @@ -403,11 +456,25 @@ async function run( option(args, '--command-sandbox') ?? process.env.LIBRECHAT_CODE_COMMAND_SANDBOX?.trim().toLowerCase() ?? (nsjailDockerMode ? 'runtime' : 'native-srt'); - if (commandSandboxMode !== 'native-srt' && commandSandboxMode !== 'runtime') { + if ( + commandSandboxMode !== 'native-srt' && + commandSandboxMode !== 'runtime' + ) { throw new Error( 'LIBRECHAT_CODE_COMMAND_SANDBOX must be native-srt or runtime', ); } + if (environments.length && commandSandboxMode !== 'native-srt') { + throw new Error('Environment definitions require native-srt'); + } + if ( + environments.some(environment => environment.definition.setup) && + !allowWorkspaceCommands + ) { + throw new Error( + 'Environment setup requires --allow-workspace-commands', + ); + } const nativeProgrammaticEnabled = allowWorkspaceCommands && commandSandboxMode === 'native-srt' && @@ -486,7 +553,8 @@ async function run( } } const mutationQuarantinePath = - (allowWorkspaceWrites || allowWorkspaceCommands) && canonicalWorkerDirectory + (allowWorkspaceWrites || allowWorkspaceCommands) && + canonicalWorkerDirectory ? workspaceQuarantinePath({ codeApiUrl, workerId, @@ -508,14 +576,27 @@ async function run( root: canonicalWorkerDirectory, writable: allowWorkspaceWrites, name: + environments[0]?.definition.name ?? option(args, '--workspace-name') ?? process.env.LIBRECHAT_CODE_WORKSPACE_NAME?.trim() ?? (useDefaultWorkspace ? workspaceId - : defaultWorkspaceName(workerDirectory!, workspaceId)), + : defaultWorkspaceName( + workerDirectory!, + workspaceId, + )), }, ] : []; + for (const environment of environments.slice(1)) { + roots.push({ + id: environment.definition.name, + name: environment.definition.name, + root: environment.definition.root, + writable: allowWorkspaceWrites, + }); + } + await assertEnvironmentDefinitionsOutsideRoots(environments, roots); for (let i = 0; i < args.length; i++) { if ( args[i] === '--workspace' && @@ -551,16 +632,18 @@ async function run( if (roots.length > 32) throw new Error('At most 32 workspace roots may be registered'); const rootIdentities = await Promise.all( - roots.map((root) => stat(root.root)), + roots.map(root => stat(root.root)), ); - const normalized = roots.map((root) => root.root); + const normalized = roots.map(root => root.root); for (let i = 0; i < roots.length; i++) for (let j = 0; j < i; j++) { const inside = (a: string, b: string): boolean => { const path = relative(a, b); return ( path === '' || - (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) + (path !== '..' && + !path.startsWith(`..${sep}`) && + !isAbsolute(path)) ); }; if ( @@ -578,7 +661,9 @@ async function run( workspaceLeaseSlots > 1 && (!allowWorkspaceCommands || commandSandboxMode !== 'native-srt') ) { - throw new Error('Concurrent workspace leases require native-srt commands'); + throw new Error( + 'Concurrent workspace leases require native-srt commands', + ); } if ( roots.length > 1 && @@ -589,7 +674,7 @@ async function run( ); } const rootQuarantinePaths = new Map( - roots.map((root) => [ + roots.map(root => [ root.id, workspaceQuarantinePath({ codeApiUrl, @@ -676,7 +761,10 @@ async function run( token: createHmac( 'sha256', pairedIdentity?.privateKey ?? - required('LIBRECHAT_CODE_WORKER_TOKEN', configuredToken), + required( + 'LIBRECHAT_CODE_WORKER_TOKEN', + configuredToken, + ), ) .update('librechat-code-file-relay-v1') .digest('hex'), @@ -712,8 +800,11 @@ async function run( image: runtimeImage, ...(nsjailDockerMode && runtimeSessionId == null ? (() => { - const { seccompProfile, packagesPath, profileRevision } = - nsjailLaunchProfile!; + const { + seccompProfile, + packagesPath, + profileRevision, + } = nsjailLaunchProfile!; return { capabilities: MACOS_NSJAIL_CAPABILITIES, securityOptions: [`seccomp=${seccompProfile}`], @@ -733,10 +824,12 @@ async function run( httpClient: 'bun' as const, environment: { SANDBOX_USE_CGROUPV2: 'false', - SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', + SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: + 'false', ...(workspaceMount ? { - SANDBOX_EXTERNAL_WORKSPACE_ENABLED: 'true', + SANDBOX_EXTERNAL_WORKSPACE_ENABLED: + 'true', SANDBOX_EXTERNAL_WORKSPACE_ROOT: workspaceMount.target, SANDBOX_EXTERNAL_WORKSPACE_TOKEN: @@ -745,15 +838,21 @@ async function run( : {}), ...(fileRelayProfile ? { - EGRESS_GATEWAY_URL: fileRelayProfile.url, + EGRESS_GATEWAY_URL: + fileRelayProfile.url, SANDBOX_PRIME_CONCURRENCY: String( - fileRelayLimits!.maxConcurrentRequests, + fileRelayLimits! + .maxConcurrentRequests, ), - SANDBOX_UPLOAD_CONCURRENCY: String( - fileRelayLimits!.maxConcurrentRequests, + SANDBOX_UPLOAD_CONCURRENCY: + String( + fileRelayLimits! + .maxConcurrentRequests, ), - SANDBOX_FILE_RELAY_TOKEN: fileRelayProfile.token, - SANDBOX_REQUIRE_EGRESS_MANIFEST: 'true', + SANDBOX_FILE_RELAY_TOKEN: + fileRelayProfile.token, + SANDBOX_REQUIRE_EGRESS_MANIFEST: + 'true', SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY: executionManifestPublicKey!, } @@ -766,8 +865,10 @@ async function run( bindMounts: [workspaceMount], environment: { SANDBOX_EXTERNAL_WORKSPACE_ENABLED: 'true', - SANDBOX_EXTERNAL_WORKSPACE_ROOT: workspaceMount.target, - SANDBOX_EXTERNAL_WORKSPACE_TOKEN: workspaceCommandToken!, + SANDBOX_EXTERNAL_WORKSPACE_ROOT: + workspaceMount.target, + SANDBOX_EXTERNAL_WORKSPACE_TOKEN: + workspaceCommandToken!, }, } : {}), @@ -781,6 +882,7 @@ async function run( commandPolicy, protectedPaths: [ identityPath, + ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), github.privateKeyPath, ].filter((path): path is string => path != null), @@ -814,7 +916,7 @@ async function run( ? roots.length > 1 || workspaceLeaseSlots > 1 ? new NativeWorkspaceCommandPool( new Map( - roots.map((root) => [ + roots.map(root => [ root.id, { ...nativeOptions, workspaceRoot: root.root }, ]), @@ -826,7 +928,7 @@ async function run( if (allowWorkspaceCommands && workspaceTools) { workspaceTools = new SandboxWorkspaceTools({ workspaceTools, - commandWorkspaces: roots.map((root) => root.id), + commandWorkspaces: roots.map(root => root.id), ...(nativeProgrammaticEnabled ? { programmaticLanguages: ['bash'] } : {}), @@ -839,6 +941,12 @@ async function run( }), }); } + if (workspaceTools && environments.length) { + workspaceTools = new EnvironmentWorkspaceTools( + workspaceTools, + environments, + ); + } const capabilities = { statefulWorkspace, sandboxProfile: @@ -854,6 +962,11 @@ async function run( policyDigest: createHash('sha256') .update(policy) .update( + environments.length + ? `\0environments\0${environments.map(environment => environment.fingerprint).join('\0')}` + : '', + ) + .update( allowWorkspaceCommands && commandSandboxMode === 'native-srt' ? `\0native-srt\0${serializeNativeSrtCommandPolicy(commandPolicy)}\0${commandAllowedDomains.join('\0')}\0${github.policyIdentity}` : '', @@ -863,7 +976,9 @@ async function run( ...(workspaceLeaseSlots > 1 ? { workspaceLeaseSlots, requiresReadyConfirmation: true } : {}), - ...(workspaceTools ? { workspaceTools: workspaceTools.capabilities } : {}), + ...(workspaceTools + ? { workspaceTools: workspaceTools.capabilities } + : {}), }; if (!isValidBridgeWorkerCapabilities(capabilities)) { await fileRelaySupervisor?.stop().catch(() => undefined); @@ -874,6 +989,39 @@ async function run( try { await github.provider?.getCredential(controller.signal); await nativeCommandSandbox?.prepare(); + for (const environment of option(args, '--reset-workspace-quarantine') == null ? environments : []) { + const setup = environment.definition.setup; + if (!setup || !nativeCommandSandbox) continue; + const id = environment.definition.name; + const guard = workspaceMutationGuard( + rootQuarantinePaths.get(id)!, + workerId, + id, + incarnationId, + ); + await guard.assertAvailable(); + await guard.arm('Environment setup did not settle', 'setup'); + const result = await nativeCommandSandbox.execute( + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: id, + command: setup.command, + timeoutMs: setup.timeoutMs, + maxOutputBytes: 8192, + }, + controller.signal, + ); + await guard.clear('setup'); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error( + `Environment ${id} setup failed; inspect the setup command before restarting`, + ); + } + process.stdout.write( + `librechat-code: environment ${id} prepared\n`, + ); + } } catch (error) { await nativeCommandSandbox?.close().catch(() => undefined); await fileRelaySupervisor?.stop().catch(() => undefined); @@ -895,7 +1043,7 @@ async function run( ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { workspaceQuarantines: new Map( - roots.map((root) => [ + roots.map(root => [ root.id, workspaceMutationGuard( rootQuarantinePaths.get(root.id)!, @@ -913,7 +1061,8 @@ async function run( roots.length === 1 ? { async assertAvailable() { - const record = await loadWorkspaceMutationQuarantine( + const record = + await loadWorkspaceMutationQuarantine( mutationQuarantinePath, ); if (record != null) { @@ -925,15 +1074,18 @@ async function run( } }, async arm(reason) { - await saveWorkspaceMutationQuarantine(mutationQuarantinePath, { + await saveWorkspaceMutationQuarantine( + mutationQuarantinePath, + { version: 1, workerId, workspaceId, ownerId: incarnationId, quarantinedAt: new Date().toISOString(), reason, - }); }, + ); + }, async clear() { await clearWorkspaceMutationQuarantine( mutationQuarantinePath, @@ -950,7 +1102,7 @@ async function run( : undefined, onIdentityChange: pairedIdentity && identityPath - ? async (identity) => { + ? async identity => { await saveBridgeIdentity(identityPath, { ...pairedIdentity, credential: identity.credential, @@ -959,10 +1111,12 @@ async function run( } : undefined, onRegistered: fileRelaySupervisor - ? async (registration) => { + ? async registration => { if ( registration.registrationGeneration == null || - !Number.isSafeInteger(registration.registrationGeneration) || + !Number.isSafeInteger( + registration.registrationGeneration, + ) || registration.registrationGeneration < 1 ) { throw new Error( @@ -975,10 +1129,14 @@ async function run( ); } : undefined, - onError: (error) => { + onError: error => { const message = - error instanceof Error ? error.message : 'unknown bridge error'; - process.stderr.write(`librechat-code: reconnecting after ${message}\n`); + error instanceof Error + ? error.message + : 'unknown bridge error'; + process.stderr.write( + `librechat-code: reconnecting after ${message}\n`, + ); }, }); if (runtimeSessionId !== undefined) { @@ -994,7 +1152,10 @@ async function run( if (resetNativeRoot != null) { await worker.refreshCredential(controller.signal); await worker.registerForMaintenance(controller.signal); - await worker.resetNativeWorkspace(resetNativeRoot, controller.signal); + await worker.resetNativeWorkspace( + resetNativeRoot, + controller.signal, + ); process.stdout.write( `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}\n`, ); diff --git a/packages/code/src/environment-live.test.ts b/packages/code/src/environment-live.test.ts new file mode 100644 index 00000000..bb84c8b5 --- /dev/null +++ b/packages/code/src/environment-live.test.ts @@ -0,0 +1,117 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, mkdir, readFile, writeFile, rm } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; + +for (const { succeeds, reset } of [ + { succeeds: true, reset: false }, + { succeeds: false, reset: false }, + { succeeds: true, reset: true }, +]) { + test( + `real CLI environment setup gates registration (success=${succeeds}, reset=${reset})`, + { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 20_000, + }, + async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-live-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + await mkdir(root); + const path = join(directory, 'environment.yaml'); + await writeFile( + path, + `name: project\nroot: project\nsetup:\n command: 'printf prepared > prepared.txt; exit ${succeeds ? 0 : 2}'\n timeoutMs: 5000\n`, + ); + let registrations = 0; + let receive: (() => void) | undefined; + const registered = new Promise(resolve => { + receive = resolve; + }); + const server = createServer(async (request, response) => { + request.resume(); + if (request.url?.endsWith('/register')) { + registrations++; + if (reset) + await assert.rejects( + readFile(join(root, 'prepared.txt')), + { code: 'ENOENT' }, + ); + else + assert.equal( + await readFile(join(root, 'prepared.txt'), 'utf8'), + 'prepared', + ); + receive?.(); + } + response.writeHead(503).end(); + }); + await new Promise(resolve => + server.listen(0, '127.0.0.1', resolve), + ); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const child = spawn( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--environment', + path, + '--allow-workspace-commands', + ...(reset + ? ['--reset-workspace-quarantine', 'project'] + : []), + ], + { + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + TMPDIR: process.env.TMPDIR, + LIBRECHAT_CODE_URL: `http://127.0.0.1:${address.port}/v1`, + LIBRECHAT_CODE_WORKER_ID: 'environment-test', + LIBRECHAT_CODE_WORKER_TOKEN: 'test-only-token', + LIBRECHAT_CODE_DEFAULT_WORKSPACE: 'false', + LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE: join( + directory, + 'quarantine.json', + ), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + const exited = once(child, 'exit'); + t.after(() => child.kill('SIGKILL')); + let stderr = ''; + child.stderr.on('data', chunk => { + stderr += chunk.toString(); + }); + if (succeeds) { + await Promise.race([ + registered, + exited.then(() => { + throw new Error(stderr); + }), + ]); + child.kill('SIGTERM'); + await exited; + assert.ok(registrations > 0); + } else { + const [code] = await exited; + assert.notEqual(code, 0); + assert.match(stderr, /Environment project setup failed/); + assert.equal(registrations, 0); + } + }, + ); +} diff --git a/packages/code/src/environment-mount.test.ts b/packages/code/src/environment-mount.test.ts new file mode 100644 index 00000000..fdd5b581 --- /dev/null +++ b/packages/code/src/environment-mount.test.ts @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { assertEnvironmentMountIsolation } from './environment-mount.js'; + +const base = '1 0 8:1 / / rw - ext4 /dev/root rw\n'; +test('mount coordinates reject definition aliases in both directions and mounted files', () => { + for (const entry of [ + '2 1 8:1 /workspace/config /operator rw - ext4 /dev/root rw', + '2 1 8:1 /operator /workspace/config rw - ext4 /dev/root rw', + '2 1 8:1 /operator/app.yaml /workspace/app.yaml rw - ext4 /dev/root rw', + '2 1 8:1 /workspace/app.yaml /operator/app.yaml rw - ext4 /dev/root rw', + ]) + assert.throws( + () => + assertEnvironmentMountIsolation( + base + entry, + ['/operator/app.yaml'], + ['/workspace'], + ), + /mount alias/, + ); +}); +test('mount coordinates retain safe separate filesystems and escaped paths', () => { + assertEnvironmentMountIsolation( + base + '2 1 9:1 / /workspace rw - ext4 /dev/other rw', + ['/operator/app.yaml'], + ['/workspace'], + ); + assertEnvironmentMountIsolation( + base, + ['/operator/app.yaml'], + ['/workspace'], + ); + assert.throws( + () => + assertEnvironmentMountIsolation( + base + + '2 1 8:1 /workspace/my\\040config /operator rw - ext4 /dev/root rw', + ['/operator/app.yaml'], + ['/workspace'], + ), + /mount alias/, + ); + assert.throws(() => assertEnvironmentMountIsolation('invalid', [], [])); + assertEnvironmentMountIsolation( + base + '2 1 9:1 / / rw - ext4 /dev/other rw', + ['/operator/app.yaml'], + ['/workspace'], + ); + assert.throws( + () => + assertEnvironmentMountIsolation( + base + '2 1 9:1 / / rw - ext4 /dev/other rw', + ['/workspace/config/app.yaml'], + ['/workspace'], + ), + /mount alias/, + ); + const many = Array.from( + { length: 257 }, + (_, index) => + `${index + 2} 1 9:1 / /workspace/m${index} rw - ext4 /dev/other rw`, + ).join('\n'); + assert.throws( + () => + assertEnvironmentMountIsolation( + base + many, + ['/operator/app.yaml'], + ['/workspace'], + ), + /Too many/, + ); +}); diff --git a/packages/code/src/environment-mount.ts b/packages/code/src/environment-mount.ts new file mode 100644 index 00000000..bbc8483e --- /dev/null +++ b/packages/code/src/environment-mount.ts @@ -0,0 +1,123 @@ +import { open } from 'node:fs/promises'; +import { posix } from 'node:path'; + +interface Mount { + device: string; + root: string; + point: string; +} +const inside = (root: string, path: string): boolean => + path === root || path.startsWith(root === '/' ? '/' : `${root}/`); +const decode = (path: string): string => { + if (!path.startsWith('/') || /\\(?!040|011|012|134)/.test(path)) + throw new Error('Invalid environment mount table'); + return path.replace(/\\(040|011|012|134)/g, (_, octal: string) => + String.fromCharCode(parseInt(octal, 8)), + ); +}; + +/** Compare filesystem coordinates, not mount aliases. Include mounted descendants of each grant. */ +export function createEnvironmentMountIsolation( + table: string, +): (controls: readonly string[], roots: readonly string[]) => void { + if (Buffer.byteLength(table) > 4 * 1024 * 1024) + throw new Error('Environment mount table exceeds limit'); + const mounts: Mount[] = table + .trimEnd() + .split('\n') + .map(line => { + const fields = line.split(' '); + const separator = fields.indexOf('-', 6); + if ( + separator < 6 || + fields.length !== separator + 4 || + !/^\d+:\d+$/.test(fields[2] ?? '') + ) + throw new Error('Invalid environment mount table'); + return { + device: fields[2], + root: decode(fields[3] ?? ''), + point: decode(fields[4] ?? ''), + }; + }); + const cache = new Map(); + const coordinate = (path: string): { device: string; path: string }[] => { + const cached = cache.get(path); + if (cached) return cached; + // Include every possible backing mapping. Hidden/stacked mounts may cause + // conservative rejection but must never hide an accessible control path. + const result = mounts + .filter(mount => inside(mount.point, path)) + .map(mount => ({ + device: mount.device, + path: posix.join(mount.root, posix.relative(mount.point, path)), + })); + if (!result.length || result.length > 256) + throw new Error( + 'Environment path has an unsupported mount mapping', + ); + cache.set(path, result); + return result; + }; + return (controls, roots) => { + const points = new Set(roots); + for (const mount of mounts) + if (roots.some(root => inside(root, mount.point))) + points.add(mount.point); + if (points.size > 256) + throw new Error('Too many workspace mount boundaries'); + const exposed = [...points].flatMap(coordinate); + if (exposed.length > 1024) + throw new Error('Too many workspace mount mappings'); + for (const control of controls) { + const target = coordinate(control); + if ( + target.some(target => + exposed.some( + root => + root.device === target.device && + inside(root.path, target.path), + ), + ) + ) { + throw new Error( + 'Environment control path is writable through a workspace mount alias', + ); + } + } + }; +} + +export function assertEnvironmentMountIsolation( + table: string, + controls: readonly string[], + roots: readonly string[], +): void { + createEnvironmentMountIsolation(table)(controls, roots); +} + +export async function readEnvironmentMountTable(): Promise { + if (process.platform !== 'linux') return undefined; + const handle = await open('/proc/self/mountinfo', 'r'); + try { + const buffer = Buffer.alloc(4 * 1024 * 1024 + 1); + let length = 0; + while (length < buffer.length) { + const result = await handle.read( + buffer, + length, + buffer.length - length, + null, + ); + if (!result.bytesRead) break; + length += result.bytesRead; + } + if (length === buffer.length) + throw new Error('Environment mount table exceeds limit'); + return new TextDecoder('utf-8', { fatal: true }).decode( + buffer.subarray(0, length), + ); + } finally { + await handle.close(); + } +} diff --git a/packages/code/src/environment.test.ts b/packages/code/src/environment.test.ts new file mode 100644 index 00000000..9080f6a2 --- /dev/null +++ b/packages/code/src/environment.test.ts @@ -0,0 +1,348 @@ +import assert from 'node:assert/strict'; +import { + mkdtemp, + mkdir, + writeFile, + rm, + symlink, + link, + open, + realpath, +} from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { + parseCodeEnvironment, + loadCodeEnvironment, + assertEnvironmentDefinitionsOutsideRoots, + EnvironmentWorkspaceTools, +} from './environment.js'; +import { LocalWorkspaceTools, SandboxWorkspaceTools } from './workspace.js'; +import { isValidBridgeWorkspaceToolCapabilities } from './protocol.js'; +import type { WorkspaceExecuteCommandRequest } from './protocol.js'; + +test('environment YAML validates setup and rejects unsupported policy or action fields', () => { + const definition = parseCodeEnvironment( + 'name: app\nroot: ./project\nsetup:\n command: npm ci\n', + ); + assert.equal(definition.setup?.timeoutMs, 300_000); + for (const suffix of [ + 'scope: { users: [anyone] }', + 'actions: [{}]', + 'unknown: true', + 'setup: { command: npm ci, timeoutMs: 600000 }', + 'setup: { command: npm ci, timeoutMs: -1 }', + 'setup: { command: npm ci, env: { SECRET: x } }', + 'name: duplicate', + 'repo: https://token@github.com/a/b', + ]) + assert.throws(() => + parseCodeEnvironment(`name: app\nroot: ./project\n${suffix}\n`), + ); + assert.throws(() => parseCodeEnvironment('name: &id app\nroot: *id')); + assert.throws(() => parseCodeEnvironment('x'.repeat(65_537))); + for (const field of ['setup', 'actions']) { + const command = '漢'.repeat(12_000); + const suffix = + field === 'setup' + ? `setup: { command: '${command}' }` + : `actions: [{ name: test, command: '${command}' }]`; + assert.throws(() => + parseCodeEnvironment(`name: app\nroot: project\n${suffix}`), + ); + } +}); + +test('named actions use the loaded definition, reject stale revisions and preserve command restrictions', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-action-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const local = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'app', root: directory }], + }); + const executed: WorkspaceExecuteCommandRequest[] = []; + const commands = new SandboxWorkspaceTools({ + workspaceTools: local, + commandWorkspaces: ['app'], + commandSandbox: { + mutationFailuresAreAtomic: true, + async execute(request) { + executed.push(request); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'app', + stdout: '', + stderr: '', + exitCode: 0, + timedOut: false, + truncated: false, + }; + }, + }, + }); + const environments = [ + { + path: '/operator/environment.yaml', + fingerprint: 'a'.repeat(64), + definition: { + name: 'app', + root: directory, + actions: [ + { name: 'test', command: 'npm test', timeoutMs: 2000 }, + ], + }, + }, + ]; + const tools = new EnvironmentWorkspaceTools(commands, environments); + assert.ok(isValidBridgeWorkspaceToolCapabilities(tools.capabilities)); + assert.deepEqual(tools.capabilities.workspaces[0].environment?.actions, [ + 'test', + ]); + assert.equal( + JSON.stringify(tools.capabilities).includes('npm test'), + false, + ); + const request: WorkspaceExecuteCommandRequest = { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'app', + command: 'untrusted placeholder', + timeoutMs: 5000, + environmentAction: { name: 'test', fingerprint: 'a'.repeat(64) }, + }; + await tools.execute(request); + assert.equal(executed[0].command, 'npm test'); + assert.equal(executed[0].timeoutMs, 2000); + assert.equal(executed[0].environmentAction, undefined); + for (const altered of [ + { ...request, workspaceId: 'other' }, + { ...request, cwd: 'nested' }, + { + ...request, + environmentAction: { name: 'test', fingerprint: 'b'.repeat(64) }, + }, + { + ...request, + environmentAction: { name: 'other', fingerprint: 'a'.repeat(64) }, + }, + ]) + await assert.rejects( + tools.execute(altered), + /unavailable or its definition changed/, + ); + await assert.rejects(commands.execute(request), /not resolved/); + const readOnly = new EnvironmentWorkspaceTools(local, environments); + assert.deepEqual( + readOnly.capabilities.workspaces[0].environment?.actions, + [], + ); + await assert.rejects(readOnly.execute(request)); + assert.equal(executed.length, 1); +}); + +test('environment roots resolve relative to the definition and fingerprints cover setup', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-definition-')); + t.after(() => rm(directory, { recursive: true, force: true })); + await mkdir(join(directory, 'project')); + const path = join(directory, 'environment.yaml'); + await writeFile( + path, + 'name: app\nroot: project\nsetup: { command: "printf first" }\n', + ); + const first = await loadCodeEnvironment(path); + assert.ok(first.definition.root.endsWith('/project')); + await assertEnvironmentDefinitionsOutsideRoots( + [first], + [{ id: 'app', root: first.definition.root }], + ); + await writeFile( + path, + 'name: app\nroot: project\nsetup: { command: "printf second" }\n', + ); + assert.notEqual( + (await loadCodeEnvironment(path)).fingerprint, + first.fingerprint, + ); + await assert.rejects(() => + assertEnvironmentDefinitionsOutsideRoots( + [first], + [ + { + id: 'parent', + root: first.definition.root.slice(0, -'/project'.length), + }, + ], + ), + ); + await symlink(path, join(directory, 'project', 'alias.yaml')); + const alias = await loadCodeEnvironment( + join(directory, 'project', 'alias.yaml'), + ); + await assert.rejects(() => + assertEnvironmentDefinitionsOutsideRoots( + [alias], + [{ id: 'app', root: first.definition.root }], + ), + ); + assert.equal( + (await loadCodeEnvironment(join(directory, 'project', 'alias.yaml'))) + .path, + first.path, + ); +}); + +test('rejects a trusted definition with an in-workspace hard link', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-hardlink-')); + t.after(() => rm(directory, { recursive: true, force: true })); + await mkdir(join(directory, 'project')); + const path = join(directory, 'environment.yaml'); + await writeFile(path, 'name: app\nroot: project\n'); + await link(path, join(directory, 'project', 'alias.yaml')); + await assert.rejects(loadCodeEnvironment(path), /one link/); +}); + +test('rejects nested aliases passing through a workspace-controlled link', async t => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'code-env-nested-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + const trusted = join(directory, 'trusted'); + await mkdir(root); + await mkdir(trusted); + await writeFile( + join(trusted, 'environment.yaml'), + `name: app\nroot: ${root}\n`, + ); + await symlink(trusted, join(root, 'pivot')); + await symlink(join(root, 'pivot'), join(directory, 'alias')); + const loaded = await loadCodeEnvironment( + join(directory, 'alias', 'environment.yaml'), + ); + await assert.rejects( + () => + assertEnvironmentDefinitionsOutsideRoots( + [loaded], + [{ id: 'app', root }], + ), + /outside|mount alias/, + ); +}); + +test('reads complete definitions despite short filesystem reads', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-short-read-')); + t.after(() => rm(directory, { recursive: true, force: true })); + await mkdir(join(directory, 'project')); + const path = join(directory, 'environment.yaml'); + await writeFile( + path, + 'name: app\nroot: project\nsetup: { command: echo prepared }\n', + ); + const sample = await open(path); + const prototype = Object.getPrototypeOf(sample); + const read = prototype.read; + await sample.close(); + t.mock.method( + prototype, + 'read', + function ( + this: unknown, + buffer: Buffer, + offset: number, + length: number, + position: number, + ) { + return read.call( + this, + buffer, + offset, + Math.min(length, 7), + position, + ); + }, + ); + assert.equal( + (await loadCodeEnvironment(path)).definition.setup?.command, + 'echo prepared', + ); +}); + +test('rejects a root routed through another workspace and malformed UTF-8', async t => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'code-env-root-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const rootA = join(directory, 'a'); + const rootB = join(directory, 'b'); + await mkdir(rootA); + await mkdir(rootB); + await symlink(rootA, join(rootB, 'pivot')); + const path = join(directory, 'environment.yaml'); + await writeFile(path, `name: a\nroot: ${join(rootB, 'pivot')}\n`); + const loaded = await loadCodeEnvironment(path); + await assert.rejects( + () => + assertEnvironmentDefinitionsOutsideRoots( + [loaded], + [ + { id: 'a', root: rootA }, + { id: 'b', root: rootB }, + ], + ), + /root traversal|mount alias/, + ); + await symlink(rootA, join(rootA, 'self-pivot')); + await writeFile(path, `name: a\nroot: ${join(rootA, 'self-pivot')}\n`); + const selfControlled = await loadCodeEnvironment(path); + await assert.rejects( + () => + assertEnvironmentDefinitionsOutsideRoots( + [selfControlled], + [{ id: 'a', root: rootA }], + ), + /root traversal|mount alias/, + ); + await writeFile( + path, + Buffer.concat([ + Buffer.from(`name: a\nroot: ${rootA}\nsetup: { command: echo `), + Buffer.from([0xff]), + Buffer.from(' }'), + ]), + ); + await assert.rejects(loadCodeEnvironment(path), /encoded data/); +}); + +test('rejects a FIFO definition without waiting for a writer', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-env-fifo-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'environment.yaml'); + execFileSync('mkfifo', ['-m', '600', path], { timeout: 2000 }); + await assert.rejects(loadCodeEnvironment(path), /Invalid environment file/); +}); + +test('rejects a filesystem-identical control directory despite a different root path', async t => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'code-env-identity-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const trusted = join(directory, 'trusted'); + const alias = join(directory, 'alias'); + const project = join(directory, 'project'); + await mkdir(trusted); + await mkdir(project); + await symlink(trusted, alias); + const path = join(trusted, 'environment.yaml'); + await writeFile(path, `name: app\nroot: ${project}\n`); + const loaded = await loadCodeEnvironment(path); + // Unlike realpath-based containment, inode comparison also covers bind-mount aliases. + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots( + [loaded], + [{ id: 'alias', root: alias }], + ), + /outside|mount alias/, + ); +}); diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts new file mode 100644 index 00000000..700fc0dc --- /dev/null +++ b/packages/code/src/environment.ts @@ -0,0 +1,399 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { open, realpath, stat } from 'node:fs/promises'; +import { dirname, isAbsolute, relative, resolve, sep } from 'node:path'; +import { parseDocument } from 'yaml'; +import { + assertPrivateStorageAcl, + assertPrivateStorageAncestors, +} from './private-storage.js'; +import { + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, +} from './protocol.js'; +import type { LocalWorkspaceConfig } from './workspace.js'; +import { WorkspaceToolError } from './workspace.js'; +import { + createEnvironmentMountIsolation, + readEnvironmentMountTable, +} from './environment-mount.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; +import type { WorkspaceToolRequest, WorkspaceToolResult } from './protocol.js'; + +export interface CodeEnvironmentDefinition { + name: string; + root: string; + repo?: string; + ref?: string; + setup?: { command: string; timeoutMs: number }; + actions?: { name: string; command: string; timeoutMs: number }[]; +} + +export interface LoadedCodeEnvironment { + path: string; + sourceParents?: string[]; + rootPaths?: string[]; + definition: CodeEnvironmentDefinition; + fingerprint: string; +} + +function record(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function text(value: unknown, max: number): value is string { + return ( + typeof value === 'string' && + value.trim().length > 0 && + value.length <= max && + !value.includes('\0') + ); +} + +export function parseCodeEnvironment( + source: string, +): CodeEnvironmentDefinition { + if (Buffer.byteLength(source) > 65_536) + throw new Error('Environment file exceeds 64 KiB'); + const document = parseDocument(source, { + schema: 'core', + uniqueKeys: true, + }); + if (document.errors.length || document.warnings.length) { + throw new Error('Invalid environment YAML'); + } + const value: unknown = document.toJS({ maxAliasCount: 0 }); + if ( + !record(value) || + Object.keys(value).some( + key => + !['name', 'root', 'repo', 'ref', 'setup', 'actions'].includes( + key, + ), + ) || + !text(value.name, 64) || + !/^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(value.name) || + !text(value.root, 4096) || + (value.repo !== undefined && + (!text(value.repo, 256) || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(value.repo))) || + (value.ref !== undefined && + (!text(value.ref, 256) || /[\r\n]/.test(value.ref))) + ) { + throw new Error( + 'Invalid environment definition: expected name, root, optional repo, ref and setup', + ); + } + let setup: CodeEnvironmentDefinition['setup']; + if (value.setup !== undefined) { + if ( + !record(value.setup) || + Object.keys(value.setup).some( + key => !['command', 'timeoutMs'].includes(key), + ) || + !text(value.setup.command, 16_384) || + Buffer.byteLength(value.setup.command) > + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + ) { + throw new Error('Invalid environment setup'); + } + const timeoutMs = + value.setup.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS; + if ( + typeof timeoutMs !== 'number' || + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 1 || + timeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS + ) { + throw new Error( + `Environment setup timeout must be between 1 and ${BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS} ms`, + ); + } + setup = { command: value.setup.command, timeoutMs }; + } + let actions: CodeEnvironmentDefinition['actions']; + if (value.actions !== undefined) { + if (!Array.isArray(value.actions) || value.actions.length > 32) + throw new Error('Invalid environment actions'); + const names = new Set(); + actions = value.actions.map((action: unknown) => { + if ( + !record(action) || + Object.keys(action).some( + key => !['name', 'command', 'timeoutMs'].includes(key), + ) || + !text(action.name, 64) || + !/^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(action.name) || + names.has(action.name) || + !text(action.command, 16_384) || + Buffer.byteLength(action.command) > + BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + ) + throw new Error('Invalid environment action'); + const timeoutMs = action.timeoutMs ?? 30_000; + if ( + typeof timeoutMs !== 'number' || + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 1 || + timeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS + ) + throw new Error('Invalid environment action timeout'); + names.add(action.name); + return { name: action.name, command: action.command, timeoutMs }; + }); + } + return { + name: value.name, + root: value.root, + ...(typeof value.repo === 'string' ? { repo: value.repo } : {}), + ...(typeof value.ref === 'string' ? { ref: value.ref } : {}), + ...(setup ? { setup } : {}), + ...(actions ? { actions } : {}), + }; +} + +/** Resolve actions only against the worker-owned snapshot, after normal command admission. */ +export class EnvironmentWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + private readonly environments: Map; + + constructor( + private readonly delegate: WorkspaceToolExecutor, + environments: LoadedCodeEnvironment[], + ) { + this.mutationFailuresAreAtomic = delegate.mutationFailuresAreAtomic; + this.environments = new Map( + environments.map(environment => [ + environment.definition.name, + environment, + ]), + ); + this.capabilities = { + ...delegate.capabilities, + workspaces: delegate.capabilities.workspaces.map(workspace => { + const environment = this.environments.get(workspace.id); + if (!environment) return workspace; + const operations = + workspace.operations ?? delegate.capabilities.operations; + return { + ...workspace, + environment: { + fingerprint: environment.fingerprint, + ...(environment.definition.repo + ? { repo: environment.definition.repo } + : {}), + ...(environment.definition.ref + ? { ref: environment.definition.ref } + : {}), + actions: operations.includes('execute_command') + ? (environment.definition.actions ?? []).map( + action => action.name, + ) + : [], + }, + }; + }), + }; + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + ): Promise { + if ( + request.operation !== 'execute_command' || + !request.environmentAction + ) { + return this.delegate.execute(request, signal); + } + const environment = this.environments.get(request.workspaceId); + const action = environment?.definition.actions?.find( + action => action.name === request.environmentAction?.name, + ); + if ( + !environment || + environment.fingerprint !== request.environmentAction.fingerprint || + !action || + (request.cwd !== undefined && request.cwd !== '.') + ) { + throw new WorkspaceToolError( + 'Environment action is unavailable or its definition changed', + 'INVALID_REQUEST', + ); + } + const { environmentAction: _action, ...commandRequest } = request; + return this.delegate.execute( + { + ...commandRequest, + command: action.command, + timeoutMs: Math.min( + request.timeoutMs ?? action.timeoutMs, + action.timeoutMs, + ), + cwd: '.', + }, + signal, + ); + } +} + +export async function loadCodeEnvironment( + path: string, +): Promise { + const sourcePath = resolve(path); + const sourceParents = await assertPrivateStorageAncestors(sourcePath); + const canonicalPath = await realpath(sourcePath); + const handle = await open( + canonicalPath, + constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW, + ); + let definition: CodeEnvironmentDefinition; + try { + const metadata = await handle.stat(); + const self = process.getuid?.(); + if ( + metadata.nlink !== 1 || + (metadata.mode & 0o022) !== 0 || + (self !== undefined && metadata.uid !== self && metadata.uid !== 0) + ) { + throw new Error( + 'Environment definitions must have one link, a trusted owner and no group or other write permissions', + ); + } + await assertPrivateStorageAcl(handle, canonicalPath); + if (!metadata.isFile() || metadata.size > 65_536) + throw new Error('Invalid environment file'); + const buffer = Buffer.alloc(65_537); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const result = await handle.read( + buffer, + bytesRead, + buffer.length - bytesRead, + bytesRead, + ); + if (result.bytesRead === 0) break; + bytesRead += result.bytesRead; + } + const after = await handle.stat(); + if ( + bytesRead !== metadata.size || + after.size !== metadata.size || + after.mtimeMs !== metadata.mtimeMs || + after.ctimeMs !== metadata.ctimeMs + ) { + throw new Error('Environment definition changed while reading'); + } + definition = parseCodeEnvironment( + new TextDecoder('utf-8', { fatal: true }).decode( + buffer.subarray(0, bytesRead), + ), + ); + } finally { + await handle.close(); + } + const rootPath = resolve(dirname(canonicalPath), definition.root); + const rootPaths = await assertPrivateStorageAncestors(rootPath); + const root = await realpath(rootPath); + if (!(await stat(root)).isDirectory()) + throw new Error('Environment root must be a directory'); + definition = { ...definition, root }; + return { + path: canonicalPath, + sourceParents, + rootPaths, + definition, + fingerprint: createHash('sha256') + .update(JSON.stringify(definition)) + .digest('hex'), + }; +} + +/** A workspace must never be able to rewrite a definition used on the next startup. */ +export async function assertEnvironmentDefinitionsOutsideRoots( + environments: readonly LoadedCodeEnvironment[], + roots: readonly LocalWorkspaceConfig[], +): Promise { + if (!environments.length) return; + const mountTable = await readEnvironmentMountTable(); + if (mountTable !== undefined) { + const assertMountIsolation = + createEnvironmentMountIsolation(mountTable); + assertMountIsolation( + environments.flatMap(environment => [ + environment.path, + ...(environment.sourceParents ?? []), + ]), + roots.map(root => root.root), + ); + for (const environment of environments) { + assertMountIsolation( + environment.rootPaths ?? [], + roots + .filter(root => root.id !== environment.definition.name) + .map(root => root.root), + ); + assertMountIsolation( + (environment.rootPaths ?? []).filter( + path => path !== environment.definition.root, + ), + roots + .filter(root => root.id === environment.definition.name) + .map(root => root.root), + ); + } + } + const identities = new Map>(); + const identity = (path: string): Promise => { + let result = identities.get(path); + if (!result) { + result = stat(path).then( + metadata => `${metadata.dev}:${metadata.ino}`, + ); + identities.set(path, result); + } + return result; + }; + for (const environment of environments) { + for (const root of roots) { + const rootIdentity = await identity(root.root); + // No granted workspace may control how this root resolves on restart. + { + for (const component of environment.rootPaths ?? []) { + const path = relative(root.root, component); + if (path === '' && root.id === environment.definition.name) + continue; + if ( + (await identity(component)) === rootIdentity || + path === '' || + (!isAbsolute(path) && + path !== '..' && + !path.startsWith(`..${sep}`)) + ) { + throw new Error( + 'Environment root traversal crosses a workspace-controlled component', + ); + } + } + } + for (const controlPath of [ + environment.path, + ...(environment.sourceParents ?? []), + ]) { + const path = relative(root.root, controlPath); + if ( + (await identity(controlPath)) === rootIdentity || + path === '' || + (!isAbsolute(path) && + path !== '..' && + !path.startsWith(`..${sep}`)) + ) { + throw new Error( + 'Environment definitions must be outside every registered workspace root', + ); + } + } + } + } +} diff --git a/packages/code/src/private-storage.ts b/packages/code/src/private-storage.ts index 25f23a2a..d896de33 100644 --- a/packages/code/src/private-storage.ts +++ b/packages/code/src/private-storage.ts @@ -48,12 +48,15 @@ export async function removePrivateStorageAcl( * links one component at a time so even intermediate link targets are checked. * Other local accounts cannot replace a checked entry: its parent is either * non-writable or sticky and the entry belongs to this account or root. + * Returns every traversed entry, including intermediate symlinks, so callers + * can also enforce containment restrictions without resolving those entries away. */ export async function assertPrivateStorageAncestors( path: string, allowMissing = false, -): Promise { +): Promise { assertPrivateStorageSupported(); + const visited: string[] = []; const uid = process.getuid!(); let current = '/'; const pending = (isAbsolute(path) ? path : `${process.cwd()}/${path}`).split('/'); @@ -63,7 +66,8 @@ export async function assertPrivateStorageAncestors( if (allowMissing && error.code === 'ENOENT') return undefined; throw error; }); - if (metadata === undefined) return; + if (metadata === undefined) return visited; + visited.push(current); if (metadata.uid !== uid && metadata.uid !== 0) { throw new BridgeProtocolError( `${current} is owned by another account (uid ${metadata.uid}), ` + @@ -102,7 +106,7 @@ export async function assertPrivateStorageAncestors( } let next = pending.shift(); while (next === '' || next === '.') next = pending.shift(); - if (next === undefined) return; + if (next === undefined) return visited; current = next === '..' ? dirname(current) : `${current === '/' ? '' : current}/${next}`; } } diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index b92d21ac..9199fcf0 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -21,7 +21,8 @@ export const BRIDGE_WORKSPACE_COMMAND_DEFAULT_OUTPUT_BYTES = 256 * 1024; export const BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES = 1024 * 1024; export const BRIDGE_WORKSPACE_COMMAND_SIGNAL_MAX_LENGTH = 32; export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES = 100; -export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES = BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES - 2; +export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES = + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILES - 2; export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_CONCURRENCY = 4; export const BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS = 30_000; @@ -41,26 +42,119 @@ export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; * locally instead of discovering the mismatch only after mutating a workspace. */ const BRIDGE_ARTIFACT_EXTENSIONS = new Set([ - '.c', '.cs', '.cpp', '.go', '.java', '.js', '.kt', '.kts', '.lua', - '.php', '.pl', '.ps1', '.py', '.r', '.rb', '.rs', '.scala', '.sh', - '.sql', '.swift', '.ts', '.jsx', '.tsx', '.groovy', - '.css', '.htm', '.html', '.less', '.sass', '.scss', '.svg', '.svelte', '.vue', - '.adoc', '.asciidoc', '.md', '.rst', '.tex', '.txt', '.wiki', - '.csv', '.json', '.bson', '.json5', '.jsonl', '.parquet', '.tsv', - '.xml', '.yaml', '.yml', - '.ics', '.ical', '.ifb', '.icalendar', - '.conf', '.env', '.gitignore', '.ini', '.properties', '.toml', - '.doc', '.docx', '.pdf', '.ppt', '.pptx', '.xls', '.xlsx', - '.odt', '.ods', '.odp', '.rtf', - '.avif', '.bmp', '.gif', '.ico', '.jpeg', '.jpg', '.png', - '.tif', '.tiff', '.webp', - '.eot', '.ttf', '.woff', '.woff2', - '.7z', '.bz2', '.gz', '.gzip', '.rar', '.tar', '.zip', - '.tf', '.tfvars', '.tfstate', '.hcl', - '.dockerfile', '.Dockerfile', '.dockerignore', - '.helmignore', '.helmfile', '.jenkinsfile', '.vagrantfile', - '.eslintrc', '.prettierrc', '.editorconfig', '.nomad', - '.bat', '.cmd', '.deb', '.log', '.rpm', '.vbs', + '.c', + '.cs', + '.cpp', + '.go', + '.java', + '.js', + '.kt', + '.kts', + '.lua', + '.php', + '.pl', + '.ps1', + '.py', + '.r', + '.rb', + '.rs', + '.scala', + '.sh', + '.sql', + '.swift', + '.ts', + '.jsx', + '.tsx', + '.groovy', + '.css', + '.htm', + '.html', + '.less', + '.sass', + '.scss', + '.svg', + '.svelte', + '.vue', + '.adoc', + '.asciidoc', + '.md', + '.rst', + '.tex', + '.txt', + '.wiki', + '.csv', + '.json', + '.bson', + '.json5', + '.jsonl', + '.parquet', + '.tsv', + '.xml', + '.yaml', + '.yml', + '.ics', + '.ical', + '.ifb', + '.icalendar', + '.conf', + '.env', + '.gitignore', + '.ini', + '.properties', + '.toml', + '.doc', + '.docx', + '.pdf', + '.ppt', + '.pptx', + '.xls', + '.xlsx', + '.odt', + '.ods', + '.odp', + '.rtf', + '.avif', + '.bmp', + '.gif', + '.ico', + '.jpeg', + '.jpg', + '.png', + '.tif', + '.tiff', + '.webp', + '.eot', + '.ttf', + '.woff', + '.woff2', + '.7z', + '.bz2', + '.gz', + '.gzip', + '.rar', + '.tar', + '.zip', + '.tf', + '.tfvars', + '.tfstate', + '.hcl', + '.dockerfile', + '.Dockerfile', + '.dockerignore', + '.helmignore', + '.helmfile', + '.jenkinsfile', + '.vagrantfile', + '.eslintrc', + '.prettierrc', + '.editorconfig', + '.nomad', + '.bat', + '.cmd', + '.deb', + '.log', + '.rpm', + '.vbs', ]); function portableBasename(name: string): string { @@ -94,7 +188,8 @@ const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { '.css': 'text/css', '.csv': 'text/csv', '.doc': 'application/msword', - '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.docx': + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', '.gif': 'image/gif', '.gz': 'application/gzip', '.gzip': 'application/gzip', @@ -123,7 +218,8 @@ const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { '.pdf': 'application/pdf', '.png': 'image/png', '.ppt': 'application/vnd.ms-powerpoint', - '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.pptx': + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', '.py': 'text/x-python', '.rst': 'text/x-rst', '.rtf': 'application/rtf', @@ -143,7 +239,8 @@ const BRIDGE_ARTIFACT_MEDIA_TYPES: Readonly> = { '.woff': 'font/woff', '.woff2': 'font/woff2', '.xls': 'application/vnd.ms-excel', - '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.xlsx': + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', '.xml': 'application/xml', '.yaml': 'application/yaml', '.yml': 'application/yaml', @@ -180,6 +277,12 @@ export interface BridgeWorkspaceDescriptor { name?: string; /** Optional per-workspace restriction. Omitted by protocol-v1 readers. */ operations?: BridgeWorkspaceToolOperation[]; + environment?: { + fingerprint: string; + repo?: string; + ref?: string; + actions: string[]; + }; } export interface BridgeWorkspaceToolCapabilities { @@ -291,7 +394,8 @@ interface WorkspaceEditFileRequestBase { expectedBaseSha256?: string; } -export interface WorkspaceSingleEditFileRequest extends WorkspaceEditFileRequestBase { +export interface WorkspaceSingleEditFileRequest + extends WorkspaceEditFileRequestBase { /** Legacy single-edit form. */ oldText: string; /** Legacy single-edit form. */ @@ -299,7 +403,8 @@ export interface WorkspaceSingleEditFileRequest extends WorkspaceEditFileRequest edits?: never; } -export interface WorkspaceBatchEditFileRequest extends WorkspaceEditFileRequestBase { +export interface WorkspaceBatchEditFileRequest + extends WorkspaceEditFileRequestBase { /** Ordered exact replacements applied atomically as one file mutation. */ edits: WorkspaceTextEdit[]; oldText?: never; @@ -307,7 +412,8 @@ export interface WorkspaceBatchEditFileRequest extends WorkspaceEditFileRequestB } export type WorkspaceEditFileRequest = - WorkspaceSingleEditFileRequest | WorkspaceBatchEditFileRequest; + | WorkspaceSingleEditFileRequest + | WorkspaceBatchEditFileRequest; export interface WorkspaceTextEdit { oldText: string; @@ -330,20 +436,23 @@ interface WorkspacePreviewEditRequestBase { path: string; } -export interface WorkspaceSinglePreviewEditRequest extends WorkspacePreviewEditRequestBase { +export interface WorkspaceSinglePreviewEditRequest + extends WorkspacePreviewEditRequestBase { oldText: string; newText: string; edits?: never; } -export interface WorkspaceBatchPreviewEditRequest extends WorkspacePreviewEditRequestBase { +export interface WorkspaceBatchPreviewEditRequest + extends WorkspacePreviewEditRequestBase { edits: WorkspaceTextEdit[]; oldText?: never; newText?: never; } export type WorkspacePreviewEditRequest = - WorkspaceSinglePreviewEditRequest | WorkspaceBatchPreviewEditRequest; + | WorkspaceSinglePreviewEditRequest + | WorkspaceBatchPreviewEditRequest; export interface WorkspacePreviewEditResult { protocolVersion: BridgeProtocolVersion; @@ -368,6 +477,7 @@ export interface WorkspaceExecuteCommandRequest { timeoutMs?: number; /** Aggregate UTF-8 stdout and stderr budget. */ maxOutputBytes?: number; + environmentAction?: { name: string; fingerprint: string }; } export interface WorkspaceExecuteCommandResult { @@ -452,6 +562,7 @@ const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ ]); const WORKSPACE_TEXT_EDIT_KEYS = new Set(['oldText', 'newText']); const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ + 'environmentAction', 'protocolVersion', 'operation', 'workspaceId', @@ -720,7 +831,8 @@ export function isWorkspaceToolErrorCode( } export type BridgeSettlement = - BridgeFulfilledSettlement | BridgeRejectedSettlement; + | BridgeFulfilledSettlement + | BridgeRejectedSettlement; export interface BridgeSettlementResponse { protocolVersion: BridgeProtocolVersion; @@ -806,7 +918,8 @@ export function isBridgeWorkspaceProgrammaticRequest( (body.transfer_timeout_ms !== undefined && (!Number.isSafeInteger(body.transfer_timeout_ms) || Number(body.transfer_timeout_ms) < 1 || - Number(body.transfer_timeout_ms) > BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS)) || + Number(body.transfer_timeout_ms) > + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS)) || (body.run_timeout !== undefined && (!Number.isSafeInteger(body.run_timeout) || Number(body.run_timeout) < 1 || @@ -826,7 +939,8 @@ export function isBridgeWorkspaceProgrammaticRequest( if ( !isSafePortableRelativePath(file.name) || file.name === '.' || - portableBasename(file.name).toLowerCase() === '_ptc_pending_result.json' || + portableBasename(file.name).toLowerCase() === + '_ptc_pending_result.json' || normalizePortableRelativePath(file.name) !== file.name || names.has(file.name) ) { @@ -875,7 +989,9 @@ export function isBridgeWorkspaceProgrammaticRequest( const segments = name.split('/'); let ancestor = ''; for (let index = 0; index < segments.length - 1; index += 1) { - ancestor = ancestor ? `${ancestor}/${segments[index]}` : segments[index]!; + ancestor = ancestor + ? `${ancestor}/${segments[index]}` + : segments[index]!; if (names.has(ancestor)) return false; } } @@ -1105,6 +1221,22 @@ export function isWorkspaceToolRequest( } if (request.operation === 'execute_command') { return ( + (request.environmentAction === undefined || + (typeof request.environmentAction === 'object' && + request.environmentAction !== null && + Object.keys(request.environmentAction).length === 2 && + typeof (request.environmentAction as { name?: unknown }) + .name === 'string' && + /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test( + (request.environmentAction as { name: string }).name, + ) && + typeof ( + request.environmentAction as { fingerprint?: unknown } + ).fingerprint === 'string' && + /^[a-f0-9]{64}$/.test( + (request.environmentAction as { fingerprint: string }) + .fingerprint, + ))) && hasOnlyKeys(request, WORKSPACE_COMMAND_REQUEST_KEYS) && typeof request.command === 'string' && request.command.trim().length > 0 && @@ -1438,11 +1570,17 @@ export function isValidBridgeWorkspaceToolCapabilities( const descriptor = workspace as Record; if ( Object.keys(descriptor).some( - key => key !== 'id' && key !== 'name' && key !== 'operations', + key => + key !== 'id' && + key !== 'name' && + key !== 'operations' && + key !== 'environment', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || workspaceIds.has(descriptor.id) || + (descriptor.environment !== undefined && + !isValidCodeEnvironmentDescriptor(descriptor.environment)) || (descriptor.name !== undefined && (typeof descriptor.name !== 'string' || descriptor.name.trim().length === 0 || @@ -1469,6 +1607,37 @@ export function isValidBridgeWorkspaceToolCapabilities( }); } +export function isValidCodeEnvironmentDescriptor( + value: unknown, +): value is NonNullable { + if (typeof value !== 'object' || value === null) return false; + const environment = value as Record; + return ( + Object.keys(environment).every(key => + ['fingerprint', 'repo', 'ref', 'actions'].includes(key), + ) && + typeof environment.fingerprint === 'string' && + /^[a-f0-9]{64}$/.test(environment.fingerprint) && + (environment.repo === undefined || + (typeof environment.repo === 'string' && + environment.repo.length <= 256 && + /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(environment.repo))) && + (environment.ref === undefined || + (typeof environment.ref === 'string' && + environment.ref.trim().length > 0 && + environment.ref.length <= 256 && + !/[\0\r\n]/.test(environment.ref))) && + Array.isArray(environment.actions) && + environment.actions.length <= 32 && + environment.actions.every( + name => + typeof name === 'string' && + /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/.test(name), + ) && + new Set(environment.actions).size === environment.actions.length + ); +} + export function isValidBridgeWorkerCapabilities( value: unknown, ): value is BridgeWorkerCapabilities { diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index ffe1b350..86e85872 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -192,6 +192,13 @@ function workspaceCapabilitiesMatch( (workspace, index) => workspace.id === executor.workspaces[index]?.id && workspace.name === executor.workspaces[index]?.name && + workspace.environment?.fingerprint === executor.workspaces[index]?.environment?.fingerprint && + workspace.environment?.repo === executor.workspaces[index]?.environment?.repo && + workspace.environment?.ref === executor.workspaces[index]?.environment?.ref && + workspace.environment?.actions.length === executor.workspaces[index]?.environment?.actions.length && + (workspace.environment?.actions.every( + (action, actionIndex) => action === executor.workspaces[index]?.environment?.actions[actionIndex], + ) ?? executor.workspaces[index]?.environment == null) && workspace.operations?.length === executor.workspaces[index]?.operations?.length && (workspace.operations?.every( @@ -236,7 +243,9 @@ function registrationCompatibleCapabilities( return []; } const { operations: _operations, ...compatibleWorkspace } = workspace; - return [compatibleWorkspace]; + return [{ ...compatibleWorkspace, ...(workspace.environment ? { + environment: { ...workspace.environment, actions: [] }, + } : {}) }]; }); if (workspaces.length === 0) { const { workspaceTools: _workspaceTools, ...compatible } = capabilities; @@ -312,13 +321,17 @@ function supportedWorkspaceCapabilities( editOperations.has(operation), ); const workspaces = desired.workspaces.flatMap((workspace) => { - if (workspace.operations == null) return [workspace]; - const workspaceOperations = workspace.operations.filter((operation) => + const workspaceOperations = (workspace.operations ?? operations).filter((operation) => operations.includes(operation), ); return workspaceOperations.length === 0 ? [] - : [{ ...workspace, operations: workspaceOperations }]; + : [{ ...workspace, + ...(workspace.operations ? { operations: workspaceOperations } : {}), + ...(workspace.environment && !workspaceOperations.includes('execute_command') ? { + environment: { ...workspace.environment, actions: [] }, + } : {}), + }]; }); if (workspaces.length === 0) return undefined; const editFileFeatures = desired.editFileFeatures?.filter((feature) => diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index f6205cd4..d97735dc 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -7,6 +7,30 @@ import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; +test('worker clears named actions when command execution is not negotiated', async () => { + const workspaceTools = { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'execute_command' as const], + workspaces: [{ id: 'primary', environment: { fingerprint: 'a'.repeat(64), actions: ['test'] } }], + }; + const registrations: Array = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], workspaceTools }, + workspaceMutationQuarantine: mutationQuarantine(), + workspaceTools: { capabilities: workspaceTools, async execute() { throw new Error('not executed'); } }, + fetchImpl: async (_input, init) => { + registrations.push(JSON.parse(String(init?.body)).capabilities.workspaceTools); + return Response.json({ protocolVersion: 1, workerId: 'vm-1', incarnationId, + registeredAt: new Date().toISOString(), leaseTtlMs: 60000, supportedWorkspaceToolOperations: ['read_file'] }); + }, + }); + await worker.register(); + assert.ok(registrations.length > 0); + for (const registration of registrations) assert.deepEqual(registration.workspaces[0].environment.actions, []); +}); + const listWorkspaceCapabilities = { protocolVersion: 1 as const, operations: [ @@ -2422,6 +2446,32 @@ test('worker refuses to advertise workspace tools without a matching executor', ); }); +test('worker refuses environment metadata that differs from its executor', () => { + const environment = { fingerprint: 'a'.repeat(64), repo: 'owner/repo', ref: 'main', actions: [] as string[] }; + const workspaceTools = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', environment }], + }; + for (const changed of [ + undefined, + { ...environment, fingerprint: 'b'.repeat(64) }, + { ...environment, repo: 'other/repo' }, + { ...environment, ref: 'other' }, + { ...environment, actions: ['test'] }, + ]) { + assert.throws(() => new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], workspaceTools }, + workspaceTools: { + capabilities: { ...workspaceTools, workspaces: [{ id: 'primary', ...(changed ? { environment: changed } : {}) }] }, + async execute() { throw new Error('not executed'); }, + }, + }), /workspace tool capabilities require a matching executor/i); + } +}); + test('worker requires durable quarantine before advertising command execution', () => { const workspaceCapabilities = { protocolVersion: 1 as const, diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index dfda49fc..91e89f54 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1693,6 +1693,9 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { if (request.operation !== 'execute_command') { return this.options.workspaceTools.execute(request, signal); } + if (request.environmentAction) { + throw new WorkspaceToolError('Environment action was not resolved by this worker', 'INVALID_REQUEST'); + } if (!this.commandWorkspaces.has(request.workspaceId)) { throw new WorkspaceToolError( 'Command execution is disabled for this workspace', From f2dcb93b78578fe0fe5eeb7cf1b6a965a5d036f1 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 20:01:54 -0400 Subject: [PATCH 24/42] fix: Allow Trusted Own-Root Environment Symlinks (#212) * fix: Allow Trusted Own-Root Environment Symlinks * fix: Check Alias Parent Ownership by Filesystem Identity * fix: Enforce Parent Ownership Across Every Environment Path --- .github/workflows/ci.yml | 2 + packages/code/src/environment.test.ts | 62 +++++++++++++++++++++++++++ packages/code/src/environment.ts | 28 +++++++++++- 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f041e2d6..bbf9e302 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -194,6 +194,8 @@ jobs: node-version: 24.16.0 - run: npm ci - run: npm run build + - name: Native environment containment tests + run: node --test dist/environment.test.js - name: Native ACL and credential lifecycle tests run: node --test dist/macos-storage.test.js dist/private-storage.test.js dist/storage.test.js dist/github.test.js diff --git a/packages/code/src/environment.test.ts b/packages/code/src/environment.test.ts index 9080f6a2..a8b5b036 100644 --- a/packages/code/src/environment.test.ts +++ b/packages/code/src/environment.test.ts @@ -193,6 +193,68 @@ test('environment roots resolve relative to the definition and fingerprints cove ); }); +test('accepts an own root through trusted external symlinks without allowing other root identities', async t => { + const directory = await realpath(await mkdtemp(join(tmpdir(), 'code-env-own-alias-'))); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + await mkdir(root); + const alias = join(directory, 'alias'); + await symlink(root, alias); + await symlink(alias, join(directory, 'nested-alias')); + const path = join(directory, 'environment.yaml'); + for (const selected of [alias, join(directory, 'nested-alias')]) { + await writeFile(path, `name: app\nroot: ${selected}\n`); + const loaded = await loadCodeEnvironment(path); + assert.equal(loaded.definition.root, root); + await assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'app', root }]); + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'other', root }]), + /root traversal|mount alias/, + ); + } +}); + +test('rejects own-root links hidden by parent aliases or filesystem casing', async t => { + const directory = await realpath(await mkdtemp(join(tmpdir(), 'code-env-parent-alias-'))); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'Project'); + await mkdir(root); + await symlink(root, join(root, 'self')); + const outside = join(directory, 'outside'); + await mkdir(outside); + await symlink(root, join(outside, 'back')); + await symlink(outside, join(root, 'pivot')); + const alias = join(directory, 'parent-alias'); + await symlink(root, alias); + const path = join(directory, 'environment.yaml'); + const selectedRoots = [join(alias, 'self'), join(alias, 'pivot', 'back')]; + try { + if (await realpath(join(directory, 'project')) === root) { + selectedRoots.push(join(directory, 'project', 'self')); + selectedRoots.push(join(directory, 'project', 'pivot', 'back')); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + for (const selected of selectedRoots) { + await writeFile(path, `name: app\nroot: ${selected}\n`); + const loaded = await loadCodeEnvironment(path); + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'app', root }]), + /root traversal|mount alias/, + ); + } + const definition = join(outside, 'environment.yaml'); + await writeFile(definition, `name: app\nroot: ${root}\n`); + for (const selected of selectedRoots.filter(path => path.endsWith('/back'))) { + const loaded = await loadCodeEnvironment(selected.replace(/back$/, 'environment.yaml')); + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots([loaded], [{ id: 'app', root }]), + /outside|mount alias/, + ); + } +}); + test('rejects a trusted definition with an in-workspace hard link', async t => { const directory = await mkdtemp(join(tmpdir(), 'code-env-hardlink-')); t.after(() => rm(directory, { recursive: true, force: true })); diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index 700fc0dc..c2b2cf1f 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -355,6 +355,25 @@ export async function assertEnvironmentDefinitionsOutsideRoots( } return result; }; + // The entry's parent, not its symlink target, determines who can replace it. + // Compare ancestor identities so casing and directory aliases cannot make a + // workspace-controlled entry look external on case-insensitive filesystems. + const canonicalParents = new Map>(); + const controlsEntry = async (component: string, rootIdentity: string): Promise => { + const directory = dirname(component); + let canonical = canonicalParents.get(directory); + if (!canonical) { + canonical = realpath(directory); + canonicalParents.set(directory, canonical); + } + let parent = await canonical; + while (true) { + if ((await identity(parent)) === rootIdentity) return true; + const next = dirname(parent); + if (next === parent) return false; + parent = next; + } + }; for (const environment of environments) { for (const root of roots) { const rootIdentity = await identity(root.root); @@ -362,10 +381,14 @@ export async function assertEnvironmentDefinitionsOutsideRoots( { for (const component of environment.rootPaths ?? []) { const path = relative(root.root, component); - if (path === '' && root.id === environment.definition.name) + const sameRoot = (await identity(component)) === rootIdentity; + const controlled = await controlsEntry(component, rootIdentity); + // A trusted external alias may select its own root, but a + // link beneath that root is still writable by the workspace. + if (sameRoot && !controlled && root.id === environment.definition.name) continue; if ( - (await identity(component)) === rootIdentity || + controlled || sameRoot || path === '' || (!isAbsolute(path) && path !== '..' && @@ -383,6 +406,7 @@ export async function assertEnvironmentDefinitionsOutsideRoots( ]) { const path = relative(root.root, controlPath); if ( + (await controlsEntry(controlPath, rootIdentity)) || (await identity(controlPath)) === rootIdentity || path === '' || (!isAbsolute(path) && From 840537b9d95a3ad6c761729a198c4e9a0d020d90 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 14 Sep 2026 20:02:05 -0400 Subject: [PATCH 25/42] fix: Retain Quarantine After Failed Environment Setup (#213) * fix: Retain Quarantine After Failed Environment Setup * test: Run Native Environment Setup Lifecycle in CI * fix: Document and Verify Local Setup Quarantine Recovery --- .github/workflows/ci.yml | 4 ++ packages/code/README.md | 11 ++++-- packages/code/src/cli.ts | 4 +- packages/code/src/environment-live.test.ts | 43 ++++++++++++++++++---- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbf9e302..2bdeda21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,6 +198,10 @@ jobs: run: node --test dist/environment.test.js - name: Native ACL and credential lifecycle tests run: node --test dist/macos-storage.test.js dist/private-storage.test.js dist/storage.test.js dist/github.test.js + - name: Native environment setup lifecycle tests + env: + LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' + run: node --test dist/environment-live.test.js lambda-microvm-provisioning: name: Lambda MicroVM Provisioning diff --git a/packages/code/README.md b/packages/code/README.md index c3bb16d3..8b98618f 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -689,9 +689,14 @@ worker runs. This inspection happens at startup, not on the command hot path. Setup is an operator-authorized startup command under the configured native sandbox policy. It requires commands to be enabled, runs once per worker startup before registration, and must be idempotent for restarts. Its timeout is bounded to five -minutes and captured output to 8 KiB. Setup failure prevents registration. A crash -or uncertain termination retains the existing workspace quarantine marker; inspect -the workspace before clearing quarantine. No setup output is sent to the model. +minutes and captured output to 8 KiB. Setup failure prevents registration. A nonzero +exit, timeout, crash or uncertain termination retains the workspace quarantine marker; +inspect the workspace before running `librechat-code clear-workspace-quarantine +--worker-dir --workspace-id ` with the same +deployment and identity configuration. Only use the separate +`--reset-workspace-quarantine ` run option afterward if a server +fence also needs clearing. Only successful setup automatically clears its marker. +No setup output is sent to the model. Named actions are fixed commands without model-supplied substitution. The bridge advertises only their names and the definition fingerprint, never their shell source diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 5498af63..00eca576 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1012,12 +1012,12 @@ async function run( }, controller.signal, ); - await guard.clear('setup'); if (result.exitCode !== 0 || result.timedOut) { throw new Error( - `Environment ${id} setup failed; inspect the setup command before restarting`, + `Environment ${id} setup failed; inspect the workspace and use clear-workspace-quarantine with its root and workspace ID before restarting`, ); } + await guard.clear('setup'); process.stdout.write( `librechat-code: environment ${id} prepared\n`, ); diff --git a/packages/code/src/environment-live.test.ts b/packages/code/src/environment-live.test.ts index bb84c8b5..9d442982 100644 --- a/packages/code/src/environment-live.test.ts +++ b/packages/code/src/environment-live.test.ts @@ -8,13 +8,14 @@ import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import test from 'node:test'; -for (const { succeeds, reset } of [ - { succeeds: true, reset: false }, - { succeeds: false, reset: false }, - { succeeds: true, reset: true }, +for (const { succeeds, reset, timesOut } of [ + { succeeds: true, reset: false, timesOut: false }, + { succeeds: false, reset: false, timesOut: false }, + { succeeds: false, reset: false, timesOut: true }, + { succeeds: true, reset: true, timesOut: false }, ]) { test( - `real CLI environment setup gates registration (success=${succeeds}, reset=${reset})`, + `real CLI environment setup gates registration (success=${succeeds}, reset=${reset}, timeout=${timesOut})`, { skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', timeout: 20_000, @@ -27,7 +28,7 @@ for (const { succeeds, reset } of [ const path = join(directory, 'environment.yaml'); await writeFile( path, - `name: project\nroot: project\nsetup:\n command: 'printf prepared > prepared.txt; exit ${succeeds ? 0 : 2}'\n timeoutMs: 5000\n`, + `name: project\nroot: project\nsetup:\n command: 'printf prepared >> prepared.txt; ${timesOut ? 'sleep 10' : `exit ${succeeds ? 0 : 2}`}'\n timeoutMs: ${timesOut ? 1000 : 5000}\n`, ); let registrations = 0; let receive: (() => void) | undefined; @@ -38,6 +39,7 @@ for (const { succeeds, reset } of [ request.resume(); if (request.url?.endsWith('/register')) { registrations++; + await assert.rejects(readFile(join(directory, 'quarantine.json')), { code: 'ENOENT' }); if (reset) await assert.rejects( readFile(join(root, 'prepared.txt')), @@ -61,10 +63,15 @@ for (const { succeeds, reset } of [ }); const address = server.address(); assert.ok(address && typeof address !== 'string'); - const child = spawn( + const start = (clear = false) => spawn( process.execPath, [ fileURLToPath(new URL('./cli.js', import.meta.url)), + ...(clear ? [ + 'clear-workspace-quarantine', + '--worker-dir', root, + '--workspace-id', 'project', + ] : [ 'run', '--environment', path, @@ -72,6 +79,7 @@ for (const { succeeds, reset } of [ ...(reset ? ['--reset-workspace-quarantine', 'project'] : []), + ]), ], { env: { @@ -90,6 +98,7 @@ for (const { succeeds, reset } of [ stdio: ['ignore', 'pipe', 'pipe'], }, ); + const child = start(); const exited = once(child, 'exit'); t.after(() => child.kill('SIGKILL')); let stderr = ''; @@ -111,6 +120,26 @@ for (const { succeeds, reset } of [ assert.notEqual(code, 0); assert.match(stderr, /Environment project setup failed/); assert.equal(registrations, 0); + const marker = await readFile(join(directory, 'quarantine.json'), 'utf8'); + assert.equal(JSON.parse(marker).workspaceId, 'project'); + const before = await readFile(join(root, 'prepared.txt'), 'utf8'); + const retry = start(); + t.after(() => retry.kill('SIGKILL')); + let retryStderr = ''; + retry.stderr.on('data', chunk => { retryStderr += chunk.toString(); }); + const [retryCode] = await once(retry, 'exit'); + assert.notEqual(retryCode, 0); + assert.match(retryStderr, /quarantined/); + assert.equal(await readFile(join(root, 'prepared.txt'), 'utf8'), before); + assert.equal(await readFile(join(directory, 'quarantine.json'), 'utf8'), marker); + assert.equal(registrations, 0); + const recovery = start(true); + t.after(() => recovery.kill('SIGKILL')); + const [recoveryCode] = await once(recovery, 'exit'); + assert.equal(recoveryCode, 0); + await assert.rejects(readFile(join(directory, 'quarantine.json')), { code: 'ENOENT' }); + assert.equal(await readFile(join(root, 'prepared.txt'), 'utf8'), before); + assert.equal(registrations, 0); } }, ); From 9a3f5dbd6fedf42462239d135c2c25ec8e737628 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 15 Sep 2026 14:25:50 -0400 Subject: [PATCH 26/42] fix: Allow Lambda MicroVM metadata in hardened mode (#215) --- api/src/secure-startup.test.ts | 12 ++++++++++++ api/src/secure-startup.ts | 9 ++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/api/src/secure-startup.test.ts b/api/src/secure-startup.test.ts index c1558e85..e1291712 100644 --- a/api/src/secure-startup.test.ts +++ b/api/src/secure-startup.test.ts @@ -113,6 +113,18 @@ describe('hardened sandbox-runner startup config', () => { expect(() => validateHardenedSandboxStartup()).toThrow('REDIS_HOST'); }); + test('allows Lambda MicroVM image metadata while rejecting AWS credentials', () => { + setValidHardenedConfig(); + process.env.AWS_LAMBDA_MICROVM_IMAGE_ARN = 'arn:aws:lambda:us-east-1:123456789012:microvm-image:codeapi'; + process.env.AWS_LAMBDA_MICROVM_IMAGE_NAME = 'codeapi'; + process.env.AWS_LAMBDA_MICROVM_IMAGE_VERSION = '3'; + process.env.AWS_REGION = 'us-east-1'; + expect(() => validateHardenedSandboxStartup()).not.toThrow(); + + process.env.AWS_ACCESS_KEY_ID = 'access-key'; + expect(() => validateHardenedSandboxStartup()).toThrow('AWS_ACCESS_KEY_ID'); + }); + test('rejects missing manifest verifier and wrong forwarding target', () => { setValidHardenedConfig(); config.egress_gateway_url = ''; diff --git a/api/src/secure-startup.ts b/api/src/secure-startup.ts index 04f5638a..326bf810 100644 --- a/api/src/secure-startup.ts +++ b/api/src/secure-startup.ts @@ -1,6 +1,13 @@ import { config } from './config'; import { workspaceIsolationConfigErrors } from './workspace-isolation'; +const ALLOWED_LAMBDA_MICROVM_AWS_ENV = new Set([ + 'AWS_LAMBDA_MICROVM_IMAGE_ARN', + 'AWS_LAMBDA_MICROVM_IMAGE_NAME', + 'AWS_LAMBDA_MICROVM_IMAGE_VERSION', + 'AWS_REGION', +]); + export class SandboxSecureStartupError extends Error { constructor(message: string) { super(message); @@ -57,7 +64,7 @@ function forbiddenEnvNames(): string[] { if (name === 'CODEAPI_HARDENED_SANDBOX_MODE') continue; if (name.startsWith('CODEAPI_')) forbidden.push(name); if (name.startsWith('REDIS_')) forbidden.push(name); - if (name.startsWith('AWS_')) forbidden.push(name); + if (name.startsWith('AWS_') && !ALLOWED_LAMBDA_MICROVM_AWS_ENV.has(name)) forbidden.push(name); if (name.startsWith('S3_')) forbidden.push(name); if (name.startsWith('MINIO_')) forbidden.push(name); if (/(SECRET|TOKEN|PASSWORD|PRIVATE_KEY)/.test(name)) forbidden.push(name); From c03d309429f152ae11a6d9503d3804dbaab9d228 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 00:38:02 -0400 Subject: [PATCH 27/42] docs: add self-hosted worker setup runbook (#219) --- docs/remote-bridge/README.md | 5 + docs/remote-bridge/worker-runbook.md | 527 +++++++++++++++++++++++++++ packages/code/README.md | 3 + 3 files changed, 535 insertions(+) create mode 100644 docs/remote-bridge/worker-runbook.md diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index b9de3ac2..f66499b2 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -3,6 +3,10 @@ Remote Code Bridge makes an operator-owned VM a stateful Code API execution environment without exposing that VM to inbound internet traffic. +For an end-to-end host setup, including pairing, named environments, systemd, +launchd, GitHub App credentials, upgrades, verification, and recovery, see the +[self-hosted worker runbook](./worker-runbook.md). + ```text LibreChat -> Code API -> Redis assignment ^ | @@ -200,6 +204,7 @@ Expose the Code API deployment as an environment under the Agents endpoint: endpoints: agents: statefulCodeSessions: + allowedEnvironments: [user, agent-user, conversation] environments: - id: my-vm name: My VM diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md new file mode 100644 index 00000000..7511a265 --- /dev/null +++ b/docs/remote-bridge/worker-runbook.md @@ -0,0 +1,527 @@ +# Self-hosted worker runbook + +This runbook attaches an operator-controlled laptop or VM to a LibreChat Code +API deployment. The worker makes an outbound HTTPS connection; it does not +open an inbound port. The same procedure works for one machine or many +principal-bound machines. + +The guide uses a named environment and the native Sandbox Runtime (SRT). It +covers a restricted personal-machine deployment and the `trusted-vm` preset, +where a separate VM boundary is responsible for most host isolation. + +## 1. Understand the boundaries + +Four independently managed components participate: + +1. LibreChat stores the environment record, resolves its principal, applies + administrator/user policy, and selects the worker for a conversation. +2. Code API authenticates the selection, queues and fences assignments, and + exposes the outbound bridge. +3. `@librechat/code` runs on the attached machine, owns local workspace + admission, rotates its bridge credential, and executes tools through SRT. +4. The machine owner controls the OS, workspace, network, credentials, and + service lifecycle. + +Pairing authenticates a worker. It does not make the host trustworthy, attest +the host policy, or replace tool approval. A `trusted-vm` worker is appropriate +only when the VM boundary is already operated as the security boundary. + +## 2. Configure LibreChat and Code API + +Deploy Code API's remote-bridge profile before pairing a machine. At minimum, +use paired authentication and dynamic routing so one bridge can serve many +principal-bound workers: + +```dotenv +CODEAPI_SANDBOX_BACKEND=remote-bridge +CODEAPI_EXECUTION_PROFILE=stateful +CODEAPI_RUNTIME_SESSION_MODE=affinity +CODEAPI_BRIDGE_AUTH_MODE=paired +CODEAPI_BRIDGE_DYNAMIC_WORKERS=true +CODEAPI_BRIDGE_TOKEN= +``` + +The administrator token belongs only on the Code API/control-plane host. Never +put it on an attached machine. Configure Redis and the remaining Code API +settings as described in the [Remote Code Bridge guide](./README.md). + +Expose that Code API endpoint to LibreChat and explicitly choose which +state-sharing scopes the deployment permits: + +```yaml +endpoints: + agents: + capabilities: + [ + deferred_tools, + execute_code, + file_search, + web_search, + artifacts, + subagents, + actions, + context, + skills, + memory, + ask_user_question, + tools, + chain, + ocr, + stateful_code_sessions, + ] + statefulCodeSessions: + allowedEnvironments: [user, agent-user, conversation] + environments: + - id: attached-workers + name: Attached machines + type: attached + baseURL: https://code.example.com/v1 + default: true +``` + +The example preserves LibreChat's default capabilities and adds the opt-in +`stateful_code_sessions` capability. Adjust the list to the deployment's +policy. Exactly one configured Code environment must be the default. The three +sharing scopes mean: + +- `user`: reuse an environment for the signed-in user; +- `agent-user`: reuse it for one agent and user; and +- `conversation`: isolate reuse to one conversation. + +These scopes determine session reuse. They do not weaken a worker's filesystem +root or share one user's principal-bound machine with another user. + +Enable stateful code sessions on the intended agent and select the attached +environment. Start with file writes and command execution set to `ask`; expose +`allow` or `deny` only when the deployment and machine policy permit them. +Worker capabilities are a ceiling: a conversation setting cannot enable a +command or write that the worker did not advertise. + +## 3. Roll out compatible consumers first + +Before enabling `--environment` on a worker: + +1. Deploy a LibreChat version that accepts named environment descriptors. +2. Deploy the matching Code API API and queue-worker processes. +3. Update `@librechat/code` on the attached machine. +4. Only then restart the worker with `--environment`. + +An old worker remains compatible with new consumers until the opt-in flag is +used. An old strict consumer can reject a new worker's environment metadata. +During a rolling deployment, update every API/queue replica before changing +workers. + +Record the exact source commit or package version at every tier. Do not infer a +worker's version from the Code API server: the worker is a separate process on +a separate machine. + +## 4. Prepare the worker host + +Install: + +- Node.js 20.11 or newer (Node.js 24 is supported); +- Git; +- `bubblewrap`, `socat`, and `ripgrep` on Linux; +- Bash 5.2 or newer and `jq` when Bash Programmatic Tool Calling is enabled; + and +- GitHub CLI and Git LFS only when the workflows need them. + +For example, install the system dependencies on Ubuntu with: + +```bash +sudo apt-get update +sudo apt-get install -y bash bubblewrap git jq ripgrep socat +``` + +On macOS, install the optional PTC and GitHub tools with: + +```bash +brew install bash gh git-lfs jq ripgrep +``` + +Install Node.js through the host's managed package source or version manager. +Bun is not required by `@librechat/code`. + +Keep source, application state, environment definitions, and credentials in +separate paths. For example: + +```text +/opt/librechat-code/releases// pinned worker source/build +/srv/code-workspaces/ coding roots +/etc/librechat-code/environments/ operator-owned YAML definitions +~/.config/librechat/code/ paired identity +~/.config/librechat-code/github-app.pem optional GitHub App key +``` + +Every ancestor of a definition, identity, key, quarantine file, or workspace +root must be owned by the worker account or root. It must not be writable by +group or other users. Sticky shared directories such as `/tmp` are handled +separately, but should not hold durable configuration. + +The workspace remains writable by its owner. For a dedicated service account, +a typical root is: + +```bash +sudo install -d -o librechat-code -g librechat-code -m 0750 /srv/code-workspaces +``` + +Do not register a home directory or another root containing credentials, +shell history, SSH keys, or unrelated projects. + +## 5. Install a pinned worker + +Use a published version when available. To install from source, keep a pinned +checkout and build only the worker package: + +```bash +git clone https://github.com/LibreChat-AI/code-interpreter.git /opt/librechat-code/source +cd /opt/librechat-code/source +git fetch origin main +git checkout --detach +npm ci --prefix packages/code +npm run build --prefix packages/code +cd packages/code +sudo npm link +``` + +Confirm that `/usr/local/bin/librechat-code` resolves to the intended build. +Do not replace a running release until the new build and its native imports +have succeeded. Keeping releases in commit-named directories makes rollback a +service-path change instead of a rebuild. + +## 6. Pair the machine + +Create a pairing in LibreChat's Code environments UI when available. The +pairing must be bound to the intended deployment, tenant, user, role, or group. +The code is single-use and expires after ten minutes. + +Redeem it on the worker machine: + +```bash +librechat-code pair https://code.example.com/v1 '' \ + --worker-id code-example123 +``` + +Run pairing as the same operating-system account that will run the worker. If +the systemd service uses `User=librechat-code`, run the command as that account +or supply an explicit identity path the account can read and replace. + +The CLI generates the Ed25519 private key locally and saves the identity under +`~/.config/librechat/code/` with owner-only permissions. Do not transmit or +copy that file through chat. The bridge credential expires after fifteen +minutes, but a running worker rotates it automatically. A normal restart does +not require re-pairing. + +For a custom location, use `--identity` during pairing and set +`LIBRECHAT_CODE_IDENTITY_FILE` in the service. Keep the worker ID stable: agent +defaults and conversations refer to the environment record associated with +that identity. + +## 7. Define named environments + +Store definitions outside every workspace root. A broad, multi-project VM can +preserve an existing `primary` binding without pretending the root is one Git +repository: + +```yaml +# /etc/librechat-code/environments/primary.yaml +name: primary +root: /srv/code-workspaces +``` + +For a single project, descriptive repository metadata and fixed actions may be +useful: + +```yaml +name: app-dev +root: /srv/code-workspaces/app +repo: example/app +ref: main +setup: + command: npm ci + timeoutMs: 300000 +actions: + - name: typecheck + command: npm run typecheck + timeoutMs: 120000 + - name: test + command: npm test + timeoutMs: 300000 +``` + +Important semantics: + +- `name` is both the workspace ID and its current display name. Preserve an + existing ID such as `primary` to preserve agent/conversation bindings. +- `repo` and `ref` are labels. They do not clone, fetch, or check out anything. +- `root` must already exist. Relative roots resolve from the definition file. +- Setup runs before registration on every worker start. It must be idempotent. +- A setup failure or timeout prevents registration and leaves a durable + quarantine marker for operator inspection. +- Actions are fixed operator commands. The model selects only the action name + and fingerprint; it cannot inject arguments, a command, or a working + directory. +- Actions still pass through LibreChat approval and worker command policy. +- Up to 32 roots may be declared, and they must not overlap. A broad parent + environment cannot coexist with child project environments. + +Definitions contain policy rather than secrets. A root-owned file may be +readable by the service account, but must not be group/other writable. For +example: + +```bash +sudo install -d -o root -g librechat-code -m 0750 /etc/librechat-code/environments +sudo install -o root -g librechat-code -m 0640 primary.yaml \ + /etc/librechat-code/environments/primary.yaml +``` + +Do not combine `--environment` with `--worker-dir`, `--default-workspace`, +`--workspace`, `--workspace-id`, or `--workspace-name`. Remove the equivalent +`LIBRECHAT_CODE_WORKER_DIR`, `LIBRECHAT_CODE_WORKSPACE_ID`, and +`LIBRECHAT_CODE_WORKSPACE_NAME` settings too. + +## 8. Choose a command policy + +For a personal machine, use the default `restricted` policy and explicitly +allow only required network destinations. + +For a separately secured VM whose outer boundary is managed by the operator, +the worker may use: + +```text +--allow-workspace-writes +--allow-workspace-commands +--command-policy-preset trusted-vm +``` + +`trusted-vm` is a policy preset, not an unsandboxed execution mode. SRT still +protects the bridge identity, GitHub credentials, worker configuration, and +control sockets. The preset deliberately permits broader workspace and network +behavior because the VM owner accepts responsibility for the host boundary. + +LibreChat's tool approval remains independent. Enabling commands on a worker +does not authorize a user or agent to bypass `ask` or `deny` policy. + +## 9. Optionally configure a GitHub App + +Prefer a GitHub App over a personal token. Install it only on repositories the +agent may use and grant the minimum permissions its workflows require. Git +clone/fetch/push generally needs repository Contents access; API-based pull +request workflows also need Pull requests access. + +Store the downloaded private key outside every workspace. Unlike an +environment definition, the key must have no group or other access and must be +readable by the service account: + +```bash +install -d -m 0700 ~/.config/librechat-code +install -m 0600 app.private-key.pem ~/.config/librechat-code/github-app.pem +``` + +Configure the worker, preferably in a separate service drop-in: + +```ini +[Service] +Environment=LIBRECHAT_CODE_GITHUB_APP_ID=12345 +Environment=LIBRECHAT_CODE_GITHUB_INSTALLATION_ID=67890 +Environment=LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/home/librechat-code/.config/librechat-code/github-app.pem +``` + +The trusted worker mints short-lived installation tokens. Sandboxed commands +receive masked Git/`gh` credentials only for the configured GitHub hosts; the +token is not written to the repository, remote URL, or Git configuration. + +## 10. Run under systemd + +Use a dedicated service account in a multi-user deployment. This example keeps +the paired identity in its default location: + +```ini +# /etc/systemd/system/librechat-code.service +[Unit] +Description=LibreChat attached code worker +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=librechat-code +Group=librechat-code +WorkingDirectory=/srv/code-workspaces +Environment=HOME=/home/librechat-code +Environment=NODE_ENV=production +Environment=LIBRECHAT_CODE_WORKER_ID=code-example123 +ExecStart=/usr/local/bin/librechat-code run \ + --environment /etc/librechat-code/environments/primary.yaml \ + --allow-workspace-writes \ + --allow-workspace-commands +Restart=always +RestartSec=5s +TimeoutStopSec=35s +KillMode=control-group +UMask=0077 +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target +``` + +For a trusted VM, append `--command-policy-preset trusted-vm` to `ExecStart`. +After installing or changing a unit or drop-in, reload it before restart: + +```bash +sudo systemd-analyze verify librechat-code.service +sudo systemctl daemon-reload +sudo systemctl enable --now librechat-code.service +``` + +`systemctl restart` alone does not load a changed unit definition. + +## 11. Run under launchd on macOS + +Use absolute executable and release paths in the property list. Keep the paired +identity in the logged-in user's private configuration directory: + +```xml +ProgramArguments + + /absolute/path/to/node + /opt/librechat-code/releases/COMMIT/packages/code/dist/cli.js + run + --environment + /Users/worker/.config/librechat/code/environments/primary.yaml + --allow-workspace-writes + --allow-workspace-commands + +EnvironmentVariables + + LIBRECHAT_CODE_WORKER_ID + code-example123 + LIBRECHAT_CODE_IDENTITY_FILE + /Users/worker/.config/librechat/code/code-example123.json + +``` + +Editing the plist does not update launchd's cached job. Reload it: + +```bash +launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/ai.librechat.code.plist +launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/ai.librechat.code.plist +``` + +`launchctl kickstart -k` restarts the already-loaded definition and therefore +continues using stale paths after a plist edit. + +## 12. Verify the complete path + +Do not stop at “the process is running.” Check: + +1. The service command points to the intended version and environment file. +2. The native executor child started. +3. The worker has an established outbound HTTPS connection to Code API. +4. Code API reports the worker `online: true` and `ready: true` with the expected + workspace IDs and operations. +5. LibreChat lists the environment for the expected principal. +6. A disposable chat can select the workspace, read a file, perform an approved + write, execute a command, and retain state on the next turn. +7. Stop/cancellation prevents a delayed mutation. +8. A denied action remains denied even when the worker uses `trusted-vm`. + +Useful host checks: + +```bash +systemctl show librechat-code.service -p ExecStart -p MainPID -p NRestarts +journalctl -u librechat-code.service --since '10 minutes ago' +ss -tpn | grep librechat-code +``` + +The Code API status endpoint requires its administrator credential. Filter the +response before sharing it; do not expose tokens, pairings, bindings, or host +paths in logs or chat. + +## 13. Upgrade and roll back + +For each upgrade: + +1. Read the release notes and confirm whether LibreChat/Code API consumers must + land first. +2. Stage and build the new worker beside the current release. +3. Run focused package/native checks. +4. Update the service path or pinned checkout. +5. Reload the service manager definition when it changed. +6. Restart once and verify the complete path above. +7. Retain the previous release until the worker has completed real work. + +For source-linked installations, verify both `git rev-parse HEAD` and the +actual executable target. Updating a Code API checkout on another host does not +update this worker. + +Rollback by restoring the previous executable/service path and restarting. Do +not roll a new-metadata worker back behind the minimum consumer version while +it still advertises named environments. + +## 14. Recover safely + +### Expired bridge credential + +A running worker refreshes its short-lived credential automatically. If a +machine is offline long enough that refresh can no longer authenticate, issue +a fresh one-time pairing for the same worker ID and redeem it with a newly +generated keypair. Reusing the worker ID preserves the LibreChat environment +record and its agent assignments; creating a new ID creates a new environment. + +### Failed environment setup or uncertain mutation + +Inspect or restore the affected workspace first. Then, with the normal worker +stopped, clear the local quarantine using the same identity/deployment context: + +```bash +librechat-code clear-workspace-quarantine \ + --worker-dir /srv/code-workspaces/app \ + --workspace-id app-dev +``` + +If Code API also retains a server-side workspace fence, run the normal worker +configuration once with `--reset-workspace-quarantine app-dev`, wait for it to +exit successfully, and then start the normal service. The reset flag does not +replace the local clear command. + +Never clear quarantine merely to make the worker start. It represents a setup, +command, cancellation, or settlement whose effects may be incomplete. + +## 15. Common failures + +- **`--environment cannot be combined...`:** remove old workspace flags and + equivalent environment variables. +- **Definition or root rejected as replaceable:** remove group/other write + permission from every path ancestor; keep owner write. +- **Worker starts but old command/path remains:** run + `systemctl daemon-reload`, or fully boot out/bootstrap a changed launchd + plist. +- **Worker online but not ready:** check native sandbox preparation, definition + validation, setup, quarantine, and readiness logs. +- **Setup repeats on restart:** setup is intentionally per-start; make it + idempotent or remove it. +- **Git works on the host but not in tools:** verify the App installation, + permissions, private-key mode/owner, and allowed GitHub domains. +- **Repository label is present but files are absent:** `repo`/`ref` are + metadata; clone or mount the repository yourself. +- **Existing chats lose their workspace:** preserve the original workspace ID + in `name`, commonly `primary`. +- **Multiple project roots are rejected:** roots cannot overlap; remove the + broad parent or keep it as the only environment. + +## Final checklist + +- [ ] LibreChat and every Code API replica support the worker protocol. +- [ ] Worker version/source commit is recorded. +- [ ] Pairing is principal-bound and the identity file is private. +- [ ] Definitions are outside roots and immutable to sandboxed tools. +- [ ] Workspace ancestors are not group/other writable. +- [ ] GitHub App is optional, least-privilege, and installed only where needed. +- [ ] Approval policy remains enforced independently of worker capability. +- [ ] Service manager uses the intended executable and configuration. +- [ ] Worker is online, ready, and advertises the expected workspace. +- [ ] Read, approved mutation, command, persistence, denial, and cancellation + are tested. +- [ ] Upgrade and quarantine-recovery procedures are recorded for the operator. diff --git a/packages/code/README.md b/packages/code/README.md index 8b98618f..3cc61677 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -3,6 +3,9 @@ Provider-neutral protocol and worker CLI for attaching a stateful, sandboxed code environment to LibreChat Code API. +For a complete machine setup and operations guide, see the +[self-hosted worker runbook](../../docs/remote-bridge/worker-runbook.md). + The CLI owns the runtime-supervisor seam. Native workspace commands use Anthropic's open-source Sandbox Runtime (SRT) on the worker machine. The bundled endpoint adapter can also connect to an already-running loopback Code From 6dbf03ba13db9e0c894d06b73412a43cfd39cb0f Mon Sep 17 00:00:00 2001 From: Jackson Riding <99007683+jacksonriding@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:09:43 +1000 Subject: [PATCH 28/42] fix: bound memory buffering for streamed file uploads (#218) --- service/src/file-server.ts | 67 +------------------- service/src/minio-client.test.ts | 101 +++++++++++++++++++++++++++++++ service/src/minio-client.ts | 69 +++++++++++++++++++++ 3 files changed, 172 insertions(+), 65 deletions(-) create mode 100644 service/src/minio-client.test.ts create mode 100644 service/src/minio-client.ts diff --git a/service/src/file-server.ts b/service/src/file-server.ts index 22302042..00b91e4a 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -11,11 +11,11 @@ import { sendFileDownload } from './file-download'; import path from 'path'; import IORedis from 'ioredis'; import express from 'express'; -import { Client } from 'minio'; +import { createMinioClient } from './minio-client'; import { nanoid } from 'nanoid'; import { PassThrough } from 'stream'; import { pipeline } from 'stream/promises'; -import type { BucketItem, BucketItemStat, ClientOptions } from 'minio'; +import type { BucketItem, BucketItemStat, Client } from 'minio'; import type { Readable } from 'stream'; import type * as tls from 'tls'; import type * as t from './types'; @@ -40,69 +40,6 @@ app.use(httpMetricsMiddleware); const bucketName = process.env.MINIO_BUCKET ?? 'test-bucket'; -type IamProviderModule = { IamAwsProvider?: new (opts: object) => unknown; default?: new (opts: object) => unknown }; - -async function createMinioClient(): Promise { - const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; - const irsaEnvVars = Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && Boolean(process.env.AWS_ROLE_ARN); - const useIrsa = irsaExplicit || irsaEnvVars; - - const baseConfig: ClientOptions = { - endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', - port: process.env.MINIO_NO_PORT?.toLowerCase() === 'true' ? undefined : parseInt(process.env.MINIO_PORT ?? '9000'), - useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', - region: process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', - }; - - if (useIrsa) { - logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { - tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, - roleArn: process.env.AWS_ROLE_ARN, - region: baseConfig.region, - }); - - /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module - * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) - */ - let IamAwsProviderClass: new (opts: object) => unknown; - try { - const mod = await import('minio/dist/main/IamAwsProvider.js') as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (primaryError) { - try { - // Fallback for bun: resolve path using require if available (CJS context) - let resolvePath = 'node_modules/minio/'; - try { - // eslint-disable-next-line @typescript-eslint/no-require-imports - resolvePath = require.resolve('minio').replace(/dist\/.*$/, ''); - } catch { - // require.resolve not available (ESM context), use default path - } - const mod = await import(`${resolvePath}dist/main/IamAwsProvider.js`) as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (fallbackError) { - logger.error('Failed to load IamAwsProvider', { primaryError, fallbackError }); - throw new Error('Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.'); - } - } - - const credentialsProvider = new IamAwsProviderClass({}); - - return new Client({ - ...baseConfig, - credentialsProvider: credentialsProvider as ClientOptions['credentialsProvider'], - }); - } - - logger.info('Using explicit credentials for MinIO/S3 authentication'); - return new Client({ - ...baseConfig, - accessKey: process.env.MINIO_ACCESS_KEY ?? '', - secretKey: process.env.MINIO_SECRET_KEY ?? '', - sessionToken: process.env.MINIO_SESSION_TOKEN, - }); -} - let minioClient: Client; let storageInitialized = false; diff --git a/service/src/minio-client.test.ts b/service/src/minio-client.test.ts new file mode 100644 index 00000000..13259d03 --- /dev/null +++ b/service/src/minio-client.test.ts @@ -0,0 +1,101 @@ +import { expect, test } from 'bun:test'; +import { createHash } from 'node:crypto'; +import { Readable } from 'node:stream'; +import { createMinioClient } from './minio-client'; + +const MiB = 1024 * 1024; + +// Exercise the real SDK against a local S3 HTTP fixture. No storage account, +// Redis, or file-server listener is needed to test the production client. +test.each([1024, 8 * MiB, 20 * MiB + 17])( + 'unknown-length upload of %i bytes uses bounded parts without losing bytes', + async size => { + const parts = new Map(); + const lengths: number[] = []; + const uploaded: { body?: Buffer; contentType?: string | null; originalFilename?: string | null } = {}; + const xml = (body: string) => new Response(body, { + headers: { 'Content-Type': 'application/xml' }, + }); + const server = Bun.serve({ + hostname: '127.0.0.1', + port: 0, + async fetch(req) { + const url = new URL(req.url); + if (req.method === 'GET' && url.searchParams.has('uploads')) { + return xml('false'); + } + if (req.method === 'POST' && url.searchParams.has('uploads')) { + uploaded.contentType = req.headers.get('content-type'); + uploaded.originalFilename = req.headers.get('x-amz-meta-original-filename'); + return xml('test-upload'); + } + if (req.method === 'PUT' && url.searchParams.has('partNumber')) { + const body = Buffer.from(await req.arrayBuffer()); + lengths.push(body.length); + if (Number(req.headers.get('content-length')) !== body.length || + req.headers.get('content-md5') !== createHash('md5').update(body).digest('base64')) { + return new Response('Invalid part length or checksum', { status: 400 }); + } + parts.set(Number(url.searchParams.get('partNumber')), body); + return new Response(null, { + headers: { ETag: `"${createHash('md5').update(body).digest('hex')}"` }, + }); + } + if (req.method === 'POST' && url.searchParams.has('uploadId')) { + const manifest = await req.text(); + const ordered = [...manifest.matchAll(/(\d+)<\/PartNumber>/g)] + .map(match => parts.get(Number(match[1]))); + if (ordered.length !== parts.size || ordered.some(part => !part)) { + return new Response('Invalid multipart completion', { status: 400 }); + } + uploaded.body = Buffer.concat(ordered as Buffer[]); + return xml('http://localhost/test-bucket/input.bintest-bucketinput.bin"complete"'); + } + return new Response('Unexpected S3 request', { status: 400 }); + }, + }); + const settings: Record = { + MINIO_ENDPOINT: '127.0.0.1', + MINIO_PORT: String(server.port), + MINIO_NO_PORT: 'false', + MINIO_USE_SSL: 'false', + MINIO_REGION: 'us-east-1', + MINIO_USE_IRSA: 'false', + AWS_WEB_IDENTITY_TOKEN_FILE: undefined, + AWS_ROLE_ARN: undefined, + MINIO_ACCESS_KEY: 'test-access', + MINIO_SECRET_KEY: 'test-secret', + MINIO_SESSION_TOKEN: undefined, + }; + const saved = Object.fromEntries(Object.keys(settings).map(key => [key, process.env[key]])); + try { + for (const [key, value] of Object.entries(settings)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + const client = await createMinioClient(); + const expected = Buffer.alloc(size); + for (let i = 0; i < expected.length; i++) expected[i] = i % 251; + function* chunks() { + for (let offset = 0; offset < size; offset += 64 * 1024) { + yield expected.subarray(offset, Math.min(size, offset + 64 * 1024)); + } + } + await client.putObject('test-bucket', 'input.bin', Readable.from(chunks()), undefined, { + 'Content-Type': 'application/octet-stream', + 'X-Amz-Meta-Original-Filename': 'input.bin', + }); + expect(lengths.length).toBe(Math.ceil(size / (8 * MiB))); + expect(lengths.every(length => length <= 8 * MiB)).toBe(true); + expect(uploaded.body?.equals(expected)).toBe(true); + expect(uploaded.contentType).toBe('application/octet-stream'); + expect(uploaded.originalFilename).toBe('input.bin'); + } finally { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + await server.stop(true); + } + }, +); diff --git a/service/src/minio-client.ts b/service/src/minio-client.ts new file mode 100644 index 00000000..07af2891 --- /dev/null +++ b/service/src/minio-client.ts @@ -0,0 +1,69 @@ +import { Client, type ClientOptions } from 'minio'; +import logger from './fileServerLogger'; + +type IamProviderModule = { IamAwsProvider?: new (opts: object) => unknown; default?: new (opts: object) => unknown }; + +export async function createMinioClient(): Promise { + const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; + const irsaEnvVars = Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && Boolean(process.env.AWS_ROLE_ARN); + const useIrsa = irsaExplicit || irsaEnvVars; + + const baseConfig: ClientOptions = { + // Unknown-length streams otherwise grow SDK parts to 528 MiB (the 5 TiB + // object limit / 10,000 parts). Bound each multipart buffer instead. + partSize: 8 * 1024 * 1024, + endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', + port: process.env.MINIO_NO_PORT?.toLowerCase() === 'true' ? undefined : parseInt(process.env.MINIO_PORT ?? '9000'), + useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', + region: process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', + }; + + if (useIrsa) { + logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { + tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, + roleArn: process.env.AWS_ROLE_ARN, + region: baseConfig.region, + }); + + /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module + * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) + */ + let IamAwsProviderClass: new (opts: object) => unknown; + try { + const mod = await import('minio/dist/main/IamAwsProvider.js') as IamProviderModule; + IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; + } catch (primaryError) { + try { + // Fallback for bun: resolve path using require if available (CJS context) + let resolvePath = 'node_modules/minio/'; + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + resolvePath = require.resolve('minio').replace(/dist\/.*$/, ''); + } catch { + // require.resolve not available (ESM context), use default path + } + const mod = await import(`${resolvePath}dist/main/IamAwsProvider.js`) as IamProviderModule; + IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; + } catch (fallbackError) { + logger.error('Failed to load IamAwsProvider', { primaryError, fallbackError }); + throw new Error('Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.'); + } + } + + const credentialsProvider = new IamAwsProviderClass({}); + + return new Client({ + ...baseConfig, + credentialsProvider: credentialsProvider as ClientOptions['credentialsProvider'], + }); + } + + logger.info('Using explicit credentials for MinIO/S3 authentication'); + return new Client({ + ...baseConfig, + accessKey: process.env.MINIO_ACCESS_KEY ?? '', + secretKey: process.env.MINIO_SECRET_KEY ?? '', + sessionToken: process.env.MINIO_SESSION_TOKEN, + }); +} + From 3a3003a4ca4e31b9c120f3bd55238a4a74a8aabd Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 01:26:03 -0400 Subject: [PATCH 29/42] ci: Automate Auditable Main Releases (#216) * fix: decouple repository release versions * ci: automate releases after successful main builds --- .github/scripts/next-release-version.sh | 65 +++++++++++++++ .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 100 +++++++++++++++++++----- CONTRIBUTING.md | 10 ++- README.md | 4 +- docs/RELEASING.md | 66 ++++++++++------ tests/release-versioning.sh | 56 +++++++++++++ 7 files changed, 255 insertions(+), 49 deletions(-) create mode 100755 .github/scripts/next-release-version.sh create mode 100755 tests/release-versioning.sh diff --git a/.github/scripts/next-release-version.sh b/.github/scripts/next-release-version.sh new file mode 100755 index 00000000..a5766479 --- /dev/null +++ b/.github/scripts/next-release-version.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +CURRENT_TAG="$1" +REVISION_RANGE="$2" + +if [[ ! "$CURRENT_TAG" =~ ^v([0-9]+)[.]([0-9]+)[.]([0-9]+)$ ]]; then + echo "current release must be a stable vMAJOR.MINOR.PATCH tag (got '$CURRENT_TAG')" >&2 + exit 2 +fi + +# Documentation, workflow, and test-only changes remain auditable in git but do +# not produce a deployable release. Any unrecognised path is treated as +# deployable so a newly added runtime component cannot silently miss a release. +DEPLOYABLE=false +while IFS= read -r path; do + case "$path" in + .github/*|docs/*|tests/*|*.md|*/README|*/README.*|*.test.*|*.spec.*) + ;; + *) + DEPLOYABLE=true + break + ;; + esac +done < <(git diff --name-only "$REVISION_RANGE") + +if [ "$DEPLOYABLE" = "false" ]; then + exit 0 +fi + +COMMIT_MESSAGES="$(git log --format='%s%n%b' "$REVISION_RANGE")" +BUMP=patch + +if grep -Eq '^[[:alnum:]_-]+(\([^)]*\))?!:' <<<"$COMMIT_MESSAGES" \ + || grep -Eq '^BREAKING([ -])CHANGE:' <<<"$COMMIT_MESSAGES"; then + BUMP=major +elif grep -Eq '^feat(\([^)]*\))?:' <<<"$COMMIT_MESSAGES"; then + BUMP=minor +fi + +VERSION="${CURRENT_TAG#v}" +IFS=. read -r MAJOR MINOR PATCH <<<"$VERSION" + +case "$BUMP" in + major) + MAJOR=$((MAJOR + 1)) + MINOR=0 + PATCH=0 + ;; + minor) + MINOR=$((MINOR + 1)) + PATCH=0 + ;; + patch) + PATCH=$((PATCH + 1)) + ;; +esac + +printf 'v%s.%s.%s\n' "$MAJOR" "$MINOR" "$PATCH" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bdeda21..07622ee7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,9 @@ jobs: - name: Compose bridge configuration run: node tests/compose-bridge-config.cjs + - name: Release versioning + run: tests/release-versioning.sh + - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 55c7a116..c47b1a0d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,11 @@ # file is inert inside the monorepo — GitHub only runs workflows from the repo # root — and becomes a root workflow in the published repo. # -# Two entry points feed one job: +# Three entry points feed one job: +# +# * successful CI on main — automatically releases deployable changes. The +# next repository version follows Conventional Commit intent; changes that +# only touch docs, workflows, or tests do not cut a release. # # * workflow_dispatch — pick a version in the Actions UI. The chart is # packaged before the tag is created, so a packaging failure aborts while @@ -18,10 +22,13 @@ name: Release on: + workflow_run: + workflows: ['CI'] + types: [completed] workflow_dispatch: inputs: version: - description: 'Version to release, e.g. v2.0.0 or v2.1.0-rc1. Must match helm/codeapi/Chart.yaml appVersion.' + description: 'Repository version to release, e.g. v1.0.0 or v1.1.0-rc1.' required: true type: string draft: @@ -36,12 +43,20 @@ permissions: contents: write concurrency: - group: release-${{ github.event.inputs.version || github.ref_name }} + # Automatic runs serialize against one another. If main advances before an + # older run starts, version resolution skips the stale SHA and the newest + # successful run releases the full range instead. + group: release-${{ github.event_name == 'workflow_run' && 'main' || github.event.inputs.version || github.ref_name }} cancel-in-progress: false jobs: release: name: Tag and publish + if: >- + github.event_name != 'workflow_run' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main') runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -50,19 +65,60 @@ jobs: # Full history and tags: resolving whether this release is the newest # stable one compares it against every other tag in the repository. fetch-depth: 0 + ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.ref }} - name: Resolve and validate version id: version env: EVENT_NAME: ${{ github.event_name }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} INPUT_VERSION: ${{ github.event.inputs.version }} INPUT_DRAFT: ${{ github.event.inputs.draft }} REF_NAME: ${{ github.ref_name }} REF_TYPE: ${{ github.ref_type }} + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + SKIP=false + if [ "$EVENT_NAME" = "workflow_run" ]; then + if [ "$(git rev-parse HEAD)" != "$HEAD_SHA" ]; then + echo "::error::Checked out SHA does not match the successful CI run" + exit 1 + fi + + git fetch --no-tags origin main:refs/remotes/origin/main + if [ "$(git rev-parse refs/remotes/origin/main)" != "$HEAD_SHA" ]; then + echo "main advanced after this CI run; the newer successful run will release the combined changes" + SKIP=true + fi + + # A rerun after tag creation but before release publication resumes + # the missing release rather than incrementing the version again. + EXACT_TAG="$({ git tag --points-at HEAD || true; } | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' | sort -V | tail -n 1)" + if [ "$SKIP" = "false" ] && [ -n "$EXACT_TAG" ]; then + if gh release view "$EXACT_TAG" >/dev/null 2>&1; then + echo "$EXACT_TAG already publishes this commit; nothing to do" + SKIP=true + else + VERSION="$EXACT_TAG" + fi + elif [ "$SKIP" = "false" ]; then + PREVIOUS_TAG="$(git tag --merged HEAD \ + | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' \ + | sort -V \ + | tail -n 1)" + if [ -z "$PREVIOUS_TAG" ]; then + echo "::error::Automatic releases require an existing stable vMAJOR.MINOR.PATCH tag" + exit 1 + fi + VERSION="$(.github/scripts/next-release-version.sh "$PREVIOUS_TAG" "$PREVIOUS_TAG..HEAD")" + if [ -z "$VERSION" ]; then + echo "Only documentation, workflow, or test files changed since $PREVIOUS_TAG; no release needed" + SKIP=true + fi + fi + elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then # Releases describe what shipped to main. Dispatching from a topic # branch would tag a commit that is not on the release line. if [ "$REF_TYPE" != "branch" ] || [ "$REF_NAME" != "main" ]; then @@ -74,6 +130,11 @@ jobs: VERSION="$REF_NAME" fi + if [ "$SKIP" = "true" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # A bare "2.0.0" typed into the dispatch box is accepted; everything # downstream works with the v-prefixed form the tag actually uses. case "$VERSION" in @@ -82,14 +143,15 @@ jobs: esac if [[ ! "$VERSION" =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+)?$ ]]; then - echo "::error::Release tags must be v.. or v..-rcN, for example v2.0.0 or v2.1.0-rc1 (got '$VERSION')" + echo "::error::Release tags must be v.. or v..-rcN, for example v1.0.0 or v1.1.0-rc1 (got '$VERSION')" exit 1 fi - # v2.1.0-rc1 -> 2.1.0. Release candidates carry the version they are - # candidates for, so they compare against the same appVersion. - BASE_VERSION="${VERSION%%-rc*}" - BASE_VERSION="${BASE_VERSION#v}" + if [ "$EVENT_NAME" = "workflow_run" ] \ + && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + echo "::error::Calculated tag $VERSION already exists on a different commit" + exit 1 + fi read_chart_field() { grep -m1 "^$1:" helm/codeapi/Chart.yaml \ @@ -98,14 +160,6 @@ jobs: APP_VERSION="$(read_chart_field appVersion)" CHART_VERSION="$(read_chart_field version)" - # The tag is the app version. Requiring the bump to have landed on - # main first keeps a deployed chart from reporting a version that no - # release ever carried. - if [ "$APP_VERSION" != "$BASE_VERSION" ]; then - echo "::error::Tag $VERSION does not match helm/codeapi/Chart.yaml appVersion ($APP_VERSION). Land the appVersion bump on main before releasing." - exit 1 - fi - if [ "$EVENT_NAME" = "workflow_dispatch" ] \ && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then echo "::error::Tag $VERSION already exists. Pick a new version, or delete the tag if it was cut in error." @@ -142,8 +196,8 @@ jobs: fi { + echo "skip=false" echo "version=$VERSION" - echo "base_version=$BASE_VERSION" echo "app_version=$APP_VERSION" echo "chart_version=$CHART_VERSION" echo "prerelease=$PRERELEASE" @@ -157,6 +211,7 @@ jobs: # ci.yml depend on it. - name: Package Helm chart id: chart + if: steps.version.outputs.skip != 'true' run: | set -euo pipefail @@ -183,17 +238,20 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Create tag - if: github.event_name == 'workflow_dispatch' + if: steps.version.outputs.skip != 'true' && github.event_name != 'push' env: VERSION: ${{ steps.version.outputs.version }} run: | set -euo pipefail git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - git tag -a "$VERSION" -m "$VERSION" - git push origin "refs/tags/$VERSION" + if ! git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + git tag -a "$VERSION" -m "$VERSION" + git push origin "refs/tags/$VERSION" + fi - name: Publish release + if: steps.version.outputs.skip != 'true' env: GH_TOKEN: ${{ github.token }} VERSION: ${{ steps.version.outputs.version }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 37ecc984..e7b95797 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,10 +25,12 @@ Practical consequences: ## Releases Tagged releases are cut from `main` as `vMAJOR.MINOR.PATCH` (with `-rcN` for -release candidates), and each one carries the packaged Helm chart. The version -comes from `helm/codeapi/Chart.yaml`'s `appVersion`, so a version bump lands on -`main` through the pull request flow above before it can be released. See -[docs/RELEASING.md](docs/RELEASING.md) for the full process. +release candidates), and each one carries the packaged Helm chart. Repository, +API, service, and chart versions advance independently; component version bumps +land on `main` through the pull request flow above before they are included in a +release. Successful `main` CI automatically releases deployable changes while +documentation, workflow, and test-only changes are skipped. See +[docs/RELEASING.md](docs/RELEASING.md) for the full process and manual path. ## Development diff --git a/README.md b/README.md index bad5b66e..3856b254 100644 --- a/README.md +++ b/README.md @@ -108,14 +108,14 @@ Deployments should pin a [tagged release](https://github.com/LibreChat-AI/code-i rather than track `main`, which moves whenever an internal snapshot is merged: ```bash -git clone --branch v2.0.0 --depth 1 https://github.com/LibreChat-AI/code-interpreter.git +git clone --branch v1.0.0 --depth 1 https://github.com/LibreChat-AI/code-interpreter.git ``` Every release attaches `codeapi-.tgz`, the packaged Helm chart with its Redis and MinIO subcharts vendored: ```bash -helm install codeapi ./codeapi-0.3.0.tgz -f my-values.yaml +helm install codeapi ./codeapi-0.3.1.tgz -f my-values.yaml ``` Versions are `vMAJOR.MINOR.PATCH`, with `-rcN` release candidates published as diff --git a/docs/RELEASING.md b/docs/RELEASING.md index dd00c96d..3b2ff3ef 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -6,44 +6,66 @@ tags are cut. ## Versioning A release is named `vMAJOR.MINOR.PATCH`, optionally with a `-rcN` suffix for a -release candidate — `v2.0.0`, `v2.1.0-rc1`. That version is the **app -version**: `helm/codeapi/Chart.yaml`'s `appVersion` is its source of truth, and -the release workflow refuses any tag that disagrees with it. A release -candidate carries the version it is a candidate for, so `v2.1.0-rc1` also -requires `appVersion: "2.1.0"`. +release candidate — `v1.0.0`, `v1.1.0-rc1`. This is the public repository's +release sequence and is independent from the versions of the components it +contains. The first public release is therefore `v1.0.0` even though the API, +service, and Helm chart already have their own version histories. -Two other version numbers are deliberately independent: +Component versions are deliberately independent: +- `helm/codeapi/Chart.yaml`'s `appVersion` identifies the API version deployed + by the chart. - `helm/codeapi/Chart.yaml`'s `version` is the **chart** version. Bump it when the chart's templates or values change, not when the app changes. It names the packaged chart attached to the release (`codeapi-.tgz`). - `service/package.json`'s `version` tracks the Lambda service package alone. -By convention `api/package.json`'s `version` is kept in step with `appVersion`, -so the API package and the tag agree. Nothing enforces it. +By convention `api/package.json`'s `version` is kept in step with `appVersion`. +Nothing enforces it. -## Cutting a release +## Automatic releases -1. Land the `appVersion` bump on `main` first. `main` takes no direct pushes - (see [CONTRIBUTING.md](../CONTRIBUTING.md)), so it arrives through a sync - pull request from the internal monorepo or a community pull request. Bump - the chart `version` too if the chart changed. +After the full **CI** workflow succeeds for the current tip of `main`, the +release workflow examines everything since the last stable repository tag. It +cuts a release when that range changes deployable files and skips ranges that +only change documentation, GitHub workflows, or tests. If several merges land +while CI is running, the newest successful run releases them together. + +The next version follows Conventional Commit intent across the unreleased +range: + +- a `BREAKING CHANGE:` footer or `type!:` subject bumps the major version; +- a `feat:` subject bumps the minor version; +- every other deployable change bumps the patch version. + +This makes the safe fallback a patch release even when a merge title does not +follow the convention. The workflow packages the Helm chart before creating +the tag, so a packaging failure leaves the version available for a retry. A +rerun also resumes publication if the tag was created before a later step +failed. + +## Manual releases + +1. Land every intended component version bump on `main` first. `main` takes no + direct pushes (see [CONTRIBUTING.md](../CONTRIBUTING.md)), so changes arrive + through a sync pull request from the internal monorepo or a community pull + request. Bump the chart `version` only if the chart changed. 2. Run the **Release** workflow from the Actions tab against `main`, entering - the version (`v2.1.0`). Tick *draft* to review the generated notes before - they go public. + the next repository version (`v1.1.0`). Tick *draft* to review the generated + notes before they go public. -The workflow validates the version, packages the Helm chart, then creates the -annotated tag and publishes the release. Packaging runs before tagging so a -failure — a rate-limited subchart pull, most likely — leaves the version -unused and the run safe to retry. +Use this path when intentionally overriding the automatically selected version, +cutting a release candidate, or recovering while automatic releases are +disabled. The workflow validates the version, packages the Helm chart, then +creates the annotated tag and publishes the release. A tag pushed by hand works as well, and takes the same path from validation onward: ```bash git checkout main && git pull -git tag -a v2.1.0 -m v2.1.0 -git push origin v2.1.0 +git tag -a v1.1.0 -m v1.1.0 +git push origin v1.1.0 ``` ## What the release contains @@ -64,7 +86,7 @@ repository, so re-cutting an older patch cannot drag it backwards. Delete the release and its tag, then re-run the workflow: ```bash -gh release delete v2.1.0 --cleanup-tag --yes +gh release delete v1.1.0 --cleanup-tag --yes ``` Republishing the same version is only safe while nobody has deployed it. Once diff --git a/tests/release-versioning.sh b/tests/release-versioning.sh new file mode 100755 index 00000000..3ab80bef --- /dev/null +++ b/tests/release-versioning.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RESOLVER="$ROOT/.github/scripts/next-release-version.sh" +TEST_REPO="$(mktemp -d)" +trap 'rm -rf "$TEST_REPO"' EXIT + +git -C "$TEST_REPO" init -q +git -C "$TEST_REPO" config user.name test +git -C "$TEST_REPO" config user.email test@example.com + +commit_file() { + local path="$1" + local content="$2" + local message="$3" + + mkdir -p "$TEST_REPO/$(dirname "$path")" + printf '%s\n' "$content" > "$TEST_REPO/$path" + git -C "$TEST_REPO" add "$path" + git -C "$TEST_REPO" commit -q -m "$message" +} + +assert_version() { + local expected="$1" + local actual + actual="$(cd "$TEST_REPO" && bash "$RESOLVER" v1.2.3 v1.2.3..HEAD)" + if [ "$actual" != "$expected" ]; then + echo "expected '$expected', got '$actual'" >&2 + exit 1 + fi +} + +commit_file api/runtime.ts initial 'chore: initial release' +git -C "$TEST_REPO" tag v1.2.3 + +commit_file docs/guide.md docs 'docs: clarify deployment' +assert_version '' + +commit_file api/runtime.ts fix 'fix: repair execution' +assert_version v1.2.4 + +git -C "$TEST_REPO" reset -q --hard v1.2.3 +commit_file api/runtime.ts feature 'feat: add execution mode' +assert_version v1.3.0 + +git -C "$TEST_REPO" reset -q --hard v1.2.3 +commit_file api/runtime.ts breaking 'feat!: replace execution protocol' +assert_version v2.0.0 + +git -C "$TEST_REPO" reset -q --hard v1.2.3 +commit_file service/config.ts config 'chore: tune runtime defaults' +assert_version v1.2.4 + +echo 'release versioning tests passed' From f6cdfb3658a0f61c81cb6c63b73eaacf03417463 Mon Sep 17 00:00:00 2001 From: Jackson Riding <99007683+jacksonriding@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:26:27 +1000 Subject: [PATCH 30/42] fix: forward input-file limit into sandbox guests (#217) --- launcher/src/main.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 7771b9b9..27f08f92 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -433,6 +433,7 @@ fn is_allowed_guest_env_key(key: &str, egress_gateway_enabled: bool) -> bool { "SANDBOX_LOG_LEVEL", "SANDBOX_MAX_CONCURRENT_JOBS", "SANDBOX_MAX_FILE_SIZE", + "SANDBOX_MAX_INPUT_FILES", "SANDBOX_MAX_NESTING_DEPTH", "SANDBOX_MAX_OPEN_FILES", "SANDBOX_MAX_OUTPUT_FILES", @@ -850,6 +851,13 @@ mod tests { } } + #[test] + fn guest_env_allowlist_forwards_input_file_limit_in_both_egress_modes() { + for egress_gateway_enabled in [false, true] { + assert!(is_allowed_guest_env_key("SANDBOX_MAX_INPUT_FILES", egress_gateway_enabled)); + } + } + #[test] fn guest_env_allowlist_preserves_legacy_file_server_url_only_without_egress_gateway() { assert!(is_allowed_guest_env_key("FILE_SERVER_URL", false)); From b35c503fd2fe7be412d95c0eef6db50a09aad280 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 01:51:42 -0400 Subject: [PATCH 31/42] feat: Inspect Local Coding Projects (#221) * feat: add bounded local project inventory * fix: report incomplete Git metadata reads * fix: preserve incomplete discovery and remote identities * fix: finalize discovery budgets and nested remote identities * fix(code): stop project traversal at filesystem budget boundaries --- docs/remote-bridge/projects.md | 69 ++++++ packages/code/README.md | 37 +++ packages/code/src/cli.ts | 11 + packages/code/src/projects.test.ts | 368 +++++++++++++++++++++++++++++ packages/code/src/projects.ts | 295 +++++++++++++++++++++++ 5 files changed, 780 insertions(+) create mode 100644 docs/remote-bridge/projects.md create mode 100644 packages/code/src/projects.test.ts create mode 100644 packages/code/src/projects.ts diff --git a/docs/remote-bridge/projects.md b/docs/remote-bridge/projects.md new file mode 100644 index 00000000..987ce112 --- /dev/null +++ b/docs/remote-bridge/projects.md @@ -0,0 +1,69 @@ +# Projects and worktrees + +The intended experience is to select a project on an attached machine, choose +the current checkout or a new worktree, and have subsequent tool calls and +approval resumes use that selection without repeating a working directory. + +## Delivery sequence + +1. Local project inventory (`librechat-code projects --root `). + Discover bounded Git metadata without changing registration or authority. +2. Negotiated project selection across the worker, Code API, and LibreChat. + Persist the selection and enforce the same project boundary in file tools, + commands, programmatic execution, environment actions, and approval resumes. +3. Worktree creation and setup with durable operation receipts. Publish a new + selection only after Git creation, setup, and registration have completed. +4. Worktree selection in the composer and an approved agent operation. Allow + an authenticated owner to narrow a selection to a newly created worktree + with a compare-and-set against the prior conversation decision. +5. Explicit listing and removal, with binding checks and recovery for uncertain + outcomes. Retention policy determines eligibility, not automatic permission + to destroy uncommitted work or unpublished commits. + +Only the first step is implemented by the inventory command. Existing explicit +workspace registration remains available for independent project directories. + +## Boundaries that must remain consistent + +- Discovery metadata is advisory. A remote is not an authorization grant, a + trusted repository identity, or automatically a codegraph repository ID. + Keep the host in normalized remotes to distinguish identically named repos. +- A discovery root may authorize enumeration while an execution root narrows + writes to one selected project. A broad parent must not remain an independent + concurrent execution lane alongside its descendants. +- Path-derived IDs must be scoped by the registered root. Admission must + validate the current directory identity; inventory cannot reserve a path + against replacement after discovery. +- Discovery must not run on every status request. A future worker catalog + needs bounded caching, coalesced refreshes, and explicit generation changes. +- A linked worktree shares Git metadata with its parent. Project IDs alone + cannot make those metadata mutations independent. Admission needs both a + filesystem boundary and coordination for the common Git directory. +- A worktree beneath its parent checkout overlaps that checkout. Either use + disjoint execution roots under a discovery grant or explicitly exclude and + coordinate descendant worktrees before relaxing root exclusion. +- Setup and dependency links must remain within the execution policy. Sharing + writable dependency directories between supposedly isolated worktrees + reintroduces overlap and requires an explicit operator decision. +- Dynamic registration requires versioned capabilities and fenced catalog + generations. Old consumers must not silently drop a project selection and + execute against its broader parent. Deploy consumers before producers. +- Create, setup, registration, and conversation binding form a recoverable + lifecycle. A network retry must find the same worktree, not create a second + one. Failed setup leaves it unavailable; uncertain mutation quarantines it. +- Approval decisions must include the exact target and operation. Creating a + branch changes repository state and follows mutation policy. An additive + operation is not automatically exempt from required approval. +- Cleanup must coordinate live bindings and active execution. A missing remote + branch alone does not prove a worktree is disposable. + +## Acceptance cases for selection and lifecycle + +Verify separate repositories under one discovery root, two chats sharing one +project, two worktrees sharing Git metadata, directory replacement, stale +catalog generations, old/new consumer combinations, and cross-principal access. +Exercise file tools, commands, programmatic execution, and environment actions +through the same persisted selection. Include pause/resume, cancellation during +creation and setup, process death before registration, retry after binding, and +removal racing an active conversation. Use disposable local fixtures before +testing the hosted deployment. diff --git a/packages/code/README.md b/packages/code/README.md index 3cc61677..48bf769d 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -14,6 +14,43 @@ container/NsJail profile. The worker connects outbound to Code API, long-polls for assignments, sends them to the local runtime, and returns fenced results. The VM does not need an inbound public port. +## Inspect local projects + +Before registering a directory containing several checkouts, inspect its Git +projects on the worker machine: + +```bash +librechat-code projects --root /srv/projects +``` + +The command prints JSON with `projects`, `truncated`, and `incomplete`. Each +project contains a path relative to the requested directory, a path-derived ID, +and the current origin, branch, and HEAD. Origins are normalized to +`host[:port]/namespace/repository`, including nested namespaces; URL credentials, +query strings, and fragments are omitted. An unsupported configured origin is +redacted to null and marks the inventory incomplete. +A detached HEAD has a null branch; an unborn branch has a null HEAD. IDs stay +stable when branches change, but moving or renaming a directory changes its ID. +IDs are local to the supplied discovery root. + +Discovery runs only when requested. Its default limits are three directory +levels, 10,000 entries, 256 projects, and a ten-second processing budget with +bounded Git subprocess output and timeouts. +The time budget starts before resolving the root and is checked between native +filesystem operations; it cannot interrupt a kernel call stalled on a filesystem. +Use a responsive local filesystem. Discovery skips hidden directories, +dependencies, symlinks, and children of an identified repository. Linked +worktrees and submodules using a `.git` file are skipped and set `incomplete`: +their shared Git metadata needs separate admission before independent execution. +An empty project list does not prevent registering a non-Git directory. + +This is a local inventory command. It does not clone, register roots, pair a +worker, change the sandbox, or automatically select a conversation workspace. +For the existing picker and independent lease slots, explicitly register the +chosen non-overlapping project directories with `--workspace` or `--environment`. +Do not also register their parent directory. Treat the inventory as a snapshot; +normal workspace admission must validate any directory selected from it. + ## Pair Hardened deployments use a one-time code instead of copying a long-lived diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 00eca576..7406e267 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -5,6 +5,7 @@ import { realpath, stat } from 'node:fs/promises'; import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; +import { discoverProjects } from './projects.js'; import { loadCodeEnvironment, assertEnvironmentDefinitionsOutsideRoots, @@ -1237,6 +1238,16 @@ async function clearMutationQuarantine(args: string[]): Promise { async function main(): Promise { const args = process.argv.slice(2); + if (args[0] === 'projects') { + const root = option(args, '--root'); + if (!root || args.slice(1).some((arg, index, rest) => + arg !== '--root' && rest[index - 1] !== '--root' && !arg.startsWith('--root='))) { + throw new Error('Usage: librechat-code projects --root '); + } + const inventory = await discoverProjects({ root }); + process.stdout.write(`${JSON.stringify(inventory, null, 2)}\n`); + return; + } if (args[0] === 'relay') { await relay(); return; diff --git a/packages/code/src/projects.test.ts b/packages/code/src/projects.test.ts new file mode 100644 index 00000000..98ce0949 --- /dev/null +++ b/packages/code/src/projects.test.ts @@ -0,0 +1,368 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { + chmod, + mkdtemp, + mkdir, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import fs from 'node:fs/promises'; +import { syncBuiltinESMExports } from 'node:module'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import type { TestContext } from 'node:test'; +import { discoverProjects, projectRemote } from './projects.js'; + +const exec = promisify(execFile); +async function fixture(t: TestContext) { + const root = await mkdtemp(join(tmpdir(), 'code-projects-')); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} +async function repo(root: string, path: string) { + const directory = join(root, path); + await mkdir(directory, { recursive: true }); + await exec('git', ['init', '--initial-branch=dev', directory]); + return directory; +} + +test('discovers real sibling repositories with stable IDs and bounded metadata', async t => { + const root = await fixture(t); + const a = await repo(root, 'a'); + await repo(root, 'nested/b'); + await exec('git', [ + '-C', + a, + 'remote', + 'add', + 'origin', + 'https://user:secret@github.com/example/app.git?token=secret', + ]); + await exec('git', [ + '-C', + a, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '--allow-empty', + '-m', + 'initial', + ]); + const before = await discoverProjects({ root }); + assert.equal(before.incomplete, false); + assert.equal(before.truncated, false); + assert.deepEqual( + before.projects.map(p => p.path), + ['a', 'nested/b'] + ); + assert.equal(before.projects[0].remote, 'github.com/example/app'); + assert.equal(before.projects[0].branch, 'dev'); + assert.match(before.projects[0].head!, /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/); + assert.equal(before.projects[1].head, null); + assert.ok(!JSON.stringify(before).includes('secret')); + await exec('git', ['-C', a, 'checkout', '-b', 'next']); + const after = await discoverProjects({ root }); + assert.deepEqual( + after.projects.map(p => p.id), + before.projects.map(p => p.id) + ); + assert.equal(after.projects[0].branch, 'next'); +}); + +test('does not walk dependencies, hidden directories, symlinks or repository children', async t => { + const root = await fixture(t); + await repo(root, 'node_modules/ignored'); + await repo(root, '.hidden/ignored'); + await repo(root, 'parent'); + await repo(root, 'parent/nested'); + const outside = await fixture(t); + await repo(outside, 'external'); + await symlink(outside, join(root, 'alias'), 'dir'); + const inventory = await discoverProjects({ root }); + assert.deepEqual( + inventory.projects.map(p => p.path), + ['parent'] + ); +}); + +test('linked worktrees are reported incomplete until shared git metadata is admitted', async t => { + const root = await fixture(t); + await mkdir(join(root, 'linked')); + await writeFile( + join(root, 'linked', '.git'), + 'gitdir: /outside/metadata\n' + ); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.deepEqual(inventory.projects, []); +}); + +test('oversized Git metadata is incomplete rather than silently reported absent', async t => { + const root = await fixture(t); + const directory = await repo(root, 'app'); + await exec('git', [ + '-C', + directory, + 'config', + 'remote.origin.url', + 'x'.repeat(10_000), + ]); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.equal(inventory.projects[0].remote, null); +}); + +test('a valid branch beyond the metadata bound reports incomplete', async t => { + const root = await fixture(t); + const directory = await repo(root, 'app'); + const branch = ['a'.repeat(100), 'b'.repeat(100), 'c'.repeat(100)].join( + '/' + ); + await exec('git', [ + '-C', + directory, + 'symbolic-ref', + 'HEAD', + `refs/heads/${branch}`, + ]); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.equal(inventory.projects[0].branch, null); +}); + +test('unreadable Git markers report incomplete', async t => { + if (process.platform === 'win32' || process.getuid?.() === 0) { + t.skip('requires POSIX permissions under an unprivileged account'); + return; + } + const root = await fixture(t); + const directory = await repo(root, 'app'); + await chmod(directory, 0o400); + try { + assert.equal( + (await discoverProjects({ root: directory })).incomplete, + true + ); + } finally { + await chmod(directory, 0o700); + } +}); + +test('root resolution consumes the processing budget and pre-abort wins', async t => { + const root = await fixture(t); + let ticks = 0; + t.mock.method(Date, 'now', () => (ticks++ === 0 ? 0 : 20_000)); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.truncated, true); + assert.deepEqual(inventory.projects, []); + const reason = new Error('cancelled before filesystem access'); + await assert.rejects( + discoverProjects({ + root: join(root, 'missing'), + signal: AbortSignal.abort(reason), + }), + error => error === reason + ); +}); + +test('empty directory completion checks a late deadline and cancellation', async t => { + const root = await fixture(t); + const original = fs.opendir; + let clock = 0; + let cancel: AbortController | undefined; + t.mock.method(Date, 'now', () => clock); + const openMock = t.mock.method( + fs, + 'opendir', + async (path: Parameters[0]) => { + const directory = await original(path); + clock = 20_000; + cancel?.abort(); + return directory; + } + ); + syncBuiltinESMExports(); + try { + assert.equal((await discoverProjects({ root })).truncated, true); + clock = 0; + cancel = new AbortController(); + await assert.rejects( + discoverProjects({ root, signal: cancel.signal }), + { name: 'AbortError' } + ); + } finally { + openMock.mock.restore(); + syncBuiltinESMExports(); + } +}); + +test('late filesystem boundaries stop before starting the next operation', async t => { + const root = await fixture(t); + for (const boundary of [3, 4, 5]) { + for (const abort of [false, true]) { + let calls = 0; + let clock = 0; + const controller = new AbortController(); + const now = t.mock.method(Date, 'now', () => clock); + const mocks = ['lstat', 'realpath', 'opendir'].map(name => { + const original = fs[name as 'lstat']; + return t.mock.method( + fs, + name as 'lstat', + async (...args: Parameters) => { + calls++; + try { + return await original(...args); + } finally { + if (calls === boundary) { + clock = 20_000; + if (abort) controller.abort(); + } + } + } + ); + }); + syncBuiltinESMExports(); + try { + const discovery = discoverProjects({ + root, + signal: controller.signal, + }); + if (abort) + await assert.rejects(discovery, { name: 'AbortError' }); + else assert.equal((await discovery).truncated, true); + assert.equal(calls, boundary); + } finally { + for (const mock of mocks) mock.mock.restore(); + now.mock.restore(); + syncBuiltinESMExports(); + } + } + } +}); + +test('unsupported configured origins are distinguishable from missing origins', async t => { + const root = await fixture(t); + const directory = await repo(root, 'app'); + await exec('git', [ + '-C', + directory, + 'config', + 'remote.origin.url', + '/private/local/repo', + ]); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.incomplete, true); + assert.equal(inventory.projects[0].remote, null); +}); + +test('project, entry and depth ceilings report partial discovery', async t => { + const root = await fixture(t); + await repo(root, 'a'); + await repo(root, 'b'); + await repo(root, 'nested/deeper/c'); + assert.equal( + (await discoverProjects({ root, maxProjects: 1 })).truncated, + true + ); + assert.equal( + (await discoverProjects({ root, maxEntries: 1 })).truncated, + true + ); + assert.equal( + (await discoverProjects({ root, maxDepth: 1 })).truncated, + true + ); + await assert.rejects(discoverProjects({ root, maxDepth: 100 }), /limit/); + await assert.rejects( + discoverProjects({ root, signal: AbortSignal.abort() }) + ); +}); + +test('root checkout uses dot and detached HEAD has no branch', async t => { + const root = await fixture(t); + await repo(root, '.'); + await exec('git', [ + '-C', + root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '--allow-empty', + '-m', + 'initial', + ]); + await exec('git', ['-C', root, 'checkout', '--detach']); + const inventory = await discoverProjects({ root }); + assert.equal(inventory.projects[0].path, '.'); + assert.equal(inventory.projects[0].branch, null); +}); + +test('repository identity retains host and drops credentials, query and fragments', () => { + assert.equal( + projectRemote('ssh://git@example.com:2222/org/repo.git'), + 'example.com:2222/org/repo' + ); + assert.equal( + projectRemote('https://example.com:8443/org/repo.git'), + 'example.com:8443/org/repo' + ); + assert.equal( + projectRemote('git@github.com:org/repo.git'), + 'github.com/org/repo' + ); + assert.equal( + projectRemote('ssh://git@example.com/org/repo.git'), + 'example.com/org/repo' + ); + assert.equal( + projectRemote('https://token@example.com/org/repo.git?secret#fragment'), + 'example.com/org/repo' + ); + assert.equal(projectRemote('/home/user/private'), null); + assert.equal(projectRemote('file:///home/user/private'), null); + assert.equal( + projectRemote('https://example.com/group/subgroup/repo.git'), + 'example.com/group/subgroup/repo' + ); + assert.equal( + projectRemote('git@example.com:group/subgroup/repo.git'), + 'example.com/group/subgroup/repo' + ); +}); + +test('CLI inventories a real checkout without pairing or starting a worker', async t => { + const root = await fixture(t); + await repo(root, 'app'); + const { stdout, stderr } = await exec( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'projects', + '--root', + root, + ], + { env: { PATH: process.env.PATH }, timeout: 15_000 } + ); + const result = JSON.parse(stdout); + assert.equal(stderr, ''); + assert.equal(result.projects[0].path, 'app'); + assert.equal(result.projects[0].branch, 'dev'); + assert.equal(result.incomplete, false); + await assert.rejects( + exec(process.execPath, [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'projects', + ]), + /Usage: librechat-code projects/ + ); +}); diff --git a/packages/code/src/projects.ts b/packages/code/src/projects.ts new file mode 100644 index 00000000..1e2fcbd3 --- /dev/null +++ b/packages/code/src/projects.ts @@ -0,0 +1,295 @@ +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { lstat, opendir, realpath } from 'node:fs/promises'; +import { isAbsolute, relative, resolve, sep } from 'node:path'; +import { promisify } from 'node:util'; + +const exec = promisify(execFile); +const skipped = new Set(['node_modules', 'vendor']); + +export interface LocalProject { + id: string; + path: string; + remote: string | null; + branch: string | null; + head: string | null; +} + +export interface ProjectInventory { + projects: LocalProject[]; + truncated: boolean; + incomplete: boolean; +} + +export interface ProjectDiscoveryOptions { + root: string; + maxDepth?: number; + maxProjects?: number; + maxEntries?: number; + timeoutMs?: number; + signal?: AbortSignal; +} + +/** Public repository identity only: never propagate credentials or URL query data. */ +export function projectRemote(value: string): string | null { + let host: string; + let path: string; + try { + const scp = /^(?:[^/@:\s]+@)?([^/:\s]+):([^\s]+)$/.exec(value); + if (scp && !value.includes('://')) { + host = scp[1]; + path = scp[2]; + } else { + const url = new URL(value); + if (!['https:', 'http:', 'ssh:', 'git:'].includes(url.protocol)) + return null; + host = url.host; + path = url.pathname.replace(/^\//, ''); + } + path = path.replace(/\.git$/, ''); + if ( + !/^[A-Za-z0-9.-]+(?::[0-9]+)?$/.test(host) || + !/^[A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)+$/.test(path) + ) + return null; + if (path.split('/').some(part => part === '.' || part === '..')) + return null; + return `${host.toLowerCase()}/${path}`; + } catch { + return null; + } +} + +function limit( + value: number | undefined, + fallback: number, + maximum: number +): number { + const resolved = value ?? fallback; + if (!Number.isSafeInteger(resolved) || resolved < 1 || resolved > maximum) + throw new Error('Invalid project discovery limit'); + return resolved; +} + +/** Bounded local inventory; it does not grant roots or mutate a checkout. */ +export async function discoverProjects( + options: ProjectDiscoveryOptions +): Promise { + const maxDepth = limit(options.maxDepth, 3, 16); + const maxProjects = limit(options.maxProjects, 256, 256); + const maxEntries = limit(options.maxEntries, 10_000, 100_000); + const timeoutMs = limit(options.timeoutMs, 10_000, 60_000); + const deadline = Date.now() + timeoutMs; + const result: ProjectInventory = { + projects: [], + truncated: false, + incomplete: false, + }; + let entries = 0; + const expired = (): boolean => { + options.signal?.throwIfAborted(); + if (Date.now() < deadline) return false; + result.truncated = true; + return true; + }; + options.signal?.throwIfAborted(); + const root = await realpath(options.root); + if (expired()) return result; + const rootStat = await lstat(root); + if (expired()) return result; + if (!rootStat.isDirectory()) + throw new Error('Project root must be a directory'); + const queue = [{ path: root, depth: 0 }]; + const git = async ( + path: string, + args: string[], + expectedExitCodes: number[] = [] + ): Promise => { + if (expired()) return null; + try { + const { stdout } = await exec( + 'git', + [ + '--no-optional-locks', + '-C', + path, + '-c', + 'core.fsmonitor=false', + ...args, + ], + { + env: { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }, + encoding: 'utf8', + maxBuffer: 4096, + timeout: Math.max(1, Math.min(1500, deadline - Date.now())), + signal: options.signal, + } + ); + return stdout.trim(); + } catch (error) { + options.signal?.throwIfAborted(); + const expected = + error instanceof Error && + 'code' in error && + typeof error.code === 'number' && + expectedExitCodes.includes(error.code); + if (!expected) result.incomplete = true; + return null; + } + }; + for (let index = 0; index < queue.length; index++) { + if (expired()) break; + const current = queue[index]; + try { + // Revalidate queued directories; never traverse a replaced symlink. + const currentStat = await lstat(current.path); + if (expired()) break; + if (currentStat.isSymbolicLink()) { + result.incomplete = true; + continue; + } + const canonical = await realpath(current.path); + if (expired()) break; + const rel = relative(root, canonical); + if (rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { + result.incomplete = true; + continue; + } + const marker = await lstat(resolve(current.path, '.git')).catch( + error => { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) + throw error; + return undefined; + } + ); + if (expired()) break; + if (marker) { + // Linked worktrees and submodules need separate shared-gitdir admission. + if (!marker.isDirectory() || marker.isSymbolicLink()) { + result.incomplete = true; + continue; + } + if (result.projects.length === maxProjects) { + result.truncated = true; + break; + } + const top = await git(current.path, [ + 'rev-parse', + '--show-toplevel', + ]); + if (expired()) break; + const canonicalTop = top + ? await realpath(top).catch(() => null) + : null; + if (expired()) break; + if (!top || canonicalTop !== canonical) { + result.incomplete = true; + continue; + } + const remote = await git( + current.path, + [ + 'config', + '--local', + '--no-includes', + '--get', + 'remote.origin.url', + ], + [1] + ); + const branch = await git( + current.path, + ['symbolic-ref', '--quiet', '--short', 'HEAD'], + [1] + ); + const head = await git( + current.path, + ['rev-parse', '--verify', 'HEAD'], + branch ? [128] : [] + ); + const path = rel.split(sep).join('/') || '.'; + const normalizedRemote = remote ? projectRemote(remote) : null; + const validBranch = + branch && + branch.length <= 256 && + !/[\x00-\x1f\x7f]/.test(branch) + ? branch + : null; + const validHead = + head && /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(head) + ? head + : null; + if ( + (remote !== null && normalizedRemote === null) || + (branch !== null && validBranch === null) || + (head !== null && validHead === null) + ) + result.incomplete = true; + result.projects.push({ + id: `project-${createHash('sha256') + .update(path) + .digest('hex') + .slice(0, 32)}`, + path, + remote: normalizedRemote, + branch: validBranch, + head: validHead, + }); + continue; + } + if (current.depth === maxDepth) { + result.truncated = true; + continue; + } + const directory = await opendir(current.path); + // Close a newly opened handle even when cancellation won during open. + try { + if (expired()) { + await directory.close(); + break; + } + } catch (error) { + await directory.close(); + throw error; + } + for await (const entry of directory) { + if (expired() || ++entries > maxEntries) { + result.truncated = true; + result.projects.sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0 + ); + return result; + } + if ( + entry.isDirectory() && + !entry.name.startsWith('.') && + !skipped.has(entry.name) + ) + queue.push({ + path: resolve(current.path, entry.name), + depth: current.depth + 1, + }); + } + } catch { + options.signal?.throwIfAborted(); + result.incomplete = true; + } + } + result.projects.sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0 + ); + expired(); + return result; +} From c688b30d2ebb0af0fc9d603766b9ebcb4e2a8c9d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 07:38:19 -0400 Subject: [PATCH 32/42] fix: make automatic release tip check read-only (#225) --- .github/workflows/release.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c47b1a0d..ea8a5364 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,7 @@ name: Release on: workflow_run: workflows: ['CI'] + branches: [main] types: [completed] workflow_dispatch: inputs: @@ -87,8 +88,12 @@ jobs: exit 1 fi - git fetch --no-tags origin main:refs/remotes/origin/main - if [ "$(git rev-parse refs/remotes/origin/main)" != "$HEAD_SHA" ]; then + REMOTE_MAIN_SHA="$(git ls-remote origin refs/heads/main | awk '{print $1}')" + if [ -z "$REMOTE_MAIN_SHA" ]; then + echo "::error::Could not resolve the current main branch tip" + exit 1 + fi + if [ "$REMOTE_MAIN_SHA" != "$HEAD_SHA" ]; then echo "main advanced after this CI run; the newer successful run will release the combined changes" SKIP=true fi From 4c7b224358ac1647416f070935931abe088008ef Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 09:01:32 -0400 Subject: [PATCH 33/42] feat: Discover Bounded Repository Instructions for Attached Workspaces (#226) * Discover bounded repository instructions for opted-in workspaces * Verify snapshot digests and cross-platform confinement --- packages/code/REPOSITORY-INSTRUCTIONS.md | 29 ++++ packages/code/src/cli.ts | 5 +- packages/code/src/instructions.test.ts | 206 +++++++++++++++++++++++ packages/code/src/instructions.ts | 89 ++++++++++ packages/code/src/protocol.ts | 40 +++++ packages/code/src/worker.ts | 26 ++- packages/code/src/workspace.ts | 25 ++- service/src/bridge/instructions.test.ts | 123 ++++++++++++++ 8 files changed, 539 insertions(+), 4 deletions(-) create mode 100644 packages/code/REPOSITORY-INSTRUCTIONS.md create mode 100644 packages/code/src/instructions.test.ts create mode 100644 packages/code/src/instructions.ts create mode 100644 service/src/bridge/instructions.test.ts diff --git a/packages/code/REPOSITORY-INSTRUCTIONS.md b/packages/code/REPOSITORY-INSTRUCTIONS.md new file mode 100644 index 00000000..5f8ce529 --- /dev/null +++ b/packages/code/REPOSITORY-INSTRUCTIONS.md @@ -0,0 +1,29 @@ +# Repository instructions + +After updating Code API and LibreChat to versions supporting repository instruction metadata, +add `--repository-instructions` to the existing `librechat-code run` command. Keep all existing +pairing, workspace, sandbox and environment arguments. This is an explicit machine-owner opt-in; +without it, no instructions are advertised or automatically read. + +Only each registered workspace root is examined. `AGENTS.md` takes precedence; `CLAUDE.md` +is considered only when `AGENTS.md` does not exist. Symlinks, directories, unreadable files, +invalid UTF-8 and binary files are omitted. There is no parent-directory walk or globbing. + +Metadata refreshes with the existing worker registration heartbeat. It does not change worker +identity, lease slots or workspace permissions. The next run after that refresh sees changed +metadata; this is not a filesystem watch or a guarantee of immediate edit visibility. + +The descriptor contains only the relative filename, bounded snapshot byte count, SHA-256 and +truncation flag. SHA-256 identifies the delivered UTF-8 snapshot, not unseen bytes beyond the +32 KiB cap. An incomplete trailing UTF-8 character is omitted. Content travels only through the +authorized `read_file` operation, using `instructionSha256` to reject reads if the snapshot changed. +This mode preserves newlines and is independent of the ordinary line-oriented reader. + +LibreChat caches verified content with bounded capacity and principal/machine/workspace scoping. +Its per-agent setting selects `prefer`, `defer` or `off`; it cannot enable discovery on a worker +that did not opt in or widen filesystem, network, or approval permissions. + +Rollout: update LibreChat and Code API first, then update workers and enable the flag. Older +workers continue unchanged. An older Code API rejecting the metadata causes the worker to retry +without it and omit metadata until restart. Do not enable this flag against an older LibreChat +instance: older clients validate workspace descriptors strictly. diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 7406e267..946b08d0 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -684,11 +684,13 @@ async function run( }), ]), ); - let workspaceTools: WorkspaceToolExecutor | undefined = workerDirectory + const localWorkspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: roots, + repositoryInstructions: args.includes('--repository-instructions'), }) : undefined; + let workspaceTools: WorkspaceToolExecutor | undefined = localWorkspaceTools; if (allowWorkspaceCommands && !canonicalWorkerDirectory) { throw new Error('Workspace commands require a registered directory'); } @@ -1030,6 +1032,7 @@ async function run( } try { const worker = new BridgeWorker({ + instructionDescriptors: () => localWorkspaceTools?.instructionDescriptors() ?? Promise.resolve(undefined), codeApiUrl, token: configuredToken, identity: workerIdentity, diff --git a/packages/code/src/instructions.test.ts b/packages/code/src/instructions.test.ts new file mode 100644 index 00000000..42baafe9 --- /dev/null +++ b/packages/code/src/instructions.test.ts @@ -0,0 +1,206 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, writeFile, rm, symlink, realpath } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { readRepositoryInstructions } from './instructions.js'; +import { LocalWorkspaceTools } from './workspace.js'; +import { isWorkspaceToolResult } from './protocol.js'; +import { BridgeWorker } from './worker.js'; + +test('instruction discovery selects one fixed name and refuses symlink fallback', async () => { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'repository-instructions-')), + ); + try { + await writeFile(join(root, 'CLAUDE.md'), 'fallback\n'); + assert.equal( + (await readRepositoryInstructions(root))?.content, + 'fallback\n', + ); + await writeFile(join(root, 'AGENTS.md'), 'preferred\n'); + assert.equal( + (await readRepositoryInstructions(root))?.content, + 'preferred\n', + ); + await rm(join(root, 'AGENTS.md')); + await symlink(join(root, 'CLAUDE.md'), join(root, 'AGENTS.md')); + assert.equal(await readRepositoryInstructions(root), undefined); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('registration refreshes instruction metadata without changing the worker incarnation', async () => { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'repository-instructions-')), + ); + try { + await writeFile(join(root, 'AGENTS.md'), 'first'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + repositoryInstructions: true, + }); + const registrations: Array<{ + incarnationId: string; + capabilities: { + workspaceTools: { + workspaces: Array<{ + instructions?: Array<{ sha256: string }>; + }>; + }; + }; + }> = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'test', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'test', + runtimes: [], + workspaceTools: tools.capabilities, + }, + workspaceTools: tools, + instructionDescriptors: () => tools.instructionDescriptors(), + fetchImpl: async (_url, init) => { + registrations.push(JSON.parse(String(init?.body))); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 60000, + supportedWorkspaceToolOperations: [ + 'read_file', + 'search_text', + 'list_files', + ], + supportedWorkspaceListFileFeatures: ['after_path'], + }); + }, + }); + await worker.register(); + const first = + registrations.at(-1)!.capabilities.workspaceTools.workspaces[0] + .instructions![0].sha256; + await writeFile(join(root, 'AGENTS.md'), 'second'); + await worker.register(); + assert.notEqual( + registrations.at(-1)!.capabilities.workspaceTools.workspaces[0] + .instructions![0].sha256, + first, + ); + assert.ok( + registrations.every( + item => item.incarnationId === 'incarnation-00000001', + ), + ); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('bounded UTF-8 snapshots refresh and hash-fence authorized reads', async () => { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'repository-instructions-')), + ); + try { + await writeFile(join(root, 'AGENTS.md'), 'a'.repeat(32767) + '🙂tail'); + const snapshot = await readRepositoryInstructions(root); + assert.equal(snapshot?.descriptor.truncated, true); + assert.equal(snapshot?.descriptor.bytes, 32767); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + repositoryInstructions: true, + }); + const request = { + protocolVersion: 1 as const, + operation: 'read_file' as const, + workspaceId: 'primary', + path: 'AGENTS.md', + instructionSha256: snapshot!.descriptor.sha256, + }; + const result = await tools.execute(request); + assert.equal(isWorkspaceToolResult(request, result), true); + assert.equal(isWorkspaceToolResult(request, { ...result, content: 'forged', endLine: 1 }), false); + assert.equal( + (await tools.instructionDescriptors())?.get('primary')?.[0]?.sha256, + snapshot?.descriptor.sha256, + ); + await writeFile(join(root, 'AGENTS.md'), 'new instructions\n'); + await assert.rejects( + tools.execute(request), + /changed or are unavailable/, + ); + assert.notEqual( + (await tools.instructionDescriptors())?.get('primary')?.[0]?.sha256, + snapshot?.descriptor.sha256, + ); + const disabled = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + }); + assert.equal(await disabled.instructionDescriptors(), undefined); + await assert.rejects(disabled.execute(request)); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test('an older bridge can reject metadata without breaking later registrations', async () => { + let metadataRequests = 0; + let accepted = 0; + const capabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary', name: 'Primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'test', + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'test', + runtimes: [], + workspaceTools: capabilities, + }, + workspaceTools: { + capabilities, + execute: async () => { + throw new Error('not expected'); + }, + }, + instructionDescriptors: async () => new Map([['primary', []]]), + fetchImpl: async (_url, init) => { + const body = JSON.parse(String(init?.body)); + if ( + body.capabilities.workspaceTools?.workspaces[0].instructions !== + undefined + ) { + metadataRequests++; + return Response.json( + { error: 'Unknown workspace field' }, + { status: 400 }, + ); + } + accepted++; + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + registeredAt: new Date().toISOString(), + leaseTtlMs: 60000, + supportedWorkspaceToolOperations: ['read_file'], + }); + }, + }); + await worker.register(); + await worker.register(); + assert.equal(metadataRequests, 1); + assert.ok(accepted >= 2); +}); diff --git a/packages/code/src/instructions.ts b/packages/code/src/instructions.ts new file mode 100644 index 00000000..ee3e8127 --- /dev/null +++ b/packages/code/src/instructions.ts @@ -0,0 +1,89 @@ +import { constants } from 'node:fs'; +import { open, lstat, realpath, stat } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { resolve, relative, isAbsolute, sep } from 'node:path'; + +import { REPOSITORY_INSTRUCTION_MAX_BYTES } from './protocol.js'; +import type { RepositoryInstructionDescriptor } from './protocol.js'; +export interface RepositoryInstructionSnapshot { + descriptor: RepositoryInstructionDescriptor; + content: string; +} + +/** Fixed-name, root-confined discovery. An unreadable AGENTS.md never selects a fallback. */ +export async function readRepositoryInstructions( + root: string, +): Promise { + let path: RepositoryInstructionDescriptor['path'] = 'AGENTS.md'; + try { + await lstat(resolve(root, path)); + } catch (error) { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) + return; + path = 'CLAUDE.md'; + } + let handle: Awaited> | undefined; + try { + const candidate = resolve(root, path); + if (!(await lstat(candidate)).isFile()) return; + handle = await open( + candidate, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const opened = await handle.stat(); + const canonical = await realpath(candidate); + const offset = relative(root, canonical); + const current = await stat(canonical); + if ( + !opened.isFile() || + isAbsolute(offset) || + offset === '..' || + offset.startsWith(`..${sep}`) || + opened.dev !== current.dev || + opened.ino !== current.ino + ) + return; + const buffer = Buffer.alloc(REPOSITORY_INSTRUCTION_MAX_BYTES + 1); + let length = 0; + while (length < buffer.length) { + const read = await handle.read( + buffer, + length, + buffer.length - length, + length, + ); + if (read.bytesRead === 0) break; + length += read.bytesRead; + } + const truncated = length > REPOSITORY_INSTRUCTION_MAX_BYTES; + const decoder = new TextDecoder('utf-8', { + fatal: true, + ignoreBOM: true, + }); + const content = decoder.decode( + buffer.subarray( + 0, + Math.min(length, REPOSITORY_INSTRUCTION_MAX_BYTES), + ), + { stream: truncated }, + ); + if (content.includes('\0')) return; + return { + descriptor: { + path, + bytes: Buffer.byteLength(content), + sha256: createHash('sha256').update(content).digest('hex'), + truncated, + }, + content, + }; + } catch { + return; + } finally { + await handle?.close().catch(() => undefined); + } +} diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 9199fcf0..c5353ae2 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -1,3 +1,5 @@ +import { createHash } from 'node:crypto'; + export const BRIDGE_PROTOCOL_VERSION = 1 as const; export const BRIDGE_WORKER_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; export const BRIDGE_SANDBOX_PROFILE_MAX_LENGTH = 128; @@ -275,6 +277,7 @@ export type WorkspaceProgrammaticLanguage = 'bash'; export interface BridgeWorkspaceDescriptor { id: string; name?: string; + instructions?: RepositoryInstructionDescriptor[]; /** Optional per-workspace restriction. Omitted by protocol-v1 readers. */ operations?: BridgeWorkspaceToolOperation[]; environment?: { @@ -308,6 +311,27 @@ export interface WorkspaceReadFileRequest { path: string; startLine?: number; maxLines?: number; + /** Requests an exact bounded instruction snapshot, not a line-oriented file read. */ + instructionSha256?: string; +} + +export const REPOSITORY_INSTRUCTION_MAX_BYTES = 32 * 1024; +export interface RepositoryInstructionDescriptor { + path: 'AGENTS.md' | 'CLAUDE.md'; + /** Bytes in the bounded UTF-8 snapshot, whose digest is sha256. */ + bytes: number; + sha256: string; + truncated: boolean; +} + +export function isRepositoryInstructionDescriptor(value: unknown): value is RepositoryInstructionDescriptor { + if (value == null || typeof value !== 'object') return false; + const descriptor = value as Record; + return Object.keys(descriptor).every(key => ['path', 'bytes', 'sha256', 'truncated'].includes(key)) && + (descriptor.path === 'AGENTS.md' || descriptor.path === 'CLAUDE.md') && + Number.isSafeInteger(descriptor.bytes) && Number(descriptor.bytes) >= 0 && Number(descriptor.bytes) <= REPOSITORY_INSTRUCTION_MAX_BYTES && + typeof descriptor.sha256 === 'string' && /^[a-f0-9]{64}$/.test(descriptor.sha256) && + typeof descriptor.truncated === 'boolean'; } export interface WorkspaceReadFileResult { @@ -510,6 +534,7 @@ export type WorkspaceToolResult = | WorkspaceExecuteCommandResult; const WORKSPACE_READ_REQUEST_KEYS = new Set([ + 'instructionSha256', 'protocolVersion', 'operation', 'workspaceId', @@ -1139,6 +1164,12 @@ export function isWorkspaceToolRequest( return false; } if (request.operation === 'read_file') { + if (request.instructionSha256 !== undefined) { + return hasOnlyKeys(request, WORKSPACE_READ_REQUEST_KEYS) && + typeof request.instructionSha256 === 'string' && /^[a-f0-9]{64}$/.test(request.instructionSha256) && + (request.path === 'AGENTS.md' || request.path === 'CLAUDE.md') && + request.startLine === undefined && request.maxLines === undefined; + } return ( hasOnlyKeys(request, WORKSPACE_READ_REQUEST_KEYS) && isSafePortableRelativePath(request.path) && @@ -1282,6 +1313,13 @@ export function isWorkspaceToolResult( } if (request.operation === 'read_file') { + if (request.instructionSha256 !== undefined) { + return hasOnlyKeys(result, WORKSPACE_READ_RESULT_KEYS) && result.path === request.path && + typeof result.content === 'string' && new TextEncoder().encode(result.content).byteLength <= REPOSITORY_INSTRUCTION_MAX_BYTES && + createHash('sha256').update(result.content).digest('hex') === request.instructionSha256 && + result.startLine === 1 && result.endLine === result.content.split('\n').length && + result.nextStartLine === undefined; + } const startLine = request.startLine ?? 1; const maxLines = request.maxLines ?? 200; const content = @@ -1574,11 +1612,13 @@ export function isValidBridgeWorkspaceToolCapabilities( key !== 'id' && key !== 'name' && key !== 'operations' && + key !== 'instructions' && key !== 'environment', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || workspaceIds.has(descriptor.id) || + (descriptor.instructions !== undefined && (!Array.isArray(descriptor.instructions) || descriptor.instructions.length > 1 || !descriptor.instructions.every(isRepositoryInstructionDescriptor))) || (descriptor.environment !== undefined && !isValidCodeEnvironmentDescriptor(descriptor.environment)) || (descriptor.name !== undefined && diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 86e85872..697a3c55 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -23,6 +23,7 @@ import type { BridgeWorkerRegistrationResponse, BridgeWorkspaceToolOperation, BridgeWorkspaceProgrammaticRequest, + RepositoryInstructionDescriptor, } from './protocol.js'; import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; import type { WorkspaceToolExecutor } from './workspace.js'; @@ -37,6 +38,7 @@ export interface BridgeWorkerOptions { runtimeSupervisor?: RuntimeSupervisor; capabilities: BridgeWorkerCapabilities; workspaceTools?: WorkspaceToolExecutor; + instructionDescriptors?: () => Promise | undefined>; workspaceProgrammatic?: { /** * True when a WorkspaceToolError without mutation uncertainty proves the @@ -396,6 +398,7 @@ export class BridgeWorker { private readonly compatibleCapabilities: BridgeWorkerCapabilities; private registrationCapabilities: BridgeWorkerCapabilities; private activeCapabilities: BridgeWorkerCapabilities; + private instructionMetadataSupported = true; private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; private maintenanceOnly = false; @@ -597,6 +600,8 @@ export class BridgeWorker { const registrationStartedAtMs = Date.now(); let registration: BridgeWorkerRegistrationResponse; try { + const instructions = this.instructionMetadataSupported ? await this.options.instructionDescriptors?.() : undefined; + let includeInstructions = this.instructionMetadataSupported; const register = (capabilities: BridgeWorkerCapabilities) => this.request( `${this.codeApiUrl}/bridge/workers/register`, @@ -609,12 +614,29 @@ export class BridgeWorker { ...capabilities, requiresReadyConfirmation: true, } - : capabilities, + : includeInstructions && instructions && capabilities.workspaceTools ? { + ...capabilities, + workspaceTools: { ...capabilities.workspaceTools, + workspaces: capabilities.workspaceTools.workspaces.map(workspace => ({ ...workspace, + ...((workspace.operations ?? capabilities.workspaceTools!.operations).includes('read_file') + ? { instructions: [...(instructions.get(workspace.id) ?? [])] } : {}), + })), + }, + } : capabilities, }, registrationController.signal, ); try { - registration = await register(this.registrationCapabilities); + try { + registration = await register(this.registrationCapabilities); + } catch (error) { + if (!(instructions && error instanceof BridgeProtocolError && error.status === 400)) { + throw error; + } + includeInstructions = false; + this.instructionMetadataSupported = false; + registration = await register(this.registrationCapabilities); + } } catch (error) { if ( !(error instanceof BridgeProtocolError) || diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 91e89f54..87b444e9 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -45,6 +45,7 @@ import type { } from './protocol.js'; export { isWorkspaceToolRequest, isWorkspaceToolResult }; +import { readRepositoryInstructions } from './instructions.js'; export type { WorkspaceReadFileRequest, WorkspaceReadFileResult, @@ -75,6 +76,7 @@ export interface LocalWorkspaceConfig { export interface LocalWorkspaceToolsOptions { workspaces: LocalWorkspaceConfig[]; + repositoryInstructions?: boolean; } export interface WorkspaceToolExecutor { @@ -1429,6 +1431,16 @@ async function withinListDeadline( export class LocalWorkspaceTools implements WorkspaceToolExecutor { readonly capabilities: BridgeWorkspaceToolCapabilities; readonly mutationFailuresAreAtomic = true as const; + private repositoryInstructions = false; + + async instructionDescriptors() { + if (!this.repositoryInstructions) return undefined; + const entries = await Promise.all([...this.roots].map(async ([id, { root }]) => { + const snapshot = await readRepositoryInstructions(root); + return [id, snapshot ? [snapshot.descriptor] : []] as const; + })); + return new Map(entries); + } private constructor( private readonly roots: ReadonlyMap, @@ -1508,7 +1520,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { writable: workspace.writable === true, }); } - return new LocalWorkspaceTools( + const tools = new LocalWorkspaceTools( roots, operations, workspaces, @@ -1517,6 +1529,8 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { capabilities.editFileFeatures, capabilities.listFileFeatures, ); + tools.repositoryInstructions = options.repositoryInstructions === true; + return tools; } async execute( @@ -1579,6 +1593,15 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ); } + if (request.instructionSha256 !== undefined) { + const snapshot = this.repositoryInstructions ? await readRepositoryInstructions(root) : undefined; + if (!snapshot || snapshot.descriptor.path !== request.path || snapshot.descriptor.sha256 !== request.instructionSha256) { + throw new WorkspaceToolError('Repository instructions changed or are unavailable', 'INVALID_PATH'); + } + return { protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'read_file', workspaceId: request.workspaceId, + path: request.path, content: snapshot.content, startLine: 1, endLine: snapshot.content.split('\n').length, + truncated: snapshot.descriptor.truncated }; + } const startLine = request.startLine ?? 1; const maxLines = request.maxLines ?? 200; if ( diff --git a/service/src/bridge/instructions.test.ts b/service/src/bridge/instructions.test.ts new file mode 100644 index 00000000..63edf614 --- /dev/null +++ b/service/src/bridge/instructions.test.ts @@ -0,0 +1,123 @@ +import { createServer } from 'node:http'; +import { mkdtemp, realpath, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { expect, test } from 'bun:test'; +import express from 'express'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import { BridgeWorker } from '../../../packages/code/src/worker'; +import { LocalWorkspaceTools } from '../../../packages/code/src/workspace'; +import { applyPrincipal } from '../auth/principal'; +import { createWorkspaceToolsRouter } from '../workspace-tools/router'; +import { createBridgeRouter } from './router'; +import { RedisBridgeStore } from './store'; +import { RedisBridgePairingStore } from './pairing'; + +test('repository snapshots traverse the HTTP bridge and real workspace executor', async () => { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'instruction-http-')), + ); + const redis = new RedisMock() as unknown as Redis; + const store = new RedisBridgeStore(redis); + const controller = new AbortController(); + const app = express(); + app.use(express.json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store, + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'local-test-only', + configuredWorkerId: 'instructions-worker', + }), + ); + app.use( + '/v1', + (req, _res, next) => { + applyPrincipal(req, { + userId: 'test-user', + tenantId: 'test-tenant', + principalSource: 'librechat_jwt', + }); + next(); + }, + createWorkspaceToolsRouter({ + backend: 'remote-bridge', + configuredWorkerId: 'instructions-worker', + dynamicWorkers: false, + store, + }), + ); + const server = createServer(app); + let running: Promise | undefined; + try { + await new Promise(resolve => + server.listen(0, '127.0.0.1', resolve), + ); + const address = server.address(); + if (!address || typeof address === 'string') + throw new Error('Missing address'); + const base = `http://127.0.0.1:${address.port}/v1`; + await writeFile( + join(root, 'AGENTS.md'), + 'Exact\r\nrepository guidance\n', + ); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root }], + repositoryInstructions: true, + }); + const worker = new BridgeWorker({ + codeApiUrl: base, + token: 'local-test-only', + workerId: 'instructions-worker', + sandboxEndpoint: 'http://127.0.0.1:1/api/v2', + leaseWaitMs: 50, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'test', + runtimes: [], + workspaceTools: tools.capabilities, + }, + workspaceTools: tools, + instructionDescriptors: () => tools.instructionDescriptors(), + }); + await worker.register(); + const status = await fetch( + `${base}/bridge/workers/instructions-worker/status`, + { headers: { Authorization: 'Bearer local-test-only' } }, + ).then(r => r.json()); + const descriptor = + status.capabilities.workspaceTools.workspaces[0].instructions[0]; + expect(descriptor.path).toBe('AGENTS.md'); + running = worker.run(controller.signal); + const request = { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'AGENTS.md', + instructionSha256: descriptor.sha256, + }; + const read = () => + fetch(`${base}/workspace-tools/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + }); + const response = await read(); + expect(response.status).toBe(200); + expect((await response.json()).content).toBe( + 'Exact\r\nrepository guidance\n', + ); + await writeFile(join(root, 'AGENTS.md'), 'changed'); + expect((await read()).status).toBe(422); + } finally { + controller.abort(); + await running; + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + redis.disconnect(); + await rm(root, { recursive: true, force: true }); + } +}, 10000); From 95bfcbd1dc03cf21e98d70e168891bb4e18980a3 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 10:37:24 -0400 Subject: [PATCH 34/42] feat: Register Selected Coding Projects (#222) * feat(code): register explicitly selected project roots * fix(code): reject shared Git metadata for selected projects * fix(code): pin selected project identity through executor admission * fix(code): preserve full filesystem identity precision * Check selected project identity before replay staging * Anchor replay copies to the verified working directory * fix: Bind Selected Project Operations to Held Directory Descriptors * test: Cover Selected Project PTC and Load Native Fixtures Before Platform Simulation * fix: Keep Native Root Bindings Worker-Local and Verify Directory Ancestry * fix: Anchor Project Admission and Preserve Search Permissions --- .github/workflows/ci.yml | 5 + packages/code/README.md | 44 ++ packages/code/src/cli.ts | 34 +- packages/code/src/instructions.ts | 2 +- packages/code/src/native-process.ts | 2 + .../code/src/native-programmatic-live.test.ts | 10 +- packages/code/src/native-sandbox.test.ts | 114 +++++ packages/code/src/native-sandbox.ts | 51 +- packages/code/src/project-roots.test.ts | 383 +++++++++++++++ packages/code/src/project-roots.ts | 167 +++++++ packages/code/src/root-access.test.ts | 359 ++++++++++++++ packages/code/src/root-access.ts | 460 ++++++++++++++++++ packages/code/src/root-exec.ts | 18 + packages/code/src/root-identity.ts | 27 + packages/code/src/workspace.ts | 28 +- 15 files changed, 1691 insertions(+), 13 deletions(-) create mode 100644 packages/code/src/project-roots.test.ts create mode 100644 packages/code/src/project-roots.ts create mode 100644 packages/code/src/root-access.test.ts create mode 100644 packages/code/src/root-access.ts create mode 100644 packages/code/src/root-exec.ts create mode 100644 packages/code/src/root-identity.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07622ee7..72f32178 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -197,6 +197,11 @@ jobs: node-version: 24.16.0 - run: npm ci - run: npm run build + - name: Selected project root containment tests + run: | + command -v rg || brew install ripgrep + node --test dist/root-access.test.js + node --test --test-name-pattern='selected command|selected replay copy|programmatic probes reject' dist/native-sandbox.test.js - name: Native environment containment tests run: node --test dist/environment.test.js - name: Native ACL and credential lifecycle tests diff --git a/packages/code/README.md b/packages/code/README.md index 48bf769d..720a0b35 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -51,6 +51,50 @@ chosen non-overlapping project directories with `--workspace` or `--environment` Do not also register their parent directory. Treat the inventory as a snapshot; normal workspace admission must validate any directory selected from it. +## Register selected projects + +After pairing, use paths from `projects --root` to register individual checkouts: + +```bash +librechat-code run --project-root /srv/projects \ + --project web --project services/api \ + --allow-workspace-writes --allow-workspace-commands +``` + +Only the explicitly listed checkouts become execution roots. The discovery +directory is not registered, and adding a new sibling repository does not grant +access to it. In LibreChat, select the project in the existing workspace picker; +the conversation stores that selection for subsequent tools and approval resumes. +An agent's default workspace and the user's recent selection work as before. + +Project IDs are derived from the canonical discovery directory and relative +project path, not the branch or selection order. Keep both paths unchanged across +restarts to retain chat bindings. Moving a checkout changes its ID. These are +registration IDs, not the root-local IDs printed by the inventory command. + +Up to 32 selected projects are supported. Each must be a standalone Git checkout; +linked worktrees, symlink traversal, overlapping roots, and duplicate selections +are rejected. Existing native sandbox, command/write permissions, lease-slot and +quarantine rules still apply. This mode cannot be combined with `--environment`, +`--worker-dir`, `--workspace`, default-workspace, or workspace ID/name settings. +Existing registrations are not migrated automatically; use a new conversation +when switching registration mode. Non-Git directories still use the existing +workspace flags. Named environment setup/actions still use `--environment`. + +Selected projects require macOS or Linux (including WSL2). Each request opens +and verifies the admitted directory, then retains that descriptor through file +access, repository-instruction loading, command startup, and replay copying. +Renaming a project cannot redirect an in-flight request to a replacement checkout; +subsequent requests reject the changed identity. Restart with an explicitly +selected replacement to admit it. Descriptors close when requests settle, and +independent workspaces do not share a current directory or global execution lock. + +This reuses the existing workspace protocol. Programmatic tool calling requires +a LibreChat version that preserves the selected workspace across initial +execution and replay, plus the worker's normal programmatic prerequisites. +Installation alone does not restart workers or change registration; update your +worker service arguments explicitly. + ## Pair Hardened deployments use a one-time code instead of copying a long-lived diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 946b08d0..524c5853 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -6,6 +6,7 @@ import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { discoverProjects } from './projects.js'; +import { loadProjectRoots, projectRootArguments } from './project-roots.js'; import { loadCodeEnvironment, assertEnvironmentDefinitionsOutsideRoots, @@ -309,6 +310,26 @@ async function run( runtimeSessionId?: string, args: string[] = [], ): Promise { + const projectArgs = projectRootArguments(args); + if ( + projectArgs && + (runtimeSessionId != null || + args.some(arg => + ['--environment', '--worker-dir', '--default-workspace', + '--workspace', '--workspace-id', '--workspace-name'].some( + flag => arg === flag || arg.startsWith(`${flag}=`), + ), + ) || + [process.env.LIBRECHAT_CODE_WORKER_DIR, + process.env.LIBRECHAT_CODE_WORKSPACE_ID, + process.env.LIBRECHAT_CODE_WORKSPACE_NAME].some(value => value?.trim()) || + process.env.LIBRECHAT_CODE_DEFAULT_WORKSPACE?.trim().toLowerCase() === 'true') + ) { + throw new Error('Project selection cannot be combined with other workspace registration settings'); + } + const projectRoots = projectArgs + ? await loadProjectRoots(projectArgs.root, projectArgs.projects) + : []; const environmentPaths: string[] = []; for (let i = 0; i < args.length; i++) { if (args[i] === '--environment') { @@ -426,11 +447,13 @@ async function run( runtimeSessionId == null && (fileRelayUpstream?.length ?? 0) > 0; const workspaceId = + projectRoots[0]?.id ?? environments[0]?.definition.name ?? option(args, '--workspace-id') ?? process.env.LIBRECHAT_CODE_WORKSPACE_ID?.trim() ?? 'primary'; const explicitWorkerDirectory = + projectRoots[0]?.root ?? environments[0]?.definition.root ?? (runtimeSessionId == null ? nonEmpty( @@ -468,6 +491,9 @@ async function run( if (environments.length && commandSandboxMode !== 'native-srt') { throw new Error('Environment definitions require native-srt'); } + if (projectRoots.length && commandSandboxMode !== 'native-srt') { + throw new Error('Project selections require native-srt'); + } if ( environments.some(environment => environment.definition.setup) && !allowWorkspaceCommands @@ -575,8 +601,10 @@ async function run( { id: workspaceId, root: canonicalWorkerDirectory, + identity: projectRoots[0]?.identity, writable: allowWorkspaceWrites, name: + projectRoots[0]?.name ?? environments[0]?.definition.name ?? option(args, '--workspace-name') ?? process.env.LIBRECHAT_CODE_WORKSPACE_NAME?.trim() ?? @@ -597,6 +625,9 @@ async function run( writable: allowWorkspaceWrites, }); } + for (const project of projectRoots.slice(1)) { + roots.push({ ...project, writable: allowWorkspaceWrites }); + } await assertEnvironmentDefinitionsOutsideRoots(environments, roots); for (let i = 0; i < args.length; i++) { if ( @@ -882,6 +913,7 @@ async function run( }); const nativeOptions: NativeProcessSandboxOptions = { workspaceRoot: canonicalWorkerDirectory!, + workspaceIdentity: roots[0]?.identity, commandPolicy, protectedPaths: [ identityPath, @@ -921,7 +953,7 @@ async function run( new Map( roots.map(root => [ root.id, - { ...nativeOptions, workspaceRoot: root.root }, + { ...nativeOptions, workspaceRoot: root.root, workspaceIdentity: root.identity }, ]), ), workspaceLeaseSlots, diff --git a/packages/code/src/instructions.ts b/packages/code/src/instructions.ts index ee3e8127..1a1e8cad 100644 --- a/packages/code/src/instructions.ts +++ b/packages/code/src/instructions.ts @@ -1,5 +1,5 @@ import { constants } from 'node:fs'; -import { open, lstat, realpath, stat } from 'node:fs/promises'; +import { open, lstat, realpath, stat } from './root-access.js'; import { createHash } from 'node:crypto'; import { resolve, relative, isAbsolute, sep } from 'node:path'; diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 81bbd9c4..70cb2bd1 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -336,6 +336,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan child.on('disconnect', lost); const { workspaceRoot, + workspaceIdentity, commandPolicy, protectedPaths, allowedDomains, @@ -348,6 +349,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan { options: { workspaceRoot, + workspaceIdentity, commandPolicy, protectedPaths, allowedDomains, diff --git a/packages/code/src/native-programmatic-live.test.ts b/packages/code/src/native-programmatic-live.test.ts index 2bc22356..92cab6ea 100644 --- a/packages/code/src/native-programmatic-live.test.ts +++ b/packages/code/src/native-programmatic-live.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { createServer } from 'node:http'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { mkdtemp, readFile, realpath, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; @@ -8,17 +8,20 @@ import type { AddressInfo } from 'node:net'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { resolveNativeSrtCommandPolicy } from './native-policy.js'; -test('real SRT prevents speculative network effects under trusted-vm', { +for (const selected of [false, true]) { +test(`real SRT prevents speculative network effects under trusted-vm${selected ? ' for a selected project' : ''}`, { skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', timeout: 30_000, }, async () => { - const root = await mkdtemp(join(tmpdir(), 'native-ptc-effects-')); + const root = await realpath(await mkdtemp(join(tmpdir(), 'native-ptc-effects-'))); + const identity = await stat(root, { bigint: true }); let effects = 0; const server = createServer((_req, res) => { effects += 1; res.end('ok'); }); await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); const port = (server.address() as AddressInfo).port; const executor = new NativeProcessWorkspaceCommandSandbox({ workspaceRoot: root, + ...(selected ? { workspaceIdentity: { path: root, dev: String(identity.dev), ino: String(identity.ino) } } : {}), commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), programmaticFileUpstream: `http://127.0.0.1:${port}`, }); @@ -39,3 +42,4 @@ test('real SRT prevents speculative network effects under trusted-vm', { } } }); +} diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 3252ee50..e610a0fc 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; import { EventEmitter } from 'node:events'; +import { mkdirSync, renameSync, writeFileSync } from 'node:fs'; import { access, chmod, @@ -236,6 +237,119 @@ test('programmatic probes use a copy-on-write workspace without mutating the pro assert.equal(await readFile(join(snapshot, 'state.txt'), 'utf8'), 'probe-only'); }); +test('selected command cwd stays bound when replacement happens while wrapping', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-project-command-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(await realpath(parent), 'project'); + await mkdir(root); + await writeFile(join(root, 'identity.txt'), 'original'); + const identity = await stat(root, { bigint: true }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + workspaceIdentity: { path: root, dev: String(identity.dev), ino: String(identity.ino) }, + manager: fakeManager({ beforeWrap: async () => { + await rename(root, `${root}.old`); + await mkdir(root); + await writeFile(join(root, 'identity.txt'), 'replacement'); + } }).manager, + }); + t.after(() => sandbox.close()); + const result = await sandbox.execute({ ...request, command: 'cat identity.txt; printf written > result.txt' }); + assert.equal(result.exitCode, 0, result.stderr); + assert.equal(result.stdout, 'original'); + assert.equal(await readFile(join(`${root}.old`, 'result.txt'), 'utf8'), 'written'); + await assert.rejects(access(join(root, 'result.txt'))); +}); + +test('selected command cancellation kills the exec trampoline process group', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-project-cancel-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = await realpath(parent); + const identity = await stat(root, { bigint: true }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + workspaceIdentity: { path: root, dev: String(identity.dev), ino: String(identity.ino) }, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const controller = new AbortController(); + const running = sandbox.execute({ ...request, timeoutMs: 5000, + command: 'printf started > started; sleep 3; printf late > late' }, controller.signal); + const rejected = assert.rejects(running, error => error instanceof WorkspaceToolError && error.code === 'EXECUTION_ABORTED'); + const deadline = Date.now() + 3000; + while (true) { + try { await access(join(root, 'started')); break; } catch { /* Wait for the actual child. */ } + if (Date.now() > deadline) throw new Error('Selected command did not start'); + await new Promise(resolve => setTimeout(resolve, 20)); + } + controller.abort(); + await rejected; + await new Promise(resolve => setTimeout(resolve, 3100)); + await assert.rejects(access(join(root, 'late'))); +}); + +test('selected replay copy stays on the verified directory after pathname replacement', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-project-copy-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(await realpath(parent), 'project'); + await mkdir(root); + await writeFile(join(root, 'identity.txt'), 'original'); + const identity = await stat(root, { bigint: true }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + workspaceIdentity: { path: root, dev: identity.dev.toString(), ino: identity.ino.toString() }, + manager: fakeManager().manager, + spawnCommand(command, args, options) { + assert.equal(command, process.execPath); + renameSync(root, `${root}.old`); + mkdirSync(root); + writeFileSync(join(root, 'identity.txt'), 'replacement'); + return spawn(command, args, options); + }, + }); + t.after(() => sandbox.close()); + const directory = await sandbox.createExecutionDirectory(); + let snapshot: string; + try { + snapshot = await sandbox.createProgrammaticProbeWorkspace(directory); + } catch (error) { + if (error instanceof CopyOnWriteCloneUnavailableError) { + t.skip('host filesystem does not support copy-on-write cloning'); + return; + } + throw error; + } + assert.equal(await readFile(join(root, 'identity.txt'), 'utf8'), 'replacement'); + assert.equal(await readFile(join(snapshot, 'identity.txt'), 'utf8'), 'original'); +}); + +test('programmatic probes reject a replaced selected project before copying', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-project-probe-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(await realpath(parent), 'project'); + await mkdir(root); + const identity = await stat(root, { bigint: true }); + let copies = 0; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + workspaceIdentity: { path: root, dev: identity.dev.toString(), ino: identity.ino.toString() }, + manager: fakeManager().manager, + spawnCommand() { + copies++; + throw new Error('must not copy a replaced project'); + }, + }); + t.after(() => sandbox.close()); + const executionDirectory = await sandbox.createExecutionDirectory(); + await rename(root, join(parent, 'original')); + await mkdir(root); + await assert.rejects( + sandbox.createProgrammaticProbeWorkspace(executionDirectory), + /Selected project changed before probe staging/, + ); + assert.equal(copies, 0); +}); + test('programmatic probes do not hide clone implementation failures as unsupported filesystems', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 550d0d52..84602a35 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -13,6 +13,9 @@ import { import { constants as fsConstants } from 'node:fs'; import { access, mkdtemp, open, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; +import { matchesWorkspaceRoot } from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import { withWorkspaceRoot, WorkspaceRootAccessError, spawnWithinWorkspace, realpath as rootedRealpath, stat as rootedStat } from './root-access.js'; import { SandboxManager } from '@anthropic-ai/sandbox-runtime'; @@ -157,6 +160,7 @@ type SpawnCommand = ( ) => ChildProcessWithoutNullStreams; export interface NativeSrtWorkspaceCommandSandboxOptions { + workspaceIdentity?: WorkspaceRootIdentity; workspaceRoot: string; commandPolicy?: NativeSrtCommandPolicy; /** Trusted worker files that must never become workspace-readable or writable. */ @@ -343,6 +347,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const root = await realpath(this.options.workspaceRoot); + if (this.options.workspaceIdentity && !await matchesWorkspaceRoot(root, this.options.workspaceIdentity)) { + throw new WorkspaceToolError('Selected project changed before sandbox admission', 'REGISTRATION_INVALID'); + } if (!(await stat(root)).isDirectory()) { throw new WorkspaceToolError( 'Native sandbox workspace is unavailable', @@ -573,7 +580,20 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox executionDirectory: string, signal?: AbortSignal, ): Promise { + try { + return await withWorkspaceRoot(this.options.workspaceRoot, this.options.workspaceIdentity, + () => this.createBoundProgrammaticProbeWorkspace(executionDirectory, signal)); + } catch (error) { + if (error instanceof WorkspaceRootAccessError) throw new WorkspaceToolError('Selected project changed before probe staging', 'REGISTRATION_INVALID'); + throw error; + } + } + + private async createBoundProgrammaticProbeWorkspace(executionDirectory: string, signal?: AbortSignal): Promise { await this.initialize(); + if (this.options.workspaceIdentity && !await matchesWorkspaceRoot(this.options.workspaceRoot, this.options.workspaceIdentity)) { + throw new WorkspaceToolError('Selected project changed before probe staging', 'REGISTRATION_INVALID'); + } const scratchDirectory = this.scratchDirectory; const root = this.canonicalRoot; let parent: string; @@ -608,8 +628,11 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox this.platform === 'darwin' ? ['-cR', root, destination] : ['--archive', '--reflink=always', root, destination]; + const identity = this.options.workspaceIdentity; + const copyArgs = identity ? [...args.slice(0, -2), '.', destination] : args; await new Promise((resolveCopy, rejectCopy) => { - const child = this.spawnCommand('/bin/cp', args, { + const child = spawnWithinWorkspace(this.spawnCommand, '/bin/cp', copyArgs, { + ...(identity ? { cwd: root } : {}), env: { PATH: this.environment.PATH, LANG: this.environment.LANG, @@ -800,6 +823,26 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox sandboxScratchDirectory?: string, workspaceRoot?: string, ): Promise { + try { + return await withWorkspaceRoot(this.options.workspaceRoot, workspaceRoot ? undefined : this.options.workspaceIdentity, + () => this.executeBound(request, signal, trustedEnvironment, customConfig, sandboxScratchDirectory, workspaceRoot)); + } catch (error) { + if (error instanceof WorkspaceRootAccessError) throw new WorkspaceToolError(error.message, 'REGISTRATION_INVALID'); + throw error; + } + } + + private async executeBound( + request: WorkspaceExecuteCommandRequest, + signal?: AbortSignal, + trustedEnvironment?: NodeJS.ProcessEnv, + customConfig?: Partial, + sandboxScratchDirectory?: string, + workspaceRoot?: string, + ): Promise { + if (this.options.workspaceIdentity && !await matchesWorkspaceRoot(this.options.workspaceRoot, this.options.workspaceIdentity)) { + throw new WorkspaceToolError('Selected project changed after sandbox admission', 'REGISTRATION_INVALID'); + } if ( !isWorkspaceToolRequest(request) || request.operation !== 'execute_command' @@ -819,8 +862,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const root = workspaceRoot ?? this.canonicalRoot!; let cwd: string; try { - cwd = await realpath(resolve(root, request.cwd ?? '.')); - if (!isWithin(root, cwd) || !(await stat(cwd)).isDirectory()) + cwd = await rootedRealpath(resolve(root, request.cwd ?? '.')); + if (!isWithin(root, cwd) || !(await rootedStat(cwd)).isDirectory()) throw new Error('invalid cwd'); } catch { throw new WorkspaceToolError( @@ -939,7 +982,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox (resolvePromise, reject) => { let child: ChildProcessWithoutNullStreams; try { - child = this.spawnCommand( + child = spawnWithinWorkspace(this.spawnCommand, wrapped.argv[0], wrapped.argv.slice(1), { diff --git a/packages/code/src/project-roots.test.ts b/packages/code/src/project-roots.test.ts new file mode 100644 index 00000000..f88e523f --- /dev/null +++ b/packages/code/src/project-roots.test.ts @@ -0,0 +1,383 @@ +import assert from 'node:assert/strict'; +import { execFile, spawn, spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { createServer } from 'node:http'; +import { + mkdtemp, + mkdir, + readFile, + rename, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import type { TestContext } from 'node:test'; +import { loadProjectRoots, projectRootArguments } from './project-roots.js'; +import { LocalWorkspaceTools } from './workspace.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; +import type { BridgeWorkerCapabilities } from './protocol.js'; +import { WorkspaceRootAccess } from './root-access.js'; + +const exec = promisify(execFile); +test('admission validates the captured root even while a replacement checkout occupies its path', async t => { + const root = await fixture(t); + const selected = join(root, 'app'); + await writeFile( + join(selected, '.git/commondir'), + '../../nested/api/.git\n', + ); + const originalOpen = WorkspaceRootAccess.open; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const held = await originalOpen(...args); + await rename(selected, `${selected}-original`); + await exec('git', ['init', '--initial-branch=dev', selected]); + const originalClose = held.close.bind(held); + held.close = async () => { + await rm(selected, { recursive: true, force: true }); + await rename(`${selected}-original`, selected); + await originalClose(); + }; + return held; + }, + ); + await assert.rejects( + loadProjectRoots(root, ['app']), + /Git common directory/, + ); + assert.equal( + await readFile(join(selected, '.git/commondir'), 'utf8'), + '../../nested/api/.git\n', + ); +}); + +test('admission cannot borrow a replacement checkout Git validity', async t => { + const root = await fixture(t); + const selected = join(root, 'app'); + await rm(join(selected, '.git/HEAD')); + const originalOpen = WorkspaceRootAccess.open; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const held = await originalOpen(...args); + await rename(selected, `${selected}-original`); + await exec('git', ['init', '--initial-branch=dev', selected]); + const originalClose = held.close.bind(held); + held.close = async () => { + await rm(selected, { recursive: true, force: true }); + await rename(`${selected}-original`, selected); + await originalClose(); + }; + return held; + }, + ); + await assert.rejects( + loadProjectRoots(root, ['app']), + /standalone Git checkout/, + ); +}); +test( + 'real worker CLI registers only explicitly selected project roots', + { timeout: 10000 }, + async t => { + const root = await fixture(t); + let accept: (capabilities: BridgeWorkerCapabilities) => void = () => {}; + const registered = new Promise(resolve => { + accept = resolve; + }); + const server = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on('data', (chunk: Buffer) => chunks.push(chunk)); + request.on('end', () => { + response.setHeader('Content-Type', 'application/json'); + if (request.url?.endsWith('/bridge/workers/register')) { + const body = JSON.parse( + Buffer.concat(chunks).toString(), + ) as { + workerId: string; + incarnationId: string; + capabilities: BridgeWorkerCapabilities; + }; + accept(body.capabilities); + response.end( + JSON.stringify({ + protocolVersion: 1, + workerId: body.workerId, + incarnationId: body.incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60000, + }), + ); + } else + response.end( + JSON.stringify({ + protocolVersion: 1, + serverElapsedMs: 0, + }), + ); + }); + }); + await new Promise(resolve => + server.listen(0, '127.0.0.1', resolve), + ); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const child = spawn( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--project-root', + root, + '--project', + 'app', + '--project', + 'nested/api', + ], + { + env: { + PATH: process.env.PATH, + LIBRECHAT_CODE_URL: `http://127.0.0.1:${address.port}/v1`, + LIBRECHAT_CODE_WORKER_ID: 'test-worker', + LIBRECHAT_CODE_WORKER_TOKEN: 'test-token', + }, + stdio: 'ignore', + }, + ); + t.after(() => { + if (child.exitCode === null) child.kill('SIGKILL'); + }); + const capabilities = await registered; + child.kill(); + await once(child, 'exit'); + assert.deepEqual( + capabilities.workspaceTools?.workspaces?.map( + workspace => workspace.name, + ), + ['app', 'nested/api'], + ); + assert.ok( + capabilities.workspaceTools?.workspaces?.every(workspace => + workspace.id.startsWith('project-'), + ), + ); + assert.equal(JSON.stringify(capabilities).includes(root), false); + }, +); + +async function fixture(t: TestContext) { + const root = await mkdtemp(join(tmpdir(), 'selected-projects-')); + t.after(() => rm(root, { recursive: true, force: true })); + for (const name of ['app', 'nested/api']) { + await mkdir(join(root, name), { recursive: true }); + await exec('git', ['init', '--initial-branch=dev', join(root, name)]); + } + return root; +} + +test('selected projects retain identity across order and branch changes without granting their parent', async t => { + const root = await fixture(t); + const selected = await loadProjectRoots(root, ['app', 'nested/api']); + assert.deepEqual( + selected.map(project => project.name), + ['app', 'nested/api'], + ); + assert.ok( + selected.every(project => project.root !== root && !project.writable), + ); + await exec('git', [ + '-C', + join(root, 'app'), + 'symbolic-ref', + 'HEAD', + 'refs/heads/next', + ]); + const restarted = await loadProjectRoots(root, ['nested/api', 'app']); + assert.equal(restarted[1].id, selected[0].id); + assert.equal(restarted[0].id, selected[1].id); + const otherRoot = await fixture(t); + assert.notEqual( + (await loadProjectRoots(otherRoot, ['app']))[0].id, + selected[0].id, + ); +}); + +test('real file operations use the selected project boundary', async t => { + const root = await fixture(t); + const selected = await loadProjectRoots(root, ['app', 'nested/api']); + const tools = await LocalWorkspaceTools.create({ + workspaces: selected.map(project => ({ ...project, writable: true })), + }); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: selected[1].id, + path: 'created.txt', + content: 'second project', + }); + assert.equal( + await readFile(join(root, 'nested/api/created.txt'), 'utf8'), + 'second project', + ); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: selected[0].id, + path: '../nested/api/created.txt', + }), + ); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'nested/api/created.txt', + }), + ); + await assert.rejects(readFile(join(root, 'app/created.txt'))); +}); + +test('rejects missing, escaped, aliased, duplicate and linked-worktree selections', async t => { + const root = await fixture(t); + await mkdir(join(root, 'linked')); + await writeFile( + join(root, 'linked/.git'), + 'gitdir: ../app/.git/worktrees/linked\n', + ); + await symlink(join(root, 'app'), join(root, 'alias'), 'dir'); + for (const projects of [ + [], + ['..'], + ['/tmp'], + ['missing'], + ['alias'], + ['linked'], + ['app', './app'], + Array(33).fill('app'), + ]) { + await assert.rejects(loadProjectRoots(root, projects)); + } + await assert.rejects( + loadProjectRoots(root, ['.']), + /standalone Git checkout/, + ); +}); + +test('project CLI arguments require explicit bounded selections', () => { + assert.equal(projectRootArguments(['run']), undefined); + assert.deepEqual( + projectRootArguments([ + 'run', + '--project-root=/srv/projects', + '--project', + 'app', + '--project=nested/api', + ]), + { root: '/srv/projects', projects: ['app', 'nested/api'] }, + ); + for (const args of [ + ['--project-root'], + ['--project-root=/srv'], + ['--project=app'], + ['--project-root=/srv', '--project-root=/other', '--project=app'], + ['--project-root=/srv', '--project', '--allow-workspace-writes'], + ]) { + assert.throws(() => projectRootArguments(args)); + } +}); + +test('rejects a directory-form git marker redirecting to shared metadata', async t => { + const root = await fixture(t); + await writeFile( + join(root, 'app/.git/commondir'), + '../../nested/api/.git\n', + ); + await assert.rejects( + loadProjectRoots(root, ['app']), + /Git common directory/, + ); +}); + +test('replacement after selection cannot become a file or native execution root', async t => { + const root = await fixture(t); + const outside = await fixture(t); + const [selected] = await loadProjectRoots(root, ['app']); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ ...selected, writable: true }], + }); + await rename(selected.root, `${selected.root}-previous`); + await symlink(join(outside, 'app'), selected.root, 'dir'); + await assert.rejects( + LocalWorkspaceTools.create({ workspaces: [selected] }), + /Invalid workspace registration/, + ); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: selected.id, + path: 'escaped.txt', + content: 'blocked', + }), + /changed after admission/, + ); + const executor = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: selected.root, + workspaceIdentity: selected.identity, + }); + try { + await assert.rejects(executor.prepare()); + } finally { + await executor.close().catch(() => undefined); + } + await assert.rejects(readFile(join(outside, 'app/escaped.txt')), { + code: 'ENOENT', + }); +}); + +test('CLI rejects mixed registration and overlapping selected projects before connecting', async t => { + const root = await fixture(t); + await exec('git', ['init', '--initial-branch=dev', root]); + for (const extra of [ + ['--worker-dir', root], + ['--project', '.'], + ]) { + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--project-root', + root, + '--project', + 'app', + ...extra, + ], + { + encoding: 'utf8', + timeout: 5000, + env: { + PATH: process.env.PATH, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1', + LIBRECHAT_CODE_WORKER_ID: 'test-worker', + LIBRECHAT_CODE_WORKER_TOKEN: 'test-token', + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /cannot be combined|must not overlap/); + } +}); diff --git a/packages/code/src/project-roots.ts b/packages/code/src/project-roots.ts new file mode 100644 index 00000000..06e910ec --- /dev/null +++ b/packages/code/src/project-roots.ts @@ -0,0 +1,167 @@ +import { createHash } from 'node:crypto'; +import { lstat, realpath } from 'node:fs/promises'; +import { basename, isAbsolute, relative, resolve, sep } from 'node:path'; +import { + lstat as rootedLstat, + spawn, + withWorkspaceRoot, +} from './root-access.js'; +import { matchesWorkspaceRoot } from './root-identity.js'; +import type { LocalWorkspaceConfig } from './workspace.js'; + +/** Validate the selected checkout itself, never rediscover it via its pathname. */ +async function validateCheckout(root: string): Promise { + const marker = await rootedLstat(resolve(root, '.git')).catch( + () => undefined, + ); + if (!marker?.isDirectory() || marker.isSymbolicLink()) + throw new Error( + 'Select a standalone Git checkout, not a parent directory or linked worktree', + ); + const common = await rootedLstat(resolve(root, '.git', 'commondir')).catch( + error => { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) + throw error; + return undefined; + }, + ); + if (common) + throw new Error( + 'Selected projects must not share a Git common directory', + ); + await new Promise((accept, reject) => { + const child = spawn( + 'git', + [ + '--no-optional-locks', + '--git-dir=.git', + '--work-tree=.', + '-c', + 'core.fsmonitor=false', + 'rev-parse', + '--is-inside-work-tree', + ], + { + cwd: root, + env: { + PATH: process.env.PATH, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + LC_ALL: 'C', + }, + }, + ); + let output = ''; + let exceeded = false; + const timer = setTimeout(() => { + exceeded = true; + child.kill('SIGKILL'); + }, 1500); + child.stdout.on('data', (chunk: Buffer) => { + if (output.length + chunk.length > 4096) { + exceeded = true; + child.kill('SIGKILL'); + } else output += chunk.toString(); + }); + child.stderr.resume(); + child.stdin.end(); + child.once('error', reject); + child.once('close', code => { + clearTimeout(timer); + if (!exceeded && code === 0 && output.trim() === 'true') accept(); + else + reject( + new Error( + 'Select a standalone Git checkout, not a parent directory or linked worktree', + ), + ); + }); + }); +} + +/** Explicit operator selections, not an automatically expanding execution grant. */ +export async function loadProjectRoots( + directory: string, + selections: string[], +): Promise { + if (!selections.length || selections.length > 32) + throw new Error('Choose between 1 and 32 projects'); + const root = await realpath(directory); + const paths = new Set(); + const projects: LocalWorkspaceConfig[] = []; + for (const selection of selections) { + if (!selection || isAbsolute(selection) || selection.includes('\0')) + throw new Error('Project paths must be relative to --project-root'); + const path = resolve(root, selection); + const rel = relative(root, path); + if (rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) + throw new Error('Project paths must stay inside --project-root'); + const canonical = await realpath(path); + const directoryIdentity = await lstat(path, { bigint: true }); + if (canonical !== path || !directoryIdentity.isDirectory()) + throw new Error( + 'Selected projects must be directories without symlink traversal', + ); + if (paths.has(canonical)) + throw new Error('Duplicate project selection'); + paths.add(canonical); + const portablePath = rel.split(sep).join('/') || '.'; + const identity = { + path: canonical, + dev: directoryIdentity.dev.toString(), + ino: directoryIdentity.ino.toString(), + }; + await withWorkspaceRoot(canonical, identity, () => + validateCheckout(canonical), + ); + if (!(await matchesWorkspaceRoot(canonical, identity))) + throw new Error('Selected project changed during admission'); + projects.push({ + identity, + id: `project-${createHash('sha256') + .update(`${root}\0${portablePath}`) + .digest('hex') + .slice(0, 32)}`, + name: (portablePath === '.' ? basename(root) : portablePath).slice( + 0, + 64, + ), + root: canonical, + }); + } + return projects; +} + +export function projectRootArguments( + args: string[], +): { root: string; projects: string[] } | undefined { + let root: string | undefined; + const projects: string[] = []; + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + const flag = arg.split('=')[0]; + if (flag !== '--project-root' && flag !== '--project') continue; + const value = arg.includes('=') + ? arg.slice(flag.length + 1) + : args[++index]; + if (!value || value.startsWith('--')) + throw new Error(`${flag} requires a value`); + if (flag === '--project') projects.push(value); + else { + if (root !== undefined) + throw new Error('Only one --project-root may be supplied'); + root = value; + } + } + if (root === undefined && !projects.length) return undefined; + if (root === undefined || !projects.length) + throw new Error( + '--project-root requires at least one --project relative/path', + ); + return { root, projects }; +} diff --git a/packages/code/src/root-access.test.ts b/packages/code/src/root-access.test.ts new file mode 100644 index 00000000..bb757464 --- /dev/null +++ b/packages/code/src/root-access.test.ts @@ -0,0 +1,359 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { constants } from 'node:fs'; +import * as fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + WorkspaceRootAccess, + withWorkspaceRoot, + open, + realpath, + stat, + lstat, + rename, + link, + unlink, + spawn, +} from './root-access.js'; +import { LocalWorkspaceTools } from './workspace.js'; +import type { WorkspaceToolRequest } from './protocol.js'; + +test('search-only directories support known files and command cwd without enumeration', async t => { + if (process.getuid?.() === 0) + return t.skip( + 'requires an unprivileged user to verify search permissions', + ); + const root = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'root-search-')), + ); + const nested = join(root, 'nested'); + await fs.mkdir(nested); + await fs.writeFile(join(nested, 'known'), 'known-value'); + const identity = await fs.stat(root, { bigint: true }); + t.after(async () => { + await fs.chmod(root, 0o700); + await fs.chmod(nested, 0o700); + await fs.rm(root, { recursive: true, force: true }); + }); + await fs.chmod(root, 0o111); + await fs.chmod(nested, 0o111); + await assert.rejects(fs.readdir(nested), { code: 'EACCES' }); + await withWorkspaceRoot( + root, + { path: root, dev: String(identity.dev), ino: String(identity.ino) }, + async () => { + const reader = await open(join(nested, 'known'), 'r'); + try { + assert.equal(await reader.readFile('utf8'), 'known-value'); + } finally { + await reader.close(); + } + assert.equal((await stat(nested)).isDirectory(), true); + assert.equal(await realpath(nested), nested); + await new Promise((accept, reject) => { + const child = spawn('/bin/cat', ['known'], { cwd: nested }); + let output = ''; + child.stdout!.on('data', chunk => { + output += chunk.toString(); + }); + child.once('error', reject); + child.once('close', code => { + try { + assert.equal(code, 0); + assert.equal(output, 'known-value'); + accept(); + } catch (error) { + reject(error); + } + }); + }); + }, + ); +}); + +test('held roots allow internal directory links and reject external ancestors', async t => { + const directory = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'root-links-')), + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const root = join(directory, 'root'); + await fs.mkdir(join(root, 'nested'), { recursive: true }); + await fs.mkdir(join(directory, 'outside')); + await fs.writeFile(join(root, 'nested', 'value'), 'inside'); + await fs.symlink('nested', join(root, 'inside')); + await fs.symlink('../outside', join(root, 'outside')); + const identity = await fs.stat(root, { bigint: true }); + const originalOpen = WorkspaceRootAccess.open; + let held: WorkspaceRootAccess | undefined; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + held = await originalOpen(...args); + return held; + }, + ); + await withWorkspaceRoot( + root, + { path: root, dev: String(identity.dev), ino: String(identity.ino) }, + async () => { + assert.equal( + (await lstat(join(root, 'inside'))).isSymbolicLink(), + true, + ); + const reader = await open(join(root, 'inside', 'value'), 'r'); + try { + assert.equal(await reader.readFile('utf8'), 'inside'); + } finally { + await reader.close(); + } + await assert.rejects( + open( + join(root, 'outside', 'escape'), + constants.O_CREAT | constants.O_WRONLY, + 0o600, + ), + { code: 'EACCES' }, + ); + }, + ); + assert.equal(held?.handle.fd, -1); + assert.deepEqual(await fs.readdir(join(directory, 'outside')), []); +}); + +test('held root file operations cannot be redirected by replacing its pathname', async () => { + const directory = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'root-access-')), + ); + const root = join(directory, 'project'); + await fs.mkdir(root); + await fs.writeFile(join(root, 'original'), 'original'); + const identity = await fs.stat(root, { bigint: true }); + try { + await withWorkspaceRoot( + root, + { + path: root, + dev: String(identity.dev), + ino: String(identity.ino), + }, + async () => { + await fs.rename(root, `${root}.old`); + await fs.mkdir(root); + await fs.writeFile(join(root, 'original'), 'replacement'); + assert.equal( + await realpath(join(root, 'original')), + join(root, 'original'), + ); + assert.equal((await stat(root)).isDirectory(), true); + assert.equal( + (await lstat(join(root, 'original'))).isFile(), + true, + ); + const reader = await open(join(root, 'original'), 'r'); + try { + assert.equal(await reader.readFile('utf8'), 'original'); + } finally { + await reader.close(); + } + const writer = await open( + join(root, 'new'), + constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, + 0o600, + ); + try { + await writer.writeFile('new'); + await writer.sync(); + } finally { + await writer.close(); + } + await link(join(root, 'new'), join(root, 'linked')); + await rename(join(root, 'new'), join(root, 'renamed')); + await unlink(join(root, 'linked')); + const child = spawn('/bin/sh', ['-c', 'cat original'], { + cwd: root, + }); + let output = ''; + let error = ''; + child.stdout.on('data', chunk => { + output += chunk; + }); + child.stderr.on('data', chunk => { + error += chunk; + }); + child.stdin.end(); + const code = await new Promise(resolve => + child.on('close', resolve), + ); + assert.equal(code, 0, error); + assert.equal(output, 'original'); + }, + ); + assert.equal( + await fs.readFile(join(root, 'original'), 'utf8'), + 'replacement', + ); + assert.equal( + await fs.readFile(join(`${root}.old`, 'renamed'), 'utf8'), + 'new', + ); + assert.deepEqual(await fs.readdir(root), ['original']); + await assert.rejects( + WorkspaceRootAccess.open(root, { + path: root, + dev: String(identity.dev), + ino: String(identity.ino), + }), + ); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } +}); + +for (const operation of [ + 'read_file', + 'write_file', + 'edit_file', + 'list_files', + 'search_text', + 'instructions', +] as const) { + test(`selected ${operation} stays bound when the root is replaced after acquisition`, async t => { + const directory = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'root-caller-')), + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + const root = join(directory, 'project'); + await fs.mkdir(root); + await fs.writeFile(join(root, 'original.txt'), 'original'); + await fs.writeFile(join(root, 'AGENTS.md'), 'Original instructions'); + const identity = await fs.stat(root, { bigint: true }); + const tools = await LocalWorkspaceTools.create({ + repositoryInstructions: true, + workspaces: [ + { + id: 'selected', + root, + writable: true, + identity: { + path: root, + dev: String(identity.dev), + ino: String(identity.ino), + }, + }, + ], + }); + const originalOpen = WorkspaceRootAccess.open; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const access = await originalOpen(...args); + await fs.rename(root, `${root}.old`); + await fs.mkdir(root); + await fs.writeFile( + join(root, 'replacement.txt'), + 'replacement', + ); + await fs.writeFile( + join(root, 'AGENTS.md'), + 'Replacement instructions', + ); + return access; + }, + ); + if (operation === 'instructions') { + const descriptors = await tools.instructionDescriptors(); + const { createHash } = await import('node:crypto'); + assert.equal( + descriptors?.get('selected')?.[0].sha256, + createHash('sha256') + .update('Original instructions') + .digest('hex'), + ); + } else { + const request = { + protocolVersion: 1, + workspaceId: 'selected', + operation, + ...(operation === 'write_file' + ? { path: 'new.txt', content: 'created', overwrite: false } + : {}), + ...(operation === 'read_file' ? { path: 'original.txt' } : {}), + ...(operation === 'edit_file' + ? { + path: 'original.txt', + oldText: 'original', + newText: 'edited', + } + : {}), + ...(operation === 'search_text' ? { query: 'original' } : {}), + } as WorkspaceToolRequest; + const result = await tools.execute(request); + assert.equal( + JSON.stringify(result).includes('replacement.txt'), + false, + ); + if (operation === 'read_file') + assert.equal( + (result as { content: string }).content, + 'original', + ); + if (operation === 'write_file') + assert.equal( + await fs.readFile(join(`${root}.old`, 'new.txt'), 'utf8'), + 'created', + ); + if (operation === 'edit_file') + assert.equal( + await fs.readFile( + join(`${root}.old`, 'original.txt'), + 'utf8', + ), + 'edited', + ); + } + assert.deepEqual((await fs.readdir(root)).sort(), [ + 'AGENTS.md', + 'replacement.txt', + ]); + }); +} + +test('simultaneous roots retain independent descriptor contexts and release on failure', async t => { + const directory = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'root-concurrency-')), + ); + t.after(() => fs.rm(directory, { recursive: true, force: true })); + await Promise.all( + ['a', 'b'].map(async name => { + const root = join(directory, name); + await fs.mkdir(root); + await fs.writeFile(join(root, 'value'), name); + const identity = await fs.stat(root, { bigint: true }); + await assert.rejects( + withWorkspaceRoot( + root, + { + path: root, + dev: String(identity.dev), + ino: String(identity.ino), + }, + async () => { + await fs.rename(root, `${root}.old`); + await fs.mkdir(root); + const file = await open(join(root, 'value'), 'r'); + try { + assert.equal(await file.readFile('utf8'), name); + } finally { + await file.close(); + } + throw new Error('cancelled request'); + }, + ), + /cancelled request/, + ); + }), + ); +}); diff --git a/packages/code/src/root-access.ts b/packages/code/src/root-access.ts new file mode 100644 index 00000000..4aeb6100 --- /dev/null +++ b/packages/code/src/root-access.ts @@ -0,0 +1,460 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { spawn as spawnProcess } from 'node:child_process'; +import { constants, closeSync, fstatSync, readlinkSync } from 'node:fs'; +import * as fs from 'node:fs/promises'; +import { + basename, + dirname, + isAbsolute, + relative, + resolve, + sep, +} from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; +import type { + SpawnOptionsWithoutStdio, + ChildProcessWithoutNullStreams, +} from 'node:child_process'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +type NativeCall = (...args: (string | number | Buffer)[]) => number; +interface NativeLibrary { + func(signature: string): NativeCall; +} +interface NativeRuntime { + load(path: null): NativeLibrary; + errno(): number; + os: { errno: Record }; +} +let nativeRuntime: NativeRuntime | undefined; +let library: NativeLibrary | undefined; +function runtime(): NativeRuntime { + // Code API imports workspace contracts without installing native worker + // dependencies. Load the POSIX implementation only for selected roots. + return (nativeRuntime ??= createRequire(import.meta.url)( + 'koffi', + ) as NativeRuntime); +} +function bind(signature: string): NativeCall | undefined { + if (!['darwin', 'linux'].includes(process.platform)) return undefined; + let call: NativeCall | undefined; + return (...args) => { + library ??= runtime().load(null); + call ??= library.func(signature); + return call(...args); + }; +} +const nativeOpenAt = bind( + 'int openat(int dirfd, const char *path, int flags, ...)', +); +const O_CLOEXEC = process.platform === 'darwin' ? 0x1000000 : 0x80000; +// Anchors need search, not directory enumeration permission. +const DIRECTORY_ACCESS = + constants.O_DIRECTORY | + (process.platform === 'darwin' + ? 0x40000000 /* O_SEARCH */ + : 0x200000) /* O_PATH */; +const openAt = nativeOpenAt + ? (fd: number, path: string, flags: number, mode: number): number => + nativeOpenAt(fd, path, flags | O_CLOEXEC, 'unsigned int', mode) + : undefined; +const renameAt = bind( + 'int renameat(int fromfd, const char *from, int tofd, const char *to)', +); +const linkAt = bind( + 'int linkat(int fromfd, const char *from, int tofd, const char *to, int flags)', +); +const unlinkAt = bind('int unlinkat(int dirfd, const char *path, int flags)'); +const getPath = + process.platform === 'darwin' + ? bind('int fcntl(int fd, int command, ...)') + : undefined; + +function nativeError(): NodeJS.ErrnoException { + const errno = runtime().errno(); + const code = + Object.entries(runtime().os.errno).find( + ([, value]) => value === errno, + )?.[0] ?? 'EIO'; + return Object.assign( + new Error(`Workspace descriptor access failed: ${code}`), + { code }, + ); +} + +function checked(fd: number): number { + if (fd < 0) throw nativeError(); + return fd; +} + +function descriptorPath(fd: number): string { + return process.platform === 'linux' + ? `/proc/self/fd/${fd}` + : `/dev/fd/${fd}`; +} + +function physicalPath(fd: number): string { + if (process.platform === 'linux') return readlinkSync(descriptorPath(fd)); + const buffer = Buffer.alloc(1024); + if (!getPath || getPath(fd, 50 /* F_GETPATH */, 'void *', buffer) !== 0) + throw nativeError(); + return buffer.subarray(0, buffer.indexOf(0)).toString(); +} + +function offset(root: string, path: string): string { + const value = relative(root, path); + if (isAbsolute(value) || value === '..' || value.startsWith(`..${sep}`)) { + throw Object.assign(new Error('Path is outside the held workspace'), { + code: 'EACCES', + }); + } + return value || '.'; +} + +/** A request owns one directory descriptor, not a replaceable pathname grant. */ +export class WorkspaceRootAccessError extends Error {} +export class WorkspaceRootAccess { + private constructor( + readonly path: string, + readonly handle: fs.FileHandle, + ) {} + + static async open( + path: string, + identity: WorkspaceRootIdentity, + ): Promise { + if (!openAt || path !== identity.path) + throw new WorkspaceRootAccessError( + 'Selected project root access is unavailable', + ); + const handle = await fs + .open(path, DIRECTORY_ACCESS | constants.O_NOFOLLOW) + .catch(() => { + throw new WorkspaceRootAccessError( + 'Selected project changed after admission', + ); + }); + try { + const current = await handle.stat({ bigint: true }); + if ( + !current.isDirectory() || + current.dev.toString() !== identity.dev || + current.ino.toString() !== identity.ino + ) { + throw new WorkspaceRootAccessError( + 'Selected project changed after admission', + ); + } + return new WorkspaceRootAccess(path, handle); + } catch (error) { + await handle.close(); + throw error; + } + } + + close(): Promise { + return this.handle.close(); + } + + directory(path: string): number { + const fd = checked( + openAt!( + this.handle.fd, + offset(this.path, path), + DIRECTORY_ACCESS, + 0, + ), + ); + try { + this.assertDirectoryAncestor(fd); + this.canonical(fd); + return fd; + } catch (error) { + closeSync(fd); + throw error; + } + } + + private canonical(fd: number): string { + return resolve( + this.path, + offset(physicalPath(this.handle.fd), physicalPath(fd)), + ); + } + + /** Paths are presentation, not proof of ancestry: a renamed root can make + * an outside symlink target temporarily occupy its old textual prefix. */ + private assertDirectoryAncestor(fd: number): void { + const root = fstatSync(this.handle.fd, { bigint: true }); + let current = fd; + try { + for (let depth = 0; depth <= 128; depth++) { + const identity = fstatSync(current, { bigint: true }); + if (identity.dev === root.dev && identity.ino === root.ino) + return; + const parent = checked( + openAt!( + current, + '..', + DIRECTORY_ACCESS | constants.O_NOFOLLOW, + 0, + ), + ); + if (current !== fd) closeSync(current); + current = parent; + const ancestor = fstatSync(parent, { bigint: true }); + if ( + ancestor.dev === identity.dev && + ancestor.ino === identity.ino + ) + break; + } + throw Object.assign( + new Error( + 'Directory is outside the held workspace or exceeds its ancestry limit', + ), + { code: 'EACCES' }, + ); + } finally { + if (current !== fd) closeSync(current); + } + } + + private parent(path: string): { fd: number; name: string } { + const local = offset(this.path, path); + const fd = checked( + openAt!(this.handle.fd, dirname(local), DIRECTORY_ACCESS, 0), + ); + try { + this.assertDirectoryAncestor(fd); + this.canonical(fd); + return { fd, name: basename(local) }; + } catch (error) { + closeSync(fd); + throw error; + } + } + + async openFile( + path: string, + flags: number, + mode = 0o666, + ): Promise { + const parent = this.parent(path); + let fd: number | undefined; + try { + fd = checked(openAt!(parent.fd, parent.name, flags, mode)); + // Node owns the duplicate, so callers retain native FileHandle semantics + // for asynchronous I/O, fsync, ownership and deterministic close. + const accessMode = flags & (constants.O_WRONLY | constants.O_RDWR); + const duplicate = await fs.open( + descriptorPath(fd), + accessMode | constants.O_NONBLOCK, + ); + try { + const expected = fstatSync(fd, { bigint: true }); + const actual = await duplicate.stat({ bigint: true }); + if (expected.dev !== actual.dev || expected.ino !== actual.ino) + throw new Error('Descriptor duplication changed identity'); + return duplicate; + } catch (error) { + await duplicate.close(); + throw error; + } + } finally { + if (fd !== undefined) closeSync(fd); + closeSync(parent.fd); + } + } + + stat(path: string, follow = true): ReturnType { + const parent = this.parent(path); + let fd: number | undefined; + try { + const flags = + process.platform === 'linux' + ? 0x200000 /* O_PATH */ | + (follow ? 0 : constants.O_NOFOLLOW) + : 0x8000 /* O_EVTONLY */ | + (follow ? 0 : 0x200000); /* O_SYMLINK */ + fd = this.metadataDescriptor(parent.fd, parent.name, flags); + return fstatSync(fd); + } finally { + if (fd !== undefined) closeSync(fd); + closeSync(parent.fd); + } + } + + private metadataDescriptor( + parent: number, + name: string, + flags: number, + ): number { + if (process.platform === 'darwin') { + // O_EVTONLY still requests read permission on a directory. Search-only + // descriptors preserve known-path metadata/cwd access without enumeration. + const directory = openAt!( + parent, + name, + DIRECTORY_ACCESS | + (flags & 0x200000 /* O_SYMLINK */ + ? constants.O_NOFOLLOW + : 0), + 0, + ); + if (directory >= 0) return directory; + const error = nativeError(); + if (error.code !== 'ENOTDIR' && error.code !== 'ELOOP') throw error; + } + return checked(openAt!(parent, name, flags, 0)); + } + + realpath(path: string): string { + const parent = this.parent(path); + let fd: number | undefined; + try { + fd = this.metadataDescriptor( + parent.fd, + parent.name, + process.platform === 'linux' + ? 0x200000 /* O_PATH */ + : 0x8000 /* O_EVTONLY */, + ); + return this.canonical(fd); + } finally { + if (fd !== undefined) closeSync(fd); + closeSync(parent.fd); + } + } + + install(from: string, to: string, link: boolean): void { + const source = this.parent(from); + let target: ReturnType | undefined; + try { + target = this.parent(to); + const result = link + ? linkAt!(source.fd, source.name, target.fd, target.name, 0) + : renameAt!(source.fd, source.name, target.fd, target.name); + if (result !== 0) throw nativeError(); + } finally { + closeSync(source.fd); + if (target) closeSync(target.fd); + } + } + + unlink(path: string): void { + const parent = this.parent(path); + try { + if (unlinkAt!(parent.fd, parent.name, 0) !== 0) throw nativeError(); + } finally { + closeSync(parent.fd); + } + } +} + +const context = new AsyncLocalStorage(); +export async function withWorkspaceRoot( + root: string, + identity: WorkspaceRootIdentity | undefined, + action: () => Promise, +): Promise { + if (!identity) return action(); + const access = await WorkspaceRootAccess.open(root, identity); + try { + return await context.run(access, action); + } finally { + await access.close(); + } +} + +type SpawnCommand = ( + command: string, + args: string[], + options: SpawnOptionsWithoutStdio, +) => ChildProcessWithoutNullStreams; +export function spawnWithinWorkspace( + spawner: SpawnCommand, + command: string, + args: string[], + options: SpawnOptionsWithoutStdio, +): ChildProcessWithoutNullStreams { + const access = context.getStore(); + if (!access) return spawner(command, args, options); + if (typeof options.cwd !== 'string') + throw new Error('Workspace process requires a working directory'); + const fd = access.directory(options.cwd); + try { + const env = { ...(options.env ?? process.env) }; + delete env.NODE_OPTIONS; + delete env.NODE_PATH; + return spawner( + process.execPath, + [ + fileURLToPath(new URL('./root-exec.js', import.meta.url)), + command, + ...args, + ], + { + ...options, + cwd: '/', + env, + // The trusted bootstrap consumes fd 3 before exec. Commands receive no + // root descriptor, bridge socket or new long-lived supervising process. + stdio: [ + ...(Array.isArray(options.stdio) + ? options.stdio.slice(0, 3) + : ['pipe', 'pipe', 'pipe']), + fd, + ], + } as SpawnOptionsWithoutStdio, + ); + } finally { + closeSync(fd); + } +} + +export const spawn = (( + command: string, + args: string[], + options: SpawnOptionsWithoutStdio, +) => + spawnWithinWorkspace( + spawnProcess, + command, + args, + options, + )) as typeof spawnProcess; + +// Only workspace filesystem consumers import these adapters. Unselected legacy +// roots retain their existing behavior; concurrent selected roots never share fd state. +export const open = async ( + path: string, + flags: number | 'r', + mode?: number, +): Promise => + context + .getStore() + ?.openFile(path, flags === 'r' ? constants.O_RDONLY : flags, mode) ?? + fs.open(path, flags, mode); +export const stat = async (path: string) => + context.getStore()?.stat(path) ?? fs.stat(path); +export const lstat = async (path: string) => + context.getStore()?.stat(path, false) ?? fs.lstat(path); +export const realpath = async (path: string): Promise => + context.getStore()?.realpath(path) ?? fs.realpath(path); +export const rename = async (from: string, to: string): Promise => { + const access = context.getStore(); + if (access) access.install(from, to, false); + else await fs.rename(from, to); +}; +export const link = async (from: string, to: string): Promise => { + const access = context.getStore(); + if (access) access.install(from, to, true); + else await fs.link(from, to); +}; +export const unlink = async (path: string): Promise => { + const access = context.getStore(); + if (access) access.unlink(path); + else await fs.unlink(path); +}; diff --git a/packages/code/src/root-exec.ts b/packages/code/src/root-exec.ts new file mode 100644 index 00000000..4cd7132c --- /dev/null +++ b/packages/code/src/root-exec.ts @@ -0,0 +1,18 @@ +import koffi from 'koffi'; + +// Private exec trampoline. The parent supplies an already validated directory +// on fd 3. fchdir is process-local here and never changes the bridge's cwd. +const lib = koffi.load(null); +const fchdir = lib.func('int fchdir(int fd)'); +const close = lib.func('int close(int fd)'); +const execvp = lib.func('int execvp(const char *file, const char **argv)'); +const fcntl = lib.func('int fcntl(int fd, int command, ...)'); +const args = process.argv.slice(2); +if (args.length === 0 || fchdir(3) !== 0 || close(3) !== 0) process.exit(125); +// Node marks its standard streams close-on-exec during startup. Preserve only +// the three conventional streams; every internal descriptor stays closed. +for (const fd of [0, 1, 2]) { + if (fcntl(fd, 2 /* F_SETFD */, 'int', 0) !== 0) process.exit(125); +} +execvp(args[0], [...args, null]); +process.exit(126); diff --git a/packages/code/src/root-identity.ts b/packages/code/src/root-identity.ts new file mode 100644 index 00000000..3ddfe453 --- /dev/null +++ b/packages/code/src/root-identity.ts @@ -0,0 +1,27 @@ +import { lstat, realpath } from 'node:fs/promises'; + +export interface WorkspaceRootIdentity { + path: string; + dev: string; + ino: string; +} + +/** Revalidation of a trusted snapshot, never a fresh grant to a replacement. */ +export async function matchesWorkspaceRoot( + root: string, + identity: WorkspaceRootIdentity +): Promise { + if (root !== identity.path) return false; + try { + const current = await lstat(root, { bigint: true }); + return ( + current.isDirectory() && + !current.isSymbolicLink() && + current.dev.toString() === identity.dev && + current.ino.toString() === identity.ino && + (await realpath(root)) === identity.path + ); + } catch { + return false; + } +} diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 87b444e9..65757d39 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,10 +1,11 @@ -import { spawn } from 'node:child_process'; import { createHash, randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { link, lstat, open, realpath, rename, stat, unlink } from 'node:fs/promises'; +import { link, lstat, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; +import { matchesWorkspaceRoot } from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; import { BRIDGE_PROTOCOL_VERSION, @@ -67,6 +68,7 @@ export type { }; export interface LocalWorkspaceConfig { + identity?: WorkspaceRootIdentity; id: string; name?: string; root: string; @@ -336,6 +338,7 @@ async function readConfinedFile( } interface WorkspaceRoot { + identity?: WorkspaceRootIdentity; root: string; writable: boolean; } @@ -1435,8 +1438,8 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { async instructionDescriptors() { if (!this.repositoryInstructions) return undefined; - const entries = await Promise.all([...this.roots].map(async ([id, { root }]) => { - const snapshot = await readRepositoryInstructions(root); + const entries = await Promise.all([...this.roots].map(async ([id, { root, identity }]) => { + const snapshot = await withWorkspaceRoot(root, identity, () => readRepositoryInstructions(root)); return [id, snapshot ? [snapshot.descriptor] : []] as const; })); return new Map(entries); @@ -1508,6 +1511,8 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { let canonicalRoot: string; try { canonicalRoot = await realpath(workspace.root); + if (workspace.identity && !await matchesWorkspaceRoot(canonicalRoot, workspace.identity)) throw new Error(); + await withWorkspaceRoot(canonicalRoot, workspace.identity, async () => undefined); if (!(await stat(canonicalRoot)).isDirectory()) throw new Error(); } catch { throw new WorkspaceToolError( @@ -1516,6 +1521,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ); } roots.set(workspace.id, { + identity: workspace.identity, root: canonicalRoot, writable: workspace.writable === true, }); @@ -1536,6 +1542,20 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { async execute( request: WorkspaceToolRequest, signal?: AbortSignal, + ): Promise { + const workspace = this.roots.get(request?.workspaceId); + if (!workspace?.identity) return this.executeBound(request, signal); + try { + return await withWorkspaceRoot(workspace.root, workspace.identity, () => this.executeBound(request, signal)); + } catch (error) { + if (error instanceof WorkspaceRootAccessError) throw new WorkspaceToolError(error.message, 'REGISTRATION_INVALID'); + throw error; + } + } + + private async executeBound( + request: WorkspaceToolRequest, + signal?: AbortSignal, ): Promise { if (signal?.aborted) { throw new WorkspaceToolError( From 672e19522900d1b78b4b32bd2e60587d8d3b1080 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 16 Sep 2026 11:51:18 -0400 Subject: [PATCH 35/42] fix: Report workspace admission capacity without ambiguous timeout errors (#227) * fix: Distinguish workspace admission capacity from execution expiry * test: Preserve execution uncertainty while classifying blocked follow-ups * fix: Classify admission expiry at the enqueue boundary --- .github/workflows/ci.yml | 7 + packages/code/README.md | 13 ++ service/src/bridge/concurrent-worker.test.ts | 2 +- service/src/bridge/fleet.test.ts | 132 ++++++++++++++++++ service/src/bridge/store.ts | 22 ++- service/src/bridge/worker-admission.test.ts | 24 +++- .../src/sandbox-backend/remote-bridge.test.ts | 1 + service/src/sandbox-backend/remote-bridge.ts | 1 + service/src/workspace-tools/router.test.ts | 2 + service/src/workspace-tools/router.ts | 2 + 10 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 service/src/bridge/fleet.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72f32178..7259ec87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,6 +145,13 @@ jobs: - name: Install Redis for ledger integration tests run: sudo apt-get update && sudo apt-get install -y redis-server + - name: Verify fleet admission with isolated Redis + run: | + redis_socket="$RUNNER_TEMP/byom-admission.sock" + redis-server --port 0 --unixsocket "$redis_socket" --save '' --appendonly no --daemonize yes + trap 'redis-cli -s "$redis_socket" shutdown nosave' EXIT + BRIDGE_TEST_REDIS_URL="$redis_socket" bun test src/bridge/fleet.test.ts + - name: Build service run: bun run build diff --git a/packages/code/README.md b/packages/code/README.md index 720a0b35..08dc54a7 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -692,6 +692,19 @@ librechat-code run \ --allow-workspace-commands ``` +Slots are per machine, not a fleet-wide execution limit. A busy machine does not +consume another machine's slots. Requests for the same root remain serialized, +including commands started through background tools. Independent checkouts can +use different slots; selecting subdirectories beneath one registered parent root +does not create separate scheduling boundaries. Linked Git worktrees share Git +metadata and are not supported by selected-project registration. + +Admission waits at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP +503, `Retry-After: 1`) means the operation was not assigned or started; wait for +capacity before submitting it again. This is distinct from `ASSIGNMENT_EXPIRED` +or a transport timeout after dispatch, where execution may have occurred and +mutations must not be blindly retried. No automatic retry is added by this policy. + Keep the existing URL, pairing/identity, and network policy configuration. The primary root keeps its configured workspace ID (default `primary`). Repeat `--workspace id=path` to add named roots, up to the protocol's 32-root limit. diff --git a/service/src/bridge/concurrent-worker.test.ts b/service/src/bridge/concurrent-worker.test.ts index ea5ac601..c107576a 100644 --- a/service/src/bridge/concurrent-worker.test.ts +++ b/service/src/bridge/concurrent-worker.test.ts @@ -281,7 +281,7 @@ for (const failure of [ ).rejects.toMatchObject({ code: failure === 'delivery-outage' - ? 'ASSIGNMENT_EXPIRED' + ? 'WORKSPACE_QUEUE_TIMEOUT' : 'WORKSPACE_QUARANTINED', }); await expect( diff --git a/service/src/bridge/fleet.test.ts b/service/src/bridge/fleet.test.ts new file mode 100644 index 00000000..3eac5088 --- /dev/null +++ b/service/src/bridge/fleet.test.ts @@ -0,0 +1,132 @@ +import { expect, test } from 'bun:test'; +import Redis from 'ioredis'; +import { randomUUID } from 'node:crypto'; +import { RedisBridgeStore } from './store'; +import type { CodeBridgeAssignment } from './store'; + +/** Opt-in integration check against a disposable Redis, never a deployment database. */ +test.skipIf(!process.env.BRIDGE_TEST_REDIS_URL)( + 'admission saturation is isolated across machines and independent roots', + async () => { + const redis = new Redis(process.env.BRIDGE_TEST_REDIS_URL!); + const store = new RedisBridgeStore(redis, 60, 1000, 2); + const prefix = `fleet-${randomUUID()}`; + const machines = [`${prefix}-a`, `${prefix}-b`]; + const incarnationId = 'fleet-incarnation'; + const pending: Promise[] = []; + const controller = new AbortController(); + const dispatch = ( + workerId: string, + workspaceId: string, + budgetMs = 3000, + ) => { + const result = store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + budgetMs, + executionTimeoutMs: 5000, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId, + path: 'probe', + }, + }); + void result.catch(() => undefined); + pending.push(result); + return result; + }; + const settle = async (assignment: CodeBridgeAssignment) => { + await store.acknowledgeLease( + assignment.workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + await store.settle(assignment.workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'probe complete', + }); + }; + try { + for (const workerId of machines) { + const generation = await store.register({ + protocolVersion: 1, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 2, + requiresReadyConfirmation: true, + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'a' }, { id: 'b' }], + }, + }, + }); + await store.confirmReady(workerId, incarnationId, generation); + } + const busy = dispatch(machines[0], 'a'); + const held = await store.lease( + machines[0], + incarnationId, + 1000, + undefined, + undefined, + 0, + ); + expect(held).toBeDefined(); + const blocked = dispatch(machines[0], 'a', 300); + const independent = dispatch(machines[0], 'b'); + const otherMachine = dispatch(machines[1], 'a'); + const root = await store.lease( + machines[0], + incarnationId, + 1000, + undefined, + undefined, + 1, + ); + const remote = await store.lease( + machines[1], + incarnationId, + 1000, + undefined, + undefined, + 0, + ); + expect(root?.request).toMatchObject({ workspaceId: 'b' }); + expect(remote?.workerId).toBe(machines[1]); + await settle(root!); + await settle(remote!); + await Promise.all([independent, otherMachine]); + await expect(blocked).rejects.toMatchObject({ + code: 'WORKSPACE_QUEUE_TIMEOUT', + }); + await settle(held!); + await busy; + expect( + await store.lease( + machines[0], + incarnationId, + 20, + undefined, + undefined, + 0, + ), + ).toBeUndefined(); + } finally { + controller.abort(); + await Promise.allSettled(pending); + await redis.quit(); + } + }, +); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index d6b91469..4eaabd7f 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -47,6 +47,7 @@ export class BridgeStoreError extends Error { | 'WORKER_UNAUTHORIZED' | 'WORKER_BUSY' | 'WORKER_QUEUE_FULL' + | 'WORKSPACE_QUEUE_TIMEOUT' | 'ASSIGNMENT_EXPIRED' | 'ASSIGNMENT_FENCED' | 'ASSIGNMENT_NOT_FOUND' @@ -838,6 +839,7 @@ export class RedisBridgeStore { ); const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; + let enqueueAttempted = false; let workspaceLeaseSlot: number | undefined; const selectedWorkspaceId = args.workspaceRequest?.workspaceId ?? args.workspaceId; @@ -1024,12 +1026,14 @@ export class RedisBridgeStore { this.assertDispatchActive(args.signal, args.deadlineAtMs); assignment.incarnationId = registration.incarnationId; queued = await this.dispatchCommand( - () => - this.enqueueForActiveIncarnation( + () => { + enqueueAttempted = true; + return this.enqueueForActiveIncarnation( assignment!, ttlSeconds, readyToken, - ), + ); + }, args, 'Bridge assignment enqueue', ); @@ -1130,6 +1134,18 @@ export class RedisBridgeStore { } throw error; } + } catch (error) { + // Once enqueue starts, even a lost Redis response may hide execution. + if ( + admission != null && !enqueueAttempted && !args.signal.aborted && + error instanceof BridgeStoreError && error.code === 'ASSIGNMENT_EXPIRED' + ) { + throw new BridgeStoreError( + 'WORKSPACE_QUEUE_TIMEOUT', + 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.', + ); + } + throw error; } finally { if (admission != null) { // Expiry remains the fallback if Redis is unavailable during cancellation. diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts index db2e5956..655748b1 100644 --- a/service/src/bridge/worker-admission.test.ts +++ b/service/src/bridge/worker-admission.test.ts @@ -1,4 +1,4 @@ -import { afterEach, expect, test } from 'bun:test'; +import { afterEach, expect, spyOn, test } from 'bun:test'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; @@ -114,7 +114,7 @@ test('an expired queued call never reaches the worker and does not strand later const assignment = await store.lease(workerId, incarnationId, 1000); await expect( dispatch('expired', new AbortController(), 25, 1000), - ).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); + ).rejects.toMatchObject({ code: 'WORKSPACE_QUEUE_TIMEOUT' }); const third = dispatch('third'); await settle(assignment); await first; @@ -170,3 +170,23 @@ test('execution expires independently of an unused queue allowance', async () => expect(Date.parse(assignment!.expiresAt) - Date.now()).toBeLessThanOrEqual(150); await expect(completion).rejects.toMatchObject({ code: 'ASSIGNMENT_EXPIRED' }); }); + +test('expiry after generation allocation but before enqueue is definitely not started', async () => { + await register(); + const now = Date.now; + const incr = redis.incr.bind(redis); + let expired = false; + const clock = spyOn(Date, 'now').mockImplementation(() => now() + (expired ? 10_000 : 0)); + const generation = spyOn(redis, 'incr').mockImplementation(async (key) => { + const value = await incr(key); + expired = true; + return value; + }); + try { + await expect(dispatch('not-enqueued')).rejects.toMatchObject({ code: 'WORKSPACE_QUEUE_TIMEOUT' }); + } finally { + clock.mockRestore(); + generation.mockRestore(); + } + expect(await store.lease(workerId, incarnationId, 20)).toBeUndefined(); +}); diff --git a/service/src/sandbox-backend/remote-bridge.test.ts b/service/src/sandbox-backend/remote-bridge.test.ts index 697271ee..4f50898e 100644 --- a/service/src/sandbox-backend/remote-bridge.test.ts +++ b/service/src/sandbox-backend/remote-bridge.test.ts @@ -108,6 +108,7 @@ describe('RemoteBridgeSandboxBackend', () => { WORKER_UNAUTHORIZED: ['BRIDGE_WORKER_UNAUTHORIZED', false, 403, 'Code environment is not authorized for this tenant'], WORKER_BUSY: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], WORKER_QUEUE_FULL: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], + WORKSPACE_QUEUE_TIMEOUT: ['BRIDGE_WORKER_BUSY', false, 409, 'Code environment is busy'], ASSIGNMENT_EXPIRED: ['BRIDGE_DEADLINE_EXCEEDED', false, 504, 'Code environment execution timed out'], ASSIGNMENT_FENCED: ['BRIDGE_ASSIGNMENT_FENCED', false, 409, 'Code environment assignment is fenced; inspect the execution before retrying'], ASSIGNMENT_NOT_FOUND: ['BRIDGE_ASSIGNMENT_NOT_FOUND', false, 409, 'Code environment assignment is no longer available; inspect the execution before retrying'], diff --git a/service/src/sandbox-backend/remote-bridge.ts b/service/src/sandbox-backend/remote-bridge.ts index 6e06eda8..2cdbd767 100644 --- a/service/src/sandbox-backend/remote-bridge.ts +++ b/service/src/sandbox-backend/remote-bridge.ts @@ -19,6 +19,7 @@ const bridgeErrorCodes = { WORKER_UNAUTHORIZED: 'BRIDGE_WORKER_UNAUTHORIZED', WORKER_BUSY: 'BRIDGE_WORKER_BUSY', WORKER_QUEUE_FULL: 'BRIDGE_WORKER_BUSY', + WORKSPACE_QUEUE_TIMEOUT: 'BRIDGE_WORKER_BUSY', ASSIGNMENT_EXPIRED: 'BRIDGE_DEADLINE_EXCEEDED', ASSIGNMENT_FENCED: 'BRIDGE_ASSIGNMENT_FENCED', ASSIGNMENT_NOT_FOUND: 'BRIDGE_ASSIGNMENT_NOT_FOUND', diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index ae291a4b..04738b8a 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -320,6 +320,7 @@ test.each([ ['ASSIGNMENT_EXPIRED', 504], ['WORKER_OFFLINE', 503], ['WORKER_BUSY', 503], + ['WORKSPACE_QUEUE_TIMEOUT', 503], ['WORKER_MISMATCH', 409], ] as const)('logs store rejection %s with actual HTTP %i', async (errorCode, expectedStatus) => { const app = express(); @@ -360,6 +361,7 @@ test.each([ }), }); expect(response.status).toBe(expectedStatus); + expect(response.headers.get('retry-after')).toBe(errorCode === 'WORKSPACE_QUEUE_TIMEOUT' ? '1' : null); await response.text(); expect(logSpy).toHaveBeenCalledTimes(1); expect(logSpy).toHaveBeenCalledWith( diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 17085963..eb89370e 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -36,6 +36,7 @@ function asyncRoute(handler: (req: AuthenticatedRequest, res: Response) => Promi } export function bridgeStoreStatus(error: BridgeStoreError): number { + if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') return 503; if (error.code === 'WORKER_QUEUE_FULL') return 429; if (error.code === 'WORKER_UNAUTHORIZED') return 403; if (error.code === 'ASSIGNMENT_INVALID') return 400; @@ -172,6 +173,7 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) } catch (error) { if (error instanceof BridgeStoreError) { outcome.errorCode = error.code; + if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') res.setHeader('Retry-After', '1'); res.status(bridgeStoreStatus(error)).json({ error: error.message, code: error.code, From fd9a4fa65e0a5189957032c0046eb286311fda62 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:33:43 -0400 Subject: [PATCH 36/42] ci: Fix Release Version Resolution for Untagged and Resumed Runs (#233) * ci: Fix Release Version Resolution for Untagged and Resumed Runs The release workflow resolved its version in one inline shell block under `set -euo pipefail`, where two paths could not succeed. Filtering tags through `grep` made a no-match fatal. On the ordinary untagged tip of `main`, `git tag --points-at HEAD | grep -E '^v[0-9]+...'` exits 1, and the step died before reaching its skip handling or `next-release-version.sh`, so a deployable commit could not obtain a release version (#228). Selecting stable tags now reads exit 1 as an empty answer while exit 2 and above still fail the release, which also lets the missing-previous-tag case report its own error. The rerun-resume path then rejected the tag it had itself chosen. With a stable tag already pointing at `HEAD` and no release published, the version comes from that tag, and the following existence check failed merely because the ref existed (#229). It now compares the tag's commit against the release commit, so only a tag on some other commit is a collision; `Create tag` already tolerates a tag that exists. The block moved into `.github/scripts/resolve-release-version.sh`, beside the `next-release-version.sh` it calls, so `tests/release-version-resolution.sh` can cover every path: automatic, resumed, skipped, dispatched, pushed-tag, and the runs that must be refused, each against a throwaway repository with a stubbed `gh`. * fix: harden release resolver execution --------- Co-authored-by: Lia Co-authored-by: Danny Avila --- .github/scripts/resolve-release-version.sh | 195 +++++++++++++++ .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 153 ++---------- tests/release-version-resolution.sh | 265 +++++++++++++++++++++ 4 files changed, 483 insertions(+), 133 deletions(-) create mode 100755 .github/scripts/resolve-release-version.sh create mode 100755 tests/release-version-resolution.sh diff --git a/.github/scripts/resolve-release-version.sh b/.github/scripts/resolve-release-version.sh new file mode 100755 index 00000000..ef5f8667 --- /dev/null +++ b/.github/scripts/resolve-release-version.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash + +# Resolves which version a release run publishes, or decides that it publishes +# nothing, and records the decision in $GITHUB_OUTPUT. Extracted from +# .github/workflows/release.yml so the four entry paths — an automatic release +# after CI, a rerun resuming a release whose tag was already cut, a dispatch, +# and a pushed tag — are covered by tests/release-version-resolution.sh. +# +# Inputs arrive as environment variables, mirroring the workflow's env block: +# +# EVENT_NAME github.event_name +# HEAD_SHA github.event.workflow_run.head_sha +# INPUT_VERSION github.event.inputs.version +# INPUT_DRAFT github.event.inputs.draft +# REF_NAME github.ref_name +# REF_TYPE github.ref_type +# GH_TOKEN a token `gh release view` can read releases with +# +# Outputs: skip, and for a real release version, app_version, chart_version, +# prerelease, latest, draft. Run from the repository root; the chart fields are +# read from helm/codeapi/Chart.yaml relative to it. + +set -euo pipefail + +EVENT_NAME="${EVENT_NAME:-}" +HEAD_SHA="${HEAD_SHA:-}" +INPUT_VERSION="${INPUT_VERSION:-}" +INPUT_DRAFT="${INPUT_DRAFT:-}" +REF_NAME="${REF_NAME:-}" +REF_TYPE="${REF_TYPE:-}" +GITHUB_OUTPUT="${GITHUB_OUTPUT:?GITHUB_OUTPUT must name the step output file}" + +STABLE_TAG_PATTERN='^v[0-9]+[.][0-9]+[.][0-9]+$' + +# `grep` exits 1 when nothing matches, and under `pipefail` that would abort the +# step. A commit with no stable tag is the ordinary state of `main`, so a +# no-match reads as an empty answer while a genuine grep failure — exit 2 and +# above — still fails the release. +select_stable_tags() { + local status=0 + grep -E "$STABLE_TAG_PATTERN" || status=$? + [ "$status" -le 1 ] +} + +# Highest stable tag among those `git tag` selects, empty when there are none. +newest_stable_tag() { + git tag "$@" | select_stable_tags | sort -V | tail -n 1 +} + +# The commit a tag resolves to. The caller first verifies that the ref exists, +# so a failure here means the tag ultimately names a non-commit object. +tag_commit() { + git rev-parse -q --verify "refs/tags/$1^{commit}" +} + +SKIP=false +VERSION="" +HEAD_COMMIT="" + +if [ "$EVENT_NAME" = "workflow_run" ]; then + HEAD_COMMIT="$(git rev-parse HEAD)" + if [ "$HEAD_COMMIT" != "$HEAD_SHA" ]; then + echo "::error::Checked out SHA does not match the successful CI run" + exit 1 + fi + + REMOTE_MAIN_SHA="$(git ls-remote origin refs/heads/main | awk '{print $1}')" + if [ -z "$REMOTE_MAIN_SHA" ]; then + echo "::error::Could not resolve the current main branch tip" + exit 1 + fi + if [ "$REMOTE_MAIN_SHA" != "$HEAD_SHA" ]; then + echo "main advanced after this CI run; the newer successful run will release the combined changes" + SKIP=true + fi + + # A rerun after tag creation but before release publication resumes the + # missing release rather than incrementing the version again. + EXACT_TAG="$(newest_stable_tag --points-at HEAD)" + if [ "$SKIP" = "false" ] && [ -n "$EXACT_TAG" ]; then + if gh release view "$EXACT_TAG" >/dev/null 2>&1; then + echo "$EXACT_TAG already publishes this commit; nothing to do" + SKIP=true + else + VERSION="$EXACT_TAG" + fi + elif [ "$SKIP" = "false" ]; then + PREVIOUS_TAG="$(newest_stable_tag --merged HEAD)" + if [ -z "$PREVIOUS_TAG" ]; then + echo "::error::Automatic releases require an existing stable vMAJOR.MINOR.PATCH tag" + exit 1 + fi + VERSION="$(.github/scripts/next-release-version.sh "$PREVIOUS_TAG" "$PREVIOUS_TAG..HEAD")" + if [ -z "$VERSION" ]; then + echo "Only documentation, workflow, or test files changed since $PREVIOUS_TAG; no release needed" + SKIP=true + fi + fi +elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then + # Releases describe what shipped to main. Dispatching from a topic branch + # would tag a commit that is not on the release line. + if [ "$REF_TYPE" != "branch" ] || [ "$REF_NAME" != "main" ]; then + echo "::error::Releases must be cut from main; this run is on '$REF_NAME'" + exit 1 + fi + VERSION="$INPUT_VERSION" +else + VERSION="$REF_NAME" +fi + +if [ "$SKIP" = "true" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 +fi + +# A bare "2.0.0" typed into the dispatch box is accepted; everything downstream +# works with the v-prefixed form the tag actually uses. +case "$VERSION" in + v*) ;; + *) VERSION="v$VERSION" ;; +esac + +if [[ ! "$VERSION" =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+)?$ ]]; then + echo "::error::Release tags must be v.. or v..-rcN, for example v1.0.0 or v1.1.0-rc1 (got '$VERSION')" + exit 1 +fi + +if [ "$EVENT_NAME" = "workflow_run" ]; then + # A tag already pointing at this commit is the resumed release above, and the + # publish steps tolerate it. Only a tag on some other commit is a collision. + if git show-ref --verify --quiet "refs/tags/$VERSION"; then + if ! EXISTING_TAG_COMMIT="$(tag_commit "$VERSION")"; then + echo "::error::Calculated tag $VERSION already exists but does not point to a commit" + exit 1 + fi + if [ "$EXISTING_TAG_COMMIT" != "$HEAD_COMMIT" ]; then + echo "::error::Calculated tag $VERSION already exists on a different commit" + exit 1 + fi + fi +fi + +read_chart_field() { + grep -m1 "^$1:" helm/codeapi/Chart.yaml \ + | sed -E "s/^$1:[[:space:]]*//; s/[[:space:]]*#.*//; s/^[\"']//; s/[\"']\$//" +} +APP_VERSION="$(read_chart_field appVersion)" +CHART_VERSION="$(read_chart_field version)" + +if [ "$EVENT_NAME" = "workflow_dispatch" ] \ + && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + echo "::error::Tag $VERSION already exists. Pick a new version, or delete the tag if it was cut in error." + exit 1 +fi + +case "$VERSION" in + *-rc*) PRERELEASE=true ;; + *) PRERELEASE=false ;; +esac + +# `latest` moves only when this is the highest stable version, so re-cutting an +# older patch cannot drag it backwards. The tag under dispatch does not exist +# yet, hence adding it to the comparison. +LATEST=false +if [ "$PRERELEASE" = "false" ]; then + HIGHEST_STABLE="$( + { + git tag --list 'v[0-9]*' + printf '%s\n' "$VERSION" + } \ + | select_stable_tags \ + | sort -V \ + | tail -n 1 + )" + if [ "$HIGHEST_STABLE" = "$VERSION" ]; then + LATEST=true + fi +fi + +DRAFT=false +if [ "$INPUT_DRAFT" = "true" ]; then + DRAFT=true +fi + +{ + echo "skip=false" + echo "version=$VERSION" + echo "app_version=$APP_VERSION" + echo "chart_version=$CHART_VERSION" + echo "prerelease=$PRERELEASE" + echo "latest=$LATEST" + echo "draft=$DRAFT" +} >> "$GITHUB_OUTPUT" + +echo "Releasing $VERSION (chart $CHART_VERSION, appVersion $APP_VERSION, prerelease=$PRERELEASE, latest=$LATEST, draft=$DRAFT)" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7259ec87..b645f4b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,9 @@ jobs: - name: Release versioning run: tests/release-versioning.sh + - name: Release version resolution + run: tests/release-version-resolution.sh + - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ea8a5364..684f1654 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,6 +61,20 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: + # `workflow_run` can be rerun for a commit older than this resolver. Save + # the helper from the revision that supplied this workflow before the + # release checkout replaces the working tree with that historical SHA. + - name: Checkout release workflow + if: github.event_name == 'workflow_run' + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 1 + ref: ${{ github.workflow_sha }} + + - name: Preserve release resolver + if: github.event_name == 'workflow_run' + run: install -m 755 .github/scripts/resolve-release-version.sh "$RUNNER_TEMP/resolve-release-version.sh" + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: # Full history and tags: resolving whether this release is the newest @@ -78,139 +92,12 @@ jobs: REF_NAME: ${{ github.ref_name }} REF_TYPE: ${{ github.ref_type }} GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - SKIP=false - if [ "$EVENT_NAME" = "workflow_run" ]; then - if [ "$(git rev-parse HEAD)" != "$HEAD_SHA" ]; then - echo "::error::Checked out SHA does not match the successful CI run" - exit 1 - fi - - REMOTE_MAIN_SHA="$(git ls-remote origin refs/heads/main | awk '{print $1}')" - if [ -z "$REMOTE_MAIN_SHA" ]; then - echo "::error::Could not resolve the current main branch tip" - exit 1 - fi - if [ "$REMOTE_MAIN_SHA" != "$HEAD_SHA" ]; then - echo "main advanced after this CI run; the newer successful run will release the combined changes" - SKIP=true - fi - - # A rerun after tag creation but before release publication resumes - # the missing release rather than incrementing the version again. - EXACT_TAG="$({ git tag --points-at HEAD || true; } | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' | sort -V | tail -n 1)" - if [ "$SKIP" = "false" ] && [ -n "$EXACT_TAG" ]; then - if gh release view "$EXACT_TAG" >/dev/null 2>&1; then - echo "$EXACT_TAG already publishes this commit; nothing to do" - SKIP=true - else - VERSION="$EXACT_TAG" - fi - elif [ "$SKIP" = "false" ]; then - PREVIOUS_TAG="$(git tag --merged HEAD \ - | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' \ - | sort -V \ - | tail -n 1)" - if [ -z "$PREVIOUS_TAG" ]; then - echo "::error::Automatic releases require an existing stable vMAJOR.MINOR.PATCH tag" - exit 1 - fi - VERSION="$(.github/scripts/next-release-version.sh "$PREVIOUS_TAG" "$PREVIOUS_TAG..HEAD")" - if [ -z "$VERSION" ]; then - echo "Only documentation, workflow, or test files changed since $PREVIOUS_TAG; no release needed" - SKIP=true - fi - fi - elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then - # Releases describe what shipped to main. Dispatching from a topic - # branch would tag a commit that is not on the release line. - if [ "$REF_TYPE" != "branch" ] || [ "$REF_NAME" != "main" ]; then - echo "::error::Releases must be cut from main; this run is on '$REF_NAME'" - exit 1 - fi - VERSION="$INPUT_VERSION" - else - VERSION="$REF_NAME" - fi - - if [ "$SKIP" = "true" ]; then - echo "skip=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # A bare "2.0.0" typed into the dispatch box is accepted; everything - # downstream works with the v-prefixed form the tag actually uses. - case "$VERSION" in - v*) ;; - *) VERSION="v$VERSION" ;; - esac - - if [[ ! "$VERSION" =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+)?$ ]]; then - echo "::error::Release tags must be v.. or v..-rcN, for example v1.0.0 or v1.1.0-rc1 (got '$VERSION')" - exit 1 - fi - - if [ "$EVENT_NAME" = "workflow_run" ] \ - && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then - echo "::error::Calculated tag $VERSION already exists on a different commit" - exit 1 - fi - - read_chart_field() { - grep -m1 "^$1:" helm/codeapi/Chart.yaml \ - | sed -E "s/^$1:[[:space:]]*//; s/[[:space:]]*#.*//; s/^[\"']//; s/[\"']\$//" - } - APP_VERSION="$(read_chart_field appVersion)" - CHART_VERSION="$(read_chart_field version)" - - if [ "$EVENT_NAME" = "workflow_dispatch" ] \ - && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then - echo "::error::Tag $VERSION already exists. Pick a new version, or delete the tag if it was cut in error." - exit 1 - fi - - case "$VERSION" in - *-rc*) PRERELEASE=true ;; - *) PRERELEASE=false ;; - esac - - # `latest` moves only when this is the highest stable version, so - # re-cutting an older patch cannot drag it backwards. The tag under - # dispatch does not exist yet, hence adding it to the comparison. - LATEST=false - if [ "$PRERELEASE" = "false" ]; then - HIGHEST_STABLE="$( - { - git tag --list 'v[0-9]*' - printf '%s\n' "$VERSION" - } \ - | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' \ - | sort -V \ - | tail -n 1 - )" - if [ "$HIGHEST_STABLE" = "$VERSION" ]; then - LATEST=true - fi - fi - - DRAFT=false - if [ "$INPUT_DRAFT" = "true" ]; then - DRAFT=true - fi - - { - echo "skip=false" - echo "version=$VERSION" - echo "app_version=$APP_VERSION" - echo "chart_version=$CHART_VERSION" - echo "prerelease=$PRERELEASE" - echo "latest=$LATEST" - echo "draft=$DRAFT" - } >> "$GITHUB_OUTPUT" - - echo "Releasing $VERSION (chart $CHART_VERSION, appVersion $APP_VERSION, prerelease=$PRERELEASE, latest=$LATEST, draft=$DRAFT)" + RESOLVER_PATH: ${{ github.event_name == 'workflow_run' && format('{0}/resolve-release-version.sh', runner.temp) || '.github/scripts/resolve-release-version.sh' }} + # The resolution itself lives in a script so that every path through it + # — automatic release, resumed release, dispatch, pushed tag, and the + # runs that must skip or fail — is covered by + # tests/release-version-resolution.sh in CI. + run: "$RESOLVER_PATH" # helm is preinstalled on ubuntu-latest, the same way the chart tests in # ci.yml depend on it. diff --git a/tests/release-version-resolution.sh b/tests/release-version-resolution.sh new file mode 100755 index 00000000..c70d50d3 --- /dev/null +++ b/tests/release-version-resolution.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash + +# Covers .github/scripts/resolve-release-version.sh: the version a release run +# publishes, and the runs that have to skip or fail instead. Every case builds a +# throwaway repository with an `origin` the resolver can query and a stubbed +# `gh`, so nothing here reaches the network or the real repository. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# The resolver is deliberately kept outside each throwaway checkout. That +# mirrors release.yml preserving the workflow revision in RUNNER_TEMP before a +# workflow_run checks out the possibly historical release commit. +RESOLVER="$ROOT/.github/scripts/resolve-release-version.sh" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +REPO="$WORK/repo" +ORIGIN="$WORK/origin.git" +OUTPUT="$WORK/github_output" +LOG="$WORK/log" +CASE='' +STATUS=0 +FAILURES=0 + +# `gh release view` is the only gh call the resolver makes. PUBLISHED lists the +# releases that already exist; anything else must not be invoked at all. +mkdir -p "$WORK/bin" +cat > "$WORK/bin/gh" <<'STUB' +#!/usr/bin/env bash +if [ "$1" = 'release' ] && [ "$2" = 'view' ]; then + for published in ${PUBLISHED:-}; do + if [ "$published" = "$3" ]; then + exit 0 + fi + done + exit 1 +fi +echo "unexpected gh invocation: $*" >&2 +exit 2 +STUB +chmod +x "$WORK/bin/gh" +PATH="$WORK/bin:$PATH" + +git_repo() { + git -C "$REPO" "$@" +} + +# A fresh repository whose layout matches what the resolver reads from the +# checkout: the version bump script it shells out to, and the chart it takes the +# app and chart versions from, trailing comments and quotes included. +new_case() { + CASE="$1" + rm -rf "$REPO" "$ORIGIN" + git init -q --bare "$ORIGIN" + git init -q -b main "$REPO" + git_repo config user.name test + git_repo config user.email test@example.com + git_repo remote add origin "$ORIGIN" + mkdir -p "$REPO/.github/scripts" "$REPO/helm/codeapi" + cp "$ROOT/.github/scripts/next-release-version.sh" "$REPO/.github/scripts/" + cat > "$REPO/helm/codeapi/Chart.yaml" <<'CHART' +apiVersion: v2 +name: codeapi +version: 0.3.1 # Chart version +appVersion: "2.0.0" # App version +CHART + commit api/runtime.ts initial 'chore: initial import' + publish_main +} + +commit() { + local path="$1" content="$2" message="$3" + mkdir -p "$REPO/$(dirname "$path")" + printf '%s\n' "$content" > "$REPO/$path" + git_repo add "$path" + git_repo commit -q -m "$message" +} + +publish_main() { + git_repo push -q origin main +} + +head_sha() { + git_repo rev-parse HEAD +} + +# Runs the resolver in the throwaway repository. Arguments are KEY=VALUE pairs +# standing in for the workflow's env block. +resolve() { + : > "$OUTPUT" + set +e + (cd "$REPO" && env GITHUB_OUTPUT="$OUTPUT" "$@" bash "$RESOLVER") > "$LOG" 2>&1 + STATUS=$? + set -e +} + +fail() { + echo "FAIL [$CASE] $1" >&2 + sed 's/^/ | /' "$LOG" >&2 + FAILURES=$((FAILURES + 1)) +} + +expect_status() { + if [ "$STATUS" != "$1" ]; then + fail "exit status: expected $1, got $STATUS" + fi +} + +expect_output() { + local actual + actual="$(sed -n "s/^$1=//p" "$OUTPUT" | tail -n 1)" + if [ "$actual" != "$2" ]; then + fail "output $1: expected '$2', got '$actual'" + fi +} + +expect_log() { + if ! grep -qF "$1" "$LOG"; then + fail "expected log to mention: $1" + fi +} + +# An untagged tip of main is the ordinary automatic-release path: the version +# comes from Conventional Commit intent since the last stable tag. Filtering +# tags with `grep` used to abort the step here, because no match under +# `pipefail` looks like a command failure. +new_case 'automatic release from an untagged commit' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 0 +expect_output skip false +expect_output version v1.2.4 +expect_output app_version 2.0.0 +expect_output chart_version 0.3.1 +expect_output prerelease false +expect_output latest true +expect_output draft false + +new_case 'documentation-only range releases nothing' +git_repo tag v1.2.3 +commit docs/guide.md docs 'docs: clarify deployment' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 0 +expect_output skip true +expect_output version '' +expect_log 'no release needed' + +new_case 'a repository without a stable tag reports why' +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 1 +expect_log 'Automatic releases require an existing stable' + +# The rerun-to-publish recovery path: a previous run created the tag and then +# failed before the release existed. +new_case 'a rerun resumes the tag that already points at HEAD' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +git_repo tag v1.2.4 +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" PUBLISHED='' +expect_status 0 +expect_output skip false +expect_output version v1.2.4 +expect_output latest true + +new_case 'a published tag at HEAD releases nothing twice' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +git_repo tag v1.2.4 +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" PUBLISHED='v1.2.4' +expect_status 0 +expect_output skip true +expect_log 'already publishes this commit' + +new_case 'a calculated tag held by another commit is a collision' +git_repo tag v1.2.3 +git_repo checkout -q -b elsewhere +commit api/runtime.ts diverged 'fix: unrelated work' +git_repo tag v1.2.4 +git_repo checkout -q main +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 1 +expect_log 'already exists on a different commit' + +new_case 'a calculated tag held by a non-commit object is a collision' +git_repo tag v1.2.3 +blob="$(printf 'not a commit\n' | git_repo hash-object -w --stdin)" +git_repo update-ref refs/tags/v1.2.4 "$blob" +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 1 +expect_log 'already exists but does not point to a commit' + +new_case 'a stale CI run defers to the newer tip' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +commit api/runtime.ts advanced 'fix: land more work' +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 0 +expect_output skip true +expect_log 'main advanced after this CI run' + +new_case 'a dispatched version may omit the v prefix' +git_repo tag v1.2.3 +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=2.0.0 INPUT_DRAFT=true +expect_status 0 +expect_output version v2.0.0 +expect_output prerelease false +expect_output latest true +expect_output draft true + +new_case 'a release candidate is a prerelease and never latest' +git_repo tag v1.2.3 +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=v1.3.0-rc1 +expect_status 0 +expect_output version v1.3.0-rc1 +expect_output prerelease true +expect_output latest false + +new_case 'dispatching from a topic branch is refused' +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=feature/x \ + INPUT_VERSION=v1.3.0 +expect_status 1 +expect_log 'Releases must be cut from main' + +new_case 'dispatching an existing version is refused' +git_repo tag v1.2.3 +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=v1.2.3 +expect_status 1 +expect_log 'already exists' + +new_case 'a malformed version is refused' +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=1.2 +expect_status 1 +expect_log 'Release tags must be' + +new_case 'a pushed older patch tag does not become latest' +git_repo tag v9.9.9 +resolve EVENT_NAME=push REF_TYPE=tag REF_NAME=v1.0.1 +expect_status 0 +expect_output version v1.0.1 +expect_output prerelease false +expect_output latest false + +if [ "$FAILURES" -ne 0 ]; then + echo "$FAILURES release version resolution assertion(s) failed" >&2 + exit 1 +fi + +echo 'release version resolution tests passed' From 54aca9262ecc1eca6ad3746e6f5b72169897afcf Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 09:50:44 -0400 Subject: [PATCH 37/42] feat: Route GitHub App credentials per repository (#236) * feat: Route GitHub App credentials per repository * test: Make repository routing assertion deterministic * fix: Harden repository credential routing * fix: Bound shared GitHub credential refreshes * fix: Bind GitHub credentials to admitted workspaces --- docs/remote-bridge/worker-runbook.md | 19 +- packages/code/README.md | 14 +- packages/code/src/cli.test.ts | 59 ++- packages/code/src/cli.ts | 52 ++- packages/code/src/github.test.ts | 442 +++++++++++++++++++++- packages/code/src/github.ts | 445 ++++++++++++++++++++--- packages/code/src/native-process.test.ts | 5 +- packages/code/src/native-process.ts | 12 +- packages/code/src/native-sandbox.test.ts | 45 +++ packages/code/src/native-sandbox.ts | 26 +- 10 files changed, 1037 insertions(+), 82 deletions(-) diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 7511a265..16c24742 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -323,13 +323,21 @@ Configure the worker, preferably in a separate service drop-in: ```ini [Service] Environment=LIBRECHAT_CODE_GITHUB_APP_ID=12345 -Environment=LIBRECHAT_CODE_GITHUB_INSTALLATION_ID=67890 Environment=LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/home/librechat-code/.config/librechat-code/github-app.pem ``` -The trusted worker mints short-lived installation tokens. Sandboxed commands -receive masked Git/`gh` credentials only for the configured GitHub hosts; the -token is not written to the repository, remote URL, or Git configuration. +Install the same App separately on every personal account or organization the +worker is allowed to use. The trusted worker resolves the correct installation +from the repository containing each command's working directory, then mints and +caches a repository-scoped token. Cross-repository work therefore does not +require changing an installation ID or restarting the worker. Set +`LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy fixed-installation +fallback. + +Sandboxed commands receive masked Git/`gh` credentials only for the configured +GitHub hosts; the token is not written to the repository, remote URL, or Git +configuration. Git commits receive the App bot's canonical no-reply identity so +GitHub renders the bot profile and avatar. ## 10. Run under systemd @@ -518,7 +526,8 @@ command, cancellation, or settlement whose effects may be incomplete. - [ ] Pairing is principal-bound and the identity file is private. - [ ] Definitions are outside roots and immutable to sandboxed tools. - [ ] Workspace ancestors are not group/other writable. -- [ ] GitHub App is optional, least-privilege, and installed only where needed. +- [ ] GitHub App is optional, least-privilege, and installed on every account + the worker is expected to use. - [ ] Approval policy remains enforced independently of worker capability. - [ ] Service manager uses the intended executable and configuration. - [ ] Worker is online, ready, and advertises the expected workspace. diff --git a/packages/code/README.md b/packages/code/README.md index 08dc54a7..69d85b16 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -283,14 +283,24 @@ repositories the agent may access: ```bash LIBRECHAT_CODE_GITHUB_APP_ID=12345 \ -LIBRECHAT_CODE_GITHUB_INSTALLATION_ID=67890 \ LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/secure/librechat-agent.pem \ librechat-code run --worker-dir /path/to/project --allow-workspace-commands ``` The private key must be an owner-only regular file outside the workspace. It is read only by the trusted worker, which mints and refreshes short-lived -installation tokens. A personal access token is supported as a fallback with +installation tokens. At startup, the worker binds each explicitly admitted +workspace root to its Git repository. Commands in those independent roots can +use simultaneous installations on personal accounts and organizations without +being restarted or reconfigured, while a command cannot gain access by changing +its workspace's remote URL. Tokens are scoped and cached per repository. For +compatibility with deployments +that intentionally bind a worker to one installation, set the optional legacy +`LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` fallback. + +App-authenticated commits use the GitHub App bot's canonical no-reply identity, +so GitHub links them to the bot profile and avatar. A personal access token is +supported as a fallback with `LIBRECHAT_CODE_GITHUB_TOKEN`, but the GitHub App is the safer default because its repository access and permissions can be narrowly installed and revoked. Native Windows credential storage is unavailable until native DACL removal and diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index 18456072..f39b28d9 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -1,5 +1,9 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; +import { generateKeyPairSync } from 'node:crypto'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import test from 'node:test'; @@ -233,6 +237,59 @@ test('CLI validates GitHub App credentials before worker registration', () => { assert.doesNotMatch(result.stderr, /fetch failed/); }); +test('CLI accepts repository-routed GitHub App authentication without a fixed installation', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'cli-github-routing-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const preload = join(directory, 'fetch.mjs'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + await writeFile( + preload, + ` + globalThis.fetch = async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia-by-librechat' }); + if (url.endsWith('/users/lia-by-librechat%5Bbot%5D')) { + return Response.json({ id: 328778573, login: 'lia-by-librechat[bot]', type: 'Bot' }); + } + throw new Error('test stopped after GitHub App validation'); + }; + `, + ); + const result = spawnSync( + process.execPath, + [ + '--import', + preload, + fileURLToPath(new URL('./cli.js', import.meta.url)), + ], + { + encoding: 'utf8', + timeout: 10_000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: directory, + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: privateKeyPath, + }, + }, + ); + assert.notEqual(result.status, 0); + assert.doesNotMatch(result.stderr, /GitHub App authentication requires/); + assert.doesNotMatch(result.stderr, /installation ID/i); +}); + test('CLI requires a runtime image for Docker supervision', () => { const result = spawnSync( process.execPath, @@ -449,6 +506,6 @@ test('CLI host-only enterprise configuration sends App JWTs to GHES, never GitHu }, }); assert.equal(result.status, 1, result.stderr); - assert.match(result.stderr, /GITHUB_REQUEST:https:\/\/github\.example\.test\/api\/v3\/app\/installations\/456\/access_tokens/); + assert.match(result.stderr, /GITHUB_REQUEST:https:\/\/github\.example\.test\/api\/v3\/app/); assert.doesNotMatch(result.stderr, /GITHUB_REQUEST:https:\/\/api\.github\.com/); }); diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 524c5853..96757d57 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -46,6 +46,8 @@ import type { LocalWorkspaceConfig } from './workspace.js'; import { GITHUB_ALLOWED_DOMAINS, GitHubAppCredentialProvider, + gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForDirectory, gitHubCommandCredentialEnvironment, gitHubMaskedCredentialVariables, StaticGitHubCredentialProvider, @@ -149,6 +151,7 @@ function githubCredentials(): { host: string; privateKeyPath?: string; mode?: 'app' | 'token'; + repositoryRouting?: boolean; policyIdentity: string; } { const token = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_TOKEN); @@ -161,9 +164,9 @@ function githubCredentials(): { ); const appValues = [appId, installationId, privateKeyPath]; const hasApp = appValues.some(Boolean); - if (hasApp && !appValues.every(Boolean)) { + if (hasApp && (!appId || !privateKeyPath)) { throw new Error( - 'GitHub App authentication requires LIBRECHAT_CODE_GITHUB_APP_ID, LIBRECHAT_CODE_GITHUB_INSTALLATION_ID, and LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE', + 'GitHub App authentication requires LIBRECHAT_CODE_GITHUB_APP_ID and LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE; LIBRECHAT_CODE_GITHUB_INSTALLATION_ID is an optional legacy fallback', ); } if (hasApp && token) { @@ -203,6 +206,7 @@ function githubCredentials(): { return { host, mode: 'app', + repositoryRouting: !installationId, policyIdentity: gitHubAuthenticationPolicyIdentity({ mode: 'app', host, @@ -212,7 +216,7 @@ function githubCredentials(): { privateKeyPath, provider: new GitHubAppCredentialProvider({ appId: appId!, - installationId: installationId!, + installationId, privateKeyPath: privateKeyPath!, host, apiUrl, @@ -715,6 +719,19 @@ async function run( }), ]), ); + // Bind credentials to immutable, explicitly admitted roots. The repository + // remote is operator input at startup, never an authorization input that a + // sandboxed command may change for its next invocation. + const admittedGitHubRepositories = github.provider && github.repositoryRouting + ? new Map( + await Promise.all( + roots.map(async root => [ + root.root, + await gitHubRepositoryForDirectory(root.root, github.host), + ] as const), + ), + ) + : undefined; const localWorkspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: roots, @@ -928,18 +945,34 @@ async function run( ...(github.provider ? { maskedEnvironment: { - variables: gitHubMaskedCredentialVariables(github.host), - async resolve(signal?: AbortSignal) { + variables: gitHubMaskedCredentialVariables( + github.host, + ), + async resolve(signal?: AbortSignal, cwd?: string) { + const repository = cwd && admittedGitHubRepositories + ? gitHubRepositoryForAdmittedDirectory( + cwd, + admittedGitHubRepositories, + ) + : undefined; + if (!repository && github.repositoryRouting) { + return {}; + } return gitHubCommandCredentialEnvironment( - await github.provider!.getCredential(signal), + await github.provider!.getCredential(signal, repository), github.host, ); }, - wrapCommand(command: string, platform: NodeJS.Platform) { + wrapCommand( + command: string, + platform: NodeJS.Platform, + environment: Readonly>, + ) { return wrapGitHubCredentialCommand( command, github.host, platform, + environment, ); }, }, @@ -1022,7 +1055,10 @@ async function run( ); } try { - await github.provider?.getCredential(controller.signal); + await github.provider?.validate?.(controller.signal); + if (github.provider && !github.provider.validate) { + await github.provider.getCredential(controller.signal); + } await nativeCommandSandbox?.prepare(); for (const environment of option(args, '--reset-workspace-quarantine') == null ? environments : []) { const setup = environment.definition.setup; diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 4b028239..73e02927 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -1,4 +1,5 @@ import { generateKeyPairSync } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; import { chmod, mkdtemp, @@ -8,12 +9,14 @@ import { writeFile, } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import test from 'node:test'; import assert from 'node:assert/strict'; import { GITHUB_ALLOWED_DOMAINS, + GITHUB_AUTHOR_EMAIL_ENV_NAME, + GITHUB_AUTHOR_NAME_ENV_NAME, GitHubAppCredentialProvider, StaticGitHubCredentialProvider, gitHubAuthenticationPolicyIdentity, @@ -22,6 +25,8 @@ import { gitHubMaskedCredentialVariables, GITHUB_CREDENTIAL_ENV_NAME, gitHubCredentialEnvironment, + gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForDirectory, normalizeGitHubHost, wrapGitHubCredentialCommand, } from './github.js'; @@ -136,6 +141,358 @@ test('mints and caches a short-lived GitHub App installation token', async (t) = assert.equal(calls, 1); }); +test('routes and scopes GitHub App tokens per repository installation', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-routing-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + const calls: Array<{ url: string; body?: string }> = []; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input, init) => { + const url = String(input); + calls.push({ url, body: typeof init?.body === 'string' ? init.body : undefined }); + if (url.endsWith('/app')) { + return Response.json({ slug: 'lia-by-librechat' }); + } + if (url.endsWith('/users/lia-by-librechat%5Bbot%5D')) { + return Response.json({ + id: 328778573, + login: 'lia-by-librechat[bot]', + type: 'Bot', + }); + } + if (url.endsWith('/repos/danny-avila/LibreChat/installation')) { + return Response.json({ id: 111 }); + } + if (url.endsWith('/repos/LibreChat-AI/code-interpreter/installation')) { + return Response.json({ id: 222 }); + } + const installation = /\/app\/installations\/(\d+)\/access_tokens$/.exec(url)?.[1]; + if (installation) { + return Response.json( + { + token: `ghs_${installation}_abcdefghijklmnopqrstuvwxyz`, + expires_at: '2030-01-01T01:00:00Z', + }, + { status: 201 }, + ); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + + await provider.validate(); + const [personal, organization] = await Promise.all([ + provider.getCredential(undefined, 'danny-avila/LibreChat'), + provider.getCredential(undefined, 'LibreChat-AI/code-interpreter'), + ]); + assert.equal( + (await provider.getCredential(undefined, 'danny-avila/LibreChat')).value, + personal.value, + ); + assert.equal(personal.value, 'ghs_111_abcdefghijklmnopqrstuvwxyz'); + assert.equal(organization.value, 'ghs_222_abcdefghijklmnopqrstuvwxyz'); + assert.deepEqual(personal.actor, { + name: 'lia-by-librechat[bot]', + email: + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + }); + assert.equal( + calls.filter(call => call.url.includes('/repos/danny-avila/')).length, + 1, + ); + assert.deepEqual( + calls + .filter(call => call.url.endsWith('/access_tokens')) + .map(call => JSON.parse(call.body ?? '{}')) + .map(body => body.repositories[0]) + .sort(), + [ + 'LibreChat', + 'code-interpreter', + ], + ); +}); + +test('keeps a shared token refresh alive when one waiter is cancelled', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-cancel-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let releaseToken!: (response: Response) => void; + const tokenResponse = new Promise(resolve => { + releaseToken = resolve; + }); + let mintCount = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/repos/acme/project/installation')) { + return Response.json({ id: 111 }); + } + if (url.endsWith('/app/installations/111/access_tokens')) { + mintCount += 1; + return tokenResponse; + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + const firstController = new AbortController(); + const first = provider.getCredential( + firstController.signal, + 'acme/project', + ); + const second = provider.getCredential(undefined, 'acme/project'); + firstController.abort(new Error('first command cancelled')); + await assert.rejects(first, /first command cancelled/); + const third = provider.getCredential(undefined, 'acme/project'); + releaseToken( + Response.json({ + token: 'ghs_shared_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }), + ); + assert.equal( + (await second).value, + 'ghs_shared_abcdefghijklmnopqrstuvwxyz', + ); + assert.equal((await third).value, 'ghs_shared_abcdefghijklmnopqrstuvwxyz'); + assert.equal(mintCount, 1); +}); + +test('refreshes a cached repository installation after App reinstallation', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-reinstall-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let now = new Date('2030-01-01T00:00:00Z'); + let lookupCount = 0; + let oldMintCount = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + privateKeyPath, + now: () => now, + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/repos/acme/project/installation')) { + lookupCount += 1; + return Response.json({ id: lookupCount === 1 ? 111 : 222 }); + } + if (url.endsWith('/app/installations/111/access_tokens')) { + oldMintCount += 1; + return oldMintCount === 1 + ? Response.json({ + token: 'ghs_old_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }) + : Response.json({}, { status: 404 }); + } + if (url.endsWith('/app/installations/222/access_tokens')) { + return Response.json({ + token: 'ghs_new_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T02:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + assert.equal( + (await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_old_abcdefghijklmnopqrstuvwxyz', + ); + now = new Date('2030-01-01T00:56:00Z'); + assert.equal( + (await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_new_abcdefghijklmnopqrstuvwxyz', + ); + assert.equal(lookupCount, 2); +}); + +test('validates a configured fixed installation by minting its token', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-fixed-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let minted = false; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/app/installations/456/access_tokens')) { + minted = true; + return Response.json({ + token: 'ghs_fixed_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + assert.equal(minted, true); +}); + +test('discovers the GitHub repository from a command working directory', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-repo-')); + t.after(() => rm(directory, { recursive: true, force: true })); + execFileSync('git', ['init', directory]); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'add', + 'origin', + 'git@github.com:LibreChat-AI/code-interpreter.git', + ]); + assert.equal( + await gitHubRepositoryForDirectory(directory), + 'LibreChat-AI/code-interpreter', + ); + assert.equal( + await gitHubRepositoryForDirectory(directory, 'github.example.test'), + undefined, + ); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'set-url', + 'origin', + 'https://github.example.test:8443/acme/project.git', + ]); + assert.equal( + await gitHubRepositoryForDirectory(directory, 'github.example.test'), + 'acme/project', + ); +}); + +test('keeps repository authorization bound to the admitted workspace root', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-binding-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const nested = join(directory, 'packages', 'app'); + await mkdir(nested, { recursive: true }); + execFileSync('git', ['init', directory]); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'add', + 'origin', + 'git@github.com:acme/allowed.git', + ]); + const admitted = new Map([ + [directory, await gitHubRepositoryForDirectory(directory)], + ]); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'set-url', + 'origin', + 'git@github.com:acme/not-authorized.git', + ]); + assert.equal( + gitHubRepositoryForAdmittedDirectory(nested, admitted), + 'acme/allowed', + ); + assert.equal( + gitHubRepositoryForAdmittedDirectory(dirname(directory), admitted), + undefined, + ); +}); + +test('uses the configured GHES host for the App bot no-reply identity', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-ghes-identity-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath, + host: 'github.example.test', + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/app/installations/456/access_tokens')) { + return Response.json({ + token: 'ghs_enterprise_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + const credential = await provider.getCredential(); + assert.deepEqual(credential.actor, { + name: 'lia[bot]', + email: '1234+lia[bot]@users.noreply.github.example.test', + }); + const wrapped = wrapGitHubCredentialCommand( + 'git commit -m test', + 'github.example.test', + 'linux', + gitHubCommandCredentialEnvironment(credential, 'github.example.test'), + ); + assert.match( + wrapped, + /user\.email=1234\+lia\[bot\]@users\.noreply\.github\.example\.test/, + ); +}); + test('builds process-scoped Git HTTPS authorization without embedding credentials in URLs', async () => { const provider = new StaticGitHubCredentialProvider( 'github_pat_abcdefghijklmnopqrstuvwxyz', @@ -177,6 +534,76 @@ test('adds a GitHub CLI token only to the command-sandbox credential bundle', as ); }); +test('binds Git commits to the GitHub App bot identity', () => { + const environment = gitHubCommandCredentialEnvironment({ + value: 'ghs_abcdefghijklmnopqrstuvwxyz', + actor: { + name: 'lia-by-librechat[bot]', + email: + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + }, + }); + assert.equal(environment[GITHUB_AUTHOR_NAME_ENV_NAME], 'lia-by-librechat[bot]'); + assert.equal( + environment[GITHUB_AUTHOR_EMAIL_ENV_NAME], + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + ); + const variables = gitHubMaskedCredentialVariables('github.com'); + assert.ok(!variables.some(variable => variable.name === GITHUB_AUTHOR_NAME_ENV_NAME)); + assert.ok(!variables.some(variable => variable.name === GITHUB_AUTHOR_EMAIL_ENV_NAME)); + const wrapped = wrapGitHubCredentialCommand( + 'git commit -m test', + 'github.com', + 'linux', + environment, + ); + assert.match(wrapped, /user\.name=/); + assert.match(wrapped, /user\.email=/); +}); + +test('records the canonical App bot as Git author and committer', async (t) => { + if (process.platform === 'win32') { + t.skip('POSIX command wrapper integration is unavailable on Windows'); + return; + } + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-author-')); + t.after(() => rm(directory, { recursive: true, force: true })); + execFileSync('git', ['init', directory]); + const environment = gitHubCommandCredentialEnvironment({ + value: 'ghs_abcdefghijklmnopqrstuvwxyz', + actor: { + name: 'lia-by-librechat[bot]', + email: + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + }, + }); + const wrapped = wrapGitHubCredentialCommand( + 'git commit --allow-empty -m test', + 'github.com', + process.platform, + environment, + ); + execFileSync('/bin/bash', ['-lc', wrapped], { + cwd: directory, + env: { PATH: process.env.PATH, ...environment }, + }); + assert.equal( + execFileSync( + 'git', + [ + '-C', + directory, + 'show', + '-s', + '--format=%an|%ae|%cn|%ce', + 'HEAD', + ], + { encoding: 'utf8' }, + ).trim(), + 'lia-by-librechat[bot]|328778573+lia-by-librechat[bot]@users.noreply.github.com|lia-by-librechat[bot]|328778573+lia-by-librechat[bot]@users.noreply.github.com', + ); +}); + test('selects the GitHub CLI token variable for public and enterprise hosts', () => { assert.equal(gitHubCliTokenEnvironmentName('github.com'), 'GH_TOKEN'); assert.equal( @@ -217,6 +644,9 @@ test('composes the masked credential with SRT Git configuration inside the sandb 'git push', 'github.com', 'darwin', + { + [GITHUB_CREDENTIAL_ENV_NAME]: 'masked-authorization', + }, ); assert.match(wrapped, /http\.proxyAuthMethod=basic/); assert.match(wrapped, /http\.https:\/\/github\.com\/\.extraheader/); @@ -227,6 +657,16 @@ test('composes the masked credential with SRT Git configuration inside the sandb assert.ok(!wrapped.includes('github_pat_')); }); +test('omits Git authorization when repository routing resolves no credential', () => { + const wrapped = wrapGitHubCredentialCommand( + 'git clone https://github.com/LibreChat-AI/LibreChat.git', + 'github.com', + 'linux', + {}, + ); + assert.doesNotMatch(wrapped, /Authorization: Basic/); +}); + test('targets GitHub CLI at an enterprise host without exposing its token', () => { const wrapped = wrapGitHubCredentialCommand( 'gh pr create', diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index c727e70d..4c79fa37 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -1,10 +1,15 @@ import { constants } from 'node:fs'; +import { execFile } from 'node:child_process'; import { createHash, createPrivateKey, sign } from 'node:crypto'; import { open } from 'node:fs/promises'; -import { dirname } from 'node:path'; +import { dirname, isAbsolute, relative, sep } from 'node:path'; +import { promisify } from 'node:util'; +import { projectRemote } from './projects.js'; import { assertPrivateStorageAcl, assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; export const GITHUB_CREDENTIAL_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHORIZATION'; +export const GITHUB_AUTHOR_NAME_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHOR_NAME'; +export const GITHUB_AUTHOR_EMAIL_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHOR_EMAIL'; export const GITHUB_ALLOWED_DOMAINS = [ 'github.com', '*.github.com', @@ -18,15 +23,24 @@ export const GITHUB_ALLOWED_DOMAINS = [ export interface GitHubCredential { value: string; expiresAt?: Date; + actor?: { + name: string; + email: string; + }; } export interface GitHubCredentialProvider { - getCredential(signal?: AbortSignal): Promise; + getCredential( + signal?: AbortSignal, + repository?: string, + ): Promise; + validate?(signal?: AbortSignal): Promise; } export interface GitHubAppCredentialProviderOptions { appId: string; - installationId: string; + /** Legacy fixed installation. Omit to resolve the installation per repository. */ + installationId?: string; privateKeyPath: string; apiUrl?: string; /** Git HTTPS hostname; non-public hosts default to the GHES /api/v3 base. */ @@ -36,6 +50,110 @@ export interface GitHubAppCredentialProviderOptions { platform?: NodeJS.Platform; } +const execFileAsync = promisify(execFile); +const GITHUB_SHARED_REQUEST_TIMEOUT_MS = 30_000; + +async function waitForShared( + promise: Promise, + signal?: AbortSignal, +): Promise { + if (!signal) return promise; + signal.throwIfAborted(); + return new Promise((resolve, reject) => { + const aborted = () => { + try { + signal.throwIfAborted(); + } catch (error) { + reject(error); + } + }; + signal.addEventListener('abort', aborted, { once: true }); + void promise.then(resolve, reject).finally(() => { + signal.removeEventListener('abort', aborted); + }); + }); +} + +function repositoryName(value: string): { owner: string; name: string } { + const match = /^([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)$/.exec(value); + if (!match) throw new Error('GitHub repository must be owner/name'); + return { owner: match[1], name: match[2] }; +} + +/** Resolve only the repository containing the admitted command cwd. */ +export async function gitHubRepositoryForDirectory( + cwd: string, + host = 'github.com', + signal?: AbortSignal, +): Promise { + let remote: string; + try { + const result = await execFileAsync( + 'git', + [ + '--no-optional-locks', + '-C', + cwd, + '-c', + 'core.fsmonitor=false', + 'config', + '--local', + '--no-includes', + '--get', + 'remote.origin.url', + ], + { + env: { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }, + encoding: 'utf8', + maxBuffer: 4096, + timeout: 1500, + signal, + }, + ); + remote = result.stdout.trim(); + } catch { + signal?.throwIfAborted(); + return undefined; + } + const normalized = projectRemote(remote); + if (!normalized) return undefined; + const separator = normalized.indexOf('/'); + const remoteHost = normalized + .slice(0, separator) + .replace(/:[1-9][0-9]*$/, ''); + if (remoteHost !== normalizeGitHubHost(host)) { + return undefined; + } + const repository = normalized.slice(separator + 1); + repositoryName(repository); + return repository; +} + +/** Return the startup-bound repository for the admitted root containing cwd. */ +export function gitHubRepositoryForAdmittedDirectory( + cwd: string, + repositories: ReadonlyMap, +): string | undefined { + for (const [root, repository] of repositories) { + const path = relative(root, cwd); + if ( + path === '' || + (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) + ) { + return repository; + } + } + return undefined; +} + function base64UrlJson(value: unknown): string { return Buffer.from(JSON.stringify(value)).toString('base64url'); } @@ -96,8 +214,13 @@ function createAppJwt(appId: string, privateKey: string, now: Date): string { } export class GitHubAppCredentialProvider implements GitHubCredentialProvider { - private cached?: GitHubCredential; + private readonly cached = new Map(); + private readonly inFlight = new Map>(); + private readonly installationIds = new Map(); + private actor?: GitHubCredential['actor']; + private actorInFlight?: Promise>; private readonly apiUrl: string; + private readonly host: string; constructor(private readonly options: GitHubAppCredentialProviderOptions) { if ((options.platform ?? process.platform) === 'win32') { @@ -106,10 +229,12 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ); } assertPositiveIdentifier('GitHub App ID', options.appId); - assertPositiveIdentifier( - 'GitHub App installation ID', - options.installationId, - ); + if (options.installationId != null) { + assertPositiveIdentifier( + 'GitHub App installation ID', + options.installationId, + ); + } const host = options.host == null ? undefined : normalizeGitHubHost(options.host); const apiUrl = new URL(options.apiUrl ?? ( host != null && host !== 'github.com' @@ -126,58 +251,246 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { if (host != null && host !== apiHost) { throw new Error('LIBRECHAT_CODE_GITHUB_HOST must match the GitHub App API hostname'); } + this.host = host ?? apiHost; this.apiUrl = apiUrl.href.replace(/\/+$/, ''); } - async getCredential(signal?: AbortSignal): Promise { - const now = (this.options.now ?? (() => new Date()))(); - if ( - this.cached?.expiresAt != null && - this.cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 - ) { - return this.cached; - } + private async appJwt(now: Date): Promise { const privateKey = await readPrivateKey(this.options.privateKeyPath); - const jwt = createAppJwt(this.options.appId, privateKey, now); + return createAppJwt(this.options.appId, privateKey, now); + } + + private async request( + path: string, + jwt: string, + signal?: AbortSignal, + init?: RequestInit, + ): Promise { const request = this.options.fetch ?? globalThis.fetch; - const response = await request( - `${this.apiUrl}/app/installations/${this.options.installationId}/access_tokens`, - { - method: 'POST', - redirect: 'error', - headers: { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${jwt}`, - 'X-GitHub-Api-Version': '2022-11-28', - }, - signal, + return request(`${this.apiUrl}${path}`, { + redirect: 'error', + ...init, + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${jwt}`, + 'X-GitHub-Api-Version': '2022-11-28', + ...init?.headers, }, + signal, + }); + } + + private async resolveActor( + jwt: string, + signal?: AbortSignal, + ): Promise> { + if (this.actor) return this.actor; + if (!this.actorInFlight) { + const pending = (async () => { + const sharedSignal = AbortSignal.timeout( + GITHUB_SHARED_REQUEST_TIMEOUT_MS, + ); + const appResponse = await this.request('/app', jwt, sharedSignal); + if (!appResponse.ok) { + throw new Error( + `GitHub App identity request failed with status ${appResponse.status}`, + ); + } + const app = (await appResponse.json()) as { slug?: unknown }; + if ( + typeof app.slug !== 'string' || + !/^[A-Za-z0-9-]+$/.test(app.slug) + ) { + throw new Error('GitHub App identity response is invalid'); + } + const login = `${app.slug}[bot]`; + const userResponse = await this.request( + `/users/${encodeURIComponent(login)}`, + jwt, + sharedSignal, + ); + if (!userResponse.ok) { + throw new Error( + `GitHub App bot identity request failed with status ${userResponse.status}`, + ); + } + const user = (await userResponse.json()) as { + id?: unknown; + login?: unknown; + type?: unknown; + }; + if ( + !Number.isSafeInteger(user.id) || + Number(user.id) <= 0 || + user.login !== login || + user.type !== 'Bot' + ) { + throw new Error('GitHub App bot identity response is invalid'); + } + return { + name: login, + email: `${user.id}+${login}@users.noreply.${this.host}`, + }; + })(); + this.actorInFlight = pending; + void pending.then( + actor => { + this.actor = actor; + if (this.actorInFlight === pending) this.actorInFlight = undefined; + }, + () => { + if (this.actorInFlight === pending) this.actorInFlight = undefined; + }, + ); + } + return waitForShared(this.actorInFlight, signal); + } + + async validate(signal?: AbortSignal): Promise { + const now = (this.options.now ?? (() => new Date()))(); + const jwt = await this.appJwt(now); + await this.resolveActor(jwt, signal); + if (this.options.installationId) { + await this.getCredential(signal); + } + } + + private async resolveInstallationId( + repository: string, + jwt: string, + signal?: AbortSignal, + ): Promise { + if (this.options.installationId) return this.options.installationId; + const cached = this.installationIds.get(repository); + if (cached) return cached; + const { owner, name } = repositoryName(repository); + const response = await this.request( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/installation`, + jwt, + signal, ); if (!response.ok) { throw new Error( - `GitHub App token request failed with status ${response.status}`, + response.status === 404 + ? `GitHub App is not installed for ${repository}` + : `GitHub App installation lookup failed with status ${response.status}`, ); } - const body = (await response.json()) as { - token?: unknown; - expires_at?: unknown; - }; - if ( - typeof body.token !== 'string' || - body.token.length < 20 || - typeof body.expires_at !== 'string' - ) { - throw new Error('GitHub App token response is invalid'); + const body = (await response.json()) as { id?: unknown }; + if (!Number.isSafeInteger(body.id) || Number(body.id) <= 0) { + throw new Error('GitHub App installation response is invalid'); + } + const installationId = String(body.id); + this.installationIds.set(repository, installationId); + return installationId; + } + + async getCredential( + signal?: AbortSignal, + repository?: string, + ): Promise { + signal?.throwIfAborted(); + if (!this.options.installationId && !repository) { + throw new Error( + 'GitHub App authentication requires a GitHub repository for this command', + ); } - const expiresAt = new Date(body.expires_at); + if (repository) repositoryName(repository); + const now = (this.options.now ?? (() => new Date()))(); + const key = this.options.installationId ?? repository!; + const cached = this.cached.get(key); if ( - !Number.isFinite(expiresAt.getTime()) || - expiresAt.getTime() <= now.getTime() + cached?.expiresAt != null && + cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 ) { - throw new Error('GitHub App token expiry is invalid'); + return cached; } - this.cached = { value: body.token, expiresAt }; - return this.cached; + const existing = this.inFlight.get(key); + if (existing) return waitForShared(existing, signal); + const pending = (async () => { + const sharedSignal = AbortSignal.timeout( + GITHUB_SHARED_REQUEST_TIMEOUT_MS, + ); + const jwt = await this.appJwt(now); + const scopedRepository = repository + ? repositoryName(repository).name + : undefined; + const installationId = await this.resolveInstallationId( + repository ?? '', + jwt, + sharedSignal, + ); + let response = await this.request( + `/app/installations/${installationId}/access_tokens`, + jwt, + sharedSignal, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + ...(this.options.installationId + ? {} + : { body: JSON.stringify({ repositories: [scopedRepository] }) }), + }, + ); + if (!this.options.installationId && response.status === 404) { + this.installationIds.delete(repository!); + const refreshedInstallationId = await this.resolveInstallationId( + repository!, + jwt, + sharedSignal, + ); + response = await this.request( + `/app/installations/${refreshedInstallationId}/access_tokens`, + jwt, + sharedSignal, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ repositories: [scopedRepository] }), + }, + ); + } + if (!response.ok) { + throw new Error( + `GitHub App token request failed with status ${response.status}`, + ); + } + const body = (await response.json()) as { + token?: unknown; + expires_at?: unknown; + }; + if ( + typeof body.token !== 'string' || + body.token.length < 20 || + typeof body.expires_at !== 'string' + ) { + throw new Error('GitHub App token response is invalid'); + } + const expiresAt = new Date(body.expires_at); + if ( + !Number.isFinite(expiresAt.getTime()) || + expiresAt.getTime() <= now.getTime() + ) { + throw new Error('GitHub App token expiry is invalid'); + } + const credential = { + value: body.token, + expiresAt, + ...(this.actor ? { actor: this.actor } : {}), + }; + this.cached.set(key, credential); + return credential; + })(); + this.inFlight.set(key, pending); + const clearPending = () => { + if (this.inFlight.get(key) === pending) this.inFlight.delete(key); + }; + void pending.then(clearPending, clearPending); + return waitForShared(pending, signal); } } @@ -211,6 +524,12 @@ export function gitHubCommandCredentialEnvironment( return { ...gitHubCredentialEnvironment(credential), [gitHubCliTokenEnvironmentName(host)]: credential.value, + ...(credential.actor + ? { + [GITHUB_AUTHOR_NAME_ENV_NAME]: credential.actor.name, + [GITHUB_AUTHOR_EMAIL_ENV_NAME]: credential.actor.email, + } + : {}), }; } @@ -260,12 +579,10 @@ export function gitHubAuthenticationPolicyIdentity(options: { return `${identity}:fingerprint:${fingerprint}`; } if (options.mode !== 'app') return identity; - if (!options.appId || !options.installationId) { - throw new Error( - 'GitHub App policy identity requires an App and installation ID', - ); + if (!options.appId) { + throw new Error('GitHub App policy identity requires an App ID'); } - return `${identity}:app:${options.appId}:installation:${options.installationId}`; + return `${identity}:app:${options.appId}:installation:${options.installationId ?? 'repository'}`; } export function normalizeGitHubHost(value: string): string { @@ -284,24 +601,48 @@ export function wrapGitHubCredentialCommand( command: string, host = 'github.com', platform: NodeJS.Platform = process.platform, + environment: Readonly> = {}, ): string { const key = `http.https://${host}/.extraheader`; const cliHost = host === 'github.com' ? undefined : host; + const hasCredential = Boolean(environment[GITHUB_CREDENTIAL_ENV_NAME]); + const actorName = environment[GITHUB_AUTHOR_NAME_ENV_NAME]; + const actorEmail = environment[GITHUB_AUTHOR_EMAIL_ENV_NAME]; + const noReplyHost = `users.noreply.${normalizeGitHubHost(host)}` + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const hasActor = + /^[A-Za-z0-9_.-]+\[bot\]$/.test(actorName ?? '') && + new RegExp( + `^[1-9][0-9]+\\+[A-Za-z0-9_.-]+\\[bot\\]@${noReplyHost}$`, + ).test(actorEmail ?? ''); if (platform === 'win32') { return [ 'set "GIT_CONFIG_GLOBAL=NUL"', 'set "GIT_CONFIG_NOSYSTEM=1"', ...(cliHost ? [`set "GH_HOST=${cliHost}"`] : []), - `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, + ...(hasCredential + ? [`set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`] + : ['set "GIT_CONFIG_PARAMETERS="']), + ...(hasActor + ? [`set "GIT_CONFIG_PARAMETERS=%GIT_CONFIG_PARAMETERS% 'user.name=${actorName}' 'user.email=${actorEmail}'"`] + : []), `set "${GITHUB_CREDENTIAL_ENV_NAME}="`, + `set "${GITHUB_AUTHOR_NAME_ENV_NAME}="`, + `set "${GITHUB_AUTHOR_EMAIL_ENV_NAME}="`, command, ].join(' && '); } return [ 'export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1', ...(cliHost ? [`export GH_HOST=${cliHost}`] : []), - `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, + ...(hasCredential + ? [`export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`] + : ['export GIT_CONFIG_PARAMETERS=']), + ...(hasActor + ? [`export GIT_CONFIG_PARAMETERS="\${GIT_CONFIG_PARAMETERS} 'user.name=${actorName}' 'user.email=${actorEmail}'"`] + : []), `unset ${GITHUB_CREDENTIAL_ENV_NAME}`, + `unset ${GITHUB_AUTHOR_NAME_ENV_NAME} ${GITHUB_AUTHOR_EMAIL_ENV_NAME}`, command, ].join(';\n'); } diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 89651845..b3016937 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -181,12 +181,14 @@ test('executor forwards the resolved command policy without worker credentials', test('executor hands credentials over IPC only for the current command', async () => { const fake = fixture(); + let credentialCwd: string | undefined; const sandbox = new NativeProcessWorkspaceCommandSandbox( { workspaceRoot: '/workspace', maskedEnvironment: { variables: [{ name: 'TOKEN', injectHosts: ['github.com'] }], - async resolve() { + async resolve(_signal, cwd) { + credentialCwd = cwd; return { TOKEN: 'per-command-secret' }; }, wrapCommand(command) { @@ -208,6 +210,7 @@ test('executor hands credentials over IPC only for the current command', async ( assert.deepEqual(fake.messages[1].credentials, { TOKEN: 'per-command-secret', }); + assert.equal(credentialCwd, '/workspace'); assert.equal(fake.messages[1].wrappedCommand, 'wrapped printf ok'); await sandbox.close(); }); diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 70cb2bd1..ba411042 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -2,7 +2,7 @@ import { execFile, fork } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { constants as fsConstants } from 'node:fs'; import { access, realpath } from 'node:fs/promises'; -import { isAbsolute, join, relative, sep } from 'node:path'; +import { isAbsolute, join, relative, resolve, sep } from 'node:path'; import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; @@ -436,11 +436,15 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan if (signal?.aborted) throw new Error('aborted'); programmaticExecutables = await this.resolveProgrammaticExecutables(); if (signal?.aborted) throw new Error('aborted'); - credentials = await this.options.maskedEnvironment?.resolve(signal); + credentials = await this.options.maskedEnvironment?.resolve( + signal, + this.options.workspaceRoot, + ); if (signal?.aborted) throw new Error('aborted'); wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( NATIVE_PROGRAMMATIC_COMMAND, process.platform, + credentials ?? {}, ); if (signal?.aborted) throw new Error('aborted'); } catch (error) { @@ -529,11 +533,13 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan try { await this.prepare(); if (signal?.aborted) throw new Error('aborted'); - credentials = await this.options.maskedEnvironment?.resolve(signal); + const cwd = resolve(this.options.workspaceRoot, request.cwd ?? '.'); + credentials = await this.options.maskedEnvironment?.resolve(signal, cwd); if (signal?.aborted) throw new Error('aborted'); wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( request.command, process.platform, + credentials ?? {}, ); if (signal?.aborted) throw new Error('aborted'); } catch (error) { diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index e610a0fc..b1e4c5df 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1031,6 +1031,51 @@ test('masks a host credential for only its injection host and restores the paren }); }); +test('keeps trusted public command context out of credential masking', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + allowedDomains: ['github.com'], + maskedEnvironment: { + variables: [ + { + name: 'LIBRECHAT_CODE_TEST_CREDENTIAL', + injectHosts: ['github.com'], + }, + ], + async resolve() { + return { + LIBRECHAT_CODE_TEST_CREDENTIAL: 'real-secret', + LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY: 'lia[bot]', + }; + }, + wrapCommand(command, _platform, environment) { + assert.equal( + environment.LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY, + 'lia[bot]', + ); + return `export LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY="${environment.LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY}"; ${command}`; + }, + }, + manager: fake.manager, + }); + + const result = await sandbox.execute({ + ...request, + command: + 'printf "%s|%s" "$LIBRECHAT_CODE_TEST_CREDENTIAL" "$LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY"', + }); + + assert.equal(result.stdout, 'Authorization: Bearer srt-sentinel|lia[bot]'); + assert.ok( + !fake.config?.credentials?.envVars?.some( + variable => variable.name === 'LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY', + ), + ); +}); + test('serializes credential handoff across concurrent sandbox instances', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 84602a35..5bf61c50 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -182,8 +182,15 @@ export interface NativeSrtWorkspaceCommandSandboxOptions { injectHosts: string[]; extract?: string; }>; - resolve(signal?: AbortSignal): Promise>; - wrapCommand?(command: string, platform: NodeJS.Platform): string; + resolve( + signal?: AbortSignal, + cwd?: string, + ): Promise>; + wrapCommand?( + command: string, + platform: NodeJS.Platform, + environment: Readonly>, + ): string; }; } @@ -872,18 +879,19 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const commandId = `librechat-code-${randomUUID()}`; - const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand - ? this.options.maskedEnvironment.wrapCommand( - request.command, - this.platform, - ) - : request.command; let wrapped: Awaited< ReturnType >; try { const credentialEnvironment = - await this.options.maskedEnvironment?.resolve(signal); + await this.options.maskedEnvironment?.resolve(signal, cwd); + const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand + ? this.options.maskedEnvironment.wrapCommand( + request.command, + this.platform, + credentialEnvironment ?? {}, + ) + : request.command; wrapped = await this.withTemporaryHostEnvironment( { ...TRUSTED_GIT_ENVIRONMENT, From dc48249741e9d9bd010d19a3d5fcf56e2c159b79 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 11:12:23 -0400 Subject: [PATCH 38/42] fix: Authenticate GitHub App bot identity lookup (#237) --- packages/code/src/github.test.ts | 61 ++++++++++++++++++++++---------- packages/code/src/github.ts | 54 +++++++++++++++++++++++----- 2 files changed, 89 insertions(+), 26 deletions(-) diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 73e02927..352bf1c3 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -104,23 +104,30 @@ test('mints and caches a short-lived GitHub App installation token', async (t) = { mode: 0o600 }, ); await chmod(privateKeyPath, 0o600); - let calls = 0; + const calls: Array<{ url: string; authorization: string | null }> = []; const request = async ( - _input: string | URL | Request, + input: string | URL | Request, init?: RequestInit, ) => { - calls += 1; - assert.match( - String(new Headers(init?.headers).get('authorization')), - /^Bearer eyJ/, - ); - return new Response( - JSON.stringify({ + const url = String(input); + const authorization = new Headers(init?.headers).get('authorization'); + calls.push({ url, authorization }); + if (url.endsWith('/app')) { + assert.match(String(authorization), /^Bearer eyJ/); + return Response.json({ slug: 'lia' }); + } + if (url.endsWith('/app/installations/456/access_tokens')) { + assert.match(String(authorization), /^Bearer eyJ/); + return Response.json({ token: 'ghs_abcdefghijklmnopqrstuvwxyz', expires_at: '2030-01-01T01:00:00Z', - }), - { status: 201 }, - ); + }, { status: 201 }); + } + if (url.endsWith('/users/lia%5Bbot%5D')) { + assert.equal(authorization, 'Bearer ghs_abcdefghijklmnopqrstuvwxyz'); + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + return Response.json({}, { status: 404 }); }; const provider = new GitHubAppCredentialProvider({ appId: '123', @@ -138,7 +145,15 @@ test('mints and caches a short-lived GitHub App installation token', async (t) = (await provider.getCredential()).value, 'ghs_abcdefghijklmnopqrstuvwxyz', ); - assert.equal(calls, 1); + assert.equal(calls.filter(call => call.url.endsWith('/app')).length, 1); + assert.equal( + calls.filter(call => call.url.endsWith('/access_tokens')).length, + 1, + ); + assert.equal( + calls.filter(call => call.url.endsWith('/users/lia%5Bbot%5D')).length, + 1, + ); }); test('routes and scopes GitHub App tokens per repository installation', async (t) => { @@ -767,16 +782,26 @@ test('App JWT requests use the resolved public or enterprise endpoint', async (t now: () => new Date('2030-01-01T00:00:00Z'), fetch: async (input, init) => { calls++; - assert.equal(String(input), `${expected}/app/installations/456/access_tokens`); - assert.equal(init?.method, 'POST'); + const url = String(input); assert.equal(init?.redirect, 'error'); - assert.match(new Headers(init?.headers).get('authorization')!, /^Bearer eyJ/); - return new Response(JSON.stringify({ token: 'ghs_abcdefghijklmnopqrstuvwxyz', expires_at: '2030-01-01T01:00:00Z' }), { status: 201 }); + const authorization = new Headers(init?.headers).get('authorization'); + if (url === `${expected}/app`) { + assert.match(authorization!, /^Bearer eyJ/); + return Response.json({ slug: 'lia' }); + } + if (url === `${expected}/app/installations/456/access_tokens`) { + assert.equal(init?.method, 'POST'); + assert.match(authorization!, /^Bearer eyJ/); + return new Response(JSON.stringify({ token: 'ghs_abcdefghijklmnopqrstuvwxyz', expires_at: '2030-01-01T01:00:00Z' }), { status: 201 }); + } + assert.equal(url, `${expected}/users/lia%5Bbot%5D`); + assert.equal(authorization, 'Bearer ghs_abcdefghijklmnopqrstuvwxyz'); + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); }, }); await provider.getCredential(); await provider.getCredential(); - assert.equal(calls, 1); + assert.equal(calls, 3); } }); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 4c79fa37..fb6b6fdc 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -217,6 +217,8 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { private readonly cached = new Map(); private readonly inFlight = new Map>(); private readonly installationIds = new Map(); + private appLogin?: string; + private appLoginInFlight?: Promise; private actor?: GitHubCredential['actor']; private actorInFlight?: Promise>; private readonly apiUrl: string; @@ -280,12 +282,12 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { }); } - private async resolveActor( + private async resolveAppLogin( jwt: string, signal?: AbortSignal, - ): Promise> { - if (this.actor) return this.actor; - if (!this.actorInFlight) { + ): Promise { + if (this.appLogin) return this.appLogin; + if (!this.appLoginInFlight) { const pending = (async () => { const sharedSignal = AbortSignal.timeout( GITHUB_SHARED_REQUEST_TIMEOUT_MS, @@ -303,10 +305,41 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ) { throw new Error('GitHub App identity response is invalid'); } - const login = `${app.slug}[bot]`; + return `${app.slug}[bot]`; + })(); + this.appLoginInFlight = pending; + void pending.then( + login => { + this.appLogin = login; + if (this.appLoginInFlight === pending) { + this.appLoginInFlight = undefined; + } + }, + () => { + if (this.appLoginInFlight === pending) { + this.appLoginInFlight = undefined; + } + }, + ); + } + return waitForShared(this.appLoginInFlight, signal); + } + + private async resolveActor( + jwt: string, + installationToken: string, + signal?: AbortSignal, + ): Promise> { + if (this.actor) return this.actor; + if (!this.actorInFlight) { + const pending = (async () => { + const sharedSignal = AbortSignal.timeout( + GITHUB_SHARED_REQUEST_TIMEOUT_MS, + ); + const login = await this.resolveAppLogin(jwt, sharedSignal); const userResponse = await this.request( `/users/${encodeURIComponent(login)}`, - jwt, + installationToken, sharedSignal, ); if (!userResponse.ok) { @@ -349,7 +382,7 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { async validate(signal?: AbortSignal): Promise { const now = (this.options.now ?? (() => new Date()))(); const jwt = await this.appJwt(now); - await this.resolveActor(jwt, signal); + await this.resolveAppLogin(jwt, signal); if (this.options.installationId) { await this.getCredential(signal); } @@ -477,10 +510,15 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ) { throw new Error('GitHub App token expiry is invalid'); } + const actor = await this.resolveActor( + jwt, + body.token, + sharedSignal, + ); const credential = { value: body.token, expiresAt, - ...(this.actor ? { actor: this.actor } : {}), + actor, }; this.cached.set(key, credential); return credential; From 277fa7742d383eb1b6606ec228cdafe36af043a4 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 14:43:11 -0400 Subject: [PATCH 39/42] feat: Advertise Workspace Command Timeout Ceiling (#238) --- packages/code/src/protocol.ts | 2 ++ service/src/bridge/index.ts | 1 + service/src/bridge/router.test.ts | 54 ++++++++++++++++++++++++++++++- service/src/bridge/router.ts | 15 +++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index c5353ae2..27220735 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -712,6 +712,8 @@ export interface BridgeWorkerStatusResponse { online: boolean; ready: boolean; leaseExpiresInMs?: number; + /** Server-owned execution ceiling for workspace commands. Omitted by legacy servers. */ + maxCommandTimeoutMs?: number; capabilities?: BridgeWorkerCapabilities; } diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index a1ec5d78..16857fad 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -21,4 +21,5 @@ export default createBridgeRouter({ adminToken: env.BRIDGE_TOKEN, configuredWorkerId: env.BRIDGE_WORKER_ID, allowDynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, + maxCommandTimeoutMs: env.JOB_TIMEOUT, }); diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index af0e765c..63640111 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -10,7 +10,10 @@ import { createBridgeIdentity, signBridgeRequest, } from '../../../packages/code/src/identity'; -import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { + BRIDGE_PROTOCOL_VERSION, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, +} from '../../../packages/code/src/protocol'; import { RedisBridgePairingStore } from './pairing'; import { createBridgeRouter } from './router'; import { RedisBridgeStore } from './store'; @@ -115,6 +118,55 @@ describe('paired bridge HTTP API', () => { expect(unauthorized.status).toBe(401); }); + test('advertises the effective server command timeout for command-capable workers', async () => { + const store = new RedisBridgeStore(redis); + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store, + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'command-worker', + maxCommandTimeoutMs: 900_000, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'command-worker', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['execute_command'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + const response = await fetch( + `http://127.0.0.1:${address.port}/v1/bridge/workers/command-worker/status`, + { headers: { Authorization: 'Bearer strong-administrator-bootstrap-token' } }, + ); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + workerId: 'command-worker', + maxCommandTimeoutMs: BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + }); + }); + test('rejects a malformed optional binding for a configured worker', async () => { const app = express(); app.use(json()); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 369b306c..73870772 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -7,6 +7,7 @@ import type { BridgePrincipalType, BridgeWorkerBinding } from './pairing'; import type { CodeBridgeAssignment, CodeBridgeSettlement } from './store'; import { + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, @@ -36,6 +37,7 @@ export interface BridgeRouterOptions { adminToken: string; configuredWorkerId?: string; allowDynamicWorkers?: boolean; + maxCommandTimeoutMs?: number; } function sameToken(left: string, right: string): boolean { @@ -132,6 +134,16 @@ function isSettlement(value: unknown): value is CodeBridgeSettlement { export function createBridgeRouter(options: BridgeRouterOptions): Router { const router = Router(); if (options.enabled === false) return router; + if ( + options.maxCommandTimeoutMs !== undefined && + (!Number.isSafeInteger(options.maxCommandTimeoutMs) || options.maxCommandTimeoutMs < 1) + ) { + throw new RangeError('Workspace command timeout must be a positive safe integer'); + } + const maxCommandTimeoutMs = + options.maxCommandTimeoutMs == null + ? undefined + : Math.min(options.maxCommandTimeoutMs, BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS); const configuredWorker = (workerId: string): boolean => options.allowDynamicWorkers === true || @@ -324,10 +336,13 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { return; } const status = await options.store.workerStatus(workerId); + const supportsCommands = + status.capabilities?.workspaceTools?.operations.includes('execute_command') === true; res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, workerId, ...status, + ...(supportsCommands && maxCommandTimeoutMs != null ? { maxCommandTimeoutMs } : {}), }); }), ); From 95ebbd3cca39948c42e92d6c8228ace6a2f44298 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 21:42:37 -0400 Subject: [PATCH 40/42] =?UTF-8?q?=F0=9F=8C=B3=20feat:=20Provision=20Conver?= =?UTF-8?q?sation-Scoped=20Code=20Worktrees=20(#239)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: provision conversation-scoped code worktrees * fix: isolate conversation checkout metadata * docs: clarify isolated conversation checkouts * fix: revalidate conversation checkout sources * fix: preserve synchronous legacy execution startup * fix: harden conversation worktree lifecycle * test: use canonical workspace isolation keys * fix: secure conversation worktree provisioning * fix: preserve isolated workspace lifecycle * fix: harden conversation worktree provisioning * fix: fence worktree setup and credential routing * fix: use kernel-backed provisioning locks * fix: load worktree locking only when provisioned * fix: retain conversation provisioning ownership through recovery * fix: reserve provisioning before launching checkout writers * fix: pin Git provisioning inputs and close instance admission gaps --- packages/code/README.md | 59 +- packages/code/src/cli.ts | 190 +++- packages/code/src/git-snapshot.ts | 343 ++++++ packages/code/src/native-pool.test.ts | 120 ++- packages/code/src/native-pool.ts | 106 +- packages/code/src/native-process.test.ts | 1 + packages/code/src/native-sandbox.test.ts | 31 + packages/code/src/process-lock.test.ts | 61 ++ packages/code/src/process-lock.ts | 59 ++ packages/code/src/protocol.test.ts | 42 +- packages/code/src/protocol.ts | 42 +- packages/code/src/root-identity.ts | 16 + packages/code/src/worker-slots.test.ts | 35 +- packages/code/src/worker.ts | 163 ++- packages/code/src/workspace-cli.test.ts | 32 + packages/code/src/workspace-instances.test.ts | 234 +++++ packages/code/src/workspace-instances.ts | 232 ++++ packages/code/src/workspace-worker.test.ts | 100 ++ packages/code/src/worktrees.test.ts | 988 ++++++++++++++++++ packages/code/src/worktrees.ts | 718 +++++++++++++ service/src/bridge/concurrent-store.test.ts | 69 +- service/src/bridge/router.ts | 1 + service/src/bridge/store.test.ts | 12 +- service/src/bridge/store.ts | 55 +- service/src/bridge/workspace-instance.test.ts | 40 + service/src/bridge/workspace-instance.ts | 17 + service/src/service/programmatic-router.ts | 34 +- .../src/service/programmatic-state.test.ts | 16 + service/src/service/programmatic-state.ts | 12 + service/src/service/replay-state.ts | 2 + service/src/types/service.ts | 4 + service/src/workspace-tools/router.test.ts | 31 + service/src/workspace-tools/router.ts | 19 +- 33 files changed, 3803 insertions(+), 81 deletions(-) create mode 100644 packages/code/src/git-snapshot.ts create mode 100644 packages/code/src/process-lock.test.ts create mode 100644 packages/code/src/process-lock.ts create mode 100644 packages/code/src/workspace-instances.test.ts create mode 100644 packages/code/src/workspace-instances.ts create mode 100644 packages/code/src/worktrees.test.ts create mode 100644 packages/code/src/worktrees.ts create mode 100644 service/src/bridge/workspace-instance.test.ts create mode 100644 service/src/bridge/workspace-instance.ts diff --git a/packages/code/README.md b/packages/code/README.md index 69d85b16..e30b80cb 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -706,8 +706,63 @@ Slots are per machine, not a fleet-wide execution limit. A busy machine does not consume another machine's slots. Requests for the same root remain serialized, including commands started through background tools. Independent checkouts can use different slots; selecting subdirectories beneath one registered parent root -does not create separate scheduling boundaries. Linked Git worktrees share Git -metadata and are not supported by selected-project registration. +does not create separate scheduling boundaries. + +To bind each conversation to an isolated checkout of the selected Git +repository, configure worker-owned conversation worktrees: + +```sh +librechat-code run \ + --worker-dir /projects/LibreChat \ + --workspace-lease-slots 4 \ + --conversation-worktree-root /var/lib/librechat-code/worktrees \ + --conversation-worktree-max 64 \ + --conversation-worktree-clone-timeout-ms 300000 \ + --allow-workspace-writes \ + --allow-workspace-commands +``` + +`LIBRECHAT_CODE_CONVERSATION_WORKTREE_ROOT` and +`LIBRECHAT_CODE_CONVERSATION_WORKTREE_MAX` are the environment equivalents; +`LIBRECHAT_CODE_CONVERSATION_WORKTREE_CLONE_TIMEOUT_MS` controls the bounded +clone budget (five minutes by default, from 30 seconds through 30 minutes). +The storage root must be owner-controlled, must not overlap a registered +workspace, and every registered source must be a Git repository. The worker +creates a deterministic branch in an isolated local checkout for the opaque +conversation identity supplied by LibreChat. Each checkout owns its writable +Git metadata and object storage, without alternates or hardlinks to the source. +Provisioning pins the source Git-directory and object-store identities. It copies +Git data through no-follow, descriptor-relative reads into private staging before +running Git; source hooks and config includes are not used. The clone budget +also bounds this snapshot. Local hardlinks only connect private staging to its +new checkout, never to the source; staging is removed before setup. Source +alternates admitted at worker startup are materialized into independent objects. +Git metadata replacement requires operator recovery, not automatic re-admission. +Host paths remain private. The configured count +is a hard per-machine quota, provisioning is serialized, and operations for one +conversation remain serialized while different conversations may occupy +different lease slots. Recognizable abandoned checkouts without a lifecycle +record are discarded before admission. New provisioning reserves its record +before starting Git or setup; a worker crash leaves that checkout reserved for +operator recovery because child processes might still be running. +Reservations count even when a crash happens before a checkout directory exists. + +Cancellation also covers waiting for the provisioning lock, cloning, and setup. +The worker waits for setup cleanup before releasing the assignment. If cleanup +cannot be confirmed, the checkout stays reserved and fails closed on restart. +A completed checkout with a changed source identity or invalid completion record +is preserved for operator recovery, including any uncommitted work. After stopping +the worker and confirming no executor still uses the checkout, an operator can +archive the affected checkout and its adjacent `.complete` record before retrying. +Also archive any adjacent `.source` staging directory. Pre-release version-1 +completion records are deliberately preserved but not admitted by this version; +they do not contain the required source Git identity binding. + +GitHub App routing is inherited from the operator-admitted source repository; +commands cannot select a different installation by rewriting a worktree remote. +Legacy requests without a conversation identity continue to use the selected +source root. Older Code API deployments do not negotiate the capability, so the +worker omits it until every request path understands the isolation boundary. Admission waits at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP 503, `Retry-After: 1`) means the operation was not assigned or started; wait for diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 96757d57..26c60e9a 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -36,6 +36,9 @@ import { import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; +import { GitWorktreeManager } from './worktrees.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; import { resolveNativeSrtCommandPolicy, serializeNativeSrtCommandPolicy, @@ -61,8 +64,9 @@ import type { WorkspaceToolExecutor } from './workspace.js'; import { BRIDGE_WORKSPACE_NAME_MAX_LENGTH, BridgeProtocolError, - isValidBridgeWorkerCapabilities, - isValidBridgeWorkerId, + isValidBridgeWorkerCapabilities, + isValidBridgeWorkerId, + workspaceIsolationKey, } from './protocol.js'; function workspaceSecurityIdentity( @@ -600,6 +604,32 @@ async function run( ); if (workspaceLeaseSlots > 8) throw new Error('Workspace lease slots cannot exceed 8'); + const conversationWorktreeRoot = + option(args, '--conversation-worktree-root') ?? + process.env.LIBRECHAT_CODE_CONVERSATION_WORKTREE_ROOT?.trim(); + const conversationWorktreeMax = positiveInteger( + 'LIBRECHAT_CODE_CONVERSATION_WORKTREE_MAX', + option(args, '--conversation-worktree-max') ?? + process.env.LIBRECHAT_CODE_CONVERSATION_WORKTREE_MAX, + 64, + ); + if (conversationWorktreeMax > 1024) { + throw new Error('Conversation worktree capacity cannot exceed 1024'); + } + const conversationWorktreeCloneTimeoutMs = positiveInteger( + 'LIBRECHAT_CODE_CONVERSATION_WORKTREE_CLONE_TIMEOUT_MS', + option(args, '--conversation-worktree-clone-timeout-ms') ?? + process.env.LIBRECHAT_CODE_CONVERSATION_WORKTREE_CLONE_TIMEOUT_MS, + 5 * 60_000, + ); + if ( + conversationWorktreeCloneTimeoutMs < 30_000 || + conversationWorktreeCloneTimeoutMs > 30 * 60_000 + ) { + throw new Error( + 'Conversation worktree clone timeout must be between 30000 and 1800000 milliseconds', + ); + } const roots: LocalWorkspaceConfig[] = canonicalWorkerDirectory ? [ { @@ -701,6 +731,16 @@ async function run( 'Concurrent workspace leases require native-srt commands', ); } + if ( + conversationWorktreeRoot && + (!allowWorkspaceCommands || + commandSandboxMode !== 'native-srt' || + workspaceLeaseSlots < 2) + ) { + throw new Error( + 'Conversation worktrees require native-srt commands and at least two workspace lease slots', + ); + } if ( roots.length > 1 && process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() @@ -732,6 +772,14 @@ async function run( ), ) : undefined; + const repositoriesByWorkspace = admittedGitHubRepositories + ? new Map( + roots.map((root) => [ + root.id, + admittedGitHubRepositories.get(root.root), + ]), + ) + : undefined; const localWorkspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: roots, @@ -981,7 +1029,7 @@ async function run( }; const nativeCommandSandbox = allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? roots.length > 1 || workspaceLeaseSlots > 1 + ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot ? new NativeWorkspaceCommandPool( new Map( roots.map(root => [ @@ -1008,6 +1056,105 @@ async function run( incarnationId, }), }); + } + const conversationWorktrees = conversationWorktreeRoot + ? new GitWorktreeManager({ + cloneTimeoutMs: conversationWorktreeCloneTimeoutMs, + maxCount: conversationWorktreeMax, + root: conversationWorktreeRoot, + ...(nativeCommandSandbox instanceof NativeWorkspaceCommandPool + ? { + prepareInstance: async (instance, signal) => { + const setup = environments.find( + (environment) => + environment.definition.name === instance.sourceWorkspaceId, + )?.definition.setup; + if (!setup) return; + if (admittedGitHubRepositories) { + admittedGitHubRepositories.set( + instance.root, + repositoriesByWorkspace?.get(instance.sourceWorkspaceId), + ); + } + const id = internalWorkspaceId(instance.sourceWorkspaceId, instance.id); + await nativeCommandSandbox.registerRoot(id, { + ...nativeOptions, + workspaceIdentity: instance.identity, + workspaceRoot: instance.root, + }); + const result = await nativeCommandSandbox.execute( + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: id, + command: setup.command, + timeoutMs: setup.timeoutMs, + maxOutputBytes: 8192, + }, + signal, + ); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error( + `Environment ${instance.sourceWorkspaceId} setup failed for its conversation worktree`, + ); + } + }, + discardInstance: async (instance) => { + await nativeCommandSandbox.unregisterRoot( + internalWorkspaceId(instance.sourceWorkspaceId, instance.id), + ); + admittedGitHubRepositories?.delete(instance.root); + }, + } + : {}), + sources: new Map( + await Promise.all( + roots.map(async (root) => [ + root.id, + { + root: root.root, + identity: + root.identity ?? + (await captureWorkspaceRootIdentity(root.root)), + }, + ] as const), + ), + ), + }) + : undefined; + let conversationWorkspaceTools: GitWorktreeWorkspaceTools | undefined; + if (conversationWorktrees && workspaceTools) { + if (!(nativeCommandSandbox instanceof NativeWorkspaceCommandPool)) { + throw new Error('Conversation worktrees require a native command pool'); + } + conversationWorkspaceTools = new GitWorktreeWorkspaceTools({ + commandPool: nativeCommandSandbox, + delegate: workspaceTools, + manager: conversationWorktrees, + onResolve(workspaceId, root) { + if (admittedGitHubRepositories) { + admittedGitHubRepositories.set( + root, + repositoriesByWorkspace?.get(workspaceId), + ); + } + }, + sources: new Map( + roots.map((root) => [ + root.id, + { + command: { + ...nativeOptions, + workspaceIdentity: root.identity, + workspaceRoot: root.root, + }, + repositoryInstructions: args.includes('--repository-instructions'), + writable: root.writable ?? false, + }, + ]), + ), + }); + workspaceTools = conversationWorkspaceTools; } if (workspaceTools && environments.length) { workspaceTools = new EnvironmentWorkspaceTools( @@ -1059,6 +1206,7 @@ async function run( if (github.provider && !github.provider.validate) { await github.provider.getCredential(controller.signal); } + await conversationWorktrees?.prepare(); await nativeCommandSandbox?.prepare(); for (const environment of option(args, '--reset-workspace-quarantine') == null ? environments : []) { const setup = environment.definition.setup; @@ -1110,7 +1258,34 @@ async function run( capabilities, workspaceTools, ...(nativeProgrammaticEnabled && nativeCommandSandbox - ? { workspaceProgrammatic: nativeCommandSandbox } + ? { + workspaceProgrammatic: + conversationWorkspaceTools ?? nativeCommandSandbox, + } + : {}), + ...(conversationWorktrees + ? { + workspaceQuarantineResolver: async ( + selectedWorkspaceId: string, + workspaceInstanceId: string, + ) => + workspaceMutationGuard( + defaultWorkspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: await conversationWorktrees.plannedRoot( + selectedWorkspaceId, + workspaceInstanceId, + ), + }), + workerId, + workspaceIsolationKey( + selectedWorkspaceId, + workspaceInstanceId, + ), + incarnationId, + ), + } : {}), ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { @@ -1222,14 +1397,19 @@ async function run( } const resetNativeRoot = option(args, '--reset-workspace-quarantine'); if (resetNativeRoot != null) { + const resetWorkspaceInstance = option( + args, + '--reset-workspace-instance', + ); await worker.refreshCredential(controller.signal); await worker.registerForMaintenance(controller.signal); await worker.resetNativeWorkspace( resetNativeRoot, controller.signal, + resetWorkspaceInstance, ); process.stdout.write( - `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}\n`, + `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}${resetWorkspaceInstance ? ` instance ${resetWorkspaceInstance}` : ''}\n`, ); return; } diff --git a/packages/code/src/git-snapshot.ts b/packages/code/src/git-snapshot.ts new file mode 100644 index 00000000..7c57beda --- /dev/null +++ b/packages/code/src/git-snapshot.ts @@ -0,0 +1,343 @@ +import { constants, close, fstat, read } from 'node:fs'; +import { mkdir, open, opendir, writeFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +const closeFd = promisify(close); +const statFd = promisify(fstat); +const readFd = promisify(read); +let binding: + | Promise<{ + openat: (fd: number, name: string, flags: number) => number; + errno: () => number; + }> + | undefined; + +async function childFd( + parent: number, + name: string +): Promise { + if (!name || name === '.' || name === '..' || name.includes('/')) + throw new Error('Invalid Git metadata entry'); + binding ??= import('koffi').then(({ default: koffi }) => ({ + openat: koffi + .load(null) + .func('int openat(int dirfd, const char *path, int flags)'), + errno: () => koffi.errno(), + })); + const native = await binding; + // Node does not expose O_CLOEXEC. Set it atomically with openat so unrelated + // concurrent executor spawns cannot inherit privileged source descriptors. + if (process.platform !== 'darwin' && process.platform !== 'linux') + throw new Error('Git snapshots require a POSIX host'); + const closeOnExec = process.platform === 'darwin' ? 0x1000000 : 0x80000; + const fd = native.openat( + parent, + name, + constants.O_RDONLY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK | + closeOnExec + ); + if (fd >= 0) return fd; + if (native.errno() === 2) return undefined; // ENOENT on supported POSIX hosts + throw new Error('Git metadata entry is unavailable or is a symbolic link'); +} + +async function withDirectory( + identity: WorkspaceRootIdentity, + operation: (fd: number) => Promise +): Promise { + const handle = await open( + identity.path, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW + ); + try { + const current = await handle.stat({ bigint: true }); + if ( + current.dev.toString() !== identity.dev || + current.ino.toString() !== identity.ino + ) { + throw new Error('Source Git metadata changed after admission'); + } + return await operation(handle.fd); + } finally { + await handle.close(); + } +} + +async function textAt( + parent: number, + name: string, + limit = 16 * 1024 +): Promise { + const fd = await childFd(parent, name); + if (fd == null) return undefined; + try { + const metadata = await statFd(fd); + if (!metadata.isFile() || metadata.size > limit) + throw new Error('Invalid Git metadata file'); + const buffer = Buffer.alloc(metadata.size); + let offset = 0; + while (offset < buffer.length) { + const { bytesRead } = await readFd( + fd, + buffer, + offset, + buffer.length - offset, + offset + ); + if (!bytesRead) throw new Error('Git metadata changed during snapshot'); + offset += bytesRead; + } + return buffer.toString('utf8'); + } finally { + await closeFd(fd); + } +} + +async function alternatesAt(parent: number): Promise { + const fd = await childFd(parent, 'info'); + if (fd == null) return undefined; + try { + if (!(await statFd(fd)).isDirectory()) + throw new Error('Invalid Git objects info directory'); + return await textAt(fd, 'alternates'); + } finally { + await closeFd(fd); + } +} + +/** Copies only regular files/directories through descriptor-relative, no-follow opens. + * Renaming a parent or replacing a child with a symlink never expands the read grant. + * No source config, hooks, object alternates, or executable helpers reach Git. + */ +async function copyEntry( + parent: number, + name: string, + destination: string, + signal: AbortSignal | undefined, + depth = 0 +): Promise { + signal?.throwIfAborted(); + if (depth > 64) + throw new Error('Git metadata nesting exceeds snapshot limit'); + const fd = await childFd(parent, name); + if (fd == null) return; + try { + const metadata = await statFd(fd); + if (metadata.isDirectory()) { + await mkdir(destination, { recursive: true, mode: 0o700 }); + const directory = await opendir(`/dev/fd/${fd}`); + for await (const entry of directory) { + await copyEntry( + fd, + entry.name, + join(destination, entry.name), + signal, + depth + 1 + ); + } + } else if (metadata.isFile()) { + const target = await open(destination, 'w', 0o600); + try { + const buffer = Buffer.alloc(128 * 1024); + let offset = 0; + while (offset < metadata.size) { + signal?.throwIfAborted(); + const { bytesRead } = await readFd( + fd, + buffer, + 0, + Math.min(buffer.length, metadata.size - offset), + offset + ); + if (!bytesRead) + throw new Error('Git metadata changed during snapshot'); + await target.writeFile(buffer.subarray(0, bytesRead)); + offset += bytesRead; + } + } finally { + await target.close(); + } + } else { + throw new Error('Git snapshot requires regular files and directories'); + } + } finally { + await closeFd(fd); + } +} + +export class GitSourceSnapshot { + private constructor( + private readonly source: WorkspaceRootIdentity, + private readonly gitDirectory: WorkspaceRootIdentity, + private readonly common: WorkspaceRootIdentity, + private readonly gitfile: string | undefined, + private readonly commondir: string | undefined, + private readonly objects: Array<{ + identity: WorkspaceRootIdentity; + alternates: string | undefined; + }> + ) {} + + get fingerprint(): string { + return createHash('sha256') + .update( + JSON.stringify([ + this.gitDirectory, + this.common, + this.gitfile, + this.commondir, + this.objects, + ]) + ) + .digest('hex'); + } + + static async admit( + source: WorkspaceRootIdentity + ): Promise { + let gitfile: string | undefined; + await withDirectory(source, async (fd) => { + const git = await childFd(fd, '.git'); + if (git == null) + throw new Error('Source workspace is not a Git repository'); + try { + if (!(await statFd(git)).isDirectory()) + gitfile = await textAt(fd, '.git'); + } finally { + await closeFd(git); + } + }); + if (gitfile != null && !/^gitdir: .+\n?$/.test(gitfile)) + throw new Error('Invalid source Git directory pointer'); + const gitDirectory = await captureWorkspaceRootIdentity( + gitfile == null + ? join(source.path, '.git') + : resolve(source.path, gitfile.slice(8).trim()) + ); + const commondir = await withDirectory(gitDirectory, (fd) => + textAt(fd, 'commondir') + ); + const common = + commondir == null + ? gitDirectory + : await captureWorkspaceRootIdentity( + resolve(gitDirectory.path, commondir.trim()) + ); + const objects: Array<{ + identity: WorkspaceRootIdentity; + alternates: string | undefined; + }> = []; + const visit = async (path: string): Promise => { + const identity = await captureWorkspaceRootIdentity(path); + if (objects.some((entry) => entry.identity.path === identity.path)) + return; + if (objects.length >= 32) + throw new Error('Too many source Git object stores'); + const alternates = await withDirectory(identity, alternatesAt); + objects.push({ identity, alternates }); + for (const alternate of alternates?.split('\n').filter(Boolean) ?? []) { + if (alternate.startsWith('"')) + throw new Error('Quoted Git alternate paths are unsupported'); + await visit(resolve(identity.path, alternate)); + } + }; + await visit(join(common.path, 'objects')); + const admitted = new GitSourceSnapshot( + source, + gitDirectory, + common, + gitfile, + commondir, + objects + ); + await admitted.validate(); + return admitted; + } + + async validate(): Promise { + await withDirectory(this.source, async (fd) => { + if (this.gitfile != null) { + if ((await textAt(fd, '.git')) !== this.gitfile) + throw new Error('Source Git metadata changed after admission'); + } else { + // Compare the directory reached from the admitted root, not just its name. + const child = await childFd(fd, '.git'); + if (child == null) + throw new Error('Source Git metadata changed after admission'); + try { + const current = await promisify(fstat)(child, { + bigint: true, + }); + if ( + current.dev.toString() !== this.gitDirectory.dev || + current.ino.toString() !== this.gitDirectory.ino + ) { + throw new Error('Source Git metadata changed after admission'); + } + } finally { + await closeFd(child); + } + } + }); + await withDirectory(this.gitDirectory, async (fd) => { + if ((await textAt(fd, 'commondir')) !== this.commondir) + throw new Error('Source Git metadata changed after admission'); + }); + await withDirectory(this.common, async () => {}); + for (const entry of this.objects) { + await withDirectory(entry.identity, async (fd) => { + if ((await alternatesAt(fd)) !== entry.alternates) + throw new Error('Source Git alternates changed after admission'); + }); + } + } + + async copyTo(destination: string, signal?: AbortSignal): Promise { + await this.validate(); + await mkdir(destination, { mode: 0o700 }); + await mkdir(join(destination, 'objects'), { mode: 0o700 }); + await mkdir(join(destination, 'refs'), { mode: 0o700 }); + await withDirectory(this.gitDirectory, (fd) => + copyEntry(fd, 'HEAD', join(destination, 'HEAD'), signal) + ); + await withDirectory(this.common, async (fd) => { + for (const name of ['refs', 'packed-refs', 'shallow']) + await copyEntry(fd, name, join(destination, name), signal); + // Kept outside Git's config name; caller may query origin with --no-includes. + const config = await textAt(fd, 'config', 1024 * 1024); + if (config != null) + await writeFile(join(destination, 'source-config'), config, { + mode: 0o600, + }); + }); + for (const entry of [...this.objects].reverse()) { + await withDirectory(entry.identity, async (fd) => { + const directory = await opendir(`/dev/fd/${fd}`); + for await (const child of directory) { + // Object info (notably alternates) never crosses into private staging. + if (child.name === 'pack' || /^[a-f0-9]{2}$/.test(child.name)) { + await copyEntry( + fd, + child.name, + join(destination, 'objects', child.name), + signal + ); + } + } + }); + } + await writeFile( + join(destination, 'config'), + '[core]\nrepositoryformatversion = 0\nbare = true\n', + { mode: 0o600 } + ); + await this.validate(); + signal?.throwIfAborted(); + } +} diff --git a/packages/code/src/native-pool.test.ts b/packages/code/src/native-pool.test.ts index cad30a52..ce25caa4 100644 --- a/packages/code/src/native-pool.test.ts +++ b/packages/code/src/native-pool.test.ts @@ -5,7 +5,7 @@ import { WorkspaceToolError } from './workspace.js'; import type { WorkspaceExecuteCommandRequest } from './protocol.js'; const roots = new Map( - ['a', 'b', 'c'].map((id) => [id, { workspaceRoot: `/fixture/${id}` }]), + ['a', 'b', 'c'].map((id) => [id, { workspaceRoot: `/fixture/${id}` }]) ); const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ protocolVersion: 1, @@ -14,6 +14,36 @@ const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ command: 'fixture', }); +test('failed provisioning roots are removed only after confirmed executor cleanup', async () => { + let failClose = true; + const pool = new NativeWorkspaceCommandPool(roots, 2, () => ({ + async prepare() {}, + async execute() { + throw new Error('not used'); + }, + async close() { + if (failClose) throw new Error('cleanup unconfirmed'); + }, + })); + await pool.registerRoot('instance', { workspaceRoot: '/fixture/instance' }); + // Allocate this executor without dispatching a command. + const controller = new AbortController(); + controller.abort(); + await assert.rejects( + pool.execute(request('instance'), controller.signal), + /cancelled/ + ); + await assert.rejects(pool.unregisterRoot('instance'), /cleanup unconfirmed/); + failClose = false; + await pool.unregisterRoot('instance'); + await assert.rejects(pool.execute(request('instance')), /unavailable/); + await pool.close(); + await assert.rejects( + pool.registerRoot('new', { workspaceRoot: '/fixture/new' }), + /unavailable/ + ); +}); + test('native pool preflights every registered root with bounded concurrency', async () => { const prepared: string[] = []; let active = 0; @@ -22,7 +52,7 @@ test('native pool preflights every registered root with bounded concurrency', as async prepare() { active += 1; peak = Math.max(peak, active); - await new Promise(resolve => setTimeout(resolve, 5)); + await new Promise((resolve) => setTimeout(resolve, 5)); prepared.push(options.workspaceRoot); active -= 1; }, @@ -38,6 +68,90 @@ test('native pool preflights every registered root with bounded concurrency', as await pool.close(); }); +test('native pool admits worker-owned roots after startup', async () => { + const created: string[] = []; + const pool = new NativeWorkspaceCommandPool( + new Map([['primary', { workspaceRoot: '/fixture/primary' }]]), + 2, + (options) => ({ + async prepare() {}, + async close() {}, + async execute(req) { + created.push(options.workspaceRoot); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: req.workspaceId, + stdout: '', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + }) + ); + await pool.registerRoot('conversation', { + workspaceRoot: '/fixture/conversation', + }); + await pool.execute(request('conversation')); + assert.deepEqual(created, ['/fixture/conversation']); + await assert.rejects( + async () => + pool.registerRoot('conversation', { + workspaceRoot: '/fixture/replaced', + }), + { code: 'REGISTRATION_INVALID' } + ); + await pool.close(); +}); + +test('native pool retires a cached executor when a root inode changes', async () => { + let created = 0; + let closed = 0; + const pool = new NativeWorkspaceCommandPool( + new Map([['primary', { workspaceRoot: '/fixture/primary' }]]), + 2, + () => { + created++; + return { + async prepare() {}, + async close() { + closed++; + }, + async execute(req) { + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: req.workspaceId, + stdout: '', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + }; + } + ); + const options = (ino: string) => ({ + workspaceRoot: '/fixture/conversation', + workspaceIdentity: { + path: '/fixture/conversation', + dev: '1', + ino, + }, + }); + await pool.registerRoot('conversation', options('1')); + await pool.execute(request('conversation')); + await pool.registerRoot('conversation', options('2')); + await pool.execute(request('conversation')); + + assert.equal(created, 2); + assert.equal(closed, 1); + await pool.close(); +}); + test('a known-clean executor failure is retired without replaying the command', async () => { let created = 0; let executed = 0; @@ -55,7 +169,7 @@ test('a known-clean executor failure is retired without replaying the command', throw new WorkspaceToolError( 'prepare failed', 'COMMAND_UNAVAILABLE', - false, + false ); return { protocolVersion: 1, diff --git a/packages/code/src/native-pool.ts b/packages/code/src/native-pool.ts index 766409b1..59db3e0a 100644 --- a/packages/code/src/native-pool.ts +++ b/packages/code/src/native-pool.ts @@ -12,9 +12,7 @@ interface Entry { NativeProcessWorkspaceCommandSandbox, 'prepare' | 'execute' | 'close' > & - Partial< - Pick - >; + Partial>; busy: boolean; } @@ -27,12 +25,12 @@ export class NativeWorkspaceCommandPool { private allocation: Promise = Promise.resolve(); private closing = false; constructor( - private readonly roots: ReadonlyMap, + roots: ReadonlyMap, private readonly capacity: number, private readonly createSandbox: ( - options: NativeProcessSandboxOptions, + options: NativeProcessSandboxOptions ) => Entry['sandbox'] = (options) => - new NativeProcessWorkspaceCommandSandbox(options), + new NativeProcessWorkspaceCommandSandbox(options) ) { if ( !Number.isSafeInteger(capacity) || @@ -42,6 +40,74 @@ export class NativeWorkspaceCommandPool { ) { throw new Error('Native executor capacity must be between 1 and 8'); } + this.roots = new Map(roots); + } + + private readonly roots: Map; + + /** Confirm executor cleanup before a failed provisioning attempt removes its root. */ + async unregisterRoot(id: string): Promise { + const pending = this.allocation.then(async () => { + const entry = this.entries.get(id); + if (entry?.busy) { + throw new WorkspaceToolError( + 'Native workspace still executing', + 'COMMAND_UNAVAILABLE' + ); + } + if (entry) await entry.sandbox.close(); + this.entries.delete(id); + this.roots.delete(id); + }); + this.allocation = pending.catch(() => undefined); + await pending; + } + + /** Add or safely replace a worker-owned isolated root. */ + async registerRoot( + id: string, + options: NativeProcessSandboxOptions + ): Promise { + const pending = this.allocation.then(async () => { + if (this.closing) { + throw new WorkspaceToolError( + 'Native workspace unavailable', + 'REGISTRATION_INVALID' + ); + } + const existing = this.roots.get(id); + if (!existing) { + this.roots.set(id, options); + return; + } + if (existing.workspaceRoot !== options.workspaceRoot) { + throw new WorkspaceToolError( + 'Native workspace identity changed', + 'REGISTRATION_INVALID' + ); + } + if ( + existing.workspaceIdentity?.dev === options.workspaceIdentity?.dev && + existing.workspaceIdentity?.ino === options.workspaceIdentity?.ino && + existing.workspaceIdentity?.path === options.workspaceIdentity?.path + ) { + return; + } + const entry = this.entries.get(id); + if (entry?.busy) { + throw new WorkspaceToolError( + 'Native workspace changed during execution', + 'REGISTRATION_INVALID' + ); + } + if (entry) { + await entry.sandbox.close(); + this.entries.delete(id); + } + this.roots.set(id, options); + }); + this.allocation = pending.catch(() => undefined); + await pending; } private allocate(root: string): Promise { @@ -50,23 +116,23 @@ export class NativeWorkspaceCommandPool { if (this.closing || !options) throw new WorkspaceToolError( 'Native workspace unavailable', - 'REGISTRATION_INVALID', + 'REGISTRATION_INVALID' ); let entry = this.entries.get(root); if (entry?.busy) throw new WorkspaceToolError( 'Native workspace already executing', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); if (!entry) { if (this.entries.size >= this.capacity) { const idle = [...this.entries].find( - ([, candidate]) => !candidate.busy, + ([, candidate]) => !candidate.busy ); if (!idle) throw new WorkspaceToolError( 'Native executor capacity reached', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); await idle[1].sandbox.close(); this.entries.delete(idle[0]); @@ -88,7 +154,7 @@ export class NativeWorkspaceCommandPool { if (error instanceof WorkspaceToolError) throw error; throw new WorkspaceToolError( 'Native executor allocation failed', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); }); this.allocation = checked.catch(() => undefined); @@ -112,14 +178,14 @@ export class NativeWorkspaceCommandPool { entry.busy = false; } } - }, - ), + } + ) ); } async execute( request: WorkspaceExecuteCommandRequest, - signal?: AbortSignal, + signal?: AbortSignal ): Promise { const entry = await this.allocate(request.workspaceId); let enteredExecutor = false; @@ -127,7 +193,7 @@ export class NativeWorkspaceCommandPool { if (signal?.aborted) throw new WorkspaceToolError( 'Command cancelled before dispatch', - 'EXECUTION_ABORTED', + 'EXECUTION_ABORTED' ); enteredExecutor = true; return await entry.sandbox.execute(request, signal); @@ -156,7 +222,7 @@ export class NativeWorkspaceCommandPool { async executeProgrammatic( workspaceId: string, request: BridgeWorkspaceProgrammaticRequest, - signal?: AbortSignal, + signal?: AbortSignal ): Promise { const entry = await this.allocate(workspaceId); let enteredExecutor = false; @@ -164,19 +230,19 @@ export class NativeWorkspaceCommandPool { if (signal?.aborted) throw new WorkspaceToolError( 'Programmatic execution cancelled before dispatch', - 'EXECUTION_ABORTED', + 'EXECUTION_ABORTED' ); enteredExecutor = true; if (!entry.sandbox.executeProgrammatic) { throw new WorkspaceToolError( 'Native programmatic executor is unavailable', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); } return await entry.sandbox.executeProgrammatic( workspaceId, request, - signal, + signal ); } catch (error) { if ( @@ -202,7 +268,7 @@ export class NativeWorkspaceCommandPool { this.closing = true; await this.allocation; const results = await Promise.allSettled( - [...this.entries.values()].map((entry) => entry.sandbox.close()), + [...this.entries.values()].map((entry) => entry.sandbox.close()) ); this.entries.clear(); const errors = results diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index b3016937..5f7eedbb 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -146,6 +146,7 @@ test('executor bootstrap excludes bridge credentials and Node injection variable assert.deepEqual(fake.options?.execArgv, []); assert.deepEqual(fake.options?.env, { PATH: '/bin' }); assert.equal(JSON.stringify(fake.messages).includes('secret'), false); + assert.equal('gitSharedObjectDirectory' in fake.messages[0].options, false); await sandbox.close(); }); diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index b1e4c5df..eab72966 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -590,6 +590,37 @@ test('trusted-vm permits unmatched egress and local development sockets', async ]); }); +test('recreated executors never grant reads through a replaced Git object directory', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-code-worktree-')); + const root = join(parent, 'worktree'); + const gitSharedObjectDirectory = join(parent, 'source.git', 'objects'); + await mkdir(join(root, '.git'), { recursive: true }); + await mkdir(gitSharedObjectDirectory, { recursive: true }); + await symlink(gitSharedObjectDirectory, join(root, '.git', 'objects')); + t.after(() => rm(parent, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + + await sandbox.prepare(); + + assert.equal( + fake.config?.filesystem.allowRead?.includes( + await realpath(gitSharedObjectDirectory), + ), + false, + ); + assert.equal( + fake.config?.filesystem.allowWrite?.includes( + await realpath(gitSharedObjectDirectory), + ), + false, + ); +}); + test('provides an isolated scratch directory to commands and restores the host environment', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/process-lock.test.ts b/packages/code/src/process-lock.test.ts new file mode 100644 index 00000000..3594a1a9 --- /dev/null +++ b/packages/code/src/process-lock.test.ts @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { withProcessLock } from './process-lock.js'; + +test( + 'kernel lock survives contention and is released when the owning process crashes', + { timeout: 10_000 }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-lock-')); + const path = join(directory, '.provision.lock'); + const moduleUrl = new URL('./process-lock.js', import.meta.url).href; + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { withProcessLock } from ${JSON.stringify(moduleUrl)}; + await withProcessLock(${JSON.stringify(path)}, async () => { + process.stdout.write('locked'); + await new Promise(() => { setInterval(() => {}, 1000); }); + }); + `, + ], + { stdio: ['ignore', 'pipe', 'pipe'] } + ); + const closed = once(child, 'close'); + t.after(async () => { + child.kill('SIGKILL'); + await closed; + await rm(directory, { recursive: true, force: true }); + }); + await once(child.stdout!, 'data'); + let entered = false; + await assert.rejects( + withProcessLock( + path, + async () => { + entered = true; + }, + AbortSignal.timeout(100) + ) + ); + assert.equal(entered, false); + child.kill('SIGKILL'); + await closed; + await withProcessLock( + path, + async () => { + entered = true; + }, + AbortSignal.timeout(1000) + ); + assert.equal(entered, true); + } +); diff --git a/packages/code/src/process-lock.ts b/packages/code/src/process-lock.ts new file mode 100644 index 00000000..aeca561a --- /dev/null +++ b/packages/code/src/process-lock.ts @@ -0,0 +1,59 @@ +import { constants } from 'node:fs'; +import { open } from 'node:fs/promises'; +import { setTimeout as delay } from 'node:timers/promises'; + +const LOCK_EX = 2; +const LOCK_NB = 4; +const LOCK_UN = 8; +let binding: + | Promise<{ + flock: (fd: number, operation: number) => number; + eagain: number; + errno: () => number; + }> + | undefined; + +async function lockBinding() { + binding ??= import('koffi').then(({ default: koffi }) => ({ + flock: koffi.load(null).func('int flock(int fd, int operation)'), + eagain: koffi.os.errno.EAGAIN, + errno: () => koffi.errno(), + })); + return await binding; +} + +/** Process-lifetime advisory lock; the kernel releases it on crash or restart. */ +export async function withProcessLock( + path: string, + operation: () => Promise, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted(); + if (process.platform !== 'darwin' && process.platform !== 'linux') { + throw new Error('Conversation worktree locking requires a POSIX host'); + } + const native = await lockBinding(); + const handle = await open( + path, + constants.O_CREAT | constants.O_RDWR | constants.O_NOFOLLOW, + 0o600 + ); + try { + for (;;) { + signal?.throwIfAborted(); + if (native.flock(handle.fd, LOCK_EX | LOCK_NB) === 0) break; + const errno = native.errno(); + if (errno !== native.eagain) { + throw new Error( + `Conversation worktree lock failed with errno ${errno}` + ); + } + await delay(50, undefined, { signal }); + } + signal?.throwIfAborted(); + return await operation(); + } finally { + native.flock(handle.fd, LOCK_UN); + await handle.close(); + } +} diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 08cba1c7..5c26c462 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -10,6 +10,7 @@ import { isValidBridgeWorkerId, isWorkspaceToolRequest, isWorkspaceToolResult, + workspaceIsolationKey, } from './protocol.js'; import type { WorkspaceEditFileRequest, @@ -90,6 +91,19 @@ test('bridgeWorkerPath encodes worker-controlled path segments', () => { ); }); +test('workspace isolation keys keep roots and instances in disjoint namespaces', () => { + const instanceId = 'a'.repeat(64); + assert.notEqual( + workspaceIsolationKey(`foo:git-worktree:${instanceId}`), + workspaceIsolationKey('foo', instanceId), + ); + assert.equal(workspaceIsolationKey('foo'), 'foo'); + assert.notEqual( + workspaceIsolationKey('foo'), + workspaceIsolationKey('workspace:foo'), + ); +}); + test('bridge worker IDs reject path, whitespace, and oversized values', () => { assert.equal(isValidBridgeWorkerId('engineering-vm:1'), true); assert.equal(isValidBridgeWorkerId('engineering/vm'), false); @@ -262,6 +276,20 @@ test('workspace file listing accepts only bounded portable requests and results' afterPath: 'src/app.ts', }; assert.equal(isWorkspaceToolRequest(request), true); + assert.equal( + isWorkspaceToolRequest({ + ...request, + workspaceInstanceId: 'a'.repeat(64), + }), + true, + ); + assert.equal( + isWorkspaceToolRequest({ + ...request, + workspaceInstanceId: 'conversation-1', + }), + false, + ); assert.equal( isWorkspaceToolRequest({ ...request, path: '../outside' }), false, @@ -596,7 +624,11 @@ test('workspace capabilities allow per-workspace operation restrictions', () => operations: ['read_file', 'write_file'], workspaces: [ { id: 'readonly', operations: ['read_file'] }, - { id: 'writable', operations: ['read_file', 'write_file'] }, + { + id: 'writable', + operations: ['read_file', 'write_file'], + workspaceInstances: ['git_worktree'], + }, ], }, }; @@ -660,6 +692,7 @@ test('workspace programmatic requests accept only stable input cache identities' max_output_files: 50, max_output_file_bytes: 10_000_000, session_id: 'session-1', + workspace_instance_id: 'a'.repeat(64), files: [ { name: 'main.sh', content: 'echo ready' }, { @@ -672,6 +705,13 @@ test('workspace programmatic requests accept only stable input cache identities' }, }; assert.equal(isBridgeWorkspaceProgrammaticRequest(request), true); + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + ...request, + body: { ...request.body, workspace_instance_id: '../escape' }, + }), + false, + ); assert.equal( isBridgeWorkspaceProgrammaticRequest({ ...request, diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 27220735..c678a1af 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -259,6 +259,16 @@ export function bridgeArtifactMediaType(name: string): string { export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; +/** Collision-free identity shared by scheduling and worker quarantine state. */ +export function workspaceIsolationKey( + workspaceId: string, + instanceId?: string, +): string { + return instanceId === undefined + ? workspaceId + : `\0git-worktree\0${workspaceId}\0${instanceId}`; +} + export type BridgeWorkspaceToolOperation = | 'read_file' | 'search_text' @@ -280,6 +290,8 @@ export interface BridgeWorkspaceDescriptor { instructions?: RepositoryInstructionDescriptor[]; /** Optional per-workspace restriction. Omitted by protocol-v1 readers. */ operations?: BridgeWorkspaceToolOperation[]; + /** Worker-owned isolation schemes available beneath this selected root. */ + workspaceInstances?: ['git_worktree']; environment?: { fingerprint: string; repo?: string; @@ -308,6 +320,7 @@ export interface WorkspaceReadFileRequest { protocolVersion: BridgeProtocolVersion; operation: 'read_file'; workspaceId: string; + workspaceInstanceId?: string; path: string; startLine?: number; maxLines?: number; @@ -350,6 +363,7 @@ export interface WorkspaceSearchTextRequest { protocolVersion: BridgeProtocolVersion; operation: 'search_text'; workspaceId: string; + workspaceInstanceId?: string; query: string; path?: string; maxResults?: number; @@ -374,6 +388,7 @@ export interface WorkspaceListFilesRequest { protocolVersion: BridgeProtocolVersion; operation: 'list_files'; workspaceId: string; + workspaceInstanceId?: string; path?: string; maxResults?: number; /** Continue strictly after this canonical path from a previous page. */ @@ -394,6 +409,7 @@ export interface WorkspaceWriteFileRequest { protocolVersion: BridgeProtocolVersion; operation: 'write_file'; workspaceId: string; + workspaceInstanceId?: string; path: string; content: string; /** False requires an atomic create and refuses to replace an existing file. */ @@ -413,6 +429,7 @@ interface WorkspaceEditFileRequestBase { protocolVersion: BridgeProtocolVersion; operation: 'edit_file'; workspaceId: string; + workspaceInstanceId?: string; path: string; /** Refuses the mutation unless current file bytes match this preview revision. */ expectedBaseSha256?: string; @@ -457,6 +474,7 @@ interface WorkspacePreviewEditRequestBase { protocolVersion: BridgeProtocolVersion; operation: 'preview_edit'; workspaceId: string; + workspaceInstanceId?: string; path: string; } @@ -494,6 +512,7 @@ export interface WorkspaceExecuteCommandRequest { protocolVersion: BridgeProtocolVersion; operation: 'execute_command'; workspaceId: string; + workspaceInstanceId?: string; /** Shell source evaluated only inside the selected sandbox runtime. */ command: string; /** Portable path relative to the workspace root; defaults to '.'. */ @@ -538,6 +557,7 @@ const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'startLine', 'maxLines', @@ -546,6 +566,7 @@ const WORKSPACE_SEARCH_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'query', 'path', 'maxResults', @@ -554,6 +575,7 @@ const WORKSPACE_LIST_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'maxResults', 'afterPath', @@ -562,6 +584,7 @@ const WORKSPACE_WRITE_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'content', 'overwrite', @@ -570,6 +593,7 @@ const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'oldText', 'newText', @@ -580,6 +604,7 @@ const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'oldText', 'newText', @@ -591,6 +616,7 @@ const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'command', 'cwd', 'timeoutMs', @@ -702,6 +728,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceListFileFeatures?: WorkspaceListFileFeature[]; /** PTC languages this Code API can safely route into a selected workspace. */ supportedWorkspaceProgrammaticLanguages?: WorkspaceProgrammaticLanguage[]; + /** Workspace isolation schemes this Code API understands and can route. */ + supportedWorkspaceInstanceTypes?: ['git_worktree']; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -748,6 +776,7 @@ export type BridgeProgrammaticPayloadFile = export interface BridgeWorkspaceProgrammaticBody { language: 'bash'; version: string; + workspace_instance_id?: string; /** Stable identity shared by every replay iteration of one execution. */ execution_id?: string; /** Declared replay tools; zero allows the worker to skip the probe pass. */ @@ -912,6 +941,9 @@ export function isBridgeWorkspaceProgrammaticRequest( typeof body.version !== 'string' || body.version.length === 0 || body.version.length > BRIDGE_RUNTIME_MAX_LENGTH || + (body.workspace_instance_id !== undefined && + (typeof body.workspace_instance_id !== 'string' || + !/^[a-f0-9]{64}$/.test(body.workspace_instance_id))) || (body.execution_id !== undefined && (typeof body.execution_id !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(body.execution_id))) || @@ -1161,7 +1193,10 @@ export function isWorkspaceToolRequest( if ( request.protocolVersion !== BRIDGE_PROTOCOL_VERSION || typeof request.workspaceId !== 'string' || - !isValidBridgeWorkerId(request.workspaceId) + !isValidBridgeWorkerId(request.workspaceId) || + (request.workspaceInstanceId !== undefined && + (typeof request.workspaceInstanceId !== 'string' || + !/^[a-f0-9]{64}$/.test(request.workspaceInstanceId))) ) { return false; } @@ -1614,12 +1649,17 @@ export function isValidBridgeWorkspaceToolCapabilities( key !== 'id' && key !== 'name' && key !== 'operations' && + key !== 'workspaceInstances' && key !== 'instructions' && key !== 'environment', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || workspaceIds.has(descriptor.id) || + (descriptor.workspaceInstances !== undefined && + (!Array.isArray(descriptor.workspaceInstances) || + descriptor.workspaceInstances.length !== 1 || + descriptor.workspaceInstances[0] !== 'git_worktree')) || (descriptor.instructions !== undefined && (!Array.isArray(descriptor.instructions) || descriptor.instructions.length > 1 || !descriptor.instructions.every(isRepositoryInstructionDescriptor))) || (descriptor.environment !== undefined && !isValidCodeEnvironmentDescriptor(descriptor.environment)) || diff --git a/packages/code/src/root-identity.ts b/packages/code/src/root-identity.ts index 3ddfe453..dd3bb2f5 100644 --- a/packages/code/src/root-identity.ts +++ b/packages/code/src/root-identity.ts @@ -6,6 +6,22 @@ export interface WorkspaceRootIdentity { ino: string; } +/** Capture the inode-bound identity of a canonical workspace grant. */ +export async function captureWorkspaceRootIdentity( + root: string, +): Promise { + const canonical = await realpath(root); + const current = await lstat(canonical, { bigint: true }); + if (!current.isDirectory() || current.isSymbolicLink()) { + throw new Error('Workspace root must be a real directory'); + } + return { + path: canonical, + dev: current.dev.toString(), + ino: current.ino.toString(), + }; +} + /** Revalidation of a trusted snapshot, never a fresh grant to a replacement. */ export async function matchesWorkspaceRoot( root: string, diff --git a/packages/code/src/worker-slots.test.ts b/packages/code/src/worker-slots.test.ts index 6e8cc8b1..fda6c659 100644 --- a/packages/code/src/worker-slots.test.ts +++ b/packages/code/src/worker-slots.test.ts @@ -5,6 +5,7 @@ import type { BridgeAssignment, BridgeWorkspaceToolCapabilities, } from './protocol.js'; +import { workspaceIsolationKey } from './protocol.js'; const capabilities: BridgeWorkspaceToolCapabilities = { protocolVersion: 1, @@ -229,7 +230,8 @@ for (const cancelled of [false, true]) { rejectUnexecutedAssignment: () => Promise; executeOwned: () => Promise; }; - internals.activeWorkspaceAssignments.set('a', { + const workspaceKey = workspaceIsolationKey('a'); + internals.activeWorkspaceAssignments.set(workspaceKey, { id: 'previous', done: new Promise(() => {}), }); @@ -257,7 +259,10 @@ for (const cancelled of [false, true]) { controller.signal, ); assert.equal(rejected, true); - assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'previous'); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceKey)?.id, + 'previous', + ); }); } @@ -281,7 +286,8 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b executeOwned: (assignment: BridgeAssignment) => Promise; }; let release!: () => void; - internals.activeWorkspaceAssignments.set('a', { + const workspaceKey = workspaceIsolationKey('a'); + internals.activeWorkspaceAssignments.set(workspaceKey, { id: 'previous', done: new Promise((resolve) => { release = resolve; @@ -290,7 +296,10 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b let executed = false; internals.executeOwned = async (assignment) => { executed = true; - assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'next'); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceKey)?.id, + 'next', + ); assert.ok(assignment.remainingMs! < 1000 && assignment.remainingMs! > 0); }; const pending = worker.executeAndSettle({ @@ -306,7 +315,7 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b } as BridgeAssignment); await new Promise((resolve) => setTimeout(resolve, 5)); assert.equal(executed, false); - internals.activeWorkspaceAssignments.delete('a'); + internals.activeWorkspaceAssignments.delete(workspaceKey); release(); await pending; assert.equal(executed, true); @@ -332,14 +341,19 @@ test('programmatic work on an independent workspace bypasses another root cleanu >; executeOwned: (assignment: BridgeAssignment) => Promise; }; - internals.activeWorkspaceAssignments.set('a', { + const workspaceAKey = workspaceIsolationKey('a'); + const workspaceBKey = workspaceIsolationKey('b'); + internals.activeWorkspaceAssignments.set(workspaceAKey, { id: 'previous', done: new Promise(() => {}), }); let executed = false; internals.executeOwned = async () => { executed = true; - assert.equal(internals.activeWorkspaceAssignments.get('b')?.id, 'next'); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceBKey)?.id, + 'next', + ); }; await worker.executeAndSettle({ assignmentId: 'next', @@ -357,6 +371,9 @@ test('programmatic work on an independent workspace bypasses another root cleanu }, } as BridgeAssignment); assert.equal(executed, true); - assert.equal(internals.activeWorkspaceAssignments.has('b'), false); - assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'previous'); + assert.equal(internals.activeWorkspaceAssignments.has(workspaceBKey), false); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceAKey)?.id, + 'previous', + ); }); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 697a3c55..2a5896fd 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -7,6 +7,7 @@ import { bridgeWorkerPath, isBridgeWorkspaceProgrammaticRequest, isWorkspaceToolResult, + workspaceIsolationKey, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; @@ -54,6 +55,13 @@ export interface BridgeWorkerOptions { workspaceMutationQuarantine?: WorkspaceMutationQuarantine; /** Required per-root durable guards when opting into concurrent workspace leases. */ workspaceQuarantines?: ReadonlyMap; + /** Resolve a durable guard for a worker-owned dynamic workspace instance. */ + workspaceQuarantineResolver?: ( + workspaceId: string, + workspaceInstanceId: string, + ) => + | WorkspaceMutationQuarantine + | Promise; leaseWaitMs?: number; leaseTransportGraceMs?: number; registrationTransportTimeoutMs?: number; @@ -208,7 +216,14 @@ function workspaceCapabilitiesMatch( operation === executor.workspaces[index]?.operations?.[operationIndex], ) ?? - executor.workspaces[index]?.operations == null), + executor.workspaces[index]?.operations == null) && + workspace.workspaceInstances?.length === + executor.workspaces[index]?.workspaceInstances?.length && + (workspace.workspaceInstances?.every( + (instanceType, instanceIndex) => + instanceType === + executor.workspaces[index]?.workspaceInstances?.[instanceIndex], + ) ?? executor.workspaces[index]?.workspaceInstances == null), ) ); } @@ -223,7 +238,8 @@ function registrationCompatibleCapabilities( (operation) => operation === 'read_file' || operation === 'search_text', ) && workspaceTools.workspaces.every( - (workspace) => workspace.operations == null, + (workspace) => + workspace.operations == null && workspace.workspaceInstances == null, )) ) { return capabilities; @@ -244,7 +260,11 @@ function registrationCompatibleCapabilities( ) { return []; } - const { operations: _operations, ...compatibleWorkspace } = workspace; + const { + operations: _operations, + workspaceInstances: _workspaceInstances, + ...compatibleWorkspace + } = workspace; return [{ ...compatibleWorkspace, ...(workspace.environment ? { environment: { ...workspace.environment, actions: [] }, } : {}) }]; @@ -329,6 +349,12 @@ function supportedWorkspaceCapabilities( return workspaceOperations.length === 0 ? [] : [{ ...workspace, + ...(workspace.workspaceInstances != null && + registration.supportedWorkspaceInstanceTypes?.includes( + 'git_worktree', + ) + ? { workspaceInstances: workspace.workspaceInstances } + : { workspaceInstances: undefined }), ...(workspace.operations ? { operations: workspaceOperations } : {}), ...(workspace.environment && !workspaceOperations.includes('execute_command') ? { environment: { ...workspace.environment, actions: [] }, @@ -479,12 +505,23 @@ export class BridgeWorker { operation === 'execute_command', ) === true && options.workspaceMutationQuarantine == null && - options.workspaceQuarantines == null + options.workspaceQuarantines == null && + options.workspaceQuarantineResolver == null ) { throw new BridgeProtocolError( 'Workspace mutation capabilities require durable quarantine storage', ); } + if ( + options.capabilities.workspaceTools?.workspaces.some( + (root) => (root.workspaceInstances?.length ?? 0) > 0, + ) && + options.workspaceQuarantineResolver == null + ) { + throw new BridgeProtocolError( + 'Workspace instance capabilities require a durable quarantine resolver', + ); + } if ((options.capabilities.workspaceLeaseSlots ?? 1) > 1) { if ( options.capabilities.requiresReadyConfirmation !== true || @@ -785,14 +822,25 @@ export class BridgeWorker { async resetNativeWorkspace( workspaceId: string, signal?: AbortSignal, + workspaceInstanceId?: string, ): Promise { - const guard = this.options.workspaceQuarantines?.get(workspaceId); + const workspace = this.options.capabilities.workspaceTools?.workspaces.find( + (root) => root.id === workspaceId, + ); + const key = workspaceIsolationKey(workspaceId, workspaceInstanceId); + const guard = + workspaceInstanceId == null + ? this.options.workspaceQuarantines?.get(workspaceId) + : await this.options.workspaceQuarantineResolver?.( + workspaceId, + workspaceInstanceId, + ); if ( !guard || this.activeWorkspaceAssignments.size > 0 || - !this.options.capabilities.workspaceTools?.workspaces.some( - (root) => root.id === workspaceId, - ) + workspace == null || + (workspaceInstanceId != null && + workspace.workspaceInstances?.includes('git_worktree') !== true) ) { throw new BridgeProtocolError( 'Native workspace reset requires an idle registered root', @@ -808,14 +856,14 @@ export class BridgeWorker { { protocolVersion: BRIDGE_PROTOCOL_VERSION, incarnationId: this.incarnationId, - runtimeSessionId: `native-workspace:${workspaceId}`, + runtimeSessionId: `native-workspace:${key}`, confirmDiscarded: true, }, this.options.resetTransportTimeoutMs ?? DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, signal, ); - this.quarantinedWorkspaces.delete(workspaceId); + this.quarantinedWorkspaces.delete(key); } async lease( @@ -1340,8 +1388,18 @@ export class BridgeWorker { private workspaceGuard( assignment: BridgeAssignment, - ): WorkspaceMutationQuarantine | undefined { - const workspaceId = this.assignmentWorkspaceId(assignment); + ): + | WorkspaceMutationQuarantine + | Promise + | undefined { + const workspaceId = this.assignmentBaseWorkspaceId(assignment); + const instanceId = this.assignmentWorkspaceInstanceId(assignment); + if (workspaceId != null && instanceId != null) { + return this.options.workspaceQuarantineResolver?.( + workspaceId, + instanceId, + ); + } return workspaceId != null ? (this.options.workspaceQuarantines?.get(workspaceId) ?? this.options.workspaceMutationQuarantine) @@ -1350,6 +1408,15 @@ export class BridgeWorker { private assignmentWorkspaceId( assignment: BridgeAssignment, + ): string | undefined { + const workspaceId = this.assignmentBaseWorkspaceId(assignment); + if (workspaceId == null) return undefined; + const instanceId = this.assignmentWorkspaceInstanceId(assignment); + return workspaceIsolationKey(workspaceId, instanceId); + } + + private assignmentBaseWorkspaceId( + assignment: BridgeAssignment, ): string | undefined { if ( assignment.executionKind === 'workspace_tool' && @@ -1366,11 +1433,33 @@ export class BridgeWorker { return undefined; } + private assignmentWorkspaceInstanceId( + assignment: BridgeAssignment, + ): string | undefined { + if ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ) { + return assignment.request.workspaceInstanceId; + } + if ( + assignment.executionKind === 'workspace_programmatic' && + isBridgeWorkspaceProgrammaticRequest(assignment.request) + ) { + return assignment.request.body.workspace_instance_id; + } + return undefined; + } + private async executeOwned( assignment: BridgeAssignment, signal?: AbortSignal, ): Promise { - const guard = this.workspaceGuard(assignment); + const unresolvedGuard = this.workspaceGuard(assignment); + const guard = + unresolvedGuard instanceof Promise + ? await unresolvedGuard + : unresolvedGuard; if (signal?.aborted === true) { throw signal.reason instanceof Error ? signal.reason @@ -1484,12 +1573,17 @@ export class BridgeWorker { throw new BridgeProtocolError('Invalid workspace tool request'); } const workspaceRequest = assignment.request; + const workspaceKey = this.assignmentWorkspaceId(assignment)!; try { - if (this.quarantinedWorkspaces.has(workspaceRequest.workspaceId)) { + if (this.quarantinedWorkspaces.has(workspaceKey)) { throw new Error('Workspace requires an explicit quarantine reset'); } - if (this.options.workspaceQuarantines != null) + if ( + this.options.workspaceQuarantines != null || + this.options.workspaceQuarantineResolver != null + ) { await guard?.assertAvailable(); + } } catch (error) { throw new BridgeWorkspaceQuarantinedError( 'Workspace is quarantined', @@ -1513,6 +1607,14 @@ export class BridgeWorker { if (workspace == null) { throw new BridgeProtocolError('Workspace is not advertised'); } + if ( + workspaceRequest.workspaceInstanceId != null && + workspace.workspaceInstances?.includes('git_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Workspace instance type is not advertised', + ); + } if ( workspace.operations != null && !workspace.operations.includes(workspaceRequest.operation) @@ -1575,7 +1677,7 @@ export class BridgeWorker { if (isMutation) { this.mutationGuardArmed = true; try { - this.armedWorkspaces.add(workspaceRequest.workspaceId); + this.armedWorkspaces.add(workspaceKey); await guard!.arm( `Workspace mutation ${workspaceRequest.operation} is pending settlement`, assignment.assignmentId, @@ -1630,12 +1732,17 @@ export class BridgeWorker { 'Worker does not provide valid selected-workspace programmatic execution', ); } + const workspaceKey = this.assignmentWorkspaceId(assignment)!; try { - if (this.quarantinedWorkspaces.has(workspaceId)) { + if (this.quarantinedWorkspaces.has(workspaceKey)) { throw new Error('Workspace requires an explicit quarantine reset'); } - if (this.options.workspaceQuarantines != null) + if ( + this.options.workspaceQuarantines != null || + this.options.workspaceQuarantineResolver != null + ) { await guard?.assertAvailable(); + } } catch (error) { throw new BridgeWorkspaceQuarantinedError( 'Workspace is quarantined', @@ -1657,9 +1764,17 @@ export class BridgeWorker { 'Selected-workspace programmatic execution is not advertised', ); } + if ( + assignment.request.body.workspace_instance_id != null && + workspace.workspaceInstances?.includes('git_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Workspace instance type is not advertised', + ); + } this.mutationGuardArmed = true; try { - this.armedWorkspaces.add(workspaceId); + this.armedWorkspaces.add(workspaceKey); await guard!.arm( 'Workspace programmatic execution is pending settlement', assignment.assignmentId, @@ -2079,11 +2194,15 @@ export class BridgeWorker { ): Promise { if (runtimeSessionId == null) { try { - await ( + const unresolvedGuard = assignment == null ? this.options.workspaceMutationQuarantine - : this.workspaceGuard(assignment) - )?.quarantine(message, cause, assignment?.assignmentId); + : this.workspaceGuard(assignment); + const guard = + unresolvedGuard instanceof Promise + ? await unresolvedGuard + : unresolvedGuard; + await guard?.quarantine(message, cause, assignment?.assignmentId); return new BridgeWorkspaceQuarantinedError(message, cause); } catch (error) { return new BridgeWorkspaceQuarantinedError( diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index c1bc7fce..1de68818 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -115,6 +115,38 @@ test('CLI supports native SRT by default and validates explicit runtime mode', a assert.match(noWorkspace.stderr, /require.*registered directory/i); }); +test('CLI requires concurrent native slots for conversation worktrees', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-conversation-')); + const workspaceRoot = join(root, 'workspace'); + const worktreeRoot = join(root, 'worktrees'); + await mkdir(workspaceRoot); + t.after(() => rm(root, { recursive: true, force: true })); + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspaceRoot, + '--allow-workspace-writes', + '--allow-workspace-commands', + '--conversation-worktree-root', + worktreeRoot, + ], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /at least two workspace lease slots/i); +}); + test('CLI advertises explicitly enabled writes without exposing the workspace root', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-cli-')); const workspaceRoot = join(root, ' '); diff --git a/packages/code/src/workspace-instances.test.ts b/packages/code/src/workspace-instances.test.ts new file mode 100644 index 00000000..a1cd6859 --- /dev/null +++ b/packages/code/src/workspace-instances.test.ts @@ -0,0 +1,234 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { GitWorktreeWorkspaceTools } from './workspace-instances.js'; +import { LocalWorkspaceTools } from './workspace.js'; +import { GitWorktreeManager } from './worktrees.js'; +import { readRepositoryInstructions } from './instructions.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; + +const execFileAsync = promisify(execFile); + +async function repository(): Promise<{ parent: string; root: string }> { + const parent = await mkdtemp(join(tmpdir(), 'librechat-instance-tools-')); + const root = join(parent, 'source'); + await execFileAsync('git', ['init', root]); + await writeFile(join(root, 'README.md'), 'source\n'); + await writeFile(join(root, 'AGENTS.md'), 'follow repository rules\n'); + await execFileAsync('git', ['-C', root, 'add', 'README.md', 'AGENTS.md']); + await execFileAsync('git', [ + '-C', + root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'initial', + ]); + return { parent, root: await realpath(root) }; +} + +async function source(root: string) { + return { root, identity: await captureWorkspaceRootIdentity(root) }; +} + +test('routes each conversation to its own writable Git worktree', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: true, + workspaces: [{ id: 'primary', root: fixture.root, writable: true }], + }); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager, + sources: new Map([ + ['primary', { repositoryInstructions: true, writable: true }], + ]), + }); + const firstId = 'a'.repeat(64); + const secondId = 'b'.repeat(64); + + assert.deepEqual(tools.capabilities.workspaces[0]?.workspaceInstances, [ + 'git_worktree', + ]); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: firstId, + path: 'conversation.txt', + content: 'first', + }); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: secondId, + path: 'conversation.txt', + content: 'second', + }); + + const first = await manager.resolve('primary', firstId); + const second = await manager.resolve('primary', secondId); + assert.equal( + await readFile(join(first.root, 'conversation.txt'), 'utf8'), + 'first', + ); + assert.equal( + await readFile(join(second.root, 'conversation.txt'), 'utf8'), + 'second', + ); + await assert.rejects(readFile(join(fixture.root, 'conversation.txt')), { + code: 'ENOENT', + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + workspaceInstanceId: firstId, + path: 'conversation.txt', + }); + assert.equal(result.workspaceId, 'primary'); + assert.equal(result.operation, 'read_file'); + assert.equal(result.content, 'first'); + + await writeFile(join(fixture.root, 'AGENTS.md'), 'local repository rules\n'); + const instructions = await readRepositoryInstructions(fixture.root); + assert.ok(instructions); + const instructionResult = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + workspaceInstanceId: firstId, + path: instructions.descriptor.path, + instructionSha256: instructions.descriptor.sha256, + }); + assert.equal(instructionResult.operation, 'read_file'); + assert.equal(instructionResult.content, 'local repository rules\n'); +}); + +test('reports provisioning rejection as an atomic workspace error', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const delegate = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root: fixture.root, writable: true }], + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager: new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }), + sources: new Map([ + ['primary', { repositoryInstructions: false, writable: true }], + ]), + }); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: 'a'.repeat(64), + path: 'first.txt', + content: 'first', + }); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: 'b'.repeat(64), + path: 'second.txt', + content: 'second', + }), + { + code: 'WRITE_UNAVAILABLE', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }, + ); +}); + +test('rebuilds file executors only after operator recovery releases the reservation', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const delegate = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root: fixture.root, writable: true }], + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager, + sources: new Map([ + ['primary', { repositoryInstructions: false, writable: true }], + ]), + }); + const instanceId = 'c'.repeat(64); + const request = { + protocolVersion: 1 as const, + operation: 'read_file' as const, + workspaceId: 'primary', + workspaceInstanceId: instanceId, + path: 'README.md', + }; + await tools.execute(request); + const initial = await manager.resolve('primary', instanceId); + await rm(initial.root, { recursive: true, force: true }); + + await assert.rejects(tools.execute(request), { + code: 'WRITE_UNAVAILABLE', + }); + await assert.rejects(tools.execute(request), /capacity is exhausted/); + // Missing checkout directories do not prove an interrupted writer is gone. + // Simulate operator recovery after confirming there is no active executor. + await rm(`${initial.root}.complete`); + const recovered = await tools.execute(request); + assert.equal(recovered.operation, 'read_file'); + assert.equal(recovered.content, 'source'); +}); + +test('leaves legacy requests on the selected source workspace', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const delegate = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root: fixture.root, writable: false }], + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager: new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }), + sources: new Map([ + ['primary', { repositoryInstructions: false, writable: false }], + ]), + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }); + assert.equal(result.operation, 'read_file'); + assert.equal(result.content, 'source'); +}); diff --git a/packages/code/src/workspace-instances.ts b/packages/code/src/workspace-instances.ts new file mode 100644 index 00000000..68e76a10 --- /dev/null +++ b/packages/code/src/workspace-instances.ts @@ -0,0 +1,232 @@ +import { createHash } from 'node:crypto'; + +import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { GitWorktreeManager } from './worktrees.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import type { + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandRequest, + WorkspaceToolRequest, + WorkspaceToolResult, +} from './protocol.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; + +interface WorkspaceInstanceSource { + command?: NativeProcessSandboxOptions; + repositoryInstructions: boolean; + writable: boolean; +} + +export interface GitWorktreeWorkspaceToolsOptions { + commandPool?: NativeWorkspaceCommandPool; + delegate: WorkspaceToolExecutor; + manager: GitWorktreeManager; + onResolve?: (workspaceId: string, root: string) => void; + sources: ReadonlyMap; +} + +export function internalWorkspaceId(workspaceId: string, instanceId: string): string { + return `instance-${createHash('sha256') + .update(`${workspaceId}\0${instanceId}`) + .digest('hex')}`; +} + +function publicResult( + result: WorkspaceToolResult, + workspaceId: string, +): WorkspaceToolResult { + return { ...result, workspaceId }; +} + +/** Resolve an opaque conversation binding into an isolated Git worktree. */ +export class GitWorktreeWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + private readonly executors = new Map< + string, + { identity: WorkspaceRootIdentity; value: Promise } + >(); + + constructor(private readonly options: GitWorktreeWorkspaceToolsOptions) { + this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; + this.capabilities = { + ...options.delegate.capabilities, + workspaces: options.delegate.capabilities.workspaces.map((workspace) => ({ + ...workspace, + ...(options.sources.has(workspace.id) + ? { workspaceInstances: ['git_worktree' as const] } + : {}), + })), + }; + } + + private async executor( + workspaceId: string, + instanceId: string, + signal?: AbortSignal, + ): Promise<{ + executor: LocalWorkspaceTools; + identity: WorkspaceRootIdentity; + internalId: string; + root: string; + }> { + const source = this.options.sources.get(workspaceId); + if (!source) { + throw new WorkspaceToolError( + 'Workspace does not allow conversation worktrees', + 'INVALID_REQUEST', + ); + } + let instance; + try { + instance = await this.options.manager.resolve( + workspaceId, + instanceId, + signal, + ); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + if ( + signal?.aborted || + (error instanceof Error && error.name === 'AbortError') + ) { + throw new WorkspaceToolError( + 'Conversation worktree provisioning aborted', + 'EXECUTION_ABORTED', + ); + } + throw new WorkspaceToolError( + error instanceof Error + ? error.message + : 'Conversation worktree provisioning failed', + 'WRITE_UNAVAILABLE', + ); + } + this.options.onResolve?.(workspaceId, instance.root); + const internalId = internalWorkspaceId(workspaceId, instanceId); + const key = `${workspaceId}\0${instanceId}`; + let cached = this.executors.get(key); + if ( + cached == null || + cached.identity.dev !== instance.identity.dev || + cached.identity.ino !== instance.identity.ino || + cached.identity.path !== instance.identity.path + ) { + cached = { + identity: instance.identity, + value: LocalWorkspaceTools.create({ + repositoryInstructions: source.repositoryInstructions, + workspaces: [ + { + id: internalId, + identity: instance.identity, + root: instance.root, + writable: source.writable, + }, + ], + }), + }; + this.executors.set(key, cached); + } + return { + executor: await cached.value, + identity: instance.identity, + internalId, + root: instance.root, + }; + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + ): Promise { + if (!request.workspaceInstanceId) { + return await this.options.delegate.execute(request, signal); + } + if ( + request.operation === 'read_file' && + request.instructionSha256 !== undefined + ) { + const { workspaceInstanceId: _workspaceInstanceId, ...sourceRequest } = + request; + return await this.options.delegate.execute(sourceRequest, signal); + } + const { workspaceInstanceId, ...baseRequest } = request; + const source = this.options.sources.get(request.workspaceId); + const resolved = await this.executor( + request.workspaceId, + workspaceInstanceId, + signal, + ); + const isolatedRequest = { + ...baseRequest, + workspaceId: resolved.internalId, + } as WorkspaceToolRequest; + if (request.operation === 'execute_command') { + if (!source?.command || !this.options.commandPool) { + throw new WorkspaceToolError( + 'Conversation worktree commands are unavailable', + 'COMMAND_DISABLED', + ); + } + await this.options.commandPool.registerRoot(resolved.internalId, { + ...source.command, + workspaceIdentity: resolved.identity, + workspaceRoot: resolved.root, + }); + return publicResult( + await this.options.commandPool.execute( + isolatedRequest as WorkspaceExecuteCommandRequest, + signal, + ), + request.workspaceId, + ); + } + return publicResult( + await resolved.executor.execute(isolatedRequest, signal), + request.workspaceId, + ); + } + + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + const instanceId = request.body.workspace_instance_id; + if (!instanceId) { + if (!this.options.commandPool) { + throw new WorkspaceToolError( + 'Workspace programmatic execution is unavailable', + 'COMMAND_DISABLED', + ); + } + return await this.options.commandPool.executeProgrammatic( + workspaceId, + request, + signal, + ); + } + const source = this.options.sources.get(workspaceId); + if (!source?.command || !this.options.commandPool) { + throw new WorkspaceToolError( + 'Conversation worktree programmatic execution is unavailable', + 'COMMAND_DISABLED', + ); + } + const resolved = await this.executor(workspaceId, instanceId, signal); + await this.options.commandPool.registerRoot(resolved.internalId, { + ...source.command, + workspaceIdentity: resolved.identity, + workspaceRoot: resolved.root, + }); + return await this.options.commandPool.executeProgrammatic( + resolved.internalId, + request, + signal, + ); + } +} diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index d97735dc..96a5d1f0 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -7,6 +7,19 @@ import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; +test('instance advertisement requires a guard resolver even for reads and with base guards', () => { + for (const operation of ['read_file', 'write_file'] as const) { + const workspaceTools = { protocolVersion: 1 as const, operations: [operation], workspaces: [{ id: 'primary', workspaceInstances: ['git_worktree'] as ['git_worktree'] }] }; + assert.throws(() => new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools }, + workspaceQuarantines: new Map([['primary', mutationQuarantine()]]), + workspaceTools: { capabilities: workspaceTools, async execute() { throw new Error('must not execute'); } }, + }), /instance capabilities require a durable quarantine resolver/); + } +}); + test('worker clears named actions when command execution is not negotiated', async () => { const workspaceTools = { protocolVersion: 1 as const, @@ -974,6 +987,93 @@ test('worker executes a workspace tool assignment locally without acquiring a sa }); }); +test('worker isolates dynamic worktree guards from collision-shaped root IDs', async () => { + const instanceId = 'a'.repeat(64); + const collisionRoot = `foo:git-worktree:${instanceId}`; + const lifecycle: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [ + { id: 'foo', workspaceInstances: ['git_worktree'] as ['git_worktree'] }, + { id: collisionRoot }, + ], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute(request) { + return { + protocolVersion: 1, + operation: 'write_file', + workspaceId: request.workspaceId, + path: 'result.txt', + created: true, + bytesWritten: 2, + }; + }, + }, + workspaceQuarantines: new Map([ + [ + collisionRoot, + mutationQuarantine( + undefined, + () => lifecycle.push('root:arm'), + () => lifecycle.push('root:clear'), + ), + ], + ]), + workspaceQuarantineResolver: async () => + mutationQuarantine( + undefined, + () => lifecycle.push('instance:arm'), + () => lifecycle.push('instance:clear'), + ), + fetchImpl: async () => + Response.json({ protocolVersion: 1, accepted: true }), + }); + const assignment = (workspaceId: string, suffix: string) => ({ + protocolVersion: 1 as const, + assignmentId: `assignment-${suffix}`, + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: `lease-token-that-is-long-enough-${suffix}`, + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool' as const, + request: { + protocolVersion: 1 as const, + operation: 'write_file' as const, + workspaceId, + path: 'result.txt', + content: 'ok', + ...(workspaceId === 'foo' ? { workspaceInstanceId: instanceId } : {}), + }, + }); + + await worker.executeAndSettle(assignment('foo', 'instance')); + await worker.executeAndSettle(assignment(collisionRoot, 'root')); + + assert.deepEqual(lifecycle, [ + 'instance:arm', + 'instance:clear', + 'root:arm', + 'root:clear', + ]); +}); + test('worker executes programmatic Bash in the selected workspace and preserves its fence', async () => { const programmaticRequests: object[] = []; const quarantineEvents: string[] = []; diff --git a/packages/code/src/worktrees.test.ts b/packages/code/src/worktrees.test.ts new file mode 100644 index 00000000..c353c24e --- /dev/null +++ b/packages/code/src/worktrees.test.ts @@ -0,0 +1,988 @@ +import assert from 'node:assert/strict'; +import { execFile, spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rename, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { promisify } from 'node:util'; +import test from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { GitWorktreeManager } from './worktrees.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { GitSourceSnapshot } from './git-snapshot.js'; + +const execFileAsync = promisify(execFile); + +test('aborting private staging releases its reservation only after cleanup', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const controller = new AbortController(); + const copy = GitSourceSnapshot.prototype.copyTo; + const mocked = t.mock.method( + GitSourceSnapshot.prototype, + 'copyTo', + async function ( + this: GitSourceSnapshot, + destination: string, + signal?: AbortSignal + ) { + await copy.call(this, destination, signal); + controller.abort(); + } + ); + const id = 'b'.repeat(64); + const path = await manager.plannedRoot('primary', id); + await assert.rejects(manager.resolve('primary', id, controller.signal), { + name: 'AbortError', + }); + for (const name of [path, `${path}.source`, `${path}.complete`]) + await assert.rejects(stat(name), { code: 'ENOENT' }); + mocked.mock.restore(); + assert.ok(await manager.resolve('primary', 'c'.repeat(64))); +}); + +test('setup cannot publish a checkout that redirects its Git metadata', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async (instance) => { + await writeFile( + join(instance.root, '.git', 'commondir'), + join(fixture.root, '.git') + ); + }, + }); + await assert.rejects( + manager.resolve('primary', 'd'.repeat(64)), + /must not redirect/ + ); +}); + +test('private snapshots preserve SHA-256 repositories and the configured origin', async (t) => { + const fixture = await repository('sha256'); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + await git( + fixture.root, + 'remote', + 'add', + 'origin', + 'https://github.com/example/repo.git' + ); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const instance = await manager.resolve('primary', 'a'.repeat(64)); + assert.equal( + await git(instance.root, 'rev-parse', '--show-object-format'), + 'sha256' + ); + assert.equal( + await git(instance.root, 'remote', 'get-url', 'origin'), + 'https://github.com/example/repo.git' + ); + assert.equal( + await readFile(join(instance.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +test('sidecar-only crash reservations consume quota after restart', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const admitted = await source(fixture.root); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', admitted]]), + }; + const manager = new GitWorktreeManager(options); + const path = await manager.plannedRoot('primary', 'a'.repeat(64)); + await mkdir(join(path, '..'), { recursive: true }); + await writeFile( + `${path}.complete`, + JSON.stringify({ + version: 2, + source: admitted.identity, + sourceGit: (await GitSourceSnapshot.admit(admitted.identity)).fingerprint, + provisioningFailed: true, + }) + ); + const restarted = new GitWorktreeManager(options); + await assert.rejects( + restarted.resolve('primary', 'b'.repeat(64)), + /capacity is exhausted/ + ); + await assert.rejects( + restarted.resolve('primary', 'a'.repeat(64)), + /operator recovery required/ + ); + await assert.rejects(stat(path), { code: 'ENOENT' }); +}); + +test('linked-worktree sources retain their own HEAD and admitted common objects', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const linked = join(fixture.parent, 'linked'); + await git(fixture.root, 'worktree', 'add', '-b', 'linked', linked); + await writeFile(join(linked, 'linked.txt'), 'linked\n'); + await git(linked, 'add', 'linked.txt'); + await git( + linked, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'linked' + ); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(linked)]]), + }); + const instance = await manager.resolve('primary', 'e'.repeat(64)); + assert.equal( + await readFile(join(instance.root, 'linked.txt'), 'utf8'), + 'linked\n' + ); +}); + +test('restart cannot rebind a completed checkout to replacement source Git metadata', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }; + const id = 'f'.repeat(64); + const instance = await new GitWorktreeManager(options).resolve('primary', id); + await writeFile(join(instance.root, 'uncommitted.txt'), 'keep'); + await rename(join(fixture.root, '.git'), join(fixture.root, '.git-original')); + await git(fixture.root, 'init'); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', id), + /source identity changed/ + ); + assert.equal( + await readFile(join(instance.root, 'uncommitted.txt'), 'utf8'), + 'keep' + ); +}); + +test('rejects source Git redirection without changing the admitted working-directory inode', async (t) => { + const fixture = await repository(); + const other = await repository(); + t.after(() => + Promise.all( + [fixture, other].map(({ parent }) => + rm(parent, { recursive: true, force: true }) + ) + ) + ); + const manager = new GitWorktreeManager({ + maxCount: 2, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + await manager.prepare(); + await rename(join(fixture.root, '.git'), join(fixture.root, '.git-original')); + await writeFile( + join(fixture.root, '.git'), + `gitdir: ${join(other.root, '.git')}\n` + ); + await assert.rejects( + manager.resolve('primary', 'c'.repeat(64)), + /Git metadata changed/ + ); +}); + +test('source replacement after validation cannot redirect the private snapshot', async (t) => { + const fixture = await repository(); + const other = await repository(); + t.after(() => + Promise.all( + [fixture, other].map(({ parent }) => + rm(parent, { recursive: true, force: true }) + ) + ) + ); + const snapshot = await GitSourceSnapshot.admit( + ( + await source(fixture.root) + ).identity + ); + const validate = snapshot.validate.bind(snapshot); + t.mock.method(snapshot, 'validate', async () => { + await validate(); + await rename( + join(fixture.root, '.git'), + join(fixture.root, '.git-original') + ); + await symlink(join(other.root, '.git'), join(fixture.root, '.git')); + }); + await assert.rejects(snapshot.copyTo(join(fixture.parent, 'snapshot'))); + await assert.rejects( + stat(join(fixture.parent, 'snapshot', 'objects', 'pack')), + { code: 'ENOENT' } + ); +}); + +test('cached instances reject object-directory redirection before executor recreation', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const id = 'd'.repeat(64); + const instance = await manager.resolve('primary', id); + await rename( + join(instance.root, '.git', 'objects'), + join(instance.root, '.git', 'objects-original') + ); + await symlink( + join(fixture.root, '.git', 'objects'), + join(instance.root, '.git', 'objects') + ); + await assert.rejects( + manager.resolve('primary', id), + /does not own its Git objects/ + ); +}); + +test('snapshot rejects symlinked refs and never copies source hooks or config includes', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + await git(fixture.root, 'config', 'include.path', '/not-readable/config'); + const snapshot = await GitSourceSnapshot.admit( + ( + await source(fixture.root) + ).identity + ); + await snapshot.copyTo(join(fixture.parent, 'snapshot')); + assert.equal( + await readFile(join(fixture.parent, 'snapshot', 'config'), 'utf8'), + '[core]\nrepositoryformatversion = 0\nbare = true\n' + ); + await assert.rejects(stat(join(fixture.parent, 'snapshot', 'hooks')), { + code: 'ENOENT', + }); + await symlink( + join(fixture.root, 'README.md'), + join(fixture.root, '.git', 'refs', 'bad') + ); + await assert.rejects( + snapshot.copyTo(join(fixture.parent, 'snapshot-bad')), + /symbolic link/ + ); +}); + +test( + 'restart preserves a checkout reserved by a crashed provisioning process', + { timeout: 10_000 }, + async (t) => { + const fixture = await repository(); + const admitted = await source(fixture.root); + const storage = join(fixture.parent, 'instances'); + const id = '9'.repeat(64); + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { GitWorktreeManager } from ${JSON.stringify( + new URL('./worktrees.js', import.meta.url).href + )}; + const manager = new GitWorktreeManager({ + maxCount: 1, root: ${JSON.stringify(storage)}, + sources: new Map([['primary', ${JSON.stringify(admitted)}]]), + prepareInstance: async () => { + process.stdout.write('setup-started'); + await new Promise(() => { setInterval(() => {}, 1000); }); + }, + }); + await manager.resolve('primary', ${JSON.stringify(id)}); + `, + ], + { stdio: ['ignore', 'pipe', 'pipe'] } + ); + const closed = once(child, 'close'); + t.after(async () => { + child.kill('SIGKILL'); + await closed; + await rm(fixture.parent, { recursive: true, force: true }); + }); + await once(child.stdout!, 'data'); + child.kill('SIGKILL'); + await closed; + const restarted = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', admitted]]), + }); + await assert.rejects( + restarted.resolve('primary', id), + /operator recovery required/ + ); + await assert.rejects( + restarted.resolve('primary', '8'.repeat(64)), + /capacity is exhausted/ + ); + assert.equal( + await readFile( + join(await restarted.plannedRoot('primary', id), 'README.md'), + 'utf8' + ), + 'source\n' + ); + } +); + +test('cached checkouts revalidate their admitted source without deleting user work', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const id = 'f'.repeat(64); + const instance = await manager.resolve('primary', id); + await writeFile(join(instance.root, 'pending.txt'), 'user work'); + await rename(fixture.root, `${fixture.root}.original`); + await mkdir(fixture.root); + await assert.rejects( + manager.resolve('primary', id), + /source changed after admission/ + ); + assert.equal( + await readFile(join(instance.root, 'pending.txt'), 'utf8'), + 'user work' + ); +}); + +test('preserves a failed setup checkout until executor cleanup is confirmed', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async () => { + throw new Error('setup failed'); + }, + discardInstance: async () => { + throw new Error('child cleanup unconfirmed'); + }, + }; + const manager = new GitWorktreeManager(options); + const id = 'a'.repeat(64); + await assert.rejects(manager.resolve('primary', id), /cleanup unconfirmed/); + const root = await manager.plannedRoot('primary', id); + assert.equal(await readFile(join(root, 'README.md'), 'utf8'), 'source\n'); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', id), + /operator recovery required/ + ); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', 'b'.repeat(64)), + /capacity is exhausted/ + ); + assert.equal(await readFile(join(root, 'README.md'), 'utf8'), 'source\n'); +}); + +test('cancellation waits for setup cleanup before releasing provisioning ownership', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + let started!: () => void; + const setupStarted = new Promise((resolve) => { + started = resolve; + }); + let cleanupFinished = false; + let setupRoot = ''; + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async (instance, signal) => { + const reservation = JSON.parse( + await readFile(`${instance.root}.complete`, 'utf8') + ); + assert.equal(reservation.provisioningFailed, true); + setupRoot = instance.root; + started(); + try { + await delay(60_000, undefined, { signal }); + await writeFile(join(instance.root, 'LATE'), 'should never happen'); + } finally { + await delay(20); + cleanupFinished = true; + } + }, + }); + const controller = new AbortController(); + const pending = manager.resolve('primary', 'a'.repeat(64), controller.signal); + const rejected = assert.rejects(pending, { name: 'AbortError' }); + await setupStarted; + controller.abort(); + await rejected; + assert.equal(cleanupFinished, true); + await assert.rejects(stat(setupRoot), { code: 'ENOENT' }); + await assert.rejects(stat(`${setupRoot}.complete`), { code: 'ENOENT' }); +}); + +test('cancels lock wait without provisioning while another caller continues', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + let started!: () => void; + const setupStarted = new Promise((resolve) => { + started = resolve; + }); + let release!: () => void; + const released = new Promise((resolve) => { + release = resolve; + }); + t.after(release); + let setups = 0; + const options = { + maxCount: 2, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async () => { + setups++; + started(); + await released; + }, + }; + const active = new GitWorktreeManager(options).resolve( + 'primary', + 'a'.repeat(64) + ); + await setupStarted; + const controller = new AbortController(); + const manager = new GitWorktreeManager(options); + const waiting = manager.resolve('primary', 'b'.repeat(64), controller.signal); + const rejected = assert.rejects(waiting, { name: 'AbortError' }); + await delay(75); + controller.abort(); + await rejected; + assert.equal(setups, 1); + release(); + await active; + await assert.rejects( + stat(await manager.plannedRoot('primary', 'b'.repeat(64))), + { code: 'ENOENT' } + ); +}); + +test('recovery preserves unknown directories and malformed completion records', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }; + const manager = new GitWorktreeManager(options); + const first = await manager.resolve('primary', 'a'.repeat(64)); + const unrelated = join(options.root, 'operator-backups', 'important'); + await mkdir(unrelated, { recursive: true }); + await writeFile(join(unrelated, 'notes'), 'keep'); + await manager.resolve('primary', 'b'.repeat(64)); + assert.equal(await readFile(join(unrelated, 'notes'), 'utf8'), 'keep'); + await writeFile(`${first.root}.complete`, '{"version":0}'); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', 'a'.repeat(64)), + /completion record is invalid/ + ); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', 'c'.repeat(64)), + /completion record is invalid/ + ); + assert.equal( + await readFile(join(first.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +async function git(root: string, ...args: string[]): Promise { + const result = await execFileAsync('git', ['-C', root, ...args], { + encoding: 'utf8', + env: { + PATH: process.env.PATH, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + LC_ALL: 'C', + }, + }); + return result.stdout.trim(); +} + +async function repository( + objectFormat = 'sha1' +): Promise<{ parent: string; root: string }> { + const parent = await mkdtemp(join(tmpdir(), 'librechat-worktrees-')); + const root = join(parent, 'source'); + await execFileAsync('git', ['init', `--object-format=${objectFormat}`, root]); + await writeFile(join(root, 'README.md'), 'source\n'); + await git(root, 'add', 'README.md'); + await git( + root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'initial' + ); + return { parent, root: await realpath(root) }; +} + +async function source(root: string) { + return { root, identity: await captureWorkspaceRootIdentity(root) }; +} + +test('creates and reuses an isolated worktree for one conversation identity', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const id = 'a'.repeat(64); + + const [first, concurrent] = await Promise.all([ + manager.resolve('primary', id), + manager.resolve('primary', id), + ]); + assert.deepEqual(concurrent, first); + assert.notEqual(first.root, fixture.root); + assert.equal( + (await realpath(join(first.root, '.git', 'objects'))).startsWith( + first.root + ), + true + ); + const instanceCommon = await realpath( + await git( + first.root, + 'rev-parse', + '--path-format=absolute', + '--git-common-dir' + ) + ); + assert.equal(instanceCommon.startsWith(first.root), true); + assert.equal( + await readFile(join(first.root, 'README.md'), 'utf8'), + 'source\n' + ); + + await writeFile(join(first.root, 'README.md'), 'conversation\n'); + assert.equal( + await readFile(join(fixture.root, 'README.md'), 'utf8'), + 'source\n' + ); + + const restarted = new GitWorktreeManager({ + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + assert.equal((await restarted.resolve('primary', id)).root, first.root); +}); + +test('replaces an incomplete checkout before admitting it after restart', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const storage = join(fixture.parent, 'instances'); + const id = 'c'.repeat(64); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const first = await manager.resolve('primary', id); + await writeFile(join(first.root, 'README.md'), 'partial mutation\n'); + await rm(`${first.root}.complete`); + + const restarted = new GitWorktreeManager({ + maxCount: 4, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const recovered = await restarted.resolve('primary', id); + assert.equal( + await readFile(join(recovered.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +test('does not count an incomplete checkout against capacity after restart', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const storage = join(fixture.parent, 'instances'); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const abandoned = await manager.resolve('primary', 'c'.repeat(64)); + await rm(`${abandoned.root}.complete`); + const staleMarker = `${abandoned.root}.complete.1.tmp`; + await writeFile(staleMarker, '1\n'); + + const restarted = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const replacement = await restarted.resolve('primary', 'd'.repeat(64)); + assert.equal((await stat(replacement.root)).isDirectory(), true); + await assert.rejects(stat(abandoned.root), { code: 'ENOENT' }); + await assert.rejects(stat(staleMarker), { code: 'ENOENT' }); +}); + +test('keeps a conversation checkout independent of source object pruning', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + await writeFile(join(fixture.root, 'SECOND.md'), 'second\n'); + await git(fixture.root, 'add', 'SECOND.md'); + await git( + fixture.root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'second' + ); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const instance = await manager.resolve('primary', 'e'.repeat(64)); + const retainedHead = await git(instance.root, 'rev-parse', 'HEAD'); + + await git(fixture.root, 'reset', '--hard', 'HEAD~1'); + await git(fixture.root, 'reflog', 'expire', '--expire=now', '--all'); + await git(fixture.root, 'gc', '--prune=now'); + + assert.equal(await git(instance.root, 'rev-parse', 'HEAD'), retainedHead); + assert.equal( + await readFile(join(instance.root, 'SECOND.md'), 'utf8'), + 'second\n' + ); + await assert.rejects( + readFile(join(instance.root, '.git', 'objects', 'info', 'alternates')), + { code: 'ENOENT' } + ); +}); + +test('dissociates a checkout from inherited source alternates', async (t) => { + const upstream = await repository(); + const sharedParent = await mkdtemp( + join(tmpdir(), 'librechat-shared-source-') + ); + const sharedRoot = join(sharedParent, 'source'); + t.after(() => + Promise.all([ + rm(upstream.parent, { recursive: true, force: true }), + rm(sharedParent, { recursive: true, force: true }), + ]) + ); + await execFileAsync('git', ['clone', '--shared', upstream.root, sharedRoot]); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(sharedParent, 'instances'), + sources: new Map([['primary', await source(await realpath(sharedRoot))]]), + }); + const instance = await manager.resolve('primary', 'f'.repeat(64)); + await rm(upstream.root, { recursive: true, force: true }); + + assert.equal( + await git(instance.root, 'rev-parse', 'HEAD^{commit}'), + await git(instance.root, 'rev-parse', 'HEAD') + ); + await assert.rejects( + readFile(join(instance.root, '.git', 'objects', 'info', 'alternates')), + { code: 'ENOENT' } + ); +}); + +test('provisions an orphan branch for a repository with an unborn HEAD', async (t) => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-empty-source-')); + const root = join(parent, 'source'); + await execFileAsync('git', ['init', root]); + t.after(() => rm(parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(parent, 'instances'), + sources: new Map([['primary', await source(await realpath(root))]]), + }); + + const instance = await manager.resolve('primary', '0'.repeat(64)); + assert.match( + await git(instance.root, 'branch', '--show-current'), + /^librechat\/conversation-/ + ); + await assert.rejects(git(instance.root, 'rev-parse', '--verify', 'HEAD')); +}); + +test('rejects replacement of the admitted worktree storage root', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const storage = join(fixture.parent, 'instances'); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + await manager.resolve('primary', '1'.repeat(64)); + await rename(storage, `${storage}.original`); + await mkdir(storage, { mode: 0o700 }); + + await assert.rejects( + manager.resolve('primary', '1'.repeat(64)), + /storage changed after admission/ + ); +}); + +test('keeps conversations and source repositories isolated', async (t) => { + const first = await repository(); + const second = await repository(); + t.after(() => + Promise.all([ + rm(first.parent, { recursive: true, force: true }), + rm(second.parent, { recursive: true, force: true }), + ]) + ); + const storage = await mkdtemp(join(tmpdir(), 'librechat-worktree-storage-')); + t.after(() => rm(storage, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: storage, + sources: new Map([ + ['first', await source(first.root)], + ['second', await source(second.root)], + ]), + }); + + const firstConversation = await manager.resolve('first', '1'.repeat(64)); + const secondConversation = await manager.resolve('first', '2'.repeat(64)); + const otherRepository = await manager.resolve('second', '1'.repeat(64)); + assert.equal( + new Set([ + firstConversation.root, + secondConversation.root, + otherRepository.root, + ]).size, + 3 + ); +}); + +test('rejects invalid identities, overlapping storage and exhausted capacity', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + assert.throws( + () => + new GitWorktreeManager({ + cloneTimeoutMs: 29_999, + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([ + [ + 'primary', + { + root: fixture.root, + identity: { + path: fixture.root, + dev: '1', + ino: '1', + }, + }, + ], + ]), + }), + /clone timeout/ + ); + const overlapping = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.root, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + await assert.rejects( + overlapping.resolve('primary', 'a'.repeat(64)), + /must not overlap/ + ); + + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + await assert.rejects(manager.resolve('primary', '../escape'), /SHA-256/); + const first = await manager.resolve('primary', 'a'.repeat(64)); + assert.equal((await stat(first.root)).isDirectory(), true); + await assert.rejects( + manager.resolve('primary', 'b'.repeat(64)), + /capacity is exhausted/ + ); +}); + +test('serializes provisioning across manager instances sharing storage', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }; + const results = await Promise.allSettled([ + new GitWorktreeManager(options).resolve('primary', 'a'.repeat(64)), + new GitWorktreeManager(options).resolve('primary', 'b'.repeat(64)), + ]); + assert.equal( + results.filter((result) => result.status === 'fulfilled').length, + 1 + ); + assert.equal( + results.filter((result) => result.status === 'rejected').length, + 1 + ); + assert.match( + ( + results.find( + (result) => result.status === 'rejected' + ) as PromiseRejectedResult + ).reason.message, + /capacity is exhausted/ + ); +}); + +test('prepares a new checkout before publishing its completion marker', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + let attempts = 0; + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async (instance: { root: string }) => { + attempts += 1; + if (attempts === 1) throw new Error('setup failed'); + await writeFile(join(instance.root, 'prepared'), 'yes\n'); + }, + }; + const id = 'c'.repeat(64); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', id), + /setup failed/ + ); + const instance = await new GitWorktreeManager(options).resolve('primary', id); + assert.equal( + await readFile(join(instance.root, 'prepared'), 'utf8'), + 'yes\n' + ); + assert.equal(attempts, 2); +}); + +test('preserves a completed checkout when its admitted source changes', async (t) => { + const first = await repository(); + const second = await repository(); + t.after(() => rm(first.parent, { recursive: true, force: true })); + t.after(() => rm(second.parent, { recursive: true, force: true })); + await writeFile(join(second.root, 'README.md'), 'replacement\n'); + await git(second.root, 'add', 'README.md'); + await git( + second.root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'replacement' + ); + const storage = join(first.parent, 'instances'); + const id = 'e'.repeat(64); + const original = await new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(first.root)]]), + }).resolve('primary', id); + await writeFile(join(original.root, 'UNCOMMITTED.md'), 'user work\n'); + await assert.rejects( + new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(second.root)]]), + }).resolve('primary', id), + /source identity changed/ + ); + assert.equal( + await readFile(join(original.root, 'UNCOMMITTED.md'), 'utf8'), + 'user work\n' + ); + assert.equal( + await readFile(join(original.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +test('rejects a source whose admitted filesystem identity changed', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const metadata = await stat(fixture.root, { bigint: true }); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([ + [ + 'primary', + { + root: fixture.root, + identity: { + path: fixture.root, + dev: metadata.dev.toString(), + ino: (metadata.ino + 1n).toString(), + }, + }, + ], + ]), + }); + + await assert.rejects( + manager.resolve('primary', 'd'.repeat(64)), + /source changed after admission/ + ); +}); diff --git a/packages/code/src/worktrees.ts b/packages/code/src/worktrees.ts new file mode 100644 index 00000000..ab19aa38 --- /dev/null +++ b/packages/code/src/worktrees.ts @@ -0,0 +1,718 @@ +import { execFile } from 'node:child_process'; +import { createHash, randomUUID } from 'node:crypto'; +import { + lstat, + mkdir, + readFile, + readdir, + realpath, + rename, + rm, + stat, + writeFile, +} from 'node:fs/promises'; +import { isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { promisify } from 'node:util'; + +import { matchesWorkspaceRoot } from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import { assertPrivateStorageAncestors } from './private-storage.js'; +import { withProcessLock } from './process-lock.js'; +import { GitSourceSnapshot } from './git-snapshot.js'; + +const execFileAsync = promisify(execFile); +const WORKTREE_INSTANCE_PATTERN = /^[a-f0-9]{64}$/; +const COMPLETION_TEMP_PATTERN = /^[a-f0-9]{64}\.complete\.[a-f0-9-]+\.tmp$/; +const GIT_TIMEOUT_MS = 30_000; +const DEFAULT_CLONE_TIMEOUT_MS = 5 * 60_000; + +export interface GitWorktreeSource { + identity: WorkspaceRootIdentity; + root: string; +} + +export interface GitWorktreeInstance { + id: string; + identity: WorkspaceRootIdentity; + root: string; + sourceWorkspaceId: string; +} + +export interface GitWorktreeManagerOptions { + cloneTimeoutMs?: number; + maxCount: number; + root: string; + sources: ReadonlyMap; + prepareInstance?: ( + instance: GitWorktreeInstance, + signal?: AbortSignal + ) => Promise; + discardInstance?: (instance: GitWorktreeInstance) => Promise | void; +} + +const PROVISIONING_LOCK = '.provision.lock'; + +function isInside(parent: string, candidate: string): boolean { + const path = relative(parent, candidate); + return ( + path === '' || + (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) + ); +} + +function gitEnvironment(): NodeJS.ProcessEnv { + return { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }; +} + +async function git( + root: string, + args: string[], + signal?: AbortSignal, + timeout = GIT_TIMEOUT_MS +): Promise { + const execution = execFileAsync( + 'git', + ['--no-optional-locks', '-C', root, ...args], + { + encoding: 'utf8', + env: gitEnvironment(), + maxBuffer: 16 * 1024, + signal, + timeout, + } + ); + const closed = new Promise((resolve) => + execution.child.once('close', () => resolve()) + ); + try { + return (await execution).stdout.trim(); + } finally { + // execFile's AbortError callback can run before its child exits. Retain the + // provisioning lock and directory until the writer is actually gone. + const killTimer = setTimeout(() => execution.child.kill('SIGKILL'), 1000); + killTimer.unref(); + try { + await closed; + } finally { + clearTimeout(killTimer); + } + } +} + +async function sourceConfig( + root: string, + key: string, + signal?: AbortSignal +): Promise { + try { + const remote = await git( + root, + [ + 'config', + '--no-includes', + '--file', + join(root, 'source-config'), + '--get', + key, + ], + signal + ); + return remote || undefined; + } catch { + signal?.throwIfAborted(); + return undefined; + } +} + +async function hasCommittedHead( + root: string, + signal?: AbortSignal +): Promise { + try { + await git(root, ['rev-parse', '--verify', 'HEAD'], signal); + return true; + } catch (error) { + signal?.throwIfAborted(); + if (error instanceof Error && 'code' in error && error.code === 128) { + return false; + } + throw error; + } +} + +async function directoryIdentity(path: string): Promise { + const metadata = await lstat(path, { bigint: true }); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new Error('Conversation worktree must be a real directory'); + } + return { + path, + dev: metadata.dev.toString(), + ino: metadata.ino.toString(), + }; +} + +export class GitWorktreeManager { + private readonly instances = new Map(); + private readonly sourceSnapshots = new Map< + string, + Promise + >(); + private canonicalRoot?: Promise<{ + identity: WorkspaceRootIdentity; + path: string; + }>; + + constructor(private readonly options: GitWorktreeManagerOptions) { + if ( + !Number.isSafeInteger(options.maxCount) || + options.maxCount < 1 || + options.maxCount > 1024 || + options.sources.size === 0 + ) { + throw new Error( + 'Conversation worktree capacity must be between 1 and 1024' + ); + } + if ( + options.cloneTimeoutMs !== undefined && + (!Number.isSafeInteger(options.cloneTimeoutMs) || + options.cloneTimeoutMs < GIT_TIMEOUT_MS || + options.cloneTimeoutMs > 30 * 60_000) + ) { + throw new Error( + 'Conversation worktree clone timeout must be between 30000 and 1800000 milliseconds' + ); + } + } + + private async root(): Promise { + this.canonicalRoot ??= (async () => { + const configuredRoot = resolve(this.options.root); + await assertPrivateStorageAncestors(configuredRoot, true); + await mkdir(configuredRoot, { + mode: 0o700, + recursive: true, + }); + await assertPrivateStorageAncestors(configuredRoot); + const root = await realpath(this.options.root); + await assertPrivateStorageAncestors(root); + const metadata = await stat(root); + if ( + !metadata.isDirectory() || + (process.platform !== 'win32' && (metadata.mode & 0o022) !== 0) + ) { + throw new Error( + 'Conversation worktree root must not be group or world writable' + ); + } + for (const source of this.options.sources.values()) { + const sourceRoot = await realpath(source.root); + if (isInside(sourceRoot, root) || isInside(root, sourceRoot)) { + throw new Error( + 'Conversation worktree storage must not overlap a source workspace' + ); + } + } + return { + identity: await directoryIdentity(root), + path: root, + }; + })(); + const root = await this.canonicalRoot; + if (!(await matchesWorkspaceRoot(root.path, root.identity))) { + throw new Error('Conversation worktree storage changed after admission'); + } + return root.path; + } + + private key(sourceWorkspaceId: string, instanceId: string): string { + return `${sourceWorkspaceId}\0${instanceId}`; + } + + private branch(sourceWorkspaceId: string, instanceId: string): string { + const source = createHash('sha256') + .update(sourceWorkspaceId) + .digest('hex') + .slice(0, 8); + return `librechat/conversation-${source}-${instanceId.slice(0, 31)}`; + } + + private async instancePath( + sourceWorkspaceId: string, + instanceId: string + ): Promise { + const sourceDirectory = createHash('sha256') + .update(sourceWorkspaceId) + .digest('hex') + .slice(0, 24); + return join(await this.root(), sourceDirectory, instanceId); + } + + async plannedRoot( + sourceWorkspaceId: string, + instanceId: string + ): Promise { + if (!WORKTREE_INSTANCE_PATTERN.test(instanceId)) { + throw new Error( + 'Conversation worktree identity must be a SHA-256 digest' + ); + } + if (!this.options.sources.has(sourceWorkspaceId)) { + throw new Error('Conversation worktree source is unavailable'); + } + return await this.instancePath(sourceWorkspaceId, instanceId); + } + + async prepare(): Promise { + await this.root(); + await Promise.all( + [...this.options.sources].map(async ([_workspaceId, source]) => { + const sourceRoot = await this.admittedSourceRoot(source); + await this.sourceSnapshot(sourceRoot, source); + }) + ); + } + + private async admittedSourceRoot(source: GitWorktreeSource): Promise { + const sourceRoot = await realpath(source.root); + if (!(await matchesWorkspaceRoot(sourceRoot, source.identity))) { + throw new Error('Conversation worktree source changed after admission'); + } + return sourceRoot; + } + + private async sourceSnapshot( + root: string, + source: GitWorktreeSource + ): Promise { + let snapshot = this.sourceSnapshots.get(root); + if (!snapshot) { + snapshot = GitSourceSnapshot.admit(source.identity); + this.sourceSnapshots.set(root, snapshot); + } + const admitted = await snapshot; + await admitted.validate(); + return admitted; + } + + private async countInstances(): Promise { + const root = await this.root(); + const sourceDirectories = await readdir(root, { withFileTypes: true }); + let count = 0; + for (const sourceDirectory of sourceDirectories) { + if (sourceDirectory.name.startsWith(PROVISIONING_LOCK)) continue; + if ( + !/^[a-f0-9]{24}$/.test(sourceDirectory.name) || + !sourceDirectory.isDirectory() || + sourceDirectory.isSymbolicLink() + ) + continue; + const entries = await readdir(join(root, sourceDirectory.name), { + withFileTypes: true, + }); + const reserved = new Set(); + for (const entry of entries) { + const id = entry.name.endsWith('.complete') + ? entry.name.slice(0, -9) + : ''; + if (!WORKTREE_INSTANCE_PATTERN.test(id)) continue; + if (!entry.isFile() || entry.isSymbolicLink()) + throw new Error('Invalid worktree reservation'); + if ( + await this.hasCompletionMarker(join(root, sourceDirectory.name, id)) + ) { + reserved.add(id); + count += 1; + } + } + for (const entry of entries) { + if (entry.isFile() && COMPLETION_TEMP_PATTERN.test(entry.name)) { + await rm(join(root, sourceDirectory.name, entry.name), { + force: true, + }); + continue; + } + if ( + !WORKTREE_INSTANCE_PATTERN.test(entry.name) || + !entry.isDirectory() || + entry.isSymbolicLink() + ) + continue; + const path = join(root, sourceDirectory.name, entry.name); + if (!reserved.has(entry.name)) { + await rm(path, { recursive: true, force: true }); + await rm(this.completionMarker(path), { force: true }); + } + } + } + return count; + } + + private async withProvisioningLock( + operation: () => Promise, + signal?: AbortSignal + ): Promise { + return await withProcessLock( + join(await this.root(), PROVISIONING_LOCK), + operation, + signal + ); + } + + private completionMarker(path: string): string { + return `${path}.complete`; + } + + private async hasCompletionMarker( + path: string, + source?: WorkspaceRootIdentity, + sourceGit?: string + ): Promise { + try { + const record = JSON.parse( + await readFile(this.completionMarker(path), 'utf8') + ) as { + version?: unknown; + source?: Partial; + provisioningFailed?: boolean; + sourceGit?: string; + }; + const valid = + record.version === 2 && + typeof record.sourceGit === 'string' && + WORKTREE_INSTANCE_PATTERN.test(record.sourceGit) && + typeof record.source?.path === 'string' && + typeof record.source.dev === 'string' && + typeof record.source.ino === 'string'; + if (!valid) + throw new Error( + 'Conversation worktree completion record is invalid; existing checkout preserved' + ); + if ( + source != null && + (record.source!.path !== source.path || + record.source!.dev !== source.dev || + record.source!.ino !== source.ino || + record.sourceGit !== sourceGit) + ) { + throw new Error( + 'Conversation worktree source identity changed; existing checkout preserved' + ); + } + if (source != null && record.provisioningFailed) { + throw new Error( + 'Conversation worktree setup cleanup is unconfirmed; operator recovery required' + ); + } + return true; + } catch (error) { + if ( + error instanceof Error && + 'code' in error && + error.code === 'ENOENT' + ) { + return false; + } + throw error; + } + } + + private async writeCompletionMarker( + path: string, + source: WorkspaceRootIdentity, + sourceGit: string, + provisioningFailed = false + ): Promise { + const marker = this.completionMarker(path); + const temporary = `${marker}.${randomUUID()}.tmp`; + try { + await writeFile( + temporary, + `${JSON.stringify({ + version: 2, + source, + sourceGit, + ...(provisioningFailed ? { provisioningFailed: true } : {}), + })}\n`, + { mode: 0o600, flag: 'wx' } + ); + await rename(temporary, marker); + } finally { + await rm(temporary, { force: true }); + } + } + + private async validateRepository( + sourceWorkspaceId: string, + instanceId: string, + path: string, + signal?: AbortSignal + ): Promise { + const canonicalPath = await realpath(path); + if (canonicalPath !== path || !isInside(await this.root(), canonicalPath)) { + throw new Error( + 'Conversation worktree escaped its configured storage root' + ); + } + signal?.throwIfAborted(); + const instanceCommon = join(canonicalPath, '.git'); + const gitMetadata = await lstat(instanceCommon); + if ( + !gitMetadata.isDirectory() || + gitMetadata.isSymbolicLink() || + (await realpath(instanceCommon)) !== instanceCommon + ) { + throw new Error('Conversation worktree does not own its Git metadata'); + } + try { + await lstat(join(instanceCommon, 'commondir')); + throw new Error( + 'Conversation worktree must not redirect its Git metadata' + ); + } catch (error) { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) + throw error; + } + const instanceObjects = await realpath(join(instanceCommon, 'objects')); + if (!isInside(canonicalPath, instanceObjects)) { + throw new Error('Conversation worktree does not own its Git objects'); + } + try { + await lstat(join(instanceObjects, 'info', 'alternates')); + throw new Error( + 'Conversation worktree must not use external Git objects' + ); + } catch (error) { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) { + throw error; + } + } + return { + id: instanceId, + identity: await directoryIdentity(canonicalPath), + root: canonicalPath, + sourceWorkspaceId, + }; + } + + private async validateExisting( + sourceWorkspaceId: string, + instanceId: string, + path: string, + source: WorkspaceRootIdentity, + sourceGit: string, + signal?: AbortSignal + ): Promise { + if (!(await this.hasCompletionMarker(path, source, sourceGit))) { + const error = new Error('Conversation worktree is incomplete'); + Object.assign(error, { code: 'EINCOMPLETE' }); + throw error; + } + return await this.validateRepository( + sourceWorkspaceId, + instanceId, + path, + signal + ); + } + + private async createLocked( + sourceWorkspaceId: string, + instanceId: string, + signal?: AbortSignal + ): Promise { + if (!WORKTREE_INSTANCE_PATTERN.test(instanceId)) { + throw new Error( + 'Conversation worktree identity must be a SHA-256 digest' + ); + } + const source = this.options.sources.get(sourceWorkspaceId); + if (!source) throw new Error('Conversation worktree source is unavailable'); + const sourceRoot = await this.admittedSourceRoot(source); + const snapshot = await this.sourceSnapshot(sourceRoot, source); + const path = await this.instancePath(sourceWorkspaceId, instanceId); + try { + return await this.validateExisting( + sourceWorkspaceId, + instanceId, + path, + source.identity, + snapshot.fingerprint, + signal + ); + } catch (error) { + if (!(error instanceof Error) || !('code' in error)) { + throw error; + } + if (error.code === 'EINCOMPLETE') { + await rm(path, { recursive: true, force: true }); + await rm(this.completionMarker(path), { force: true }); + } else if (error.code !== 'ENOENT') { + throw error; + } + } + if ((await this.countInstances()) >= this.options.maxCount) { + throw new Error('Conversation worktree capacity is exhausted'); + } + await mkdir(resolve(path, '..'), { mode: 0o700, recursive: true }); + const branch = this.branch(sourceWorkspaceId, instanceId); + let instance: GitWorktreeInstance | undefined; + const staging = `${path}.source`; + try { + // Reserve before launching any writer. A worker crash may leave a Git + // child or setup executor alive after the parent's kernel lock releases. + // Recovery must not sweep or reuse that uncertain directory. + await this.writeCompletionMarker( + path, + source.identity, + snapshot.fingerprint, + true + ); + const cloneSignal = AbortSignal.any([ + ...(signal ? [signal] : []), + AbortSignal.timeout( + this.options.cloneTimeoutMs ?? DEFAULT_CLONE_TIMEOUT_MS + ), + ]); + await snapshot.copyTo(staging, cloneSignal); + const remote = await sourceConfig( + staging, + 'remote.origin.url', + cloneSignal + ); + const objectFormat = await sourceConfig( + staging, + 'extensions.objectformat', + cloneSignal + ); + if ( + objectFormat && + objectFormat !== 'sha1' && + objectFormat !== 'sha256' + ) { + throw new Error('Unsupported source Git object format'); + } + if (objectFormat === 'sha256') { + await writeFile( + join(staging, 'config'), + '[core]\nrepositoryformatversion = 1\nbare = true\n[extensions]\nobjectformat = sha256\n', + { mode: 0o600 } + ); + } + await git( + resolve(path, '..'), + ['clone', '--local', '--no-checkout', '--no-tags', staging, path], + cloneSignal, + this.options.cloneTimeoutMs ?? DEFAULT_CLONE_TIMEOUT_MS + ); + await rm(staging, { recursive: true, force: true }); + const sourceHasHead = await hasCommittedHead(path, signal); + if (remote) { + await git(path, ['remote', 'set-url', 'origin', remote], signal); + } else { + await git(path, ['remote', 'remove', 'origin'], signal); + } + await git( + path, + sourceHasHead + ? ['checkout', '--force', '-b', branch, 'HEAD'] + : ['checkout', '--orphan', branch], + signal + ); + instance = await this.validateRepository( + sourceWorkspaceId, + instanceId, + path, + signal + ); + await this.options.prepareInstance?.(instance, signal); + signal?.throwIfAborted(); + if (!(await matchesWorkspaceRoot(instance.root, instance.identity))) { + throw new Error('Conversation worktree changed during setup'); + } + await this.validateRepository( + sourceWorkspaceId, + instanceId, + path, + signal + ); + await this.admittedSourceRoot(source); + await snapshot.validate(); + await this.writeCompletionMarker( + path, + source.identity, + snapshot.fingerprint + ); + return instance; + } catch (error) { + if (instance) { + // The reservation remains until executor cleanup is confirmed. + await this.options.discardInstance?.(instance); + } + await rm(path, { recursive: true, force: true }); + await rm(staging, { recursive: true, force: true }); + await rm(this.completionMarker(path), { force: true }); + throw error; + } + } + + private async create( + sourceWorkspaceId: string, + instanceId: string, + signal?: AbortSignal + ): Promise { + return await this.withProvisioningLock( + () => this.createLocked(sourceWorkspaceId, instanceId, signal), + signal + ); + } + + async resolve( + sourceWorkspaceId: string, + instanceId: string, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted(); + const key = this.key(sourceWorkspaceId, instanceId); + const cached = this.instances.get(key); + if (cached) { + await this.root(); + const source = this.options.sources.get(sourceWorkspaceId)!; + await this.sourceSnapshot(await this.admittedSourceRoot(source), source); + if (!(await matchesWorkspaceRoot(cached.root, cached.identity))) { + this.instances.delete(key); + throw new Error('Conversation worktree changed after admission'); + } + await this.validateRepository( + sourceWorkspaceId, + instanceId, + cached.root, + signal + ); + return cached; + } + // The same kernel lock coordinates callers and processes. Keep cancellation + // attached through setup and cleanup; never release a lease while detached + // provisioning is still mutating the checkout. + const instance = await this.create(sourceWorkspaceId, instanceId, signal); + this.instances.set(key, instance); + return instance; + } +} diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index 736b7216..33f3d082 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -2,7 +2,10 @@ import { afterEach, expect, test } from 'bun:test'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import { RedisBridgeStore } from './store'; -import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { + BRIDGE_PROTOCOL_VERSION, + workspaceIsolationKey, +} from '../../../packages/code/src/protocol'; import type { CodeBridgeAssignment } from './store'; const redis = new RedisMock() as unknown as Redis; @@ -26,13 +29,20 @@ async function register(workspaceLeaseSlots = 2) { workspaceTools: { protocolVersion: BRIDGE_PROTOCOL_VERSION, operations: ['read_file'], - workspaces: [{ id: 'a' }, { id: 'b' }], + workspaces: [ + { id: 'a', workspaceInstances: ['git_worktree'] }, + { id: 'b' }, + ], }, }, }); await store.confirmReady(workerId, incarnationId, generation); } -function dispatch(workspaceId: string, signal = new AbortController().signal) { +function dispatch( + workspaceId: string, + signal = new AbortController().signal, + workspaceInstanceId?: string, +) { const promise = store.dispatchWorkspaceTool({ workerId, signal, @@ -41,6 +51,7 @@ function dispatch(workspaceId: string, signal = new AbortController().signal) { protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'read_file', workspaceId, + ...(workspaceInstanceId == null ? {} : { workspaceInstanceId }), path: 'file.txt', }, }); @@ -346,6 +357,52 @@ test('same-root work waits while another root progresses', async () => { await Promise.all([nextA, b]); }); +test('conversation worktrees on one source use independent capacity lanes', async () => { + await register(); + const firstId = 'a'.repeat(64); + const secondId = 'b'.repeat(64); + const firstPending = dispatch('a', undefined, firstId); + const first = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + const samePending = dispatch('a', undefined, firstId); + const secondPending = dispatch('a', undefined, secondId); + const second = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 1, + ))!; + expect(second.request).toMatchObject({ + workspaceId: 'a', + workspaceInstanceId: secondId, + }); + await settle(first); + await firstPending; + const same = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + expect(same.request).toMatchObject({ + workspaceId: 'a', + workspaceInstanceId: firstId, + }); + await settle(second); + await settle(same); + await Promise.all([samePending, secondPending]); +}); + test('queued cancellation never leases and does not block another root', async () => { await register(); const a = dispatch('a'); @@ -548,7 +605,11 @@ test('post-settlement fences are authenticated, idempotent, and invalidated by r await expect(dispatch('a')).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED', }); - await store.resetWorkspace(workerId, incarnationId, 'native-workspace:a'); + await store.resetWorkspace( + workerId, + incarnationId, + `native-workspace:${workspaceIsolationKey('a')}`, + ); await expect( store.settle( workerId, diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 73870772..5b319733 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -430,6 +430,7 @@ router.post( supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], supportedWorkspaceListFileFeatures: ['after_path'], supportedWorkspaceProgrammaticLanguages: ['bash'], + supportedWorkspaceInstanceTypes: ['git_worktree'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/bridge/store.test.ts b/service/src/bridge/store.test.ts index 9d271551..354108af 100644 --- a/service/src/bridge/store.test.ts +++ b/service/src/bridge/store.test.ts @@ -4,7 +4,7 @@ import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import type * as t from '../types'; import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; -import { RedisBridgeStore } from './store'; +import { RedisBridgeStore, workspaceAdmissionId } from './store'; import type { RegisteredBridgeWorker } from './store'; @@ -27,6 +27,16 @@ afterEach(async () => { }); describe('RedisBridgeStore', () => { + test('uses disjoint admission identities for roots and worktree instances', () => { + const instanceId = 'a'.repeat(64); + expect( + workspaceAdmissionId(`foo:git-worktree:${instanceId}`), + ).not.toBe(workspaceAdmissionId('foo', instanceId)); + expect(workspaceAdmissionId('foo')).not.toBe( + workspaceAdmissionId('workspace:foo'), + ); + }); + test('reports an atomic, capability-limited worker status snapshot', async () => { const store = new RedisBridgeStore(redis); const capabilities = { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 4eaabd7f..2ffeec66 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -15,8 +15,9 @@ import { BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, - isWorkspaceToolRequest, - isWorkspaceToolResult, + isWorkspaceToolRequest, + isWorkspaceToolResult, + workspaceIsolationKey, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; import { BridgeAdmissionQueue } from './admission'; @@ -124,6 +125,12 @@ function supportsWorkspaceTool( if (!supportsOperation) { return supportsOperation; } + if ( + request.workspaceInstanceId !== undefined && + workspace.workspaceInstances?.includes('git_worktree') !== true + ) { + return false; + } if (request.operation === 'list_files' && request.afterPath !== undefined) { return capabilities?.listFileFeatures?.includes('after_path') === true; } @@ -156,6 +163,7 @@ function supportsWorkspaceProgrammatic( registration: RegisteredBridgeWorker, workspaceId: string, language: string, + workspaceInstanceId?: string, ): boolean { const capabilities = registration.capabilities.workspaceTools; const workspace = capabilities?.workspaces.find( @@ -163,6 +171,8 @@ function supportsWorkspaceProgrammatic( ); return ( workspace != null && + (workspaceInstanceId === undefined || + workspace.workspaceInstances?.includes('git_worktree') === true) && capabilities?.operations.includes('execute_command') === true && (workspace.operations == null || workspace.operations.includes('execute_command')) && @@ -172,6 +182,25 @@ function supportsWorkspaceProgrammatic( ); } +function workspaceInstanceId(body: t.PayloadBody): string | undefined { + if ( + typeof body === 'object' && + body != null && + 'workspace_instance_id' in body && + typeof body.workspace_instance_id === 'string' + ) { + return body.workspace_instance_id; + } + return undefined; +} + +export function workspaceAdmissionId( + workspaceId: string, + instanceId?: string, +): string { + return workspaceIsolationKey(workspaceId, instanceId); +} + function workerKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}`; } @@ -807,6 +836,7 @@ export class RedisBridgeStore { registration, args.workspaceId, args.body.language, + workspaceInstanceId(args.body), ) ) { throw new BridgeStoreError( @@ -843,6 +873,15 @@ export class RedisBridgeStore { let workspaceLeaseSlot: number | undefined; const selectedWorkspaceId = args.workspaceRequest?.workspaceId ?? args.workspaceId; + const selectedWorkspaceInstanceId = + args.workspaceRequest?.workspaceInstanceId ?? workspaceInstanceId(args.body); + const selectedWorkspaceAdmissionId = + selectedWorkspaceId == null + ? undefined + : workspaceAdmissionId( + selectedWorkspaceId, + selectedWorkspaceInstanceId, + ); const workspaceSlots = selectedWorkspaceId != null && (registration.capabilities.workspaceLeaseSlots ?? 1) > 1 @@ -864,7 +903,7 @@ export class RedisBridgeStore { args.deadlineAtMs, workspaceSlots == null ? undefined - : selectedWorkspaceId, + : selectedWorkspaceAdmissionId, ), args, 'Bridge admission enqueue', @@ -899,7 +938,7 @@ export class RedisBridgeStore { workerId: args.workerId, incarnationId: lockIncarnationId, assignmentId, - workspaceId: selectedWorkspaceId!, + workspaceId: selectedWorkspaceAdmissionId!, capacity: registration.capabilities.workspaceLeaseSlots!, expiresAtMs: Date.now() + ttlSeconds * 1000, }), @@ -961,6 +1000,7 @@ export class RedisBridgeStore { current.registration, args.workspaceId, args.body.language, + workspaceInstanceId(args.body), )) ) { throw new BridgeStoreError( @@ -986,14 +1026,14 @@ export class RedisBridgeStore { generation, leaseToken, leaseTokenHash: tokenHash(leaseToken), - ...(selectedWorkspaceId == null ? {} : { - workspaceFence: `native-workspace:${selectedWorkspaceId}`, + ...(selectedWorkspaceAdmissionId == null ? {} : { + workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId}`, }), ...(workspaceLeaseSlot === undefined ? {} : { workspaceLeaseSlot, - workspaceFence: `native-workspace:${selectedWorkspaceId!}`, + workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId!}`, }), ...(registration.identityId != null ? { workerIdentityId: registration.identityId } @@ -1082,6 +1122,7 @@ export class RedisBridgeStore { replacement.registration, args.workspaceId, args.body.language, + workspaceInstanceId(args.body), ) ) { throw new BridgeStoreError( diff --git a/service/src/bridge/workspace-instance.test.ts b/service/src/bridge/workspace-instance.test.ts new file mode 100644 index 00000000..44e93300 --- /dev/null +++ b/service/src/bridge/workspace-instance.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'bun:test'; +import { principalWorkspaceInstanceId } from './workspace-instance'; + +describe('principalWorkspaceInstanceId', () => { + it('keeps principal components distinct even when identifiers contain delimiters', () => { + const instanceId = 'a'.repeat(64); + expect(principalWorkspaceInstanceId({ instanceId, tenantId: 'tenant\0user', principalId: 'a' })) + .not.toBe(principalWorkspaceInstanceId({ instanceId, tenantId: 'tenant', principalId: 'user\0a' })); + }); + it('is stable only within the same authenticated principal', () => { + const instanceId = 'a'.repeat(64); + const first = principalWorkspaceInstanceId({ + instanceId, + tenantId: 'tenant', + principalId: 'user-a', + }); + expect(first).toMatch(/^[a-f0-9]{64}$/); + expect( + principalWorkspaceInstanceId({ + instanceId, + tenantId: 'tenant', + principalId: 'user-a', + }) + ).toBe(first); + expect( + principalWorkspaceInstanceId({ + instanceId, + tenantId: 'tenant', + principalId: 'user-b', + }) + ).not.toBe(first); + expect( + principalWorkspaceInstanceId({ + instanceId, + tenantId: 'other', + principalId: 'user-a', + }) + ).not.toBe(first); + }); +}); diff --git a/service/src/bridge/workspace-instance.ts b/service/src/bridge/workspace-instance.ts new file mode 100644 index 00000000..4c49afbb --- /dev/null +++ b/service/src/bridge/workspace-instance.ts @@ -0,0 +1,17 @@ +import { createHash } from 'node:crypto'; + +/** Bind a caller-selected conversation identity to the authenticated principal. */ +export function principalWorkspaceInstanceId(args: { + instanceId: string; + tenantId: string; + principalId: string; +}): string { + return createHash('sha256') + .update(JSON.stringify([ + 'codeapi-workspace-instance-v1', + args.tenantId, + args.principalId, + args.instanceId, + ])) + .digest('hex'); +} diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 1063fbfe..a646b9ad 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -44,6 +44,7 @@ import { SessionKeyResolutionError, } from '../session-key'; import { getCredentialId, getPrincipalOrReject } from '../auth/principal'; +import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; import { getExecutionIdentity } from '../execution-identity'; import { PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION } from '../runtime-session/job-policy'; import { @@ -81,6 +82,7 @@ import { authorizeRequestedFiles, } from './file-authorization'; import { + bindReplayWorkspaceInstance, buildReplayExecutionState, resolveReplayStateSandboxBackend, } from './programmatic-state'; @@ -335,7 +337,7 @@ function buildReplayPayload( state: ExecutionState, history: Record, ): t.PayloadBody { - return createProgrammaticPayload({ + const payload = createProgrammaticPayload({ req, session_id: state.session_id, execution_id: state.execution_id, @@ -347,6 +349,7 @@ function buildReplayPayload( filesOverride: state.files, language: state.language ?? 'python', }); + return bindReplayWorkspaceInstance(payload, state); } async function runReplayIteration( @@ -501,10 +504,17 @@ async function handleReplayInitial( userId: string; bridgeWorkerId?: string; workspaceId?: string; + workspaceInstanceId?: string; }, cancellation: ReplayRequestCancellation, ): Promise { - const { apiKeyId, userId, bridgeWorkerId, workspaceId } = params; + const { + apiKeyId, + userId, + bridgeWorkerId, + workspaceId, + workspaceInstanceId, + } = params; const { code, tools, user_id, files } = req.body as t.ProgrammaticRequestBody; let timeout: number; @@ -660,6 +670,7 @@ async function handleReplayInitial( language, bridgeWorkerId, workspaceId, + workspaceInstanceId, executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: resolveReplayStateSandboxBackend({ @@ -1279,6 +1290,7 @@ router.post( const requestedLanguage: unknown = rawBody.language ?? rawBody.lang; let bridgeWorkerId: string | undefined; let workspaceId: string | undefined; + let workspaceInstanceId: string | undefined; if (continuation_token == null || continuation_token === '') { try { const bridgeSelection = resolveBridgeWorkerSelection({ @@ -1312,6 +1324,23 @@ router.post( } workspaceId = requestedWorkspaceId; } + const requestedWorkspaceInstanceId = rawBody.workspace_instance_id; + if (requestedWorkspaceInstanceId !== undefined) { + if ( + workspaceId == null || + typeof requestedWorkspaceInstanceId !== 'string' || + !/^[a-f0-9]{64}$/.test(requestedWorkspaceInstanceId) + ) { + return res.status(400).json({ + error: 'Invalid code workspace instance ID', + }); + } + workspaceInstanceId = principalWorkspaceInstanceId({ + instanceId: requestedWorkspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }); + } } catch (error) { if (error instanceof BridgeWorkerSelectionError) { return res @@ -1428,6 +1457,7 @@ router.post( userId, bridgeWorkerId, workspaceId, + workspaceInstanceId, }, cancellation); } if (workspaceId != null) { diff --git a/service/src/service/programmatic-state.test.ts b/service/src/service/programmatic-state.test.ts index 81405021..b0bff413 100644 --- a/service/src/service/programmatic-state.test.ts +++ b/service/src/service/programmatic-state.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test'; import type { CodeApiAuthContext, RequestFile } from '../types'; import type { LCTool } from '../preamble'; import { + bindReplayWorkspaceInstance, buildReplayExecutionState, resolveReplayStateSandboxBackend, } from './programmatic-state'; @@ -84,6 +85,7 @@ describe('buildReplayExecutionState', () => { authContext, bridgeWorkerId: 'code-user_123', workspaceId: 'project-a', + workspaceInstanceId: 'a'.repeat(64), sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', @@ -104,6 +106,7 @@ describe('buildReplayExecutionState', () => { apiKeyId: 'key_legacy', bridgeWorkerId: 'code-user_123', workspaceId: 'project-a', + workspaceInstanceId: 'a'.repeat(64), sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', @@ -119,6 +122,19 @@ describe('buildReplayExecutionState', () => { }); }); + test('binds a selected conversation checkout into every replay payload', () => { + const payload = { language: 'bash', version: '5.2', files: [] }; + expect( + bindReplayWorkspaceInstance(payload, { + workspaceInstanceId: 'b'.repeat(64), + }), + ).toEqual({ + ...payload, + workspace_instance_id: 'b'.repeat(64), + }); + expect(bindReplayWorkspaceInstance(payload, {})).toBe(payload); + }); + test('falls back to JWT identity only when no managed auth context exists', () => { const state = build({ authContext: undefined, userId: 'user_api_key' }); diff --git a/service/src/service/programmatic-state.ts b/service/src/service/programmatic-state.ts index e6bda59a..38c72486 100644 --- a/service/src/service/programmatic-state.ts +++ b/service/src/service/programmatic-state.ts @@ -39,6 +39,7 @@ export interface BuildReplayExecutionStateParams { language: 'python' | 'bash'; bridgeWorkerId?: string; workspaceId?: string; + workspaceInstanceId?: string; sandboxBackend?: SandboxBackendName; executionProfile: ExecutionProfile; executionProfileSource: ExecutionProfileSource; @@ -68,6 +69,7 @@ export function buildReplayExecutionState( apiKeyId: params.apiKeyId, bridgeWorkerId: params.bridgeWorkerId, workspaceId: params.workspaceId, + workspaceInstanceId: params.workspaceInstanceId, sandboxBackend: params.sandboxBackend, executionProfile: params.executionProfile, executionProfileSource: params.executionProfileSource, @@ -83,3 +85,13 @@ export function buildReplayExecutionState( language: params.language, }; } + +/** Bind the authenticated conversation checkout to every replay iteration. */ +export function bindReplayWorkspaceInstance( + payload: t.PayloadBody, + state: Pick, +): t.PayloadBody { + return state.workspaceInstanceId == null + ? payload + : { ...payload, workspace_instance_id: state.workspaceInstanceId }; +} diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 3b65cedf..fd4e90e3 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -118,6 +118,8 @@ export interface ExecutionState { bridgeWorkerId?: string; /** Selected workspace retained and bound across every replay iteration. */ workspaceId?: string; + /** Selected conversation checkout retained across every replay iteration. */ + workspaceInstanceId?: string; /** Original queue/backend target retained across replay continuations. */ sandboxBackend?: SandboxBackendName; /** Original producer profile retained so continuations use the same queue. */ diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 0404ad16..91c3ed89 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -204,6 +204,8 @@ export type PayloadFileRef = { export interface PayloadBody { language: string; version: string; + /** Opaque conversation checkout selected and authenticated by the API. */ + workspace_instance_id?: string; /** Stable identity shared by all replay iterations of one execution. */ execution_id?: string; replay_tool_count?: number; @@ -393,6 +395,8 @@ export interface ProgrammaticRequestBody { * legacy `/exec` sandbox body), so the router accepts either key and * normalizes to `language`. If both are present, `language` wins. */ lang?: 'python' | 'bash'; + /** Opaque conversation checkout binding for a selected native workspace. */ + workspace_instance_id?: string; } export interface ProgrammaticToolCall { diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index 04738b8a..a52b3f0b 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -13,6 +13,7 @@ import { executionProfileMiddleware } from '../middleware/execution-profile'; import { hostedAppPreviewGateway } from '../hosted-app/preview-gateway'; import { applyPrincipal } from '../auth/principal'; import { BridgeStoreError } from '../bridge/store'; +import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; import { bridgeStoreStatus, createWorkspaceToolsRouter } from './router'; import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; @@ -36,6 +37,36 @@ test('maps invalid worker results to an upstream failure', () => { expect(bridgeStoreStatus(new BridgeStoreError('WORKER_QUEUE_FULL', 'queue full'))).toBe(429); }); +test('binds instance admission to the authenticated tenant and user while preserving legacy requests', async () => { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'user-worker' }); + next(); + }); + const dispatched: WorkspaceToolRequest[] = []; + app.use(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: false, + store: { async dispatchWorkspaceTool(args) { + dispatched.push(args.request); + return { protocolVersion: 1, generation: 1, leaseToken: 'lease', incarnationId: 'incarnation', status: 'rejected', error: 'fixture' }; + } }, + })); + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + for (const workspaceInstanceId of ['a'.repeat(64), undefined]) { + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', workspaceInstanceId, path: 'README.md' }), + }); + await response.json(); + } + expect(dispatched[0]?.workspaceInstanceId).toBe(principalWorkspaceInstanceId({ instanceId: 'a'.repeat(64), tenantId: 'tenant-1', principalId: 'user-1' })); + expect(dispatched[1]?.workspaceInstanceId).toBeUndefined(); +}); + test.each<[WorkspaceToolRequest, number, number?]>([ [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined], [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined], diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index eb89370e..3e1f9fee 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -18,6 +18,7 @@ import { BridgeWorkerSelectionError, resolveBridgeWorkerSelection, } from '../bridge/selection'; +import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; interface WorkspaceToolsRouterOptions { store: Pick; @@ -82,12 +83,22 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) return; } outcome.operation = req.body.operation; - const request: WorkspaceToolRequest = req.body.operation === 'execute_command' - ? { ...req.body, timeoutMs: Math.min( - req.body.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null + ? req.body + : { + ...req.body, + workspaceInstanceId: principalWorkspaceInstanceId({ + instanceId: req.body.workspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }), + }; + const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' + ? { ...principalRequest, timeoutMs: Math.min( + principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, options.timeoutMs ?? Number.MAX_SAFE_INTEGER, ) } - : req.body; + : principalRequest; const executionBudgetMs = request.operation === 'execute_command' ? request.timeoutMs! + 5_000 : Math.min(options.timeoutMs ?? 30_000, 30_000); From c8b3e1490416753b6f633e4cdbf9100e93aeb849 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 20 Sep 2026 05:44:11 -0400 Subject: [PATCH 41/42] fix: Close native scratch directory streams (#240) --- docs/remote-bridge/README.md | 5 ++++- packages/code/src/native-scratch.ts | 30 +++++++++++++++++------------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index f66499b2..efe84b84 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -235,7 +235,10 @@ execution. - Remote bridge deployments use backend-specific BullMQ queues and serialize the expected backend on every new job, preventing Lambda or HTTP consumers from accepting attached-worker executions. -- Code API permits one active assignment per worker. +- Code API negotiates a bounded number of active workspace assignments per + worker. The lower API or worker slot ceiling wins, and assignments sharing + the same workspace isolation key remain serialized while independent + conversation worktrees may run concurrently. - Dynamic workers are fenced to their server-issued tenant before assignment. - Each assignment has an absolute deadline, generation, and random lease token. - Settlements with the wrong worker, generation, token, or expired deadline are diff --git a/packages/code/src/native-scratch.ts b/packages/code/src/native-scratch.ts index 0ecd330e..8394c1ae 100644 --- a/packages/code/src/native-scratch.ts +++ b/packages/code/src/native-scratch.ts @@ -151,19 +151,25 @@ export async function restoreScratchTraversal( if (directoryFd === undefined) continue; try { const directory = await opendir(descriptorPath(directoryFd)); - for await (const entry of directory) { - entriesInspected += 1; - if (entriesInspected > MAX_SCRATCH_ENTRIES) { - throw new Error('Native sandbox scratch cleanup exceeded its entry limit'); + try { + while (true) { + const entry = await directory.read(); + if (entry === null) break; + entriesInspected += 1; + if (entriesInspected > MAX_SCRATCH_ENTRIES) { + throw new Error('Native sandbox scratch cleanup exceeded its entry limit'); + } + if (!entry.isDirectory()) continue; + if (components.length >= MAX_SCRATCH_DEPTH) { + throw new Error('Native sandbox scratch cleanup exceeded its depth limit'); + } + if (pending.length >= MAX_SCRATCH_DIRECTORIES) { + throw new Error('Native sandbox scratch cleanup exceeded its directory limit'); + } + pending.push([...components, entry.name]); } - if (!entry.isDirectory()) continue; - if (components.length >= MAX_SCRATCH_DEPTH) { - throw new Error('Native sandbox scratch cleanup exceeded its depth limit'); - } - if (pending.length >= MAX_SCRATCH_DIRECTORIES) { - throw new Error('Native sandbox scratch cleanup exceeded its directory limit'); - } - pending.push([...components, entry.name]); + } finally { + await directory.close(); } } finally { if (directoryFd !== root.fd) closeDirectory(directoryFd); From 67d75d859aee891923c40cde1db073489d74a424 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 23 Sep 2026 19:56:56 -0400 Subject: [PATCH 42/42] feat(code): route trusted VM GitHub App tokens by checkout (#248) * feat(code): route trusted VM GitHub App tokens by checkout * fix(code): resolve checkout credentials within canonical root * fix(code): reject cross-root credential aliases --- docs/remote-bridge/worker-runbook.md | 19 ++++-- packages/code/README.md | 26 +++++++-- packages/code/src/cli.test.ts | 86 +++++++++++++++++++++++++--- packages/code/src/cli.ts | 39 ++++++++++--- packages/code/src/github.test.ts | 79 +++++++++++++++++++++++++ packages/code/src/github.ts | 47 ++++++++++++--- 6 files changed, 261 insertions(+), 35 deletions(-) diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 16c24742..3043c541 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -327,12 +327,19 @@ Environment=LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/home/librechat-code/.config/ ``` Install the same App separately on every personal account or organization the -worker is allowed to use. The trusted worker resolves the correct installation -from the repository containing each command's working directory, then mints and -caches a repository-scoped token. Cross-repository work therefore does not -require changing an installation ID or restarting the worker. Set -`LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy fixed-installation -fallback. +worker is allowed to use. By default, the worker binds each admitted workspace +root to its repository at startup, then mints and caches repository-scoped +tokens. Different admitted roots can use different installations without +restarting the worker. For a trusted VM with multiple checkouts under one root, +set `LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING=checkout` and use the `trusted-vm` +command policy. This opt-in resolves the local `origin` URL of each command's +current checkout, including linked worktrees. It remains inside the admitted +filesystem root, but anyone able to alter a checkout's remote can select any +repository where the App is installed; keep the App's installation scope narrow. +Pass the checkout as the command working directory; changing directories only +inside the shell cannot change the token chosen before command launch. +Set `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy +fixed-installation fallback; it cannot be combined with checkout routing. Sandboxed commands receive masked Git/`gh` credentials only for the configured GitHub hosts; the token is not written to the repository, remote URL, or Git diff --git a/packages/code/README.md b/packages/code/README.md index e30b80cb..5a06b92a 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -293,9 +293,23 @@ installation tokens. At startup, the worker binds each explicitly admitted workspace root to its Git repository. Commands in those independent roots can use simultaneous installations on personal accounts and organizations without being restarted or reconfigured, while a command cannot gain access by changing -its workspace's remote URL. Tokens are scoped and cached per repository. For -compatibility with deployments -that intentionally bind a worker to one installation, set the optional legacy +its workspace's remote URL. Tokens are scoped and cached per repository. + +For trusted VMs that intentionally work in multiple Git checkouts beneath one +admitted root, opt in to `--github-repository-routing checkout` (or +`LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING=checkout`) together with the +`trusted-vm` command policy. Each command then uses the repository identified +by its current checkout's local `origin` URL, including linked worktrees. +The command must set its working directory to that checkout; a shell `cd` +inside a command does not change which credential was selected before launch. +This does not widen the admitted filesystem roots, but a command able to alter +a checkout's remote can obtain a token for **any repository where the App is +installed**. Use this mode only where the machine operator trusts the VM and +the App's installation scope; the default `admitted` mode keeps the startup +binding. Checkout routing requires an App without a fixed installation ID. + +For compatibility with deployments that intentionally bind a worker to one +installation, set the optional legacy `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` fallback. App-authenticated commits use the GitHub App bot's canonical no-reply identity, @@ -758,8 +772,10 @@ Also archive any adjacent `.source` staging directory. Pre-release version-1 completion records are deliberately preserved but not admitted by this version; they do not contain the required source Git identity binding. -GitHub App routing is inherited from the operator-admitted source repository; -commands cannot select a different installation by rewriting a worktree remote. +By default, GitHub App routing is inherited from the operator-admitted source +repository; commands cannot select a different installation by rewriting a +worktree remote. On trusted VMs, the opt-in checkout routing mode above instead +uses the current worktree's local `origin` URL, within the admitted root. Legacy requests without a conversation identity continue to use the selected source root. Older Code API deployments do not negotiate the capability, so the worker omits it until every request path understands the isolation boundary. diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index f39b28d9..f95d7eee 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; import { generateKeyPairSync } from 'node:crypto'; -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -242,6 +242,8 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in t.after(() => rm(directory, { recursive: true, force: true })); const privateKeyPath = join(directory, 'app.pem'); const preload = join(directory, 'fetch.mjs'); + const workspace = join(directory, 'workspace'); + await mkdir(workspace); const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); await writeFile( privateKeyPath, @@ -251,13 +253,8 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in await writeFile( preload, ` - globalThis.fetch = async (input) => { - const url = String(input); - if (url.endsWith('/app')) return Response.json({ slug: 'lia-by-librechat' }); - if (url.endsWith('/users/lia-by-librechat%5Bbot%5D')) { - return Response.json({ id: 328778573, login: 'lia-by-librechat[bot]', type: 'Bot' }); - } - throw new Error('test stopped after GitHub App validation'); + globalThis.fetch = async () => { + throw new Error('test reached GitHub App validation'); }; `, ); @@ -276,7 +273,7 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', - LIBRECHAT_CODE_WORKER_DIR: directory, + LIBRECHAT_CODE_WORKER_DIR: workspace, LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', LIBRECHAT_CODE_GITHUB_TOKEN: undefined, LIBRECHAT_CODE_GITHUB_APP_ID: '123', @@ -288,6 +285,77 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in assert.notEqual(result.status, 0); assert.doesNotMatch(result.stderr, /GitHub App authentication requires/); assert.doesNotMatch(result.stderr, /installation ID/i); + assert.match(result.stderr, /test reached GitHub App validation/); + + const checkout = spawnSync( + process.execPath, + ['--import', preload, fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + timeout: 10_000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: workspace, + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm', + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: privateKeyPath, + LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING: 'checkout', + }, + }, + ); + assert.notEqual(checkout.status, 0); + assert.doesNotMatch( + checkout.stderr, + /Checkout GitHub repository routing requires/, + ); + assert.match(checkout.stderr, /test reached GitHub App validation/); +}); + +test('CLI rejects checkout routing outside a trusted VM or without repository-scoped App auth', () => { + const base = { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: process.cwd(), + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: '/does/not/matter', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING: 'checkout', + }; + const cli = fileURLToPath(new URL('./cli.js', import.meta.url)); + const restricted = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: base, + }); + assert.notEqual(restricted.status, 0); + assert.match(restricted.stderr, /requires the trusted-vm command policy/); + + const fixed = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { + ...base, + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: '456', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm', + }, + }); + assert.notEqual(fixed.status, 0); + assert.match(fixed.stderr, /without a fixed installation ID/); + + const invalid = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { ...base, LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING: 'unknown' }, + }); + assert.notEqual(invalid.status, 0); + assert.match(invalid.stderr, /must be admitted or checkout/); }); test('CLI requires a runtime image for Docker supervision', () => { diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 26c60e9a..841164fb 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -49,7 +49,7 @@ import type { LocalWorkspaceConfig } from './workspace.js'; import { GITHUB_ALLOWED_DOMAINS, GitHubAppCredentialProvider, - gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForCommand, gitHubRepositoryForDirectory, gitHubCommandCredentialEnvironment, gitHubMaskedCredentialVariables, @@ -150,12 +150,13 @@ function nonEmpty(value: string | undefined): string | undefined { return value?.trim().length ? value : undefined; } -function githubCredentials(): { +function githubCredentials(args: string[]): { provider?: GitHubCredentialProvider; host: string; privateKeyPath?: string; mode?: 'app' | 'token'; repositoryRouting?: boolean; + checkoutRouting?: boolean; policyIdentity: string; } { const token = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_TOKEN); @@ -163,6 +164,13 @@ function githubCredentials(): { const installationId = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_INSTALLATION_ID, ); + const routing = + option(args, '--github-repository-routing')?.trim().toLowerCase() ?? + process.env.LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING?.trim().toLowerCase() ?? + 'admitted'; + if (routing !== 'admitted' && routing !== 'checkout') { + throw new Error('GitHub repository routing must be admitted or checkout'); + } const privateKeyPath = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE, ); @@ -178,6 +186,11 @@ function githubCredentials(): { 'Configure either GitHub App authentication or a GitHub token, not both', ); } + if (routing === 'checkout' && (!hasApp || installationId)) { + throw new Error( + 'Checkout GitHub repository routing requires a GitHub App without a fixed installation ID', + ); + } const configuredHostValue = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_HOST, ); @@ -211,12 +224,13 @@ function githubCredentials(): { host, mode: 'app', repositoryRouting: !installationId, + checkoutRouting: routing === 'checkout', policyIdentity: gitHubAuthenticationPolicyIdentity({ mode: 'app', host, appId, installationId, - }), + }) + (routing === 'checkout' ? ':routing:checkout' : ''), privateKeyPath, provider: new GitHubAppCredentialProvider({ appId: appId!, @@ -530,9 +544,11 @@ async function run( } const github = runtimeSessionId == null - ? githubCredentials() + ? githubCredentials(args) : { host: 'github.com', + repositoryRouting: false, + checkoutRouting: false, policyIdentity: gitHubAuthenticationPolicyIdentity({ host: 'github.com', }), @@ -547,6 +563,11 @@ async function run( 'GitHub authentication currently requires the native-srt command sandbox', ); } + if (github.checkoutRouting && commandPolicy.preset !== 'trusted-vm') { + throw new Error( + 'Checkout GitHub repository routing requires the trusted-vm command policy', + ); + } const githubDomains = github.provider ? github.host === 'github.com' ? [...GITHUB_ALLOWED_DOMAINS] @@ -759,9 +780,8 @@ async function run( }), ]), ); - // Bind credentials to immutable, explicitly admitted roots. The repository - // remote is operator input at startup, never an authorization input that a - // sandboxed command may change for its next invocation. + // Keep an admission boundary even when trusted-VM checkout routing uses a + // nested repository's remote for the current command. const admittedGitHubRepositories = github.provider && github.repositoryRouting ? new Map( await Promise.all( @@ -998,9 +1018,12 @@ async function run( ), async resolve(signal?: AbortSignal, cwd?: string) { const repository = cwd && admittedGitHubRepositories - ? gitHubRepositoryForAdmittedDirectory( + ? await gitHubRepositoryForCommand( cwd, admittedGitHubRepositories, + github.checkoutRouting ? 'checkout' : 'admitted', + github.host, + signal, ) : undefined; if (!repository && github.repositoryRouting) { diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 352bf1c3..b13d70f7 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -4,6 +4,7 @@ import { chmod, mkdtemp, mkdir, + realpath, rm, symlink, writeFile, @@ -26,6 +27,7 @@ import { GITHUB_CREDENTIAL_ENV_NAME, gitHubCredentialEnvironment, gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForCommand, gitHubRepositoryForDirectory, normalizeGitHubHost, wrapGitHubCredentialCommand, @@ -459,6 +461,83 @@ test('keeps repository authorization bound to the admitted workspace root', asyn ); }); +test('checkout routing follows nested repositories only inside an admitted root', async (t) => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-github-checkout-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const nested = join(directory, 'worktrees', 'other'); + await mkdir(nested, { recursive: true }); + execFileSync('git', ['init', directory]); + execFileSync('git', [ + '-C', directory, 'remote', 'add', 'origin', 'git@github.com:acme/outer.git', + ]); + execFileSync('git', ['init', nested]); + execFileSync('git', [ + '-C', nested, 'remote', 'add', 'origin', 'git@github.com:acme/inner.git', + ]); + const admitted = new Map([[directory, 'acme/outer']]); + + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'admitted'), + 'acme/outer', + ); + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'checkout'), + 'acme/inner', + ); + execFileSync('git', [ + '-C', nested, 'remote', 'set-url', 'origin', 'git@github.com:acme/changed.git', + ]); + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'checkout'), + 'acme/changed', + ); + assert.equal( + await gitHubRepositoryForCommand(dirname(directory), admitted, 'checkout'), + undefined, + ); + const outside = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-github-outside-')), + ); + t.after(() => rm(outside, { recursive: true, force: true })); + execFileSync('git', ['init', outside]); + execFileSync('git', [ + '-C', outside, 'remote', 'add', 'origin', 'git@github.com:acme/outside.git', + ]); + const escaped = join(directory, 'worktrees', 'escaped'); + await symlink(outside, escaped); + assert.equal( + await gitHubRepositoryForCommand(escaped, admitted, 'checkout'), + undefined, + ); + admitted.set(outside, 'acme/outside'); + assert.equal( + await gitHubRepositoryForCommand(escaped, admitted, 'checkout'), + undefined, + ); + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'checkout', 'github.example.test'), + undefined, + ); + + const linked = join(directory, 'worktrees', 'linked'); + execFileSync('git', [ + '-C', nested, '-c', 'user.name=Test', '-c', 'user.email=test@example.com', + 'commit', '--allow-empty', '-m', 'initial', + ]); + execFileSync('git', ['-C', nested, 'worktree', 'add', '--detach', linked]); + assert.equal( + await gitHubRepositoryForCommand(linked, admitted, 'checkout'), + 'acme/changed', + ); + admitted.set(linked, 'acme/linked'); + assert.equal( + await gitHubRepositoryForCommand(linked, admitted, 'admitted'), + 'acme/linked', + ); +}); + test('uses the configured GHES host for the App bot no-reply identity', async (t) => { const directory = await mkdtemp(join(tmpdir(), 'librechat-code-ghes-identity-')); t.after(() => rm(directory, { recursive: true, force: true })); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index fb6b6fdc..19b8e639 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -1,7 +1,7 @@ import { constants } from 'node:fs'; import { execFile } from 'node:child_process'; import { createHash, createPrivateKey, sign } from 'node:crypto'; -import { open } from 'node:fs/promises'; +import { open, realpath } from 'node:fs/promises'; import { dirname, isAbsolute, relative, sep } from 'node:path'; import { promisify } from 'node:util'; import { projectRemote } from './projects.js'; @@ -137,21 +137,54 @@ export async function gitHubRepositoryForDirectory( return repository; } -/** Return the startup-bound repository for the admitted root containing cwd. */ -export function gitHubRepositoryForAdmittedDirectory( +function admittedRepositoryEntry( cwd: string, repositories: ReadonlyMap, -): string | undefined { - for (const [root, repository] of repositories) { +): readonly [string, string | undefined] | undefined { + let closest: readonly [string, string | undefined] | undefined; + for (const entry of repositories) { + const [root] = entry; const path = relative(root, cwd); if ( path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) ) { - return repository; + if (!closest || root.length > closest[0].length) closest = entry; } } - return undefined; + return closest; +} + +/** Return the startup-bound repository for the admitted root containing cwd. */ +export function gitHubRepositoryForAdmittedDirectory( + cwd: string, + repositories: ReadonlyMap, +): string | undefined { + return admittedRepositoryEntry(cwd, repositories)?.[1]; +} + +/** Resolve a checkout repository only when its cwd remains inside an admitted root. */ +export async function gitHubRepositoryForCommand( + cwd: string, + repositories: ReadonlyMap, + routing: 'admitted' | 'checkout', + host = 'github.com', + signal?: AbortSignal, +): Promise { + const admitted = admittedRepositoryEntry(cwd, repositories); + if (!admitted) return undefined; + if (routing === 'admitted') return admitted[1]; + let canonicalCwd: string; + try { + canonicalCwd = await realpath(cwd); + } catch { + signal?.throwIfAborted(); + return undefined; + } + if (admittedRepositoryEntry(canonicalCwd, repositories)?.[0] !== admitted[0]) { + return undefined; + } + return gitHubRepositoryForDirectory(canonicalCwd, host, signal); } function base64UrlJson(value: unknown): string {