diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 99ab9284..189bf1bd 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -429,6 +429,13 @@ egressGateway: console-staging.optale.com: httpsPassthrough: true httpsPassthroughTotalTimeoutMs: 300000 + # The Console API the workspace CLI calls (OPT-1586): its own header wait and + # per-run budget, metered apart from internet fetches. Stays under the sandbox + # proxy's 60 s idle cap so a slow call ends in a named FETCH_TIMEOUT. + httpsPassthroughLimits: + headersTimeoutMs: 55000 + maxFetchesPerGrant: 500 + maxAggregateBytesPerGrant: 268435456 # W730 §3 PENDING rows — do NOT widen this policy by inference. Each row # needs PRIMARY's rev-B journey evidence before it becomes a config entry: # 1. image attachment MIME on this host — PENDING diff --git a/service/config/external-fetch-policy.json b/service/config/external-fetch-policy.json index a1e86ab2..28ef177d 100644 --- a/service/config/external-fetch-policy.json +++ b/service/config/external-fetch-policy.json @@ -17,7 +17,12 @@ }, "console-staging.optale.com": { "httpsPassthrough": true, - "httpsPassthroughTotalTimeoutMs": 300000 + "httpsPassthroughTotalTimeoutMs": 300000, + "httpsPassthroughLimits": { + "headersTimeoutMs": 55000, + "maxFetchesPerGrant": 500, + "maxAggregateBytesPerGrant": 268435456 + } } } } diff --git a/service/src/egress-gateway.test.ts b/service/src/egress-gateway.test.ts index bf06df4f..72a6435b 100644 --- a/service/src/egress-gateway.test.ts +++ b/service/src/egress-gateway.test.ts @@ -2210,4 +2210,72 @@ describe('egress gateway routes', () => { }, ); + test('accepts more passthroughs than the grant fetch budget for a host with its own budget', async () => { + // service/config/external-fetch-policy.json grants console-staging.optale.com its own budget + // (httpsPassthroughLimits) while the grant-wide fetch budget stays 8. + const policyPath = path.resolve(__dirname, '../config/external-fetch-policy.json'); + env.EXTERNAL_FETCH_POLICY_FILE = policyPath; + const policy = parseExternalFetchPolicy(JSON.parse(fs.readFileSync(policyPath, 'utf8'))); + expect(policy.limits.maxFetchesPerGrant).toBe(8); + const host = 'console-staging.optale.com'; + const grant = claims({ + network_policy: serializeExternalFetchPolicy(policy), + network_policy_digest: externalFetchPolicyDigest(policy), + grant_id: 'grant_console_budget', + }); + const redis = new RedisMock(); + env.EGRESS_LEDGER_REQUIRED = true; + setEgressLedgerRedisForTest(redis); + await createEgressLedger(grant); + setHttpsPassthroughOpenForTest(async args => { + const target = validateHttpsPassthroughUrl(args.url, args.policy, args.method); + await args.beforeRequest?.(target); + const response = Object.assign(Readable.from([Buffer.from('{"ok":true}')]), { + statusCode: 200, + headers: { 'content-type': 'application/json' }, + rawHeaders: ['Content-Type', 'application/json'], + complete: false, + }) as unknown as IncomingMessage; + return { + response, + target, + responseBytes: 0, + redirects: 0, + timedOut: () => false, + close: () => response.destroy(), + }; + }); + try { + const envelope = JSON.stringify({ + url: `https://${host}/api/optale/mcp`, + method: 'POST', + headers: {}, + bodyBase64: '', + }); + for (let i = 0; i < 12; i += 1) { + const result = await gatewayPostWithTrailers( + '/https-passthrough', + { ...grantHeader(grant), 'Content-Type': 'application/json' }, + envelope, + ); + expect(result.status).toBe(200); + expect(result.body).toBe('{"ok":true}'); + } + const ledger = await assertEgressGrantActive(grant); + expect(ledger.fetch_count).toBe(0); + // Every call settled its byte reservation (the success path commits it) in the host budget. + expect(ledger.passthrough_budgets[host]).toEqual({ + fetch_count: 12, + fetched_bytes: 12 * '{"ok":true}'.length, + fetch_reservations: {}, + }); + expect(ledger.fetched_bytes).toBe(0); + expect(ledger.request_count).toBe(12); + } finally { + env.EGRESS_LEDGER_REQUIRED = false; + setEgressLedgerRedisForTest(null); + await redis.quit(); + } + }); + }); diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index bb06487e..7533e2e5 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -1890,6 +1890,7 @@ app.post('/https-passthrough', async (req, res) => { let reservedBytes = 0; let reservationId: string | undefined; let opened: OpenHttpsPassthroughResponse | undefined; + let passthroughHost: string | undefined; try { let envelope: HttpsPassthroughEnvelope; try { @@ -1912,10 +1913,17 @@ app.post('/https-passthrough', async (req, res) => { beforeRequest: async target => { await consumeEgressFetchAttempt({ grant: activeGrant, - maxFetches: target.policy.limits.maxFetchesPerGrant, + ...(target.policy.httpsPassthroughLimits + ? { + maxFetches: target.policy.httpsPassthroughLimits.maxFetchesPerGrant, + passthroughHost: target.host, + } + : { maxFetches: target.policy.limits.maxFetchesPerGrant }), }); }, }); + const ownBudget = opened.target.policy.httpsPassthroughLimits; + passthroughHost = ownBudget ? opened.target.host : undefined; const fields = auditFields(res); fields.destinationHost = opened.target.host; fields.pathHash = opened.target.pathHash; @@ -1951,7 +1959,9 @@ app.post('/https-passthrough', async (req, res) => { ? declaredBytes : opened.target.policy.limits.maxResponseBytes, maxAggregateBytes: + ownBudget?.maxAggregateBytesPerGrant ?? opened.target.policy.limits.maxAggregateBytesPerGrant, + passthroughHost, }); if (declaredBytes !== undefined && declaredBytes > reservedBytes) { opened.close(); @@ -1972,6 +1982,7 @@ app.post('/https-passthrough', async (req, res) => { reservationId, reservedBytes, responseBytes, + passthroughHost, }); reservedBytes = 0; fields.responseBytes = responseBytes; @@ -1991,6 +2002,7 @@ app.post('/https-passthrough', async (req, res) => { reservationId, reservedBytes, responseBytes, + passthroughHost, }); reservedBytes = 0; } catch { @@ -2023,7 +2035,7 @@ app.post('/https-passthrough', async (req, res) => { } finally { if (grant && reservationId && reservedBytes > 0) { try { - await egressFetchRelease({ grant, reservationId, reservedBytes }); + await egressFetchRelease({ grant, reservationId, reservedBytes, passthroughHost }); } catch { logger.error( 'Failed to release HTTPS passthrough byte reservation', diff --git a/service/src/egress-ledger.test.ts b/service/src/egress-ledger.test.ts index 8bc29464..7f8b4bdf 100644 --- a/service/src/egress-ledger.test.ts +++ b/service/src/egress-ledger.test.ts @@ -241,6 +241,50 @@ describe('egress Redis ledger', () => { expect(record.fetch_count).toBe(1); }); + test('meters a passthrough host with its own limits apart from the grant-wide fetch budget', async () => { + const claims = grant({ max_requests: 100 }); + await createEgressLedger(claims); + + for (let i = 0; i < 20; i += 1) { + await consumeEgressFetchAttempt({ grant: claims, maxFetches: 50, passthroughHost: 'console.test' }); + } + // Twenty Console calls leave the internet-fetch budget of 8 untouched. + for (let i = 0; i < 8; i += 1) { + await consumeEgressFetchAttempt({ grant: claims, maxFetches: 8 }); + } + await expectFetchError( + () => consumeEgressFetchAttempt({ grant: claims, maxFetches: 8 }), + 'FETCH_BUDGET_EXCEEDED', + ); + // An exhausted internet budget does not stop the Console host. + await consumeEgressFetchAttempt({ grant: claims, maxFetches: 50, passthroughHost: 'console.test' }); + + const reserved = await reserveEgressFetchBytes({ + grant: claims, + reservationId: 'console-reservation', + maxBytes: 60, + maxAggregateBytes: 100, + passthroughHost: 'console.test', + }); + await commitEgressFetchBytes({ + grant: claims, + reservationId: 'console-reservation', + reservedBytes: reserved, + responseBytes: 25, + passthroughHost: 'console.test', + }); + + const record = await assertEgressGrantActive(claims); + expect(record.fetch_count).toBe(8); + expect(record.fetched_bytes).toBe(0); + expect(record.passthrough_budgets['console.test']).toEqual({ + fetch_count: 21, + fetched_bytes: 25, + fetch_reservations: {}, + }); + expect(record.request_count).toBe(29); + }); + test('reserves aggregate bytes, commits actual bytes, and releases failed reservations', async () => { const claims = grant({ max_requests: 20 }); await createEgressLedger(claims); diff --git a/service/src/egress-ledger.ts b/service/src/egress-ledger.ts index 85fff02d..a6f7c500 100644 --- a/service/src/egress-ledger.ts +++ b/service/src/egress-ledger.ts @@ -10,7 +10,19 @@ import { redisKeepAliveOptions } from './redis-options'; type LedgerStatus = 'active' | 'revoked'; -export interface EgressLedgerRecord { +/** Requests and response bytes metered against one fetch budget. */ +export interface EgressFetchBudget { + fetch_count: number; + fetched_bytes: number; + fetch_reservations: Record; +} + +/** + * The record's own fetch fields are the grant-wide external-fetch budget. A passthrough host with + * its own limits is metered in `passthrough_budgets[host]` instead, so Console API calls from the + * workspace CLI neither use up nor are capped by the internet-fetch budget. + */ +export interface EgressLedgerRecord extends EgressFetchBudget { grant_id: string; exec_id: string; status: LedgerStatus; @@ -28,9 +40,7 @@ export interface EgressLedgerRecord { upload_count: number; tool_call_count: number; uploaded_bytes: number; - fetch_count: number; - fetched_bytes: number; - fetch_reservations: Record; + passthrough_budgets: Record; output_file_ids: string[]; } @@ -175,6 +185,7 @@ function recordFromGrant(grant: EgressGrantClaims): EgressLedgerRecord { fetch_count: 0, fetched_bytes: 0, fetch_reservations: {}, + passthrough_budgets: {}, uploaded_bytes: 0, output_file_ids: [], }; @@ -207,19 +218,38 @@ export async function ensureEgressLedger(grant: EgressGrantClaims): Promise { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +function normalizeBudget(budget: T): T { + if (!Number.isSafeInteger(budget.fetch_count) || budget.fetch_count < 0) budget.fetch_count = 0; + if (!Number.isSafeInteger(budget.fetched_bytes) || budget.fetched_bytes < 0) budget.fetched_bytes = 0; + if (!isPlainObject(budget.fetch_reservations)) budget.fetch_reservations = {}; + return budget; +} + function normalizeRecord(record: EgressLedgerRecord): EgressLedgerRecord { - if (!Number.isSafeInteger(record.fetch_count) || record.fetch_count < 0) record.fetch_count = 0; - if (!Number.isSafeInteger(record.fetched_bytes) || record.fetched_bytes < 0) record.fetched_bytes = 0; - if ( - !record.fetch_reservations - || typeof record.fetch_reservations !== 'object' - || Array.isArray(record.fetch_reservations) - ) { - record.fetch_reservations = {}; + normalizeBudget(record); + if (!isPlainObject(record.passthrough_budgets)) record.passthrough_budgets = {}; + for (const [host, budget] of Object.entries(record.passthrough_budgets)) { + record.passthrough_budgets[host] = normalizeBudget( + (isPlainObject(budget) ? budget : {}) as unknown as EgressFetchBudget, + ); } return record; } +/** The grant-wide budget, or the named passthrough host's own budget. */ +function fetchBudget(record: EgressLedgerRecord, passthroughHost: string | undefined): EgressFetchBudget { + if (passthroughHost === undefined) return record; + const existing = record.passthrough_budgets[passthroughHost]; + if (existing) return existing; + const created: EgressFetchBudget = { fetch_count: 0, fetched_bytes: 0, fetch_reservations: {} }; + record.passthrough_budgets[passthroughHost] = created; + return created; +} + async function loadRecord(grantId: string): Promise { const raw = await redisConnection().get(ledgerKey(grantId)); if (!raw) { @@ -350,16 +380,22 @@ export async function recordEgressToolCall(grantId: string | undefined, executio }); } +/** + * `passthroughHost` names a passthrough host whose policy carries its own limits; its requests and + * bytes are then metered in that host's budget. Omitted, the grant-wide budget is used. + */ export async function consumeEgressFetchAttempt(args: { grant: EgressGrantClaims; maxFetches: number; + passthroughHost?: string; }): Promise { return mutateRecord(args.grant, record => { - if (!Number.isSafeInteger(args.maxFetches) || args.maxFetches < 1 || record.fetch_count >= args.maxFetches) { + const budget = fetchBudget(record, args.passthroughHost); + if (!Number.isSafeInteger(args.maxFetches) || args.maxFetches < 1 || budget.fetch_count >= args.maxFetches) { throw new ExternalFetchError('FETCH_BUDGET_EXCEEDED'); } record.request_count += 1; - record.fetch_count += 1; + budget.fetch_count += 1; }); } @@ -368,13 +404,15 @@ export async function reserveEgressFetchBytes(args: { reservationId: string; maxBytes: number; maxAggregateBytes: number; + passthroughHost?: string; }): Promise { if (!/^[A-Za-z0-9_-]{1,128}$/.test(args.reservationId)) { throw new ExternalFetchError('FETCH_FAILED'); } let reservedBytes = 0; await mutateRecord(args.grant, record => { - const existing = record.fetch_reservations[args.reservationId]; + const budget = fetchBudget(record, args.passthroughHost); + const existing = budget.fetch_reservations[args.reservationId]; if (existing !== undefined) { reservedBytes = existing; return; @@ -387,11 +425,11 @@ export async function reserveEgressFetchBytes(args: { ) { throw new ExternalFetchError('FETCH_BUDGET_EXCEEDED'); } - const remaining = args.maxAggregateBytes - record.fetched_bytes; + const remaining = args.maxAggregateBytes - budget.fetched_bytes; if (remaining < 1) throw new ExternalFetchError('FETCH_BUDGET_EXCEEDED'); reservedBytes = Math.min(args.maxBytes, remaining); - record.fetched_bytes += reservedBytes; - record.fetch_reservations[args.reservationId] = reservedBytes; + budget.fetched_bytes += reservedBytes; + budget.fetch_reservations[args.reservationId] = reservedBytes; }); return reservedBytes; } @@ -401,6 +439,7 @@ export async function commitEgressFetchBytes(args: { reservationId: string; reservedBytes: number; responseBytes: number; + passthroughHost?: string; }): Promise { if ( !/^[A-Za-z0-9_-]{1,128}$/.test(args.reservationId) @@ -413,13 +452,14 @@ export async function commitEgressFetchBytes(args: { throw new ExternalFetchError('FETCH_FAILED'); } await mutateRecord(args.grant, record => { - const reservedBytes = record.fetch_reservations[args.reservationId]; + const budget = fetchBudget(record, args.passthroughHost); + const reservedBytes = budget.fetch_reservations[args.reservationId]; if (reservedBytes === undefined) return; if (reservedBytes !== args.reservedBytes) { throw new ExternalFetchError('FETCH_FAILED'); } - record.fetched_bytes = Math.max(0, record.fetched_bytes - (reservedBytes - args.responseBytes)); - delete record.fetch_reservations[args.reservationId]; + budget.fetched_bytes = Math.max(0, budget.fetched_bytes - (reservedBytes - args.responseBytes)); + delete budget.fetch_reservations[args.reservationId]; }); } @@ -427,6 +467,7 @@ export async function releaseEgressFetchBytes(args: { grant: EgressGrantClaims; reservationId: string; reservedBytes: number; + passthroughHost?: string; }): Promise { await commitEgressFetchBytes({ ...args, responseBytes: 0 }); } diff --git a/service/src/external-fetch-policy.test.ts b/service/src/external-fetch-policy.test.ts index fe817958..a846afdc 100644 --- a/service/src/external-fetch-policy.test.ts +++ b/service/src/external-fetch-policy.test.ts @@ -2,6 +2,7 @@ import crypto from 'node:crypto'; import { describe, expect, test } from 'bun:test'; import { HARD_EXTERNAL_FETCH_LIMITS, + HARD_HTTPS_PASSTHROUGH_LIMITS, HARD_MAX_EXTERNAL_FETCH_HOSTS, effectiveExternalFetchPolicy, externalFetchPolicyDigest, @@ -652,3 +653,78 @@ describe('compatibility with the policies the Console signs today', () => { expect([...effective.hosts.keys()].sort()).toEqual([CONSOLE_HOST, 'files.pythonhosted.org', 'pypi.org']); }); }); + +const CONSOLE_BUDGET = { + headersTimeoutMs: 120_000, + maxFetchesPerGrant: 500, + maxAggregateBytesPerGrant: 268_435_456, +}; + +describe('Console API passthrough budget (httpsPassthroughLimits)', () => { + const deployment = parseExternalFetchPolicy({ + version: 1, + limits: { ...PRODUCTION_SHAPED_DEPLOYMENT.limits }, + hosts: { + [CONSOLE_HOST]: { + httpsPassthrough: true, + httpsPassthroughTotalTimeoutMs: 300_000, + httpsPassthroughLimits: CONSOLE_BUDGET, + limits: { maxFetchesPerGrant: 8, maxResponseBytes: 2_097_152 }, + }, + [FROZEN_HOST]: { contentTypes: ['application/pdf'] }, + }, + }); + + test('applies only where the signature asks for it and the deployment grants it, at the smaller values', () => { + const signed = { + version: 1, + limits: { ...PRODUCTION_SHAPED_DEPLOYMENT.limits }, + hosts: { + [CONSOLE_HOST]: { + httpsPassthrough: true, + httpsPassthroughTotalTimeoutMs: 60_000, + httpsPassthroughLimits: { ...CONSOLE_BUDGET, headersTimeoutMs: 60_000, maxFetchesPerGrant: 1_000 }, + limits: PASSTHROUGH_LIMITS, + }, + [FROZEN_HOST]: { contentTypes: ['application/pdf'], limits: { ...PRODUCTION_SHAPED_DEPLOYMENT.limits } }, + }, + }; + const effective = effectiveExternalFetchPolicy(signed as never, consoleDigest(signed), deployment); + expect(effective.hosts.get(CONSOLE_HOST)?.httpsPassthroughLimits).toEqual({ + ...CONSOLE_BUDGET, + headersTimeoutMs: 60_000, + }); + // The internet host keeps the frozen limits. + expect(effective.hosts.get(FROZEN_HOST)?.httpsPassthroughLimits).toBeUndefined(); + expect(effective.hosts.get(FROZEN_HOST)?.limits.maxFetchesPerGrant).toBe(8); + expect(effective.hosts.get(FROZEN_HOST)?.limits.headersTimeoutMs).toBe(5_000); + + // A snapshot from a Console that does not sign the budget still verifies and keeps the old limits. + const legacy = { + version: 1, + limits: { ...PRODUCTION_SHAPED_DEPLOYMENT.limits }, + hosts: { [CONSOLE_HOST]: { httpsPassthrough: true, httpsPassthroughTotalTimeoutMs: 300_000, limits: PASSTHROUGH_LIMITS } }, + }; + const legacyEffective = effectiveExternalFetchPolicy(legacy as never, consoleDigest(legacy), deployment); + expect(legacyEffective.hosts.get(CONSOLE_HOST)?.httpsPassthroughLimits).toBeUndefined(); + + // A deployment that does not grant the budget caps a signature that asks for it. + const signedBudget = { ...signed, hosts: { [CONSOLE_HOST]: signed.hosts[CONSOLE_HOST] } }; + expect( + effectiveExternalFetchPolicy(signedBudget as never, consoleDigest(signedBudget), PRODUCTION_SHAPED_DEPLOYMENT) + .hosts.get(CONSOLE_HOST)?.httpsPassthroughLimits, + ).toBeUndefined(); + }); + + test.each([ + ['without HTTPS passthrough', { contentTypes: ['application/json'], httpsPassthroughLimits: CONSOLE_BUDGET }], + ['a missing key', { httpsPassthrough: true, httpsPassthroughLimits: { headersTimeoutMs: 1_000, maxFetchesPerGrant: 10 } }], + ['an unknown key', { httpsPassthrough: true, httpsPassthroughLimits: { ...CONSOLE_BUDGET, maxRedirects: 1 } }], + ['a headers timeout beyond the passthrough total', { httpsPassthrough: true, httpsPassthroughTotalTimeoutMs: 30_000, httpsPassthroughLimits: { ...CONSOLE_BUDGET, headersTimeoutMs: 30_001 } }], + ['a fetch count above the ceiling', { httpsPassthrough: true, httpsPassthroughLimits: { ...CONSOLE_BUDGET, maxFetchesPerGrant: HARD_HTTPS_PASSTHROUGH_LIMITS.maxFetchesPerGrant + 1 } }], + ])('rejects a budget %s', (_label, host) => { + expect(() => + parseExternalFetchPolicy({ version: 1, limits: { ...HARD_EXTERNAL_FETCH_LIMITS }, hosts: { [CONSOLE_HOST]: host } }), + ).toThrow(); + }); +}); diff --git a/service/src/external-fetch-policy.ts b/service/src/external-fetch-policy.ts index c55e82df..95155655 100644 --- a/service/src/external-fetch-policy.ts +++ b/service/src/external-fetch-policy.ts @@ -33,11 +33,26 @@ export interface HttpsPassthroughRoute { path: string; } +/** + * A passthrough host's own budget. When a host carries it, its HTTPS passthrough requests wait + * for response headers up to `headersTimeoutMs` and are metered in a per-host ledger budget of + * `maxFetchesPerGrant` requests and `maxAggregateBytesPerGrant` response bytes, apart from the + * grant's external-fetch budget. Without it, passthrough shares the host's `limits` and the + * grant-wide budget. Meant for the Console API the workspace CLI calls: a connector call there + * can take tens of seconds and a batch program makes hundreds of them. + */ +export interface HttpsPassthroughLimits { + headersTimeoutMs: number; + maxFetchesPerGrant: number; + maxAggregateBytesPerGrant: number; +} + export interface ExternalFetchHostPolicy { contentTypes: Set; httpsPassthrough: boolean; httpsPassthroughTotalTimeoutMs?: number; httpsPassthroughRoutes?: HttpsPassthroughRoute[]; + httpsPassthroughLimits?: HttpsPassthroughLimits; packageTransport: boolean; limits: ExternalFetchLimits; } @@ -47,6 +62,7 @@ export interface ExternalFetchPolicySnapshotHost { httpsPassthrough?: true; httpsPassthroughTotalTimeoutMs?: number; httpsPassthroughRoutes?: HttpsPassthroughRoute[]; + httpsPassthroughLimits?: HttpsPassthroughLimits; packageTransport?: true; limits: ExternalFetchLimits; } @@ -77,6 +93,17 @@ export interface ResolvedExternalAddress { } export const HARD_HTTPS_PASSTHROUGH_TOTAL_TIMEOUT_MS = 300_000; +export const HARD_HTTPS_PASSTHROUGH_LIMITS: Readonly = + Object.freeze({ + headersTimeoutMs: HARD_HTTPS_PASSTHROUGH_TOTAL_TIMEOUT_MS, + maxFetchesPerGrant: 1_000, + maxAggregateBytesPerGrant: 536_870_912, + }); +const HTTPS_PASSTHROUGH_LIMIT_KEYS: readonly (keyof HttpsPassthroughLimits)[] = [ + 'headersTimeoutMs', + 'maxFetchesPerGrant', + 'maxAggregateBytesPerGrant', +]; export const HARD_MAX_EXTERNAL_FETCH_HOSTS = 256; export const HARD_EXTERNAL_FETCH_LIMITS: Readonly = @@ -260,6 +287,29 @@ function parseLimits( return parsed; } +/** Every key is required, so a signed snapshot has one spelling and the digest is unambiguous. */ +function parseHttpsPassthroughLimits( + value: unknown, + host: string, + totalTimeoutMs: number, +): HttpsPassthroughLimits { + const label = `External fetch host ${host} httpsPassthroughLimits`; + const raw = objectValue(value, label); + assertOnlyKeys(raw, HTTPS_PASSTHROUGH_LIMIT_KEYS, label); + const parsed = {} as HttpsPassthroughLimits; + for (const key of HTTPS_PASSTHROUGH_LIMIT_KEYS) { + if (!(key in raw)) throw new Error(`${label}.${key} is required`); + parsed[key] = positiveInteger( + raw[key], + `${label}.${key}`, + key === 'headersTimeoutMs' + ? Math.min(totalTimeoutMs, HARD_HTTPS_PASSTHROUGH_LIMITS.headersTimeoutMs) + : HARD_HTTPS_PASSTHROUGH_LIMITS[key], + ); + } + return parsed; +} + function validatePolicyHostname(host: string): void { if ( host.length < 1 || @@ -377,6 +427,7 @@ export function parseExternalFetchPolicy(value: unknown): ExternalFetchPolicy { 'httpsPassthrough', 'httpsPassthroughTotalTimeoutMs', 'httpsPassthroughRoutes', + 'httpsPassthroughLimits', 'packageTransport', 'limits', ], @@ -426,6 +477,20 @@ export function parseExternalFetchPolicy(value: unknown): ExternalFetchPolicy { rawHost.httpsPassthroughRoutes === undefined ? undefined : parseHttpsPassthroughRoutes(rawHost.httpsPassthroughRoutes, host); + if (rawHost.httpsPassthroughLimits !== undefined && !httpsPassthrough) { + throw new Error( + `External fetch host ${host} cannot set httpsPassthroughLimits without HTTPS passthrough`, + ); + } + const httpsPassthroughLimits = + rawHost.httpsPassthroughLimits === undefined || + httpsPassthroughTotalTimeoutMs === undefined + ? undefined + : parseHttpsPassthroughLimits( + rawHost.httpsPassthroughLimits, + host, + httpsPassthroughTotalTimeoutMs, + ); if ( rawHost.contentTypes !== undefined && !Array.isArray(rawHost.contentTypes) @@ -465,6 +530,7 @@ export function parseExternalFetchPolicy(value: unknown): ExternalFetchPolicy { ? {} : { httpsPassthroughTotalTimeoutMs }), ...(httpsPassthroughRoutes === undefined ? {} : { httpsPassthroughRoutes }), + ...(httpsPassthroughLimits === undefined ? {} : { httpsPassthroughLimits }), limits: rawHost.limits === undefined ? { ...limits } @@ -661,6 +727,9 @@ export function serializeExternalFetchPolicy( route => ({ ...route }), ), }), + ...(entry.httpsPassthroughLimits === undefined + ? {} + : { httpsPassthroughLimits: { ...entry.httpsPassthroughLimits } }), ...(entry.packageTransport ? { packageTransport: true as const } : {}), @@ -695,6 +764,34 @@ function intersectHttpsPassthroughRoutes( return requested.map(route => ({ ...route })); } +/** + * A host's own passthrough budget applies only when the signed policy asks for it AND the + * deployment grants it, at the smaller value of each; otherwise passthrough to the host stays on + * the host's ordinary limits and the grant-wide budget. + */ +function intersectHttpsPassthroughLimits( + requested: HttpsPassthroughLimits | undefined, + upper: HttpsPassthroughLimits | undefined, + totalTimeoutMs: number, +): HttpsPassthroughLimits | undefined { + if (!requested || !upper) return undefined; + return { + headersTimeoutMs: Math.min( + requested.headersTimeoutMs, + upper.headersTimeoutMs, + totalTimeoutMs, + ), + maxFetchesPerGrant: Math.min( + requested.maxFetchesPerGrant, + upper.maxFetchesPerGrant, + ), + maxAggregateBytesPerGrant: Math.min( + requested.maxAggregateBytesPerGrant, + upper.maxAggregateBytesPerGrant, + ), + }; +} + export function intersectExternalFetchPolicies( signed: ExternalFetchPolicy, deployment: ExternalFetchPolicy, @@ -731,20 +828,30 @@ export function intersectExternalFetchPolicies( upper.httpsPassthroughRoutes, ) : undefined; + const httpsPassthroughTotalTimeoutMs = requested.httpsPassthrough + ? Math.min( + requested.httpsPassthroughTotalTimeoutMs ?? + HARD_HTTPS_PASSTHROUGH_TOTAL_TIMEOUT_MS, + upper.httpsPassthroughTotalTimeoutMs ?? + HARD_HTTPS_PASSTHROUGH_TOTAL_TIMEOUT_MS, + ) + : undefined; + const httpsPassthroughLimits = + httpsPassthroughTotalTimeoutMs === undefined + ? undefined + : intersectHttpsPassthroughLimits( + requested.httpsPassthroughLimits, + upper.httpsPassthroughLimits, + httpsPassthroughTotalTimeoutMs, + ); hosts.set(host, { contentTypes: new Set(requested.contentTypes), httpsPassthrough: requested.httpsPassthrough, - ...(requested.httpsPassthrough - ? { - httpsPassthroughTotalTimeoutMs: Math.min( - requested.httpsPassthroughTotalTimeoutMs ?? - HARD_HTTPS_PASSTHROUGH_TOTAL_TIMEOUT_MS, - upper.httpsPassthroughTotalTimeoutMs ?? - HARD_HTTPS_PASSTHROUGH_TOTAL_TIMEOUT_MS, - ), - } - : {}), + ...(httpsPassthroughTotalTimeoutMs === undefined + ? {} + : { httpsPassthroughTotalTimeoutMs }), ...(httpsPassthroughRoutes ? { httpsPassthroughRoutes } : {}), + ...(httpsPassthroughLimits ? { httpsPassthroughLimits } : {}), packageTransport: requested.packageTransport, limits: hostLimits, }); diff --git a/service/src/external-fetch.ts b/service/src/external-fetch.ts index d5738d5f..53bfb058 100644 --- a/service/src/external-fetch.ts +++ b/service/src/external-fetch.ts @@ -396,7 +396,8 @@ async function requestPinnedPassthrough( headersTimer = setTimeout(() => { timedOut = true; request.destroy(new Error('HTTPS passthrough headers timeout')); - }, target.policy.limits.headersTimeoutMs); + }, target.policy.httpsPassthroughLimits?.headersTimeoutMs ?? + target.policy.limits.headersTimeoutMs); }); }); request.once('response', response => {