From 7b4feb9aaf9fc2684f65c4e8ace5aaa20da84229 Mon Sep 17 00:00:00 2001 From: Lia Date: Sat, 26 Sep 2026 04:50:59 +0000 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=94=90=20feat:=20Recover=20Code=20API?= =?UTF-8?q?=20Machine=20Credentials=20After=20Outages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/adr/001-stateful-code-environments.md | 7 +- docs/remote-bridge/README.md | 49 ++- docs/remote-bridge/worker-runbook.md | 15 +- packages/code/package.json | 4 + packages/code/src/identity.ts | 48 +++ packages/code/src/protocol.ts | 15 + service/src/bridge/index.ts | 17 +- service/src/bridge/pairing.ts | 398 ++++++++++++++++++++- service/src/bridge/recovery-router.test.ts | 116 ++++++ service/src/bridge/recovery.test.ts | 336 +++++++++++++++++ service/src/bridge/router.ts | 71 ++++ service/src/config.ts | 13 + 12 files changed, 1071 insertions(+), 18 deletions(-) create mode 100644 service/src/bridge/recovery-router.test.ts create mode 100644 service/src/bridge/recovery.test.ts diff --git a/docs/adr/001-stateful-code-environments.md b/docs/adr/001-stateful-code-environments.md index cfa44bf4..7992ded8 100644 --- a/docs/adr/001-stateful-code-environments.md +++ b/docs/adr/001-stateful-code-environments.md @@ -52,8 +52,11 @@ worker replacement; the UI and operator documentation must not imply otherwise. - The VM requires no inbound internet listener. - Code API, not the worker, authenticates LibreChat users and normalizes work. - A stolen short-lived credential is insufficient without the worker private - key; a stolen private key is insufficient after credential expiry or - revocation. + key. In the original pairing-only model, a stolen private key is insufficient + after credential expiry or revocation. With optional durable machine + enrollment and signed credential recovery, the private key itself remains + a revocable long-lived credential: access-credential expiry alone does not + protect against theft of that key. Revocation invalidates both. - Pairing codes and credentials are stored by digest where lookup permits. - One configured worker has at most one active fenced assignment. - Sandbox isolation and default-deny egress remain the mandatory default; diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index efe84b84..42ddad5f 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -31,6 +31,36 @@ CODEAPI_BRIDGE_TOKEN= CODEAPI_BRIDGE_AUTH_MODE=paired ``` +To opt in to durable machine authorization on every Code API replica, set a +single stable public **Code API** origin (not the LibreChat URL): + +```dotenv +CODEAPI_BRIDGE_RECOVERY_SERVER_ID=https://code.example.com +# 0 (default): enrolled machine keys remain authorized until revoked. +# CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=0 +# CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=60 +# CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=12 +# CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=30 +``` + +Omitting the server ID retains the existing pairing and refresh behavior and +hides the recovery routes. Deploy the compatible Code API version to **all** +replicas before setting this value and enrolling workers again. Older Code API +replicas can still pair or refresh a worker but do not write durable enrollment; +they must not serve device login or recovery requests. Only a pairing redeemed +after this option is enabled has a recoverable key. Updating Code API alone +does not make old workers reconnect automatically: the CLI must also implement +this recovery protocol in the later worker release. + +Store Redis state durably across restarts (for example, persistent Redis with +AOF enabled, a managed persistent Redis service, and backups). Revocation and +machine enrollment share that state across replicas; do not configure eviction +of authorization keys. If enrollment state is missing, credentials minted under +that enrollment fail closed, and the worker must be explicitly enrolled again. +Restoring a backup from *before* a revocation can revive trust; reconcile +revocations after recovery from backup. Use a distinct server ID for each Code +API deployment and keep it stable when the endpoint changes behind a proxy. + Use `strict` instead of `affinity` if every request must include a runtime session hint. In hardened mode, startup requires the bridge token to be at least 32 bytes. `PTC_MODE=blocking` is rejected; replay mode is required because a @@ -226,7 +256,24 @@ execution. atomically on their first redemption attempt. - Worker credentials expire after fifteen minutes and are bound to an Ed25519 public key. Exact-request signatures include the HTTP method, path, body - digest, timestamp, nonce, and credential. + digest, timestamp, nonce, and credential. With recovery enabled, redeeming a + pairing also persists a separate machine authorization and its public key in + Redis without a TTL by default; an operator can instead set a bounded + enrollment lifetime. +- `POST /v1/bridge/workers/:workerId/credentials/challenge` accepts + `{ "protocolVersion": 1 }` without an administrator token and returns a + single-use, short-lived challenge with the configured server ID, worker ID, + enrollment generation, operation and expiry. The enrolled key signs that + entire challenge using `signBridgeRecovery` from `@librechat/code`'s identity + module. `POST .../credentials/recover` accepts the challenge fields plus + `signature` and returns a new short-lived credential. Creation and proof + attempts are bounded in shared Redis; HTTP 429 means back off. +- Recovery and revocation are atomic Redis transitions across API replicas. + A missing, revoked, expired or superseded enrollment never creates new + credentials. Recovery only restores transport authentication. It does not + clear assignment fences, worker or workspace quarantine, or uncertain + execution state. The worker private key is a durable, revocable credential; + expiry of an access credential alone does **not** protect against key theft. - Accepted proof nonces cannot be replayed, credentials rotate before expiry, and an administrator can revoke the active worker identity immediately. - Assignment leases bind to a stable paired identity rather than an individual diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 3043c541..f4c5eead 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -479,11 +479,16 @@ it still advertises named environments. ### Expired bridge credential -A running worker refreshes its short-lived credential automatically. If a -machine is offline long enough that refresh can no longer authenticate, issue -a fresh one-time pairing for the same worker ID and redeem it with a newly -generated keypair. Reusing the worker ID preserves the LibreChat environment -record and its agent assignments; creating a new ID creates a new environment. +A running worker refreshes its short-lived credential automatically. With +Code API durable enrollment enabled, a worker that still has its enrolled +private key can request a short-lived challenge and recover a new access +credential without manual re-pairing. The current CLI does **not** yet invoke +that endpoint automatically; update it when worker reconnect support ships. +Until then, or if enrollment is missing or revoked, use the one-time operator +pairing fallback. A new pairing replaces the Code API worker identity and may +require LibreChat environment reauthorization; reusing a worker ID alone does +not guarantee preservation of its LibreChat environment or agent assignments. +Never clear quarantine or workspace fences as part of credential recovery. ### Failed environment setup or uncertain mutation diff --git a/packages/code/package.json b/packages/code/package.json index b00ca807..d3cf714b 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -15,6 +15,10 @@ "types": "./dist/protocol.d.ts", "import": "./dist/protocol.js" }, + "./identity": { + "types": "./dist/identity.d.ts", + "import": "./dist/identity.js" + }, "./worker": { "types": "./dist/worker.d.ts", "import": "./dist/worker.js" diff --git a/packages/code/src/identity.ts b/packages/code/src/identity.ts index ab11c865..4b60e4f7 100644 --- a/packages/code/src/identity.ts +++ b/packages/code/src/identity.ts @@ -19,6 +19,16 @@ export interface BridgeRequestProofInput { body: string; } +/** Signed separately from access-credential requests, so neither proof can be reused for the other. */ +export interface BridgeRecoveryProofInput { + operation: 'credential.recover'; + serverId: string; + workerId: string; + enrollmentGeneration: string; + challenge: string; + expiresAt: string; +} + export function createBridgeIdentity(): BridgeIdentity { const { publicKey, privateKey } = generateKeyPairSync('ed25519', { publicKeyEncoding: { type: 'spki', format: 'pem' }, @@ -64,3 +74,41 @@ export function verifyBridgeRequest( return false; } } + +function canonicalBridgeRecovery(input: BridgeRecoveryProofInput): string { + return [ + 'librechat-code:bridge-recovery:v1', + input.operation, + input.serverId, + input.workerId, + input.enrollmentGeneration, + input.challenge, + input.expiresAt, + ].join('\n'); +} + +export function signBridgeRecovery( + privateKey: string, + input: BridgeRecoveryProofInput, +): string { + return sign(null, Buffer.from(canonicalBridgeRecovery(input)), privateKey).toString( + 'base64url', + ); +} + +export function verifyBridgeRecovery( + publicKey: string, + input: BridgeRecoveryProofInput, + signature: string, +): boolean { + try { + return verify( + null, + Buffer.from(canonicalBridgeRecovery(input)), + publicKey, + Buffer.from(signature, 'base64url'), + ); + } catch { + return false; + } +} diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index c678a1af..5bd6934a 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -759,6 +759,21 @@ export interface BridgeWorkerCredentialResponse { expiresAt: string; } +/** A short-lived, single-use challenge for an already enrolled machine key. */ +export interface BridgeRecoveryChallengeResponse { + protocolVersion: BridgeProtocolVersion; + operation: 'credential.recover'; + serverId: string; + workerId: string; + enrollmentGeneration: string; + challenge: string; + expiresAt: string; +} + +export interface BridgeRecoveryRequest extends BridgeRecoveryChallengeResponse { + signature: string; +} + export interface BridgeSandboxRequest { body: TBody; headers: Record; diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index 16857fad..2ad3c04f 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -11,7 +11,22 @@ export const bridgeStore = new RedisBridgeStore( undefined, env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, ); -export const bridgePairings = new RedisBridgePairingStore(connection); +export const bridgePairings = new RedisBridgePairingStore( + connection, + undefined, + undefined, + undefined, + undefined, + env.BRIDGE_RECOVERY_SERVER_ID + ? { + serverId: env.BRIDGE_RECOVERY_SERVER_ID, + enrollmentTtlSeconds: env.BRIDGE_ENROLLMENT_TTL_SECONDS, + challengeTtlSeconds: env.BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS, + maxChallengesPerMinute: env.BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE, + maxAttemptsPerMinute: env.BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, + } + : undefined, +); export default createBridgeRouter({ enabled: isBridgeEnabled(), diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index 9eb6d76c..bbd91f88 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -6,13 +6,17 @@ import { import type Redis from 'ioredis'; -import { verifyBridgeRequest } from '../../../packages/code/src/identity'; +import { verifyBridgeRecovery, verifyBridgeRequest } from '../../../packages/code/src/identity'; +import type { BridgeRecoveryProofInput } from '../../../packages/code/src/identity'; const PREFIX = 'codeapi:bridge:v1'; const DEFAULT_PAIRING_TTL_SECONDS = 10 * 60; const DEFAULT_CREDENTIAL_TTL_SECONDS = 15 * 60; const PROOF_NONCE_TTL_SECONDS = 2 * 60; const PROOF_CLOCK_SKEW_MS = 60_000; +const DEFAULT_CHALLENGE_TTL_SECONDS = 60; +const DEFAULT_CHALLENGES_PER_MINUTE = 12; +const DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE = 30; const LEGACY_SCAN_CLAIM_TTL_MS = 5_000; const LEGACY_SCAN_POLL_INTERVAL_MS = 25; const LEGACY_SCAN_PENDING = 'pending'; @@ -41,6 +45,7 @@ elseif (generation or '0') ~= ARGV[2] then redis.call('DEL', KEYS[1]) return 0 end +if ARGV[7] ~= '' and (generation or '0') ~= ARGV[9] then return 0 end redis.call('DEL', KEYS[1]) if redis.call('GET', KEYS[5]) == KEYS[1] then redis.call('DEL', KEYS[5]) @@ -48,9 +53,26 @@ end redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) redis.call('SET', KEYS[6], ARGV[6], 'EX', ARGV[4]) +if ARGV[7] ~= '' then + if tonumber(ARGV[8]) > 0 then + redis.call('SET', KEYS[7], ARGV[7], 'EX', ARGV[8]) + else + redis.call('SET', KEYS[7], ARGV[7]) + end + redis.call('SET', KEYS[8], ARGV[10]) +else + redis.call('DEL', KEYS[7], KEYS[8]) +end return 1 `; const ROTATE_CREDENTIAL_SCRIPT = ` +if ARGV[6] ~= '' then + if redis.call('GET', KEYS[5]) ~= ARGV[6] or (redis.call('GET', KEYS[6]) or '0') ~= ARGV[7] or redis.call('GET', KEYS[7]) ~= ARGV[8] then + return 0 + end +elseif redis.call('EXISTS', KEYS[7]) == 1 then + return 0 +end local activeDigest = redis.call('GET', KEYS[1]) local previous = redis.call('GET', KEYS[2]) if not activeDigest or not previous then @@ -82,12 +104,40 @@ if credential then redis.call('DEL', KEYS[3]) redis.call('DEL', ARGV[1] .. credential) end -redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7]) +redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8]) if activeIncarnation then redis.call('SET', ARGV[2] .. activeIncarnation .. ':fenced', '1') end return 1 `; +const CREATE_RECOVERY_CHALLENGE_SCRIPT = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] or (redis.call('GET', KEYS[4]) or '0') ~= ARGV[4] or redis.call('GET', KEYS[5]) ~= ARGV[6] then + return 0 +end +local count = redis.call('INCR', KEYS[2]) +if count == 1 then redis.call('EXPIRE', KEYS[2], 60) end +if count > tonumber(ARGV[2]) then return -1 end +if redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[5], 'NX') ~= 'OK' then return 0 end +return 1 +`; +const LIMIT_RECOVERY_ATTEMPTS_SCRIPT = ` +local count = redis.call('INCR', KEYS[1]) +if count == 1 then redis.call('EXPIRE', KEYS[1], 60) end +if count > tonumber(ARGV[1]) then return 0 end +return 1 +`; +const COMPLETE_RECOVERY_CHALLENGE_SCRIPT = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] or redis.call('GET', KEYS[2]) ~= ARGV[2] or (redis.call('GET', KEYS[6]) or '0') ~= ARGV[3] or redis.call('GET', KEYS[7]) ~= ARGV[8] then + return 0 +end +local stableIdentity = redis.call('GET', KEYS[5]) +if stableIdentity and stableIdentity ~= ARGV[4] then return 0 end +redis.call('DEL', KEYS[2]) +redis.call('SET', KEYS[3], ARGV[5], 'EX', ARGV[6]) +redis.call('SET', KEYS[4], ARGV[7], 'EX', ARGV[6]) +redis.call('SET', KEYS[5], ARGV[4], 'EX', ARGV[6]) +return 1 +`; const RELEASE_LEGACY_SCAN_CLAIM_SCRIPT = ` if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) @@ -147,8 +197,32 @@ interface StoredCredential { publicKey: string; expiresAt: string; binding?: BridgeWorkerBinding; + /** Present only for credentials backed by durable machine enrollment. */ + enrollmentGeneration?: string; +} + +interface StoredEnrollment { + workerId: string; + serverId: string; + identityId: string; + generation: string; + pairingGeneration: number; + publicKey: string; + binding?: BridgeWorkerBinding; +} + +export interface BridgeRecoveryOptions { + /** Stable HTTPS origin of this Code API deployment, identical on every replica. */ + serverId: string; + /** Zero (the default) keeps enrollment until explicit revocation. */ + enrollmentTtlSeconds?: number; + challengeTtlSeconds?: number; + maxChallengesPerMinute?: number; + maxAttemptsPerMinute?: number; } +export type BridgeRecoveryChallenge = BridgeRecoveryProofInput; + export interface BridgePairing { workerId: string; code: string; @@ -168,7 +242,10 @@ export class BridgePairingError extends Error { | 'PUBLIC_KEY_INVALID' | 'CREDENTIAL_INVALID' | 'PROOF_INVALID' - | 'PROOF_REPLAYED', + | 'PROOF_REPLAYED' + | 'ENROLLMENT_INVALID' + | 'CHALLENGE_INVALID' + | 'RECOVERY_RATE_LIMITED', message: string, ) { super(message); @@ -196,6 +273,22 @@ function workerStableIdentityKey(workerId: string): string { return `${PREFIX}:stable-identity:${workerId}`; } +function workerEnrollmentKey(workerId: string): string { + return `${PREFIX}:enrollment:${workerId}`; +} + +function workerEnrollmentRequiredKey(workerId: string): string { + return `${PREFIX}:enrollment-required:${workerId}`; +} + +function recoveryChallengeKey(challenge: string): string { + return `${PREFIX}:recovery:challenge:${digest(challenge)}`; +} + +function recoveryRateKey(workerId: string, operation: 'start' | 'complete'): string { + return `${PREFIX}:recovery:rate:${operation}:${workerId}`; +} + function workerPairingGenerationKey(workerId: string): string { return `${PREFIX}:pairing-generation:${workerId}`; } @@ -236,8 +329,100 @@ export class RedisBridgePairingStore { private readonly credentialTtlSeconds = DEFAULT_CREDENTIAL_TTL_SECONDS, private readonly legacyScanClaimTtlMs = LEGACY_SCAN_CLAIM_TTL_MS, private readonly rollbackEpoch = - process.env.CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH?.trim() ?? '', - ) {} + process.env.CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH?.trim() ?? '', + private readonly recovery?: BridgeRecoveryOptions, + ) { + if (recovery == null) return; + let server: URL; + try { + server = new URL(recovery.serverId); + } catch { + throw new Error('Bridge recovery requires a stable HTTPS server origin'); + } + if ( + server.protocol !== 'https:' || + server.username !== '' || + server.password !== '' || + server.pathname !== '/' || + server.search !== '' || + server.hash !== '' || + server.origin !== recovery.serverId + ) { + throw new Error('Bridge recovery requires a stable HTTPS server origin'); + } + for (const [value, max] of [ + [recovery.enrollmentTtlSeconds ?? 0, 10 * 365 * 24 * 3600], + [recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS, 300], + [recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE, 120], + [recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE, 120], + ]) { + if (!Number.isSafeInteger(value) || value < 0 || value > max) { + throw new RangeError('Invalid bridge recovery lifetime or rate limit'); + } + } + if ( + (recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS) === 0 || + (recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE) === 0 || + (recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE) === 0 + ) { + throw new RangeError('Bridge recovery challenge lifetime and rate limits must be positive'); + } + } + + get recoveryEnabled(): boolean { + return this.recovery != null; + } + + private checkedEnrollment( + workerId: string, + raw: string | null, + pairingGeneration: string | null, + requiredGeneration: string | null, + ): StoredEnrollment { + let parsed: unknown; + try { + parsed = raw == null ? undefined : JSON.parse(raw); + } catch { + parsed = undefined; + } + const enrollment = ( + typeof parsed === 'object' && parsed != null && !Array.isArray(parsed) + ? parsed + : {} + ) as Partial; + if ( + this.recovery == null || + enrollment.workerId !== workerId || + enrollment.serverId !== this.recovery.serverId || + typeof enrollment.identityId !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(enrollment.identityId) || + typeof enrollment.generation !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(enrollment.generation) || + enrollment.generation !== requiredGeneration || + typeof enrollment.publicKey !== 'string' || + !validEd25519PublicKey(enrollment.publicKey) || + !Number.isSafeInteger(enrollment.pairingGeneration) || + String(enrollment.pairingGeneration) !== (pairingGeneration ?? '0') + ) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine enrollment is unavailable or revoked'); + } + return enrollment as StoredEnrollment; + } + + private checkEnrolledCredential( + credential: StoredCredential, + enrollment: StoredEnrollment, + ): void { + if ( + credential.workerId !== enrollment.workerId || + credential.identityId !== enrollment.identityId || + credential.publicKey !== enrollment.publicKey || + credential.enrollmentGeneration !== enrollment.generation || + JSON.stringify(credential.binding ?? null) !== JSON.stringify(enrollment.binding ?? null) + ) { + throw new BridgePairingError('CREDENTIAL_INVALID', 'Worker credential does not match its machine enrollment'); + } + } async issue( workerId: string, @@ -314,28 +499,49 @@ export class RedisBridgePairingStore { Date.now() + this.credentialTtlSeconds * 1000, ).toISOString(); const identityId = randomBytes(18).toString('base64url'); + const pairingGeneration = pairing.generation ?? Number( + (await this.redis.get(workerPairingGenerationKey(args.workerId))) ?? '0', + ); + const enrollment: StoredEnrollment | undefined = this.recovery == null + ? undefined + : { + workerId: args.workerId, + serverId: this.recovery.serverId, + identityId, + generation: randomBytes(18).toString('base64url'), + pairingGeneration, + publicKey: args.publicKey, + binding: pairing.binding, + }; const stored: StoredCredential = { workerId: args.workerId, identityId, publicKey: args.publicKey, expiresAt, binding: pairing.binding, + ...(enrollment == null ? {} : { enrollmentGeneration: enrollment.generation }), }; const accepted = await this.redis.eval( REDEEM_PAIRING_SCRIPT, - 6, + 8, codeKey, workerPairingGenerationKey(pairing.workerId), credentialDigestKey(credentialDigest), workerIdentityKey(args.workerId), workerPairingIndexKey(args.workerId), workerStableIdentityKey(args.workerId), + workerEnrollmentKey(args.workerId), + workerEnrollmentRequiredKey(args.workerId), raw, pairing.generation == null ? '' : String(pairing.generation), JSON.stringify(stored), String(this.credentialTtlSeconds), credentialDigest, identityId, + enrollment == null ? '' : JSON.stringify(enrollment), + String(this.recovery?.enrollmentTtlSeconds ?? 0), + String(pairingGeneration), + enrollment?.generation ?? '', ); if (accepted !== 1) { throw new BridgePairingError( @@ -374,10 +580,12 @@ export class RedisBridgePairingStore { ); } const credentialDigest = digest(args.credential); - const [raw, activeDigest, pairingGeneration] = await this.redis.mget( + const [raw, activeDigest, pairingGeneration, enrollmentRaw, requiredGeneration] = await this.redis.mget( credentialDigestKey(credentialDigest), workerIdentityKey(args.workerId), workerPairingGenerationKey(args.workerId), + workerEnrollmentKey(args.workerId), + workerEnrollmentRequiredKey(args.workerId), ); if (raw == null || activeDigest == null) { throw new BridgePairingError( @@ -386,6 +594,12 @@ export class RedisBridgePairingStore { ); } const stored = JSON.parse(raw) as StoredCredential; + if (stored.enrollmentGeneration != null || requiredGeneration != null || enrollmentRaw != null) { + this.checkEnrolledCredential( + stored, + this.checkedEnrollment(args.workerId, enrollmentRaw, pairingGeneration, requiredGeneration), + ); + } if (activeDigest !== credentialDigest) { const activeRaw = await this.redis.get( credentialDigestKey(activeDigest), @@ -439,6 +653,148 @@ export class RedisBridgePairingStore { }; } + async createRecoveryChallenge(workerId: string): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const [raw, pairingGeneration, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + const enrollment = this.checkedEnrollment(workerId, raw, pairingGeneration, requiredGeneration); + const challenge: BridgeRecoveryChallenge = { + operation: 'credential.recover', + serverId: enrollment.serverId, + workerId, + enrollmentGeneration: enrollment.generation, + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date( + Date.now() + (this.recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS) * 1000, + ).toISOString(), + }; + const created = await this.redis.eval( + CREATE_RECOVERY_CHALLENGE_SCRIPT, + 5, + workerEnrollmentKey(workerId), + recoveryRateKey(workerId, 'start'), + recoveryChallengeKey(challenge.challenge), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + raw!, + String(this.recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE), + JSON.stringify(challenge), + String(enrollment.pairingGeneration), + String(this.recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS), + enrollment.generation, + ); + if (created === -1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery challenges'); + } + if (created !== 1) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine enrollment is unavailable or revoked'); + } + return challenge; + } + + async recoverCredential( + workerId: string, + proof: BridgeRecoveryChallenge, + signature: string, + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const allowed = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + recoveryRateKey(workerId, 'complete'), + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), + ); + if (allowed !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery attempts'); + } + const challengeKey = recoveryChallengeKey(proof.challenge); + const [enrollmentRaw, challengeRaw, pairingGeneration, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + challengeKey, + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + const enrollment = this.checkedEnrollment( + workerId, enrollmentRaw, pairingGeneration, requiredGeneration, + ); + let parsed: unknown; + try { + parsed = challengeRaw == null ? undefined : JSON.parse(challengeRaw); + } catch { + parsed = undefined; + } + const saved = ( + typeof parsed === 'object' && parsed != null && !Array.isArray(parsed) + ? parsed + : {} + ) as Partial; + if ( + saved.operation !== 'credential.recover' || + saved.serverId !== enrollment.serverId || + saved.workerId !== workerId || + saved.enrollmentGeneration !== enrollment.generation || + saved.challenge !== proof.challenge || + saved.expiresAt !== proof.expiresAt || + String(proof.operation) !== saved.operation || + saved.serverId !== proof.serverId || + saved.workerId !== proof.workerId || + saved.enrollmentGeneration !== proof.enrollmentGeneration || + typeof saved.expiresAt !== 'string' || + !Number.isFinite(Date.parse(saved.expiresAt)) || + Date.parse(saved.expiresAt) <= Date.now() + ) { + throw new BridgePairingError('CHALLENGE_INVALID', 'Machine recovery challenge is invalid or expired'); + } + if (!verifyBridgeRecovery(enrollment.publicKey, saved as BridgeRecoveryChallenge, signature)) { + throw new BridgePairingError('PROOF_INVALID', 'Machine recovery signature is invalid'); + } + + const credential = randomBytes(32).toString('base64url'); + const credentialDigest = digest(credential); + const expiresAt = new Date(Date.now() + this.credentialTtlSeconds * 1000).toISOString(); + const stored: StoredCredential = { + workerId, + identityId: enrollment.identityId, + publicKey: enrollment.publicKey, + expiresAt, + binding: enrollment.binding, + enrollmentGeneration: enrollment.generation, + }; + // The same Redis decision consumes the proof and issues the credential. + // A revoke or replacement on another replica wins by invalidating the + // enrollment/generation comparison, with no window to recreate trust. + const issued = await this.redis.eval( + COMPLETE_RECOVERY_CHALLENGE_SCRIPT, + 7, + workerEnrollmentKey(workerId), + challengeKey, + credentialDigestKey(credentialDigest), + workerIdentityKey(workerId), + workerStableIdentityKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + enrollmentRaw!, + challengeRaw!, + String(enrollment.pairingGeneration), + enrollment.identityId, + JSON.stringify(stored), + String(this.credentialTtlSeconds), + credentialDigest, + enrollment.generation, + ); + if (issued !== 1) { + throw new BridgePairingError('CHALLENGE_INVALID', 'Machine recovery challenge is invalid or expired'); + } + return { workerId, credential, expiresAt }; + } + async revoke(workerId: string): Promise { await this.removeLegacyPairings(workerId); // Fence redemption and consume the currently indexed code atomically. An @@ -446,7 +802,7 @@ export class RedisBridgePairingStore { // that linearizes afterward installs a distinct generation and code. await this.redis.eval( REVOKE_PAIRING_SCRIPT, - 7, + 8, workerPairingIndexKey(workerId), workerPairingGenerationKey(workerId), workerIdentityKey(workerId), @@ -454,6 +810,7 @@ export class RedisBridgePairingStore { `${PREFIX}:worker:${encodeURIComponent(workerId)}`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:ready`, + workerEnrollmentKey(workerId), `${PREFIX}:credential:`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:`, ); @@ -652,12 +1009,26 @@ export class RedisBridgePairingStore { ); } const previous = JSON.parse(previousRaw) as StoredCredential; + let enrollment: StoredEnrollment | undefined; + let enrollmentRaw: string | null = null; + const [raw, generation, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + if (previous.enrollmentGeneration != null || requiredGeneration != null || raw != null) { + enrollment = this.checkedEnrollment(workerId, raw, generation, requiredGeneration); + this.checkEnrolledCredential(previous, enrollment); + enrollmentRaw = raw; + } return await this.issueCredential( workerId, previous.publicKey, previousDigest, previous.binding, previous.identityId ?? null, + enrollment, + enrollmentRaw, ); } @@ -667,6 +1038,8 @@ export class RedisBridgePairingStore { previousDigest?: string, binding?: BridgeWorkerBinding, identityId?: string | null, + enrollment?: StoredEnrollment, + enrollmentRaw?: string | null, ): Promise { const credential = randomBytes(32).toString('base64url'); const credentialDigest = digest(credential); @@ -683,20 +1056,27 @@ export class RedisBridgePairingStore { publicKey, expiresAt, binding, + ...(enrollment == null ? {} : { enrollmentGeneration: enrollment.generation }), }; if (previousDigest !== undefined) { const rotated = await this.redis.eval( ROTATE_CREDENTIAL_SCRIPT, - 4, + 7, workerIdentityKey(workerId), credentialDigestKey(previousDigest), credentialDigestKey(credentialDigest), workerStableIdentityKey(workerId), + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), previousDigest, credentialDigest, JSON.stringify(stored), String(this.credentialTtlSeconds), stableIdentityId ?? '', + enrollmentRaw ?? '', + String(enrollment?.pairingGeneration ?? ''), + enrollment?.generation ?? '', ); if (rotated !== 1) { throw new BridgePairingError( diff --git a/service/src/bridge/recovery-router.test.ts b/service/src/bridge/recovery-router.test.ts new file mode 100644 index 00000000..d7c3f7ec --- /dev/null +++ b/service/src/bridge/recovery-router.test.ts @@ -0,0 +1,116 @@ +import { createServer, type Server } from 'http'; + +import { afterEach, describe, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; +import type { BridgeRecoveryChallengeResponse } from '../../../packages/code/src/protocol'; + +import { createBridgeIdentity, signBridgeRecovery } from '../../../packages/code/src/identity'; +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { RedisBridgePairingStore } from './pairing'; +import { createBridgeRouter } from './router'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const workerId = 'http-recovery-worker'; +let server: Server | undefined; + +async function startRouter(pairings: RedisBridgePairingStore): Promise { + const app = express(); + app.use(json()); + app.use('/v1/bridge', createBridgeRouter({ + store: new RedisBridgeStore(redis), + pairings, + authMode: 'paired', + adminToken: 'operator-only', + configuredWorkerId: workerId, + })); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + return `http://127.0.0.1:${address.port}/v1/bridge/workers/${workerId}/credentials`; +} + +function post(url: string, body: object): Promise { + return fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); +} + +afterEach(async () => { + server?.close(); + server = undefined; + await redis.flushall(); +}); + +describe('machine credential recovery HTTP API', () => { + test('does not expose recovery until the server identity is enabled', async () => { + const baseUrl = await startRouter(new RedisBridgePairingStore(redis)); + const response = await post(`${baseUrl}/challenge`, { protocolVersion: BRIDGE_PROTOCOL_VERSION }); + expect(response.status).toBe(404); + }); + + test('recovers using the stored key without administrator authentication and never leaks the binding', async () => { + const pairings = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId: 'https://code.example.test', + }); + const identity = createBridgeIdentity(); + const pairing = await pairings.issue(workerId, { + tenantId: 'tenant-one', principal: { type: 'user', id: 'owner-one' }, + }); + await pairings.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const baseUrl = await startRouter(pairings); + const challengeResponse = await post(`${baseUrl}/challenge`, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + }); + expect(challengeResponse.status).toBe(200); + const challenge = (await challengeResponse.json()) as BridgeRecoveryChallengeResponse; + expect(challenge).toMatchObject({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + serverId: 'https://code.example.test', + workerId, + operation: 'credential.recover', + }); + expect(JSON.stringify(challenge)).not.toMatch(/tenant-one|owner-one|privateKey/); + + const signature = signBridgeRecovery(identity.privateKey, challenge); + const rejected = await post(`${baseUrl}/recover`, { + ...challenge, + serverId: 'https://unrelated.example.test', + signature, + }); + expect(rejected.status).toBe(401); + await expect(rejected.json()).resolves.toMatchObject({ code: 'CHALLENGE_INVALID' }); + + const recovered = await post(`${baseUrl}/recover`, { ...challenge, signature }); + expect(recovered.status).toBe(200); + const credential = (await recovered.json()) as { workerId: string; credential: string }; + expect(credential.workerId).toBe(workerId); + expect(credential.credential.length).toBeGreaterThanOrEqual(32); + const replay = await post(`${baseUrl}/recover`, { ...challenge, signature }); + expect(replay.status).toBe(401); + await expect(replay.json()).resolves.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('limits recovery challenges across two API routers sharing Redis', async () => { + const first = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId: 'https://code.example.test', maxChallengesPerMinute: 1, + }); + const second = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId: 'https://code.example.test', maxChallengesPerMinute: 1, + }); + const identity = createBridgeIdentity(); + const pairing = await first.issue(workerId); + await first.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const baseUrl = await startRouter(second); + await first.createRecoveryChallenge(workerId); + const denied = await post(`${baseUrl}/challenge`, { protocolVersion: BRIDGE_PROTOCOL_VERSION }); + expect(denied.status).toBe(429); + await expect(denied.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); +}); diff --git a/service/src/bridge/recovery.test.ts b/service/src/bridge/recovery.test.ts new file mode 100644 index 00000000..a514b32b --- /dev/null +++ b/service/src/bridge/recovery.test.ts @@ -0,0 +1,336 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { createHash } from 'crypto'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; +import type { BridgeRecoveryProofInput } from '../../../packages/code/src/identity'; +import type { + BridgeRecoveryChallenge, + BridgeRecoveryOptions, + BridgeWorkerBinding, + BridgeWorkerCredential, +} from './pairing'; + +import { + createBridgeIdentity, + signBridgeRecovery, + signBridgeRequest, +} from '../../../packages/code/src/identity'; +import { RedisBridgePairingStore } from './pairing'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const serverId = 'https://code.example.test'; +const workerId = 'durable-worker'; +const binding: BridgeWorkerBinding = { + tenantId: 'tenant-one', + principal: { type: 'user', id: 'owner-one' }, +}; + +function recoverableStore(options: Partial = {}): RedisBridgePairingStore { + return new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, + ...options, + }); +} + +function authorizedRequest( + privateKey: string, + credential: string, + nonce: string, +): Parameters[0] { + const proof = { + credential, + method: 'POST', + path: '/v1/bridge/workers/register', + timestamp: new Date().toISOString(), + nonce, + body: JSON.stringify({ protocolVersion: 1, workerId }), + }; + return { + ...proof, + workerId, + signature: signBridgeRequest(privateKey, proof), + }; +} + +async function enroll( + store: RedisBridgePairingStore, + publicKey: string, +): Promise { + const pairing = await store.issue(workerId, binding); + return store.redeem({ workerId, code: pairing.code, publicKey }); +} + +async function recover( + store: RedisBridgePairingStore, + privateKey: string, +): Promise<{ challenge: BridgeRecoveryChallenge; credential: BridgeWorkerCredential }> { + const challenge = await store.createRecoveryChallenge(workerId); + const credential = await store.recoverCredential( + workerId, + challenge, + signBridgeRecovery(privateKey, challenge), + ); + return { challenge, credential }; +} + +afterEach(async () => { + await redis.flushall(); +}); + +describe('durable bridge enrollment', () => { + test('keeps legacy pairing and refresh compatible until recovery is enabled', async () => { + const legacy = new RedisBridgePairingStore(redis); + const identity = createBridgeIdentity(); + const issued = await enroll(legacy, identity.publicKey); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); + + const replica = recoverableStore(); + await expect(replica.createRecoveryChallenge(workerId)).rejects.toMatchObject({ + code: 'ENROLLMENT_INVALID', + }); + const rotated = await replica.rotate(workerId); + await expect( + replica.authorize(authorizedRequest(identity.privateKey, rotated.credential, 'legacy-proof')), + ).resolves.toMatchObject({ workerId, binding }); + expect(issued.credential).not.toBe(rotated.credential); + }); + + test('recovers after access expiry and restart with the same identity and binding', async () => { + const identity = createBridgeIdentity(); + const firstReplica = recoverableStore(); + const issued = await enroll(firstReplica, identity.publicKey); + const original = await firstReplica.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'before-outage'), + ); + const digest = createHash('sha256').update(issued.credential).digest('hex'); + await redis.del( + `codeapi:bridge:v1:credential:${digest}`, + `codeapi:bridge:v1:identity:${workerId}`, + `codeapi:bridge:v1:stable-identity:${workerId}`, + ); + await expect(firstReplica.rotate(workerId)).rejects.toMatchObject({ + code: 'CREDENTIAL_INVALID', + }); + + const restartedReplica = recoverableStore(); + const { challenge, credential } = await recover(restartedReplica, identity.privateKey); + expect(challenge).toMatchObject({ + operation: 'credential.recover', + serverId, + workerId, + }); + const restored = await firstReplica.authorize( + authorizedRequest(identity.privateKey, credential.credential, 'after-outage'), + ); + expect(restored).toMatchObject({ identityId: original.identityId, binding }); + expect(credential.expiresAt).toBeString(); + const rotated = await restartedReplica.rotate(workerId, restored.credentialId); + await expect(firstReplica.authorize( + authorizedRequest(identity.privateKey, rotated.credential, 'after-rotation'), + )).resolves.toMatchObject({ identityId: original.identityId, binding }); + }); + + test('requires the enrolled key and binds proofs to server, worker, generation, operation, and expiry', async () => { + const enrolled = createBridgeIdentity(); + const outsider = createBridgeIdentity(); + const store = recoverableStore(); + await enroll(store, enrolled.publicKey); + const challenge = await store.createRecoveryChallenge(workerId); + + await expect(store.recoverCredential( + workerId, challenge, signBridgeRecovery(outsider.privateKey, challenge), + )).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + for (const modified of [ + { ...challenge, serverId: 'https://other.example.test' }, + { ...challenge, workerId: 'another-worker' }, + { ...challenge, enrollmentGeneration: '0'.repeat(24) }, + { ...challenge, operation: 'credential.recover-other' as 'credential.recover' }, + { ...challenge, expiresAt: new Date(Date.now() + 120_000).toISOString() }, + ]) { + await expect(store.recoverCredential( + workerId, + modified, + signBridgeRecovery(enrolled.privateKey, modified as BridgeRecoveryProofInput), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + } + const signature = signBridgeRecovery(enrolled.privateKey, challenge); + await store.recoverCredential(workerId, challenge, signature); + await expect(store.recoverCredential(workerId, challenge, signature)) + .rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('rejects a signed challenge after its short-lived Redis window expires', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore({ challengeTtlSeconds: 1 }); + await enroll(store, identity.publicKey); + const challenge = await store.createRecoveryChallenge(workerId); + await new Promise((resolve) => setTimeout(resolve, 1_100)); + await expect(store.recoverCredential( + workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('retries a lost recovery response without changing the enrolled identity', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const originalCredential = await enroll(first, identity.publicKey); + const original = await first.authorize( + authorizedRequest(identity.privateKey, originalCredential.credential, 'before-lost-response'), + ); + await recover(first, identity.privateKey); // The caller lost this credential response. + const second = recoverableStore(); + const retried = await recover(second, identity.privateKey); + const auth = await second.authorize( + authorizedRequest(identity.privateKey, retried.credential.credential, 'after-lost-response'), + ); + expect(auth.identityId).toBe(original.identityId); + expect(auth.binding).toEqual(binding); + const enrolled = JSON.parse((await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`))!) as { + generation: string; + }; + expect(await redis.get(`codeapi:bridge:v1:enrollment-required:${workerId}`)) + .toBe(enrolled.generation); + }); + + test('rejects expired and missing enrollment even when an access credential remains live', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore({ enrollmentTtlSeconds: 1 }); + const issued = await enroll(store, identity.publicKey); + await new Promise((resolve) => setTimeout(resolve, 1_100)); + await expect(store.createRecoveryChallenge(workerId)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.rotate(workerId)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'expired-enrollment'), + )).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('never treats an unmarked access token as legacy after authorization state is lost', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore(); + const issued = await enroll(store, identity.publicKey); + const credentialKey = `codeapi:bridge:v1:credential:${createHash('sha256').update(issued.credential).digest('hex')}`; + const raw = JSON.parse((await redis.get(credentialKey))!) as { enrollmentGeneration?: string }; + delete raw.enrollmentGeneration; // Simulate a refresh by a pre-recovery replica. + await redis.set(credentialKey, JSON.stringify(raw), 'EX', 300); + await redis.del(`codeapi:bridge:v1:enrollment:${workerId}`); + + expect(await redis.get(`codeapi:bridge:v1:enrollment-required:${workerId}`)).not.toBeNull(); + await expect(store.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'lost-authorization'), + )).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.rotate(workerId)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.createRecoveryChallenge(workerId)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await redis.set(`codeapi:bridge:v1:enrollment:${workerId}`, 'null'); + await expect(store.createRecoveryChallenge(workerId)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('re-pairing explicitly supersedes a previous key and binds to the new principal', async () => { + const first = createBridgeIdentity(); + const second = createBridgeIdentity(); + const store = recoverableStore(); + const initial = await enroll(store, first.publicKey); + const pending = await store.createRecoveryChallenge(workerId); + const replacement = await store.issue(workerId, { + tenantId: 'tenant-two', principal: { type: 'user', id: 'owner-two' }, + }); + await store.redeem({ workerId, code: replacement.code, publicKey: second.publicKey }); + + await expect(store.recoverCredential( + workerId, pending, signBridgeRecovery(first.privateKey, pending), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + await expect(store.authorize( + authorizedRequest(first.privateKey, initial.credential, 'superseded-key'), + )).rejects.toMatchObject({ code: 'CREDENTIAL_INVALID' }); + const { credential } = await recover(store, second.privateKey); + await expect(store.authorize( + authorizedRequest(second.privateKey, credential.credential, 'new-owner'), + )).resolves.toMatchObject({ + binding: { tenantId: 'tenant-two', principal: { type: 'user', id: 'owner-two' } }, + }); + }); + + test('revocation beats a signed recovery pending on a different replica', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const second = recoverableStore(); + await enroll(first, identity.publicKey); + const challenge = await first.createRecoveryChallenge(workerId); + const originalEval = redis.eval.bind(redis); + let release!: () => void; + let enter!: () => void; + const paused = new Promise((resolve) => { enter = resolve; }); + const resume = new Promise((resolve) => { release = resolve; }); + redis.eval = (async (script: string, ...args: unknown[]) => { + if (script.includes('local stableIdentity = redis.call')) { + enter(); + await resume; + } + return (originalEval as (...evalArgs: unknown[]) => Promise)(script, ...args); + }) as Redis['eval']; + try { + const pending = first.recoverCredential( + workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), + ); + await paused; + await second.revoke(workerId); + release(); + await expect(pending).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + expect(await redis.get(`codeapi:bridge:v1:identity:${workerId}`)).toBeNull(); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); + } finally { + redis.eval = originalEval as Redis['eval']; + release(); + } + }); + + test('rate limits challenge creation and signing attempts across replicas', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1 }); + const second = recoverableStore({ maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1 }); + await enroll(first, identity.publicKey); + const challenge = await first.createRecoveryChallenge(workerId); + await expect(second.createRecoveryChallenge(workerId)) + .rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + await expect(first.recoverCredential(workerId, challenge, 'invalid')) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + await expect(second.recoverCredential( + workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), + )).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('recovering credentials never clears worker or workspace quarantine', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore(); + await enroll(store, identity.publicKey); + const workerQuarantine = `codeapi:bridge:v1:worker:${workerId}:incarnation:incarnation-00000001:quarantined`; + const workspaceQuarantine = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('session-one').digest('hex')}:quarantined`; + await redis.set(workerQuarantine, '1'); + await redis.set(workspaceQuarantine, '1'); + const { credential } = await recover(store, identity.privateKey); + expect(await redis.get(workerQuarantine)).toBe('1'); + expect(await redis.get(workspaceQuarantine)).toBe('1'); + const auth = await store.authorize( + authorizedRequest(identity.privateKey, credential.credential, 'quarantined-machine'), + ); + await expect(new RedisBridgeStore(redis).register({ + protocolVersion: 1, + workerId, + incarnationId: 'incarnation-00000001', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'] }, + }, auth)).rejects.toMatchObject({ code: 'WORKER_QUARANTINED' }); + }); + + test('refuses non-HTTPS or non-origin deployment identity and unbounded recovery policy', () => { + for (const invalid of ['http://code.example.test', 'https://code.example.test/path', 'https://user@code.example.test']) { + expect(() => recoverableStore({ serverId: invalid })).toThrow(); + } + expect(() => recoverableStore({ maxAttemptsPerMinute: 0 })).toThrow(); + expect(() => recoverableStore({ challengeTtlSeconds: 301 })).toThrow(); + }); +}); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 5b319733..468a2e33 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -310,6 +310,77 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { } })); + router.post('/workers/:workerId/credentials/challenge', asyncRoute(async (req, res) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + res.status(404).json({ error: 'Machine recovery is disabled' }); + return; + } + const workerId = req.params.workerId; + if (!validWorkerId(workerId) || !configuredWorker(workerId) || + !isRecord(req.body) || req.body.protocolVersion !== BRIDGE_PROTOCOL_VERSION) { + res.status(400).json({ error: 'Invalid machine recovery challenge request' }); + return; + } + try { + const challenge = await options.pairings.createRecoveryChallenge(workerId); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...challenge }); + } catch (error) { + if (error instanceof BridgePairingError) { + res.status(error.code === 'RECOVERY_RATE_LIMITED' ? 429 : 401) + .json({ error: error.message, code: error.code }); + return; + } + throw error; + } + })); + + router.post('/workers/:workerId/credentials/recover', asyncRoute(async (req, res) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + res.status(404).json({ error: 'Machine recovery is disabled' }); + return; + } + const workerId = req.params.workerId; + const body = isRecord(req.body) ? req.body : {}; + if ( + !validWorkerId(workerId) || !configuredWorker(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + body.operation !== 'credential.recover' || + typeof body.serverId !== 'string' || body.serverId.length > 256 || + typeof body.enrollmentGeneration !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(body.enrollmentGeneration) || + typeof body.challenge !== 'string' || + !/^[A-Za-z0-9_-]{43}$/.test(body.challenge) || + typeof body.expiresAt !== 'string' || body.expiresAt.length > 64 || + typeof body.signature !== 'string' || + !/^[A-Za-z0-9_-]{86}$/.test(body.signature) + ) { + res.status(400).json({ error: 'Invalid machine recovery proof' }); + return; + } + try { + const credential = await options.pairings.recoverCredential( + workerId, + { + operation: 'credential.recover', + serverId: body.serverId, + workerId, + enrollmentGeneration: body.enrollmentGeneration, + challenge: body.challenge, + expiresAt: body.expiresAt, + }, + body.signature, + ); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...credential }); + } catch (error) { + if (error instanceof BridgePairingError) { + res.status(error.code === 'RECOVERY_RATE_LIMITED' ? 429 : 401) + .json({ error: error.message, code: error.code }); + return; + } + throw error; + } + })); + router.post( '/workers/:workerId/revoke', adminAuth, diff --git a/service/src/config.ts b/service/src/config.ts index 55570dfe..162b8ec3 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -426,6 +426,19 @@ export const env = { BRIDGE_AUTH_MODE: bridgeAuthMode, /** Enrollment and lease credential shared only with the configured worker. */ BRIDGE_TOKEN: process.env.CODEAPI_BRIDGE_TOKEN ?? '', + /** Opt-in stable HTTPS origin of this Code API deployment (shared by all replicas). */ + BRIDGE_RECOVERY_SERVER_ID: process.env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? '', + /** Zero preserves machine authorization until explicit revocation. */ + BRIDGE_ENROLLMENT_TTL_SECONDS: Number(process.env.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS ?? 0), + BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS ?? 60, + ), + BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE ?? 12, + ), + BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? 30, + ), /** * Runtime session affinity for stateful sandbox backends. * - `stateless` (default): no runtime sessions; `runtime_session_hint` ignored. From 84608a27da8fbac9952803339bd019e626333b40 Mon Sep 17 00:00:00 2001 From: Lia Date: Mon, 28 Sep 2026 00:23:01 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=94=90=20fix:=20Protect=20Recovery=20?= =?UTF-8?q?Budgets=20and=20Persist=20Compose=20Enrollment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yaml | 15 +- docs/remote-bridge/README.md | 33 ++-- packages/code/src/identity.test.ts | 43 +++++ packages/code/src/identity.ts | 46 +++++ packages/code/src/protocol.ts | 11 ++ service/src/bridge/pairing.ts | 88 +++++++-- service/src/bridge/recovery-router.test.ts | 43 ++++- service/src/bridge/recovery.test.ts | 199 ++++++++++++++++++--- service/src/bridge/router.ts | 26 ++- tests/compose-bridge-config.cjs | 20 ++- 10 files changed, 464 insertions(+), 60 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index ab37fd24..59bffffa 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,6 +15,11 @@ services: - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} + - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} + - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} + - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60} + - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12} + - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_ALLOW_AUTH_PROVIDER_NONE=${CODEAPI_ALLOW_AUTH_PROVIDER_NONE:-} - CODEAPI_JWT_ISSUER=${CODEAPI_JWT_ISSUER:-} @@ -64,6 +69,11 @@ services: - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} + - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} + - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} + - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60} + - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12} + - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_JWT_SINGLE_TENANT_ID=${CODEAPI_JWT_SINGLE_TENANT_ID:-} - CODEAPI_TENANT_ISOLATION_STRICT=${CODEAPI_TENANT_ISOLATION_STRICT:-} @@ -213,9 +223,11 @@ services: redis: image: redis:7-alpine container_name: redis - command: redis-server --requirepass localdev + command: redis-server --requirepass localdev --appendonly yes ports: - ${CODEAPI_REDIS_PORT:-16379}:6379 + volumes: + - redis_data:/data minio: image: quay.io/minio/minio @@ -232,3 +244,4 @@ services: volumes: minio_data: + redis_data: diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 42ddad5f..a19457fc 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -52,8 +52,13 @@ after this option is enabled has a recoverable key. Updating Code API alone does not make old workers reconnect automatically: the CLI must also implement this recovery protocol in the later worker release. -Store Redis state durably across restarts (for example, persistent Redis with -AOF enabled, a managed persistent Redis service, and backups). Revocation and +Store Redis state durably across restarts. The primary `docker-compose.yaml` +now uses Redis AOF and a named `/data` volume; preserve that volume when +recreating the stack. If upgrading a running stack with an in-memory Redis, +migrate its state before recreating the container: mounting an empty volume +does **not** preserve active assignments, fences, or earlier revocations. Other +deployments must provide equivalent durable Redis (for example, a managed +persistent Redis service and backups). Revocation and machine enrollment share that state across replicas; do not configure eviction of authorization keys. If enrollment state is missing, credentials minted under that enrollment fail closed, and the worker must be explicitly enrolled again. @@ -260,14 +265,22 @@ execution. pairing also persists a separate machine authorization and its public key in Redis without a TTL by default; an operator can instead set a bounded enrollment lifetime. -- `POST /v1/bridge/workers/:workerId/credentials/challenge` accepts - `{ "protocolVersion": 1 }` without an administrator token and returns a - single-use, short-lived challenge with the configured server ID, worker ID, - enrollment generation, operation and expiry. The enrolled key signs that - entire challenge using `signBridgeRecovery` from `@librechat/code`'s identity - module. `POST .../credentials/recover` accepts the challenge fields plus - `signature` and returns a new short-lived credential. Creation and proof - attempts are bounded in shared Redis; HTTP 429 means back off. +- `POST /v1/bridge/workers/:workerId/credentials/challenge` does not require + an administrator token or an existing access credential, but **does** require + the enrolled key. Its JSON body contains `protocolVersion: 1`, + `operation: "credential.challenge"`, the configured `serverId`, the matching + `workerId`, a fresh UTC ISO `timestamp`, a random 32-byte base64url `nonce`, + and `signature` computed with `signBridgeRecoveryStart(privateKey, fields)` + from `@librechat/code/identity`. Code API verifies the signed fields and + consumes the nonce once before charging the machine's shared challenge + budget; a fabricated request cannot exhaust another worker's budget. +- The response is a short-lived, single-use challenge with the server ID, + worker ID, enrollment generation, operation and expiry. Sign those fields + with `signBridgeRecovery(privateKey, challenge)` and send the fields plus + `signature` to `POST .../credentials/recover` to obtain a new short-lived + credential. Invalid proofs are limited per high-entropy challenge; only + successfully verified, unused proofs consume the machine's shared recovery + budget. Both limits live in shared Redis; HTTP 429 means back off. - Recovery and revocation are atomic Redis transitions across API replicas. A missing, revoked, expired or superseded enrollment never creates new credentials. Recovery only restores transport authentication. It does not diff --git a/packages/code/src/identity.test.ts b/packages/code/src/identity.test.ts index 4e92b91f..bef3b673 100644 --- a/packages/code/src/identity.test.ts +++ b/packages/code/src/identity.test.ts @@ -1,9 +1,14 @@ import assert from 'node:assert/strict'; +import { randomBytes } from 'node:crypto'; import test from 'node:test'; import { createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, signBridgeRequest, + verifyBridgeRecovery, + verifyBridgeRecoveryStart, verifyBridgeRequest, } from './identity.js'; @@ -33,3 +38,41 @@ test('worker identity proves possession for the exact HTTP request', () => { false, ); }); + +test('signed recovery starts bind operation, server, worker, time and nonce', () => { + const identity = createBridgeIdentity(); + const start = { + operation: 'credential.challenge' as const, + serverId: 'https://code.example.test', + workerId: 'vm-1', + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + const signature = signBridgeRecoveryStart(identity.privateKey, start); + assert.equal(verifyBridgeRecoveryStart(identity.publicKey, start, signature), true); + for (const modified of [ + { ...start, operation: 'credential.recover' as 'credential.challenge' }, + { ...start, serverId: 'https://other.example.test' }, + { ...start, workerId: 'vm-2' }, + { ...start, timestamp: new Date(Date.now() + 60_000).toISOString() }, + { ...start, nonce: randomBytes(32).toString('base64url') }, + ]) { + assert.equal(verifyBridgeRecoveryStart(identity.publicKey, modified, signature), false); + } + + const challenge = { + operation: 'credential.recover' as const, + serverId: start.serverId, + workerId: start.workerId, + enrollmentGeneration: randomBytes(18).toString('base64url'), + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }; + assert.equal(verifyBridgeRecovery(identity.publicKey, challenge, signature), false); + assert.equal( + verifyBridgeRecoveryStart( + identity.publicKey, start, signBridgeRecovery(identity.privateKey, challenge), + ), + false, + ); +}); diff --git a/packages/code/src/identity.ts b/packages/code/src/identity.ts index 4b60e4f7..3556520c 100644 --- a/packages/code/src/identity.ts +++ b/packages/code/src/identity.ts @@ -19,6 +19,15 @@ export interface BridgeRequestProofInput { body: string; } +/** Signed without an access credential before requesting a server recovery challenge. */ +export interface BridgeRecoveryStartProofInput { + operation: 'credential.challenge'; + serverId: string; + workerId: string; + timestamp: string; + nonce: string; +} + /** Signed separately from access-credential requests, so neither proof can be reused for the other. */ export interface BridgeRecoveryProofInput { operation: 'credential.recover'; @@ -75,6 +84,43 @@ export function verifyBridgeRequest( } } +function canonicalBridgeRecoveryStart(input: BridgeRecoveryStartProofInput): string { + return [ + 'librechat-code:bridge-recovery-start:v1', + input.operation, + input.serverId, + input.workerId, + input.timestamp, + input.nonce, + ].join('\n'); +} + +export function signBridgeRecoveryStart( + privateKey: string, + input: BridgeRecoveryStartProofInput, +): string { + return sign(null, Buffer.from(canonicalBridgeRecoveryStart(input)), privateKey).toString( + 'base64url', + ); +} + +export function verifyBridgeRecoveryStart( + publicKey: string, + input: BridgeRecoveryStartProofInput, + signature: string, +): boolean { + try { + return verify( + null, + Buffer.from(canonicalBridgeRecoveryStart(input)), + publicKey, + Buffer.from(signature, 'base64url'), + ); + } catch { + return false; + } +} + function canonicalBridgeRecovery(input: BridgeRecoveryProofInput): string { return [ 'librechat-code:bridge-recovery:v1', diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 5bd6934a..7657fc9c 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -759,6 +759,17 @@ export interface BridgeWorkerCredentialResponse { expiresAt: string; } +/** The enrolled machine signs this request before Code API issues a challenge. */ +export interface BridgeRecoveryChallengeRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'credential.challenge'; + serverId: string; + workerId: string; + timestamp: string; + nonce: string; + signature: string; +} + /** A short-lived, single-use challenge for an already enrolled machine key. */ export interface BridgeRecoveryChallengeResponse { protocolVersion: BridgeProtocolVersion; diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index bbd91f88..0f603720 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -6,8 +6,15 @@ import { import type Redis from 'ioredis'; -import { verifyBridgeRecovery, verifyBridgeRequest } from '../../../packages/code/src/identity'; -import type { BridgeRecoveryProofInput } from '../../../packages/code/src/identity'; +import { + verifyBridgeRecovery, + verifyBridgeRecoveryStart, + verifyBridgeRequest, +} from '../../../packages/code/src/identity'; +import type { + BridgeRecoveryProofInput, + BridgeRecoveryStartProofInput, +} from '../../../packages/code/src/identity'; const PREFIX = 'codeapi:bridge:v1'; const DEFAULT_PAIRING_TTL_SECONDS = 10 * 60; @@ -17,6 +24,7 @@ const PROOF_CLOCK_SKEW_MS = 60_000; const DEFAULT_CHALLENGE_TTL_SECONDS = 60; const DEFAULT_CHALLENGES_PER_MINUTE = 12; const DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE = 30; +const RECOVERY_START_NONCE_TTL_SECONDS = 3 * 60; const LEGACY_SCAN_CLAIM_TTL_MS = 5_000; const LEGACY_SCAN_POLL_INTERVAL_MS = 25; const LEGACY_SCAN_PENDING = 'pending'; @@ -114,10 +122,12 @@ const CREATE_RECOVERY_CHALLENGE_SCRIPT = ` if redis.call('GET', KEYS[1]) ~= ARGV[1] or (redis.call('GET', KEYS[4]) or '0') ~= ARGV[4] or redis.call('GET', KEYS[5]) ~= ARGV[6] then return 0 end +if redis.call('EXISTS', KEYS[6]) == 1 then return -2 end local count = redis.call('INCR', KEYS[2]) if count == 1 then redis.call('EXPIRE', KEYS[2], 60) end if count > tonumber(ARGV[2]) then return -1 end if redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[5], 'NX') ~= 'OK' then return 0 end +redis.call('SET', KEYS[6], '1', 'EX', ARGV[7]) return 1 `; const LIMIT_RECOVERY_ATTEMPTS_SCRIPT = ` @@ -132,6 +142,9 @@ if redis.call('GET', KEYS[1]) ~= ARGV[1] or redis.call('GET', KEYS[2]) ~= ARGV[2 end local stableIdentity = redis.call('GET', KEYS[5]) if stableIdentity and stableIdentity ~= ARGV[4] then return 0 end +local count = redis.call('INCR', KEYS[8]) +if count == 1 then redis.call('EXPIRE', KEYS[8], 60) end +if count > tonumber(ARGV[9]) then return -1 end redis.call('DEL', KEYS[2]) redis.call('SET', KEYS[3], ARGV[5], 'EX', ARGV[6]) redis.call('SET', KEYS[4], ARGV[7], 'EX', ARGV[6]) @@ -285,8 +298,20 @@ function recoveryChallengeKey(challenge: string): string { return `${PREFIX}:recovery:challenge:${digest(challenge)}`; } -function recoveryRateKey(workerId: string, operation: 'start' | 'complete'): string { - return `${PREFIX}:recovery:rate:${operation}:${workerId}`; +function recoveryStartNonceKey(workerId: string, nonce: string): string { + return `${PREFIX}:recovery:nonce:${workerId}:${digest(nonce)}`; +} + +function recoveryChallengeAttemptKey(challenge: string): string { + return `${PREFIX}:recovery:attempt:${digest(challenge)}`; +} + +function recoveryRateKey( + workerId: string, + enrollmentGeneration: string, + operation: 'start' | 'complete', +): string { + return `${PREFIX}:recovery:rate:${operation}:${workerId}:${enrollmentGeneration}`; } function workerPairingGenerationKey(workerId: string): string { @@ -653,7 +678,11 @@ export class RedisBridgePairingStore { }; } - async createRecoveryChallenge(workerId: string): Promise { + async createRecoveryChallenge( + workerId: string, + request: BridgeRecoveryStartProofInput, + signature: string, + ): Promise { if (this.recovery == null) { throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); } @@ -663,6 +692,18 @@ export class RedisBridgePairingStore { workerEnrollmentRequiredKey(workerId), ); const enrollment = this.checkedEnrollment(workerId, raw, pairingGeneration, requiredGeneration); + const proofTime = Date.parse(request.timestamp); + if ( + String(request.operation) !== 'credential.challenge' || + request.serverId !== enrollment.serverId || + request.workerId !== workerId || + !/^[A-Za-z0-9_-]{43}$/.test(request.nonce) || + !Number.isFinite(proofTime) || + Math.abs(Date.now() - proofTime) > PROOF_CLOCK_SKEW_MS || + !verifyBridgeRecoveryStart(enrollment.publicKey, request, signature) + ) { + throw new BridgePairingError('PROOF_INVALID', 'Machine recovery challenge proof is invalid'); + } const challenge: BridgeRecoveryChallenge = { operation: 'credential.recover', serverId: enrollment.serverId, @@ -675,19 +716,24 @@ export class RedisBridgePairingStore { }; const created = await this.redis.eval( CREATE_RECOVERY_CHALLENGE_SCRIPT, - 5, + 6, workerEnrollmentKey(workerId), - recoveryRateKey(workerId, 'start'), + recoveryRateKey(workerId, enrollment.generation, 'start'), recoveryChallengeKey(challenge.challenge), workerPairingGenerationKey(workerId), workerEnrollmentRequiredKey(workerId), + recoveryStartNonceKey(workerId, request.nonce), raw!, String(this.recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE), JSON.stringify(challenge), String(enrollment.pairingGeneration), String(this.recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS), enrollment.generation, + String(RECOVERY_START_NONCE_TTL_SECONDS), ); + if (created === -2) { + throw new BridgePairingError('PROOF_REPLAYED', 'Machine recovery challenge proof was already used'); + } if (created === -1) { throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery challenges'); } @@ -705,15 +751,6 @@ export class RedisBridgePairingStore { if (this.recovery == null) { throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); } - const allowed = await this.redis.eval( - LIMIT_RECOVERY_ATTEMPTS_SCRIPT, - 1, - recoveryRateKey(workerId, 'complete'), - String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), - ); - if (allowed !== 1) { - throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery attempts'); - } const challengeKey = recoveryChallengeKey(proof.challenge); const [enrollmentRaw, challengeRaw, pairingGeneration, requiredGeneration] = await this.redis.mget( workerEnrollmentKey(workerId), @@ -752,6 +789,18 @@ export class RedisBridgePairingStore { ) { throw new BridgePairingError('CHALLENGE_INVALID', 'Machine recovery challenge is invalid or expired'); } + // Invalid signatures can only exhaust the high-entropy challenge they know, + // never the enrolled worker's shared quota. The worker-wide counter is + // charged inside completion, after the key proof and replay checks. + const attempts = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + recoveryChallengeAttemptKey(proof.challenge), + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), + ); + if (attempts !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery attempts'); + } if (!verifyBridgeRecovery(enrollment.publicKey, saved as BridgeRecoveryChallenge, signature)) { throw new BridgePairingError('PROOF_INVALID', 'Machine recovery signature is invalid'); } @@ -772,7 +821,7 @@ export class RedisBridgePairingStore { // enrollment/generation comparison, with no window to recreate trust. const issued = await this.redis.eval( COMPLETE_RECOVERY_CHALLENGE_SCRIPT, - 7, + 8, workerEnrollmentKey(workerId), challengeKey, credentialDigestKey(credentialDigest), @@ -780,6 +829,7 @@ export class RedisBridgePairingStore { workerStableIdentityKey(workerId), workerPairingGenerationKey(workerId), workerEnrollmentRequiredKey(workerId), + recoveryRateKey(workerId, enrollment.generation, 'complete'), enrollmentRaw!, challengeRaw!, String(enrollment.pairingGeneration), @@ -788,7 +838,11 @@ export class RedisBridgePairingStore { String(this.credentialTtlSeconds), credentialDigest, enrollment.generation, + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), ); + if (issued === -1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recoveries'); + } if (issued !== 1) { throw new BridgePairingError('CHALLENGE_INVALID', 'Machine recovery challenge is invalid or expired'); } diff --git a/service/src/bridge/recovery-router.test.ts b/service/src/bridge/recovery-router.test.ts index d7c3f7ec..3bdc41c5 100644 --- a/service/src/bridge/recovery-router.test.ts +++ b/service/src/bridge/recovery-router.test.ts @@ -1,3 +1,4 @@ +import { randomBytes } from 'crypto'; import { createServer, type Server } from 'http'; import { afterEach, describe, expect, test } from 'bun:test'; @@ -5,9 +6,13 @@ import express, { json } from 'express'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; -import type { BridgeRecoveryChallengeResponse } from '../../../packages/code/src/protocol'; +import type { BridgeRecoveryChallengeRequest, BridgeRecoveryChallengeResponse } from '../../../packages/code/src/protocol'; -import { createBridgeIdentity, signBridgeRecovery } from '../../../packages/code/src/identity'; +import { + createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, +} from '../../../packages/code/src/identity'; import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; import { RedisBridgePairingStore } from './pairing'; import { createBridgeRouter } from './router'; @@ -15,8 +20,24 @@ import { RedisBridgeStore } from './store'; const redis = new RedisMock() as unknown as Redis; const workerId = 'http-recovery-worker'; +const serverId = 'https://code.example.test'; let server: Server | undefined; +function signedStart(privateKey: string): BridgeRecoveryChallengeRequest { + const request = { + operation: 'credential.challenge' as const, + serverId, + workerId, + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + ...request, + signature: signBridgeRecoveryStart(privateKey, request), + }; +} + async function startRouter(pairings: RedisBridgePairingStore): Promise { const app = express(); app.use(json()); @@ -57,7 +78,7 @@ describe('machine credential recovery HTTP API', () => { test('recovers using the stored key without administrator authentication and never leaks the binding', async () => { const pairings = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { - serverId: 'https://code.example.test', + serverId, }); const identity = createBridgeIdentity(); const pairing = await pairings.issue(workerId, { @@ -65,14 +86,21 @@ describe('machine credential recovery HTTP API', () => { }); await pairings.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); const baseUrl = await startRouter(pairings); - const challengeResponse = await post(`${baseUrl}/challenge`, { + const unsigned = await post(`${baseUrl}/challenge`, { protocolVersion: BRIDGE_PROTOCOL_VERSION, }); + expect(unsigned.status).toBe(400); + const outsider = createBridgeIdentity(); + const forged = await post(`${baseUrl}/challenge`, signedStart(outsider.privateKey)); + expect(forged.status).toBe(401); + await expect(forged.json()).resolves.toMatchObject({ code: 'PROOF_INVALID' }); + + const challengeResponse = await post(`${baseUrl}/challenge`, signedStart(identity.privateKey)); expect(challengeResponse.status).toBe(200); const challenge = (await challengeResponse.json()) as BridgeRecoveryChallengeResponse; expect(challenge).toMatchObject({ protocolVersion: BRIDGE_PROTOCOL_VERSION, - serverId: 'https://code.example.test', + serverId, workerId, operation: 'credential.recover', }); @@ -108,8 +136,9 @@ describe('machine credential recovery HTTP API', () => { const pairing = await first.issue(workerId); await first.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); const baseUrl = await startRouter(second); - await first.createRecoveryChallenge(workerId); - const denied = await post(`${baseUrl}/challenge`, { protocolVersion: BRIDGE_PROTOCOL_VERSION }); + const initial = signedStart(identity.privateKey); + await first.createRecoveryChallenge(workerId, initial, initial.signature); + const denied = await post(`${baseUrl}/challenge`, signedStart(identity.privateKey)); expect(denied.status).toBe(429); await expect(denied.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); }); diff --git a/service/src/bridge/recovery.test.ts b/service/src/bridge/recovery.test.ts index a514b32b..5dd9ae0d 100644 --- a/service/src/bridge/recovery.test.ts +++ b/service/src/bridge/recovery.test.ts @@ -1,9 +1,12 @@ import { afterEach, describe, expect, test } from 'bun:test'; -import { createHash } from 'crypto'; +import { createHash, randomBytes } from 'crypto'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; -import type { BridgeRecoveryProofInput } from '../../../packages/code/src/identity'; +import type { + BridgeRecoveryProofInput, + BridgeRecoveryStartProofInput, +} from '../../../packages/code/src/identity'; import type { BridgeRecoveryChallenge, BridgeRecoveryOptions, @@ -14,6 +17,7 @@ import type { import { createBridgeIdentity, signBridgeRecovery, + signBridgeRecoveryStart, signBridgeRequest, } from '../../../packages/code/src/identity'; import { RedisBridgePairingStore } from './pairing'; @@ -62,11 +66,35 @@ async function enroll( return store.redeem({ workerId, code: pairing.code, publicKey }); } +function recoveryStart( + privateKey: string, + nonce = randomBytes(32).toString('base64url'), + overrides: Partial = {}, +): { proof: BridgeRecoveryStartProofInput; signature: string } { + const proof: BridgeRecoveryStartProofInput = { + operation: 'credential.challenge', + serverId, + workerId, + timestamp: new Date().toISOString(), + nonce, + ...overrides, + }; + return { proof, signature: signBridgeRecoveryStart(privateKey, proof) }; +} + +async function challengeFor( + store: RedisBridgePairingStore, + privateKey: string, +): Promise { + const { proof, signature } = recoveryStart(privateKey); + return store.createRecoveryChallenge(workerId, proof, signature); +} + async function recover( store: RedisBridgePairingStore, privateKey: string, ): Promise<{ challenge: BridgeRecoveryChallenge; credential: BridgeWorkerCredential }> { - const challenge = await store.createRecoveryChallenge(workerId); + const challenge = await challengeFor(store, privateKey); const credential = await store.recoverCredential( workerId, challenge, @@ -87,9 +115,9 @@ describe('durable bridge enrollment', () => { expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); const replica = recoverableStore(); - await expect(replica.createRecoveryChallenge(workerId)).rejects.toMatchObject({ - code: 'ENROLLMENT_INVALID', - }); + const { proof, signature } = recoveryStart(identity.privateKey); + await expect(replica.createRecoveryChallenge(workerId, proof, signature)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); const rotated = await replica.rotate(workerId); await expect( replica.authorize(authorizedRequest(identity.privateKey, rotated.credential, 'legacy-proof')), @@ -137,7 +165,7 @@ describe('durable bridge enrollment', () => { const outsider = createBridgeIdentity(); const store = recoverableStore(); await enroll(store, enrolled.publicKey); - const challenge = await store.createRecoveryChallenge(workerId); + const challenge = await challengeFor(store, enrolled.privateKey); await expect(store.recoverCredential( workerId, challenge, signBridgeRecovery(outsider.privateKey, challenge), @@ -165,7 +193,7 @@ describe('durable bridge enrollment', () => { const identity = createBridgeIdentity(); const store = recoverableStore({ challengeTtlSeconds: 1 }); await enroll(store, identity.publicKey); - const challenge = await store.createRecoveryChallenge(workerId); + const challenge = await challengeFor(store, identity.privateKey); await new Promise((resolve) => setTimeout(resolve, 1_100)); await expect(store.recoverCredential( workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), @@ -199,7 +227,7 @@ describe('durable bridge enrollment', () => { const store = recoverableStore({ enrollmentTtlSeconds: 1 }); const issued = await enroll(store, identity.publicKey); await new Promise((resolve) => setTimeout(resolve, 1_100)); - await expect(store.createRecoveryChallenge(workerId)) + await expect(challengeFor(store, identity.privateKey)) .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); await expect(store.rotate(workerId)) .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); @@ -223,10 +251,10 @@ describe('durable bridge enrollment', () => { authorizedRequest(identity.privateKey, issued.credential, 'lost-authorization'), )).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); await expect(store.rotate(workerId)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); - await expect(store.createRecoveryChallenge(workerId)) + await expect(challengeFor(store, identity.privateKey)) .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); await redis.set(`codeapi:bridge:v1:enrollment:${workerId}`, 'null'); - await expect(store.createRecoveryChallenge(workerId)) + await expect(challengeFor(store, identity.privateKey)) .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); }); @@ -235,7 +263,7 @@ describe('durable bridge enrollment', () => { const second = createBridgeIdentity(); const store = recoverableStore(); const initial = await enroll(store, first.publicKey); - const pending = await store.createRecoveryChallenge(workerId); + const pending = await challengeFor(store, first.privateKey); const replacement = await store.issue(workerId, { tenantId: 'tenant-two', principal: { type: 'user', id: 'owner-two' }, }); @@ -255,12 +283,39 @@ describe('durable bridge enrollment', () => { }); }); + test('concurrent replicas consume a recovery challenge and count it only once', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 2 }); + const second = recoverableStore({ maxAttemptsPerMinute: 2 }); + await enroll(first, identity.publicKey); + const challenge = await challengeFor(first, identity.privateKey); + const signature = signBridgeRecovery(identity.privateKey, challenge); + const attempts = await Promise.allSettled([ + first.recoverCredential(workerId, challenge, signature), + second.recoverCredential(workerId, challenge, signature), + ]); + const issued = attempts.filter( + (result): result is PromiseFulfilledResult => + result.status === 'fulfilled', + ); + const rejected = attempts.filter( + (result): result is PromiseRejectedResult => result.status === 'rejected', + ); + expect(issued).toHaveLength(1); + expect(rejected).toHaveLength(1); + expect(rejected[0]).toMatchObject({ reason: { code: 'CHALLENGE_INVALID' } }); + const digest = createHash('sha256').update(issued[0].value.credential).digest('hex'); + expect(await redis.get(`codeapi:bridge:v1:identity:${workerId}`)).toBe(digest); + expect(await redis.get(`codeapi:bridge:v1:recovery:rate:complete:${workerId}:${challenge.enrollmentGeneration}`)) + .toBe('1'); + }); + test('revocation beats a signed recovery pending on a different replica', async () => { const identity = createBridgeIdentity(); const first = recoverableStore(); const second = recoverableStore(); await enroll(first, identity.publicKey); - const challenge = await first.createRecoveryChallenge(workerId); + const challenge = await challengeFor(first, identity.privateKey); const originalEval = redis.eval.bind(redis); let release!: () => void; let enter!: () => void; @@ -289,19 +344,123 @@ describe('durable bridge enrollment', () => { } }); - test('rate limits challenge creation and signing attempts across replicas', async () => { + test('only a signed, fresh, unused start request consumes the worker challenge budget', async () => { const identity = createBridgeIdentity(); - const first = recoverableStore({ maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1 }); - const second = recoverableStore({ maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1 }); + const outsider = createBridgeIdentity(); + const first = recoverableStore({ maxChallengesPerMinute: 2 }); + const second = recoverableStore({ maxChallengesPerMinute: 2 }); await enroll(first, identity.publicKey); - const challenge = await first.createRecoveryChallenge(workerId); - await expect(second.createRecoveryChallenge(workerId)) + + for (let index = 0; index < 20; index += 1) { + const forged = recoveryStart(outsider.privateKey); + await expect(first.createRecoveryChallenge(workerId, forged.proof, forged.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + } + const otherServer = recoveryStart(identity.privateKey, undefined, { + serverId: 'https://unrelated.example.test', + }); + await expect(first.createRecoveryChallenge(workerId, otherServer.proof, otherServer.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + const stale = recoveryStart(identity.privateKey, undefined, { + timestamp: new Date(Date.now() - 5 * 60_000).toISOString(), + }); + await expect(first.createRecoveryChallenge(workerId, stale.proof, stale.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + + const valid = recoveryStart(identity.privateKey); + await first.createRecoveryChallenge(workerId, valid.proof, valid.signature); + await expect(second.createRecoveryChallenge(workerId, valid.proof, valid.signature)) + .rejects.toMatchObject({ code: 'PROOF_REPLAYED' }); + await challengeFor(second, identity.privateKey); + await expect(challengeFor(first, identity.privateKey)) .rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); - await expect(first.recoverCredential(workerId, challenge, 'invalid')) + }); + + test('fabricated completions never spend the worker budget or block a fresh signed recovery', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 1 }); + const second = recoverableStore({ maxAttemptsPerMinute: 1 }); + await enroll(first, identity.publicKey); + const real = await challengeFor(first, identity.privateKey); + + for (let index = 0; index < 35; index += 1) { + const fake = { ...real, challenge: randomBytes(32).toString('base64url') }; + await expect(first.recoverCredential(workerId, fake, 'forged')) + .rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + } + await first.recoverCredential(workerId, real, signBridgeRecovery(identity.privateKey, real)); + const next = await challengeFor(second, identity.privateKey); + await expect(second.recoverCredential( + workerId, next, signBridgeRecovery(identity.privateKey, next), + )).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('invalid signatures exhaust only their own challenge, not the enrolled worker', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 1 }); + const second = recoverableStore({ maxAttemptsPerMinute: 1 }); + await enroll(first, identity.publicKey); + const attacked = await challengeFor(first, identity.privateKey); + await expect(first.recoverCredential(workerId, attacked, 'forged')) .rejects.toMatchObject({ code: 'PROOF_INVALID' }); await expect(second.recoverCredential( - workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), + workerId, attacked, signBridgeRecovery(identity.privateKey, attacked), )).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + const fresh = await challengeFor(second, identity.privateKey); + await expect(first.recoverCredential( + workerId, fresh, signBridgeRecovery(identity.privateKey, fresh), + )).resolves.toMatchObject({ workerId }); + }); + + test('key replacement starts new signed challenge and recovery budgets', async () => { + const oldKey = createBridgeIdentity(); + const newKey = createBridgeIdentity(); + const store = recoverableStore({ maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1 }); + await enroll(store, oldKey.publicKey); + const oldChallenge = await challengeFor(store, oldKey.privateKey); + await store.recoverCredential( + workerId, oldChallenge, signBridgeRecovery(oldKey.privateKey, oldChallenge), + ); + const replacement = await store.issue(workerId, binding); + await store.redeem({ workerId, code: replacement.code, publicKey: newKey.publicKey }); + + await expect(challengeFor(store, oldKey.privateKey)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + const newChallenge = await challengeFor(store, newKey.privateKey); + await expect(store.recoverCredential( + workerId, newChallenge, signBridgeRecovery(newKey.privateKey, newChallenge), + )).resolves.toMatchObject({ workerId }); + }); + + test('revocation fences a previously verified start proof before a challenge is written', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const second = recoverableStore(); + await enroll(first, identity.publicKey); + const { proof, signature } = recoveryStart(identity.privateKey); + const originalEval = redis.eval.bind(redis); + let release!: () => void; + let enter!: () => void; + const paused = new Promise((resolve) => { enter = resolve; }); + const resume = new Promise((resolve) => { release = resolve; }); + redis.eval = (async (script: string, ...args: unknown[]) => { + if (script.includes('KEYS[6]) == 1 then return -2')) { + enter(); + await resume; + } + return (originalEval as (...evalArgs: unknown[]) => Promise)(script, ...args); + }) as Redis['eval']; + try { + const pending = first.createRecoveryChallenge(workerId, proof, signature); + await paused; + await second.revoke(workerId); + release(); + await expect(pending).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.keys('codeapi:bridge:v1:recovery:challenge:*')).toEqual([]); + } finally { + redis.eval = originalEval as Redis['eval']; + release(); + } }); test('recovering credentials never clears worker or workspace quarantine', async () => { diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 468a2e33..da04245c 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -316,13 +316,32 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { return; } const workerId = req.params.workerId; - if (!validWorkerId(workerId) || !configuredWorker(workerId) || - !isRecord(req.body) || req.body.protocolVersion !== BRIDGE_PROTOCOL_VERSION) { + const body = isRecord(req.body) ? req.body : {}; + if ( + !validWorkerId(workerId) || !configuredWorker(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + body.operation !== 'credential.challenge' || + typeof body.serverId !== 'string' || body.serverId.length > 256 || + body.workerId !== workerId || + typeof body.timestamp !== 'string' || body.timestamp.length > 64 || + typeof body.nonce !== 'string' || !/^[A-Za-z0-9_-]{43}$/.test(body.nonce) || + typeof body.signature !== 'string' || !/^[A-Za-z0-9_-]{86}$/.test(body.signature) + ) { res.status(400).json({ error: 'Invalid machine recovery challenge request' }); return; } try { - const challenge = await options.pairings.createRecoveryChallenge(workerId); + const challenge = await options.pairings.createRecoveryChallenge( + workerId, + { + operation: 'credential.challenge', + serverId: body.serverId, + workerId, + timestamp: body.timestamp, + nonce: body.nonce, + }, + body.signature, + ); res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...challenge }); } catch (error) { if (error instanceof BridgePairingError) { @@ -839,6 +858,5 @@ router.post( }), ); - return router; } diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index de14b553..162e9554 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -15,6 +15,11 @@ function render(overrides) { CODEAPI_BRIDGE_WORKER_ID: '', CODEAPI_BRIDGE_TOKEN: '', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '', + CODEAPI_BRIDGE_RECOVERY_SERVER_ID: '', + CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '', + CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '', + CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: '', + CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: '', ...overrides, }, })); @@ -28,6 +33,11 @@ for (const overrides of [ CODEAPI_BRIDGE_DYNAMIC_WORKERS: 'false', CODEAPI_BRIDGE_WORKER_ID: 'test-worker', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '4', + CODEAPI_BRIDGE_RECOVERY_SERVER_ID: 'https://code.example.test', + CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '86400', + CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '90', + CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: '8', + CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: '16', }, ]) { const config = render(overrides); @@ -39,10 +49,18 @@ for (const overrides of [ assert.equal(env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, overrides.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? '1'); assert.equal(env.CODEAPI_BRIDGE_DYNAMIC_WORKERS, overrides.CODEAPI_BRIDGE_DYNAMIC_WORKERS ?? 'true'); assert.equal(env.CODEAPI_BRIDGE_WORKER_ID, overrides.CODEAPI_BRIDGE_WORKER_ID ?? ''); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, overrides.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? ''); + assert.equal(env.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS, overrides.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS ?? '0'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS, overrides.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS ?? '60'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE ?? '12'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? '30'); } for (const name of ['egress_gateway', 'sandbox-runner']) { assert.equal(config.services[name].environment.CODEAPI_BRIDGE_TOKEN, undefined); + assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, undefined); } + assert.match(JSON.stringify(config.services.redis.command), /--appendonly.*yes/); + assert.ok(config.services.redis.volumes.some(volume => volume.target === '/data' && volume.type === 'volume')); } assert.equal(render({}).services.api.environment.CODEAPI_BRIDGE_TOKEN, ''); -console.log('Compose bridge configuration passed (dynamic/fixed pairing, no default secret).'); +console.log('Compose bridge configuration passed (pairing, opt-in recovery, durable Redis).'); From f14811531236d49c98a246ce9795a0215a85832b Mon Sep 17 00:00:00 2001 From: Lia Date: Mon, 28 Sep 2026 02:01:21 +0000 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20fix:=20Limit=20Untr?= =?UTF-8?q?usted=20Code=20API=20Recovery=20Traffic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yaml | 2 + docs/remote-bridge/README.md | 11 ++- service/src/bridge/index.ts | 1 + service/src/bridge/pairing.ts | 30 +++++++- service/src/bridge/recovery-router.test.ts | 82 ++++++++++++++++++++-- service/src/bridge/recovery.test.ts | 23 ++++++ service/src/bridge/router.ts | 22 +++++- service/src/config.ts | 3 + tests/compose-bridge-config.cjs | 4 ++ 9 files changed, 167 insertions(+), 11 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 59bffffa..1d7d28cf 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -20,6 +20,7 @@ services: - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60} - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12} - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30} + - CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE:-240} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_ALLOW_AUTH_PROVIDER_NONE=${CODEAPI_ALLOW_AUTH_PROVIDER_NONE:-} - CODEAPI_JWT_ISSUER=${CODEAPI_JWT_ISSUER:-} @@ -74,6 +75,7 @@ services: - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60} - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12} - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30} + - CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE:-240} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_JWT_SINGLE_TENANT_ID=${CODEAPI_JWT_SINGLE_TENANT_ID:-} - CODEAPI_TENANT_ISOLATION_STRICT=${CODEAPI_TENANT_ISOLATION_STRICT:-} diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index a19457fc..b29c0a19 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -41,6 +41,7 @@ CODEAPI_BRIDGE_RECOVERY_SERVER_ID=https://code.example.com # CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=60 # CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=12 # CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=30 +# CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=240 ``` Omitting the server ID retains the existing pairing and refresh behavior and @@ -280,7 +281,15 @@ execution. `signature` to `POST .../credentials/recover` to obtain a new short-lived credential. Invalid proofs are limited per high-entropy challenge; only successfully verified, unused proofs consume the machine's shared recovery - budget. Both limits live in shared Redis; HTTP 429 means back off. + budget. Separately, both recovery endpoints limit all incoming requests per + connection peer *before* key verification, including well-formed JSON with + malformed or forged proofs; forged headers and worker IDs cannot bypass + that limit or consume the signed machine budget. All limits live in shared + Redis; HTTP 429 means back off. When a reverse proxy connects to Code API, + its clients share that peer's limit. Restrict direct backend access and apply + client-IP and global + abuse limits at the trusted ingress to keep one proxy peer from becoming a + shared bottleneck; do not trust an arbitrary `X-Forwarded-For` on Code API. - Recovery and revocation are atomic Redis transitions across API replicas. A missing, revoked, expired or superseded enrollment never creates new credentials. Recovery only restores transport authentication. It does not diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index 2ad3c04f..b04de55f 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -24,6 +24,7 @@ export const bridgePairings = new RedisBridgePairingStore( challengeTtlSeconds: env.BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS, maxChallengesPerMinute: env.BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE, maxAttemptsPerMinute: env.BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, + maxUntrustedRequestsPerMinute: env.BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, } : undefined, ); diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index 0f603720..b8a32162 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -24,6 +24,7 @@ const PROOF_CLOCK_SKEW_MS = 60_000; const DEFAULT_CHALLENGE_TTL_SECONDS = 60; const DEFAULT_CHALLENGES_PER_MINUTE = 12; const DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE = 30; +const DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE = 240; const RECOVERY_START_NONCE_TTL_SECONDS = 3 * 60; const LEGACY_SCAN_CLAIM_TTL_MS = 5_000; const LEGACY_SCAN_POLL_INTERVAL_MS = 25; @@ -232,6 +233,8 @@ export interface BridgeRecoveryOptions { challengeTtlSeconds?: number; maxChallengesPerMinute?: number; maxAttemptsPerMinute?: number; + /** Shared per-connection-peer cap, separate from machine-signed budgets. */ + maxUntrustedRequestsPerMinute?: number; } export type BridgeRecoveryChallenge = BridgeRecoveryProofInput; @@ -306,6 +309,11 @@ function recoveryChallengeAttemptKey(challenge: string): string { return `${PREFIX}:recovery:attempt:${digest(challenge)}`; } +function untrustedRecoveryRateKey(peer: string, operation: 'challenge' | 'recover'): string { + // Never partition by a caller-supplied worker ID or an untrusted forwarded IP. + return `${PREFIX}:recovery:untrusted:${operation}:${digest(peer)}`; +} + function recoveryRateKey( workerId: string, enrollmentGeneration: string, @@ -380,6 +388,7 @@ export class RedisBridgePairingStore { [recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS, 300], [recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE, 120], [recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE, 120], + [recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE, 1200], ]) { if (!Number.isSafeInteger(value) || value < 0 || value > max) { throw new RangeError('Invalid bridge recovery lifetime or rate limit'); @@ -388,7 +397,8 @@ export class RedisBridgePairingStore { if ( (recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS) === 0 || (recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE) === 0 || - (recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE) === 0 + (recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE) === 0 || + (recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE) === 0 ) { throw new RangeError('Bridge recovery challenge lifetime and rate limits must be positive'); } @@ -398,6 +408,24 @@ export class RedisBridgePairingStore { return this.recovery != null; } + async limitUntrustedRecovery( + peer: string, + operation: 'challenge' | 'recover', + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const allowed = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + untrustedRecoveryRateKey(peer, operation), + String(this.recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE), + ); + if (allowed !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many recovery requests from this peer'); + } + } + private checkedEnrollment( workerId: string, raw: string | null, diff --git a/service/src/bridge/recovery-router.test.ts b/service/src/bridge/recovery-router.test.ts index 3bdc41c5..28ea4b1e 100644 --- a/service/src/bridge/recovery-router.test.ts +++ b/service/src/bridge/recovery-router.test.ts @@ -21,7 +21,7 @@ import { RedisBridgeStore } from './store'; const redis = new RedisMock() as unknown as Redis; const workerId = 'http-recovery-worker'; const serverId = 'https://code.example.test'; -let server: Server | undefined; +const servers: Server[] = []; function signedStart(privateKey: string): BridgeRecoveryChallengeRequest { const request = { @@ -40,6 +40,7 @@ function signedStart(privateKey: string): BridgeRecoveryChallengeRequest { async function startRouter(pairings: RedisBridgePairingStore): Promise { const app = express(); + app.set('trust proxy', 1); app.use(json()); app.use('/v1/bridge', createBridgeRouter({ store: new RedisBridgeStore(redis), @@ -48,24 +49,30 @@ async function startRouter(pairings: RedisBridgePairingStore): Promise { adminToken: 'operator-only', configuredWorkerId: workerId, })); - server = createServer(app); - await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const server = createServer(app); + servers.push(server); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); const address = server.address(); if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); return `http://127.0.0.1:${address.port}/v1/bridge/workers/${workerId}/credentials`; } -function post(url: string, body: object): Promise { +function post( + url: string, + body: object, + headers: Record = {}, +): Promise { return fetch(url, { method: 'POST', - headers: { 'Content-Type': 'application/json' }, + headers: { 'Content-Type': 'application/json', ...headers }, body: JSON.stringify(body), }); } afterEach(async () => { - server?.close(); - server = undefined; + await Promise.all(servers.splice(0).map(server => new Promise((resolve, reject) => { + server.close(error => error ? reject(error) : resolve()); + }))); await redis.flushall(); }); @@ -125,6 +132,67 @@ describe('machine credential recovery HTTP API', () => { await expect(replay.json()).resolves.toMatchObject({ code: 'CHALLENGE_INVALID' }); }); + test('limits forged starts before signature work across replicas despite spoofed proxy headers', async () => { + const first = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, maxUntrustedRequestsPerMinute: 2, + }); + const second = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, maxUntrustedRequestsPerMinute: 2, + }); + const identity = createBridgeIdentity(); + const attacker = createBridgeIdentity(); + const pairing = await first.issue(workerId); + await first.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const firstUrl = await startRouter(first); + const secondUrl = await startRouter(second); + let signatureChecks = 0; + for (const store of [first, second]) { + const original = store.createRecoveryChallenge.bind(store); + store.createRecoveryChallenge = async ( + ...args + ): ReturnType => { + signatureChecks += 1; + return original(...args); + }; + } + + const forged = async (url: string, suffix: number): Promise => post( + `${url}/challenge`, signedStart(attacker.privateKey), + { 'X-Forwarded-For': `198.51.100.${suffix}` }, + ); + const firstResponse = await forged(firstUrl, 1); + const secondResponse = await forged(secondUrl, 2); + const blocked = await forged( + firstUrl.replace(`/workers/${workerId}/`, '/workers/attacker-selected-worker/'), + 3, + ); + expect([firstResponse.status, secondResponse.status, blocked.status]).toEqual([401, 401, 429]); + await expect(blocked.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + expect(Number(blocked.headers.get('retry-after'))).toBeGreaterThan(0); + expect(signatureChecks).toBe(2); + expect(await redis.keys('codeapi:bridge:v1:recovery:rate:start:*')).toEqual([]); + + // Bogus completions use another untrusted bucket, not the worker's signed quota. + const fake = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'credential.recover', + serverId, + workerId, + enrollmentGeneration: 'a'.repeat(24), + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + signature: 'a'.repeat(86), + }; + const rejected = await post(`${secondUrl}/recover`, fake); + expect(rejected.status).toBe(401); + expect(await redis.keys('codeapi:bridge:v1:recovery:rate:complete:*')).toEqual([]); + const recovered = await post(`${secondUrl}/recover`, fake); + expect(recovered.status).toBe(401); + const blockedCompletion = await post(`${secondUrl}/recover`, fake); + expect(blockedCompletion.status).toBe(429); + await expect(blockedCompletion.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + test('limits recovery challenges across two API routers sharing Redis', async () => { const first = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { serverId: 'https://code.example.test', maxChallengesPerMinute: 1, diff --git a/service/src/bridge/recovery.test.ts b/service/src/bridge/recovery.test.ts index 5dd9ae0d..443e7277 100644 --- a/service/src/bridge/recovery.test.ts +++ b/service/src/bridge/recovery.test.ts @@ -344,6 +344,27 @@ describe('durable bridge enrollment', () => { } }); + test('isolates untrusted recovery limits by socket peer and endpoint from signed machine quotas', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ + maxUntrustedRequestsPerMinute: 1, maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1, + }); + const second = recoverableStore({ + maxUntrustedRequestsPerMinute: 1, maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1, + }); + await enroll(first, identity.publicKey); + await first.limitUntrustedRecovery('192.0.2.1', 'challenge'); + await expect(second.limitUntrustedRecovery('192.0.2.1', 'challenge')) + .rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + await expect(second.limitUntrustedRecovery('192.0.2.2', 'challenge')) + .resolves.toBeUndefined(); + await expect(second.limitUntrustedRecovery('192.0.2.1', 'recover')) + .resolves.toBeUndefined(); + expect((await redis.keys('codeapi:bridge:v1:recovery:rate:start:*')).length).toBe(0); + const { credential } = await recover(second, identity.privateKey); + expect(credential.workerId).toBe(workerId); + }); + test('only a signed, fresh, unused start request consumes the worker challenge budget', async () => { const identity = createBridgeIdentity(); const outsider = createBridgeIdentity(); @@ -490,6 +511,8 @@ describe('durable bridge enrollment', () => { expect(() => recoverableStore({ serverId: invalid })).toThrow(); } expect(() => recoverableStore({ maxAttemptsPerMinute: 0 })).toThrow(); + expect(() => recoverableStore({ maxUntrustedRequestsPerMinute: 0 })).toThrow(); + expect(() => recoverableStore({ maxUntrustedRequestsPerMinute: 1201 })).toThrow(); expect(() => recoverableStore({ challengeTtlSeconds: 301 })).toThrow(); }); }); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index da04245c..b3d549b1 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -310,7 +310,25 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { } })); - router.post('/workers/:workerId/credentials/challenge', asyncRoute(async (req, res) => { + const untrustedRecoveryLimit = (operation: 'challenge' | 'recover'): RequestHandler => + (req, res, next) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + next(); + return; + } + // req.ip trusts X-Forwarded-For in our server. Use the connection peer so + // untrusted headers and arbitrary worker IDs cannot create new buckets. + void options.pairings.limitUntrustedRecovery(req.socket.remoteAddress ?? '', operation) + .then(() => next(), (error: unknown) => { + if (error instanceof BridgePairingError && error.code === 'RECOVERY_RATE_LIMITED') { + res.set('Retry-After', '60').status(429).json({ error: error.message, code: error.code }); + return; + } + next(error); + }); + }; + + router.post('/workers/:workerId/credentials/challenge', untrustedRecoveryLimit('challenge'), asyncRoute(async (req, res) => { if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { res.status(404).json({ error: 'Machine recovery is disabled' }); return; @@ -353,7 +371,7 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { } })); - router.post('/workers/:workerId/credentials/recover', asyncRoute(async (req, res) => { + router.post('/workers/:workerId/credentials/recover', untrustedRecoveryLimit('recover'), asyncRoute(async (req, res) => { if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { res.status(404).json({ error: 'Machine recovery is disabled' }); return; diff --git a/service/src/config.ts b/service/src/config.ts index 162b8ec3..29ae854d 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -439,6 +439,9 @@ export const env = { BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: Number( process.env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? 30, ), + BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE ?? 240, + ), /** * Runtime session affinity for stateful sandbox backends. * - `stateless` (default): no runtime sessions; `runtime_session_hint` ignored. diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index 162e9554..8b3abd1d 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -20,6 +20,7 @@ function render(overrides) { CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '', CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: '', CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: '', + CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: '', ...overrides, }, })); @@ -38,6 +39,7 @@ for (const overrides of [ CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '90', CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: '8', CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: '16', + CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: '400', }, ]) { const config = render(overrides); @@ -54,10 +56,12 @@ for (const overrides of [ assert.equal(env.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS, overrides.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS ?? '60'); assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE ?? '12'); assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? '30'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE ?? '240'); } for (const name of ['egress_gateway', 'sandbox-runner']) { assert.equal(config.services[name].environment.CODEAPI_BRIDGE_TOKEN, undefined); assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, undefined); + assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, undefined); } assert.match(JSON.stringify(config.services.redis.command), /--appendonly.*yes/); assert.ok(config.services.redis.volumes.some(volume => volume.target === '/data' && volume.type === 'volume'));