🔐 feat: Recover Code API Machine Credentials After Outages

docker-compose.yaml

@@ -15,6 +15,12 @@ services:
             - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true}
             - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1}
             - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-}
+            - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-}
+            - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0}
+            - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60}
+            - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12}
+            - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30}
+            - CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE:-240}
             - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-}
             - CODEAPI_ALLOW_AUTH_PROVIDER_NONE=${CODEAPI_ALLOW_AUTH_PROVIDER_NONE:-}
             - CODEAPI_JWT_ISSUER=${CODEAPI_JWT_ISSUER:-}
@@ -64,6 +70,12 @@ services:
             - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true}
             - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1}
             - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-}
+            - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-}
+            - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0}
+            - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60}
+            - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12}
+            - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30}
+            - CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE:-240}
             - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-}
             - CODEAPI_JWT_SINGLE_TENANT_ID=${CODEAPI_JWT_SINGLE_TENANT_ID:-}
             - CODEAPI_TENANT_ISOLATION_STRICT=${CODEAPI_TENANT_ISOLATION_STRICT:-}
@@ -213,9 +225,11 @@ services:
     redis:
         image: redis:7-alpine
         container_name: redis
-        command: redis-server --requirepass localdev
+        command: redis-server --requirepass localdev --appendonly yes
         ports:
             - ${CODEAPI_REDIS_PORT:-16379}:6379
+        volumes:
+            - redis_data:/data
 
     minio:
         image: quay.io/minio/minio
@@ -232,3 +246,4 @@ services:
 
 volumes:
     minio_data:
+    redis_data:

docs/adr/001-stateful-code-environments.md

@@ -52,8 +52,11 @@ worker replacement; the UI and operator documentation must not imply otherwise.
 - The VM requires no inbound internet listener.
 - Code API, not the worker, authenticates LibreChat users and normalizes work.
 - A stolen short-lived credential is insufficient without the worker private
-  key; a stolen private key is insufficient after credential expiry or
-  revocation.
+  key. In the original pairing-only model, a stolen private key is insufficient
+  after credential expiry or revocation. With optional durable machine
+  enrollment and signed credential recovery, the private key itself remains
+  a revocable long-lived credential: access-credential expiry alone does not
+  protect against theft of that key. Revocation invalidates both.
 - Pairing codes and credentials are stored by digest where lookup permits.
 - One configured worker has at most one active fenced assignment.
 - Sandbox isolation and default-deny egress remain the mandatory default;

docs/remote-bridge/README.md

@@ -31,6 +31,42 @@ CODEAPI_BRIDGE_TOKEN=<strong-administrator-bootstrap-secret>
 CODEAPI_BRIDGE_AUTH_MODE=paired
 ```
 
+To opt in to durable machine authorization on every Code API replica, set a
+single stable public **Code API** origin (not the LibreChat URL):
+
+```dotenv
+CODEAPI_BRIDGE_RECOVERY_SERVER_ID=https://code.example.com
+# 0 (default): enrolled machine keys remain authorized until revoked.
+# CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=0
+# CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=60
+# CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=12
+# CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=30
+# CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=240
+```
+
+Omitting the server ID retains the existing pairing and refresh behavior and
+hides the recovery routes. Deploy the compatible Code API version to **all**
+replicas before setting this value and enrolling workers again. Older Code API
+replicas can still pair or refresh a worker but do not write durable enrollment;
+they must not serve device login or recovery requests. Only a pairing redeemed
+after this option is enabled has a recoverable key. Updating Code API alone
+does not make old workers reconnect automatically: the CLI must also implement
+this recovery protocol in the later worker release.
+
+Store Redis state durably across restarts. The primary `docker-compose.yaml`
+now uses Redis AOF and a named `/data` volume; preserve that volume when
+recreating the stack. If upgrading a running stack with an in-memory Redis,
+migrate its state before recreating the container: mounting an empty volume
+does **not** preserve active assignments, fences, or earlier revocations. Other
+deployments must provide equivalent durable Redis (for example, a managed
+persistent Redis service and backups). Revocation and
+machine enrollment share that state across replicas; do not configure eviction
+of authorization keys. If enrollment state is missing, credentials minted under
+that enrollment fail closed, and the worker must be explicitly enrolled again.
+Restoring a backup from *before* a revocation can revive trust; reconcile
+revocations after recovery from backup. Use a distinct server ID for each Code
+API deployment and keep it stable when the endpoint changes behind a proxy.
+
 Use `strict` instead of `affinity` if every request must include a runtime
 session hint. In hardened mode, startup requires the bridge token to be at least
 32 bytes. `PTC_MODE=blocking` is rejected; replay mode is required because a
@@ -226,7 +262,40 @@ execution.
   atomically on their first redemption attempt.
 - Worker credentials expire after fifteen minutes and are bound to an Ed25519
   public key. Exact-request signatures include the HTTP method, path, body
-  digest, timestamp, nonce, and credential.
+  digest, timestamp, nonce, and credential. With recovery enabled, redeeming a
+  pairing also persists a separate machine authorization and its public key in
+  Redis without a TTL by default; an operator can instead set a bounded
+  enrollment lifetime.
+- `POST /v1/bridge/workers/:workerId/credentials/challenge` does not require
+  an administrator token or an existing access credential, but **does** require
+  the enrolled key. Its JSON body contains `protocolVersion: 1`,
+  `operation: "credential.challenge"`, the configured `serverId`, the matching
+  `workerId`, a fresh UTC ISO `timestamp`, a random 32-byte base64url `nonce`,
+  and `signature` computed with `signBridgeRecoveryStart(privateKey, fields)`
+  from `@librechat/code/identity`. Code API verifies the signed fields and
+  consumes the nonce once before charging the machine's shared challenge
+  budget; a fabricated request cannot exhaust another worker's budget.
+- The response is a short-lived, single-use challenge with the server ID,
+  worker ID, enrollment generation, operation and expiry. Sign those fields
+  with `signBridgeRecovery(privateKey, challenge)` and send the fields plus
+  `signature` to `POST .../credentials/recover` to obtain a new short-lived
+  credential. Invalid proofs are limited per high-entropy challenge; only
+  successfully verified, unused proofs consume the machine's shared recovery
+  budget. Separately, both recovery endpoints limit all incoming requests per
+  connection peer *before* key verification, including well-formed JSON with
+  malformed or forged proofs; forged headers and worker IDs cannot bypass
+  that limit or consume the signed machine budget. All limits live in shared
+  Redis; HTTP 429 means back off. When a reverse proxy connects to Code API,
+  its clients share that peer's limit. Restrict direct backend access and apply
+  client-IP and global
+  abuse limits at the trusted ingress to keep one proxy peer from becoming a
+  shared bottleneck; do not trust an arbitrary `X-Forwarded-For` on Code API.
+- Recovery and revocation are atomic Redis transitions across API replicas.
+  A missing, revoked, expired or superseded enrollment never creates new
+  credentials. Recovery only restores transport authentication. It does not
+  clear assignment fences, worker or workspace quarantine, or uncertain
+  execution state. The worker private key is a durable, revocable credential;
+  expiry of an access credential alone does **not** protect against key theft.
 - Accepted proof nonces cannot be replayed, credentials rotate before expiry,
   and an administrator can revoke the active worker identity immediately.
 - Assignment leases bind to a stable paired identity rather than an individual

docs/remote-bridge/worker-runbook.md

@@ -479,11 +479,16 @@ it still advertises named environments.
 
 ### Expired bridge credential
 
-A running worker refreshes its short-lived credential automatically. If a
-machine is offline long enough that refresh can no longer authenticate, issue
-a fresh one-time pairing for the same worker ID and redeem it with a newly
-generated keypair. Reusing the worker ID preserves the LibreChat environment
-record and its agent assignments; creating a new ID creates a new environment.
+A running worker refreshes its short-lived credential automatically. With
+Code API durable enrollment enabled, a worker that still has its enrolled
+private key can request a short-lived challenge and recover a new access
+credential without manual re-pairing. The current CLI does **not** yet invoke
+that endpoint automatically; update it when worker reconnect support ships.
+Until then, or if enrollment is missing or revoked, use the one-time operator
+pairing fallback. A new pairing replaces the Code API worker identity and may
+require LibreChat environment reauthorization; reusing a worker ID alone does
+not guarantee preservation of its LibreChat environment or agent assignments.
+Never clear quarantine or workspace fences as part of credential recovery.
 
 ### Failed environment setup or uncertain mutation
 

packages/code/package.json

@@ -15,6 +15,10 @@
       "types": "./dist/protocol.d.ts",
       "import": "./dist/protocol.js"
     },
+    "./identity": {
+      "types": "./dist/identity.d.ts",
+      "import": "./dist/identity.js"
+    },
     "./worker": {
       "types": "./dist/worker.d.ts",
       "import": "./dist/worker.js"

packages/code/src/identity.test.ts

@@ -1,9 +1,14 @@
 import assert from 'node:assert/strict';
+import { randomBytes } from 'node:crypto';
 import test from 'node:test';
 
 import {
   createBridgeIdentity,
+  signBridgeRecovery,
+  signBridgeRecoveryStart,
   signBridgeRequest,
+  verifyBridgeRecovery,
+  verifyBridgeRecoveryStart,
   verifyBridgeRequest,
 } from './identity.js';
 
@@ -33,3 +38,41 @@ test('worker identity proves possession for the exact HTTP request', () => {
     false,
   );
 });
+
+test('signed recovery starts bind operation, server, worker, time and nonce', () => {
+  const identity = createBridgeIdentity();
+  const start = {
+    operation: 'credential.challenge' as const,
+    serverId: 'https://code.example.test',
+    workerId: 'vm-1',
+    timestamp: new Date().toISOString(),
+    nonce: randomBytes(32).toString('base64url'),
+  };
+  const signature = signBridgeRecoveryStart(identity.privateKey, start);
+  assert.equal(verifyBridgeRecoveryStart(identity.publicKey, start, signature), true);
+  for (const modified of [
+    { ...start, operation: 'credential.recover' as 'credential.challenge' },
+    { ...start, serverId: 'https://other.example.test' },
+    { ...start, workerId: 'vm-2' },
+    { ...start, timestamp: new Date(Date.now() + 60_000).toISOString() },
+    { ...start, nonce: randomBytes(32).toString('base64url') },
+  ]) {
+    assert.equal(verifyBridgeRecoveryStart(identity.publicKey, modified, signature), false);
+  }
+
+  const challenge = {
+    operation: 'credential.recover' as const,
+    serverId: start.serverId,
+    workerId: start.workerId,
+    enrollmentGeneration: randomBytes(18).toString('base64url'),
+    challenge: randomBytes(32).toString('base64url'),
+    expiresAt: new Date(Date.now() + 60_000).toISOString(),
+  };
+  assert.equal(verifyBridgeRecovery(identity.publicKey, challenge, signature), false);
+  assert.equal(
+    verifyBridgeRecoveryStart(
+      identity.publicKey, start, signBridgeRecovery(identity.privateKey, challenge),
+    ),
+    false,
+  );
+});

packages/code/src/identity.ts

@@ -19,6 +19,25 @@ export interface BridgeRequestProofInput {
   body: string;
 }
 
+/** Signed without an access credential before requesting a server recovery challenge. */
+export interface BridgeRecoveryStartProofInput {
+  operation: 'credential.challenge';
+  serverId: string;
+  workerId: string;
+  timestamp: string;
+  nonce: string;
+}
+
+/** Signed separately from access-credential requests, so neither proof can be reused for the other. */
+export interface BridgeRecoveryProofInput {
+  operation: 'credential.recover';
+  serverId: string;
+  workerId: string;
+  enrollmentGeneration: string;
+  challenge: string;
+  expiresAt: string;
+}
+
 export function createBridgeIdentity(): BridgeIdentity {
   const { publicKey, privateKey } = generateKeyPairSync('ed25519', {
     publicKeyEncoding: { type: 'spki', format: 'pem' },
@@ -64,3 +83,78 @@ export function verifyBridgeRequest(
     return false;
   }
 }
+
+function canonicalBridgeRecoveryStart(input: BridgeRecoveryStartProofInput): string {
+  return [
+    'librechat-code:bridge-recovery-start:v1',
+    input.operation,
+    input.serverId,
+    input.workerId,
+    input.timestamp,
+    input.nonce,
+  ].join('\n');
+}
+
+export function signBridgeRecoveryStart(
+  privateKey: string,
+  input: BridgeRecoveryStartProofInput,
+): string {
+  return sign(null, Buffer.from(canonicalBridgeRecoveryStart(input)), privateKey).toString(
+    'base64url',
+  );
+}
+
+export function verifyBridgeRecoveryStart(
+  publicKey: string,
+  input: BridgeRecoveryStartProofInput,
+  signature: string,
+): boolean {
+  try {
+    return verify(
+      null,
+      Buffer.from(canonicalBridgeRecoveryStart(input)),
+      publicKey,
+      Buffer.from(signature, 'base64url'),
+    );
+  } catch {
+    return false;
+  }
+}
+
+function canonicalBridgeRecovery(input: BridgeRecoveryProofInput): string {
+  return [
+    'librechat-code:bridge-recovery:v1',
+    input.operation,
+    input.serverId,
+    input.workerId,
+    input.enrollmentGeneration,
+    input.challenge,
+    input.expiresAt,
+  ].join('\n');
+}
+
+export function signBridgeRecovery(
+  privateKey: string,
+  input: BridgeRecoveryProofInput,
+): string {
+  return sign(null, Buffer.from(canonicalBridgeRecovery(input)), privateKey).toString(
+    'base64url',
+  );
+}
+
+export function verifyBridgeRecovery(
+  publicKey: string,
+  input: BridgeRecoveryProofInput,
+  signature: string,
+): boolean {
+  try {
+    return verify(
+      null,
+      Buffer.from(canonicalBridgeRecovery(input)),
+      publicKey,
+      Buffer.from(signature, 'base64url'),
+    );
+  } catch {
+    return false;
+  }
+}

packages/code/src/protocol.ts

@@ -759,6 +759,32 @@ export interface BridgeWorkerCredentialResponse {
   expiresAt: string;
 }
 
+/** The enrolled machine signs this request before Code API issues a challenge. */
+export interface BridgeRecoveryChallengeRequest {
+  protocolVersion: BridgeProtocolVersion;
+  operation: 'credential.challenge';
+  serverId: string;
+  workerId: string;
+  timestamp: string;
+  nonce: string;
+  signature: string;
+}
+
+/** A short-lived, single-use challenge for an already enrolled machine key. */
+export interface BridgeRecoveryChallengeResponse {
+  protocolVersion: BridgeProtocolVersion;
+  operation: 'credential.recover';
+  serverId: string;
+  workerId: string;
+  enrollmentGeneration: string;
+  challenge: string;
+  expiresAt: string;
+}
+
+export interface BridgeRecoveryRequest extends BridgeRecoveryChallengeResponse {
+  signature: string;
+}
+
 export interface BridgeSandboxRequest<TBody = object> {
   body: TBody;
   headers: Record<string, string>;

service/src/bridge/index.ts

@@ -11,7 +11,23 @@ export const bridgeStore = new RedisBridgeStore(
   undefined,
   env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS,
 );
-export const bridgePairings = new RedisBridgePairingStore(connection);
+export const bridgePairings = new RedisBridgePairingStore(
+  connection,
+  undefined,
+  undefined,
+  undefined,
+  undefined,
+  env.BRIDGE_RECOVERY_SERVER_ID
+    ? {
+      serverId: env.BRIDGE_RECOVERY_SERVER_ID,
+      enrollmentTtlSeconds: env.BRIDGE_ENROLLMENT_TTL_SECONDS,
+      challengeTtlSeconds: env.BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS,
+      maxChallengesPerMinute: env.BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE,
+      maxAttemptsPerMinute: env.BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE,
+      maxUntrustedRequestsPerMinute: env.BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE,
+    }
+    : undefined,
+);
 
 export default createBridgeRouter({
   enabled: isBridgeEnabled(),