Isolate SaaS tenant runtime state per async execution

docs/reliability-repair-progress.md

@@ -16,18 +16,38 @@ Evidence:
 
 ## D03 containment: overlapping tenant runtime state
 
-Status: containment implemented on `codex/tenant-runtime-containment`; immutable provider/client context refactor still required before D03 can be marked complete.
+Status: implemented and proven by upstream pull-request CI on head `ee377775030d27117b2568bc10426da8304c7a24`.
 
-The Cloudflare scheduled and authenticated `/tick` job-drain entry points now share one per-isolate exclusive run gate. This prevents two overlapping SaaS drains in the same Worker isolate from entering `withTenantRuntime` concurrently. Separate Worker isolates do not share process globals.
+The Cloudflare scheduled and authenticated `/tick` job-drain entry points share one per-isolate exclusive run gate. This prevents two overlapping SaaS drains in the same Worker isolate from entering the mutable tenant runtime concurrently. Separate Worker isolates do not share process globals.
 
-This is deliberately a containment layer, not the target architecture. The remaining D03 work is to remove tenant-specific mutation of shared `config` and shared token-persistence callbacks and pass immutable tenant/connection context to provider and generation clients.
+The containment regression deliberately interleaves two executions, proves maximum concurrent execution is one, and proves a rejected execution releases the gate.
 
-Acceptance evidence required before this containment is considered proven:
+## D03 runtime isolation: process-global config and token callbacks
 
-- deliberately interleaved async runs never overlap inside the gate;
-- a failed run does not poison the next queued run;
-- the complete CI gate passes on the exact branch SHA.
+Status: implemented on `codex/immutable-tenant-runtime`; upstream CI evidence is still required before this layer can be marked proven.
+
+The Worker now installs async-scoped accessors on the existing config object only after Cloudflare bindings have been copied into `process.env`. Each scheduled/authenticated SaaS drain then runs inside its own `AsyncLocalStorage` context.
+
+Within that context:
+
+- tenant config writes made by the existing `withTenantRuntime` path are copy-on-write and remain inside the current async execution instead of mutating process-global values;
+- OpenAI, Cloudinary, Instagram, Facebook and provider modules that already read the shared config object transparently resolve the scoped values without a flag-day call-signature rewrite;
+- Threads, LinkedIn and X token-persistence setters use scope-local callback slots when a SaaS runtime scope exists;
+- token rotation updates only the current scoped config snapshot while the Supabase persistence callback remains attached to that same async execution;
+- outside a SaaS runtime scope, the existing local single-tenant behaviour is preserved.
+
+The current `processPendingSupabaseJobs()` implementation remains serial, so tenant runtime mutation is restored between jobs inside a drain. The earlier exclusive run gate remains defence-in-depth but is no longer the only boundary preventing overlapping Worker invocations from sharing config or token callbacks.
+
+Acceptance evidence required before this layer is considered proven:
+
+- two deliberately overlapping runtime scopes resolve different OpenAI/provider credentials;
+- Threads, LinkedIn and X token rotations invoke only the persistence callback belonging to their own scope;
+- rotated credentials remain visible inside the originating scope but do not change the base config or the other scope;
+- a failed scoped execution cannot leak its config into the next execution;
+- the complete `npm run ci` gate passes on the exact branch SHA.
+
+No provider is re-enabled and no deployment is performed by this refactor.
 
 ## Next bounded repair
 
-After containment is green, replace shared tenant runtime mutation with immutable request-scoped/provider-scoped context. Do not re-enable Meta publication as part of that refactor.
+After runtime isolation is green, make missing tenant platform settings fail closed (D22), then proceed to atomic database claims/fencing and publication-attempt identity. Meta publication remains disabled until the publication ledger and provider-specific restoration work are ready.

package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "build": "tsx scripts/build.ts",
     "typecheck": "tsc --noEmit --project tsconfig.json",
-    "test": "npm run build && node dist/test/security-hardening.test.js && node dist/test/source-ssrf.test.js && node dist/test/browser-collector-ingest.test.js && node dist/test/slot-scheduler.test.js && node dist/test/daily-inventory-planner.test.js && node dist/test/refresh-queue-finalization.test.js && node dist/test/publish-summary.test.js && node dist/test/threads-refresh.test.js && node dist/test/linkedin-refresh.test.js && node dist/test/instagram-image-timeout.test.js && node dist/test/cost-control.test.js && node dist/test/recovery-scheduler.test.js && node dist/test/social-connector.test.js && node dist/test/meta-publication-boundary.test.js && node dist/test/cloudflare-health.test.js && node dist/test/exclusive-run-gate.test.js",
+    "test": "npm run build && node dist/test/security-hardening.test.js && node dist/test/source-ssrf.test.js && node dist/test/browser-collector-ingest.test.js && node dist/test/slot-scheduler.test.js && node dist/test/daily-inventory-planner.test.js && node dist/test/refresh-queue-finalization.test.js && node dist/test/publish-summary.test.js && node dist/test/threads-refresh.test.js && node dist/test/linkedin-refresh.test.js && node dist/test/instagram-image-timeout.test.js && node dist/test/cost-control.test.js && node dist/test/recovery-scheduler.test.js && node dist/test/social-connector.test.js && node dist/test/meta-publication-boundary.test.js && node dist/test/cloudflare-health.test.js && node dist/test/exclusive-run-gate.test.js && node dist/test/runtime-scope.test.js",
     "smoke:dist": "node dist/src/cli.js status",
     "ci": "npm run typecheck && npm test && npm run smoke:dist",
     "dev": "tsx src/agent.ts",

src/cloudflare-worker.ts

@@ -1,4 +1,5 @@
 import { createExclusiveRunGate } from './exclusive-run-gate';
+import { installScopedConfig, runWithRuntimeScope } from './runtime-scope';
 
 interface WorkerVersionMetadata {
   id: string;
@@ -120,18 +121,28 @@ function applyCloudflareEnv(env: Env): void {
 async function executeScheduledTick(env: Env): Promise<Response> {
   applyCloudflareEnv(env);
 
-  const [{ processPendingSupabaseJobs, runSupabaseAutomationScheduler }, logger] = await Promise.all([
-    import('./supabase-worker'),
-    import('./logger'),
-  ]);
-
-  const schedulerStats = await runSupabaseAutomationScheduler();
-  const stats = await processPendingSupabaseJobs();
-  logger.info(
-    `Cloudflare scheduled worker tick | scheduled_fetch:${schedulerStats.fetchJobsEnqueued} scheduled_fill:${schedulerStats.slotFillJobsEnqueued} scheduled_publish:${schedulerStats.publishJobsEnqueued} inventory_plans:${schedulerStats.inventoryPlansChecked} inventory_alerts:${schedulerStats.inventoryAlerts} stale_failed:${schedulerStats.staleJobsFailed} claimed:${stats.claimed} completed:${stats.completed} failed:${stats.failed}`
-  );
-
-  return Response.json({ ok: true, schedulerStats, stats });
+  // Config must be constructed after Worker bindings are copied into process.env.
+  // Instrument the shared object once, then keep every tenant mutation inside this
+  // async execution scope rather than process-global state.
+  const { default: config } = await import('../config');
+  installScopedConfig(config);
+
+  return runWithRuntimeScope(async () => {
+    const [{ processPendingSupabaseJobs, runSupabaseAutomationScheduler }, logger] = await Promise.all([
+      import('./supabase-worker'),
+      import('./logger'),
+    ]);
+
+    const schedulerStats = await runSupabaseAutomationScheduler();
+    const stats = await processPendingSupabaseJobs();
+    logger.info(
+      `Cloudflare scheduled worker tick | scheduled_fetch:${schedulerStats.fetchJobsEnqueued} scheduled_fill:${schedulerStats.slotFillJobsEnqueued} scheduled_publish:${schedulerStats.publishJobsEnqueued} inventory_plans:${schedulerStats.inventoryPlansChecked} inventory_alerts:${schedulerStats.inventoryAlerts} stale_failed:${schedulerStats.staleJobsFailed} claimed:${stats.claimed} completed:${stats.completed} failed:${stats.failed}`
+    );
+
+    return Response.json({ ok: true, schedulerStats, stats });
+  }, {
+    execution: 'cloudflare_saas_tick',
+  });
 }
 
 function runScheduledTick(env: Env): Promise<Response> {

src/linkedin.ts

@@ -6,6 +6,7 @@ import {
   PlatformPublishError,
   safeBodySnippet,
 } from './platform-errors';
+import { getScopedHandler, setScopedHandler } from './runtime-scope';
 
 interface LinkedInPublishSuccess {
   id?: string;
@@ -47,6 +48,7 @@ type LinkedInOAuthTokenPersistence = (
 ) => void | Promise<void>;
 
 const REFRESH_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
+const LINKEDIN_TOKEN_PERSISTENCE_HANDLER = 'linkedin_oauth2_token_persistence';
 let persistOAuth2TokensHandler: LinkedInOAuthTokenPersistence = persistOAuth2TokensToLocalRuntime;
 
 export function hasRefreshConfig(): boolean {
@@ -109,6 +111,12 @@ export async function refreshOAuth2AccessToken(
 }
 
 export function setOAuth2TokenPersistence(handler: LinkedInOAuthTokenPersistence): () => void {
+  const scopedRestore = setScopedHandler<LinkedInOAuthTokenPersistence>(
+    LINKEDIN_TOKEN_PERSISTENCE_HANDLER,
+    handler
+  );
+  if (scopedRestore) return scopedRestore;
+
   const previous = persistOAuth2TokensHandler;
   persistOAuth2TokensHandler = handler;
   return () => {
@@ -118,7 +126,9 @@ export function setOAuth2TokenPersistence(handler: LinkedInOAuthTokenPersistence
 
 export async function persistOAuth2Tokens(tokens: LinkedInOAuthTokenSet): Promise<void> {
   applyOAuth2TokensToConfig(tokens);
-  await persistOAuth2TokensHandler(tokens);
+  const handler = getScopedHandler<LinkedInOAuthTokenPersistence>(LINKEDIN_TOKEN_PERSISTENCE_HANDLER)
+    || persistOAuth2TokensHandler;
+  await handler(tokens);
 }
 
 export async function refreshAndPersistOAuth2AccessToken(): Promise<LinkedInOAuthTokenSet> {

src/runtime-scope.ts

@@ -0,0 +1,118 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+type RuntimeValueMap = Record<string, unknown>;
+type RuntimeHandler = (...args: any[]) => unknown;
+
+interface RuntimeScopeStore {
+  configOverlay: Readonly<RuntimeValueMap>;
+  handlers: Readonly<Record<string, RuntimeHandler>>;
+  metadata: Readonly<Record<string, string>>;
+}
+
+const runtimeScope = new AsyncLocalStorage<RuntimeScopeStore>();
+const installedConfigObjects = new WeakMap<object, Map<string, unknown>>();
+
+function frozenCopy<T extends Record<string, unknown>>(value: T): Readonly<T> {
+  return Object.freeze({ ...value });
+}
+
+function hasOwn(value: object, key: PropertyKey): boolean {
+  return Object.prototype.hasOwnProperty.call(value, key);
+}
+
+export function hasRuntimeScope(): boolean {
+  return Boolean(runtimeScope.getStore());
+}
+
+export function runWithRuntimeScope<T>(
+  fn: () => T,
+  metadata: Record<string, string> = {}
+): T {
+  const store: RuntimeScopeStore = {
+    configOverlay: frozenCopy({}),
+    handlers: Object.freeze({}),
+    metadata: frozenCopy(metadata),
+  };
+  return runtimeScope.run(store, fn);
+}
+
+export function getScopedConfigValue(key: string): { found: boolean; value?: unknown } {
+  const store = runtimeScope.getStore();
+  if (!store || !hasOwn(store.configOverlay, key)) {
+    return { found: false };
+  }
+  return { found: true, value: store.configOverlay[key] };
+}
+
+export function setScopedConfigValue(key: string, value: unknown): boolean {
+  const store = runtimeScope.getStore();
+  if (!store) return false;
+  store.configOverlay = frozenCopy({
+    ...store.configOverlay,
+    [key]: value,
+  });
+  return true;
+}
+
+export function getScopedHandler<T extends RuntimeHandler>(key: string): T | undefined {
+  const store = runtimeScope.getStore();
+  return store?.handlers[key] as T | undefined;
+}
+
+export function setScopedHandler<T extends RuntimeHandler>(
+  key: string,
+  handler: T
+): (() => void) | undefined {
+  const store = runtimeScope.getStore();
+  if (!store) return undefined;
+
+  const hadPrevious = hasOwn(store.handlers, key);
+  const previous = store.handlers[key];
+  store.handlers = Object.freeze({
+    ...store.handlers,
+    [key]: handler,
+  });
+
+  return () => {
+    const next = { ...store.handlers } as Record<string, RuntimeHandler>;
+    if (hadPrevious && previous) next[key] = previous;
+    else delete next[key];
+    store.handlers = Object.freeze(next);
+  };
+}
+
+export function installScopedConfig<T extends object>(config: T): T {
+  if (installedConfigObjects.has(config)) return config;
+
+  const baseValues = new Map<string, unknown>();
+  installedConfigObjects.set(config, baseValues);
+
+  for (const key of Object.keys(config)) {
+    baseValues.set(key, (config as Record<string, unknown>)[key]);
+    Object.defineProperty(config, key, {
+      configurable: true,
+      enumerable: true,
+      get() {
+        const scoped = getScopedConfigValue(key);
+        return scoped.found ? scoped.value : baseValues.get(key);
+      },
+      set(value: unknown) {
+        if (!setScopedConfigValue(key, value)) {
+          baseValues.set(key, value);
+        }
+      },
+    });
+  }
+
+  return config;
+}
+
+export function runtimeScopeMetadata(): Readonly<Record<string, string>> | null {
+  return runtimeScope.getStore()?.metadata || null;
+}
+
+export const __test__ = {
+  getScopedConfigValue,
+  getScopedHandler,
+  hasRuntimeScope,
+};

src/threads.ts

@@ -5,6 +5,7 @@ import {
   safeBodySnippet,
 } from './platform-errors';
 import { assertCanonicalMetaPublicationPath } from './meta-publication-boundary';
+import { getScopedHandler, setScopedHandler } from './runtime-scope';
 
 interface ThreadsTokenResponse extends GraphErrorResponse {
   access_token?: string;
@@ -44,6 +45,7 @@ interface GraphErrorResponse {
   id?: string;
 }
 
+const THREADS_TOKEN_PERSISTENCE_HANDLER = 'threads_token_persistence';
 let persistThreadsTokenHandler: ThreadsTokenPersistence = persistThreadsTokenToLocalRuntime;
 
 export async function refreshLongLivedAccessToken(): Promise<ThreadsTokenSet> {
@@ -224,6 +226,12 @@ export async function prepareAccessTokenForPublish(): Promise<ThreadsCredentialP
 }
 
 export function setTokenPersistence(handler: ThreadsTokenPersistence): () => void {
+  const scopedRestore = setScopedHandler<ThreadsTokenPersistence>(
+    THREADS_TOKEN_PERSISTENCE_HANDLER,
+    handler
+  );
+  if (scopedRestore) return scopedRestore;
+
   const previous = persistThreadsTokenHandler;
   persistThreadsTokenHandler = handler;
   return () => {
@@ -233,7 +241,9 @@ export function setTokenPersistence(handler: ThreadsTokenPersistence): () => voi
 
 export async function persistLongLivedAccessToken(tokens: ThreadsTokenSet): Promise<void> {
   config.THREADS_ACCESS_TOKEN = tokens.accessToken;
-  await persistThreadsTokenHandler(tokens);
+  const handler = getScopedHandler<ThreadsTokenPersistence>(THREADS_TOKEN_PERSISTENCE_HANDLER)
+    || persistThreadsTokenHandler;
+  await handler(tokens);
 }
 
 function persistThreadsTokenToLocalRuntime(tokens: ThreadsTokenSet): void {

src/x.ts

@@ -2,6 +2,7 @@ import * as crypto from 'node:crypto';
 import config from '../config';
 import { requestJson } from './http-client';
 import { PlatformPublishError, safeBodySnippet } from './platform-errors';
+import { getScopedHandler, setScopedHandler } from './runtime-scope';
 
 interface XApiErrorDetail {
   message?: string;
@@ -61,6 +62,7 @@ export type XAuthMode = 'oauth1-user' | 'oauth2-user' | 'unconfigured';
 export type XSafeAuthMode = 'x_oauth1_user_context' | 'x_oauth2_user_context' | 'unconfigured';
 export type XErrorKind = 'publish-access-tier' | 'project-required' | 'auth' | 'other';
 
+const X_TOKEN_PERSISTENCE_HANDLER = 'x_oauth2_token_persistence';
 let persistOAuth2TokensHandler: XOAuth2TokenPersistence = persistOAuth2TokensToLocalRuntime;
 
 function encodeOAuthComponent(value: string): string {
@@ -558,6 +560,12 @@ function persistOAuth2TokensToLocalRuntime(tokens: XOAuth2TokenSet): void {
 }
 
 export function setOAuth2TokenPersistence(handler: XOAuth2TokenPersistence): () => void {
+  const scopedRestore = setScopedHandler<XOAuth2TokenPersistence>(
+    X_TOKEN_PERSISTENCE_HANDLER,
+    handler
+  );
+  if (scopedRestore) return scopedRestore;
+
   const previous = persistOAuth2TokensHandler;
   persistOAuth2TokensHandler = handler;
   return () => {
@@ -567,5 +575,7 @@ export function setOAuth2TokenPersistence(handler: XOAuth2TokenPersistence): ()
 
 export async function persistOAuth2Tokens(tokens: XOAuth2TokenSet): Promise<void> {
   applyOAuth2TokensToConfig(tokens);
-  await persistOAuth2TokensHandler(tokens);
+  const handler = getScopedHandler<XOAuth2TokenPersistence>(X_TOKEN_PERSISTENCE_HANDLER)
+    || persistOAuth2TokensHandler;
+  await handler(tokens);
 }