Contain overlapping SaaS tenant runtime execution

docs/reliability-repair-progress.md

@@ -0,0 +1,33 @@
+# Reliability repair progress
+
+This file records only implementation status that is evidenced by repository changes and CI. It does not claim production deployment or provider readiness.
+
+## Sequence 1: release identity and complete CI gate
+
+Status: implemented on `codex/reliability-foundation-1` and proven by upstream pull-request CI on head `a73d1d9202f5013a944b9a2320ebb22be7caf4b1`.
+
+Evidence:
+
+- `npm run ci` gates typecheck, the complete repository test suite, and the compiled runtime smoke check.
+- the deploy workflow runs that same gate before Wrangler deployment.
+- Cloudflare version metadata and the deployment Git SHA are exposed separately from provider readiness.
+- hosted Threads and Instagram publication remain explicitly unavailable; Facebook remains paused; LinkedIn compatibility remains unverified; X remains tenant-scoped.
+- no production deployment is claimed by this change.
+
+## D03 containment: overlapping tenant runtime state
+
+Status: containment implemented on `codex/tenant-runtime-containment`; immutable provider/client context refactor still required before D03 can be marked complete.
+
+The Cloudflare scheduled and authenticated `/tick` job-drain entry points now share one per-isolate exclusive run gate. This prevents two overlapping SaaS drains in the same Worker isolate from entering `withTenantRuntime` concurrently. Separate Worker isolates do not share process globals.
+
+This is deliberately a containment layer, not the target architecture. The remaining D03 work is to remove tenant-specific mutation of shared `config` and shared token-persistence callbacks and pass immutable tenant/connection context to provider and generation clients.
+
+Acceptance evidence required before this containment is considered proven:
+
+- deliberately interleaved async runs never overlap inside the gate;
+- a failed run does not poison the next queued run;
+- the complete CI gate passes on the exact branch SHA.
+
+## Next bounded repair
+
+After containment is green, replace shared tenant runtime mutation with immutable request-scoped/provider-scoped context. Do not re-enable Meta publication as part of that refactor.

package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "build": "tsx scripts/build.ts",
     "typecheck": "tsc --noEmit --project tsconfig.json",
-    "test": "npm run build && node dist/test/security-hardening.test.js && node dist/test/source-ssrf.test.js && node dist/test/browser-collector-ingest.test.js && node dist/test/slot-scheduler.test.js && node dist/test/daily-inventory-planner.test.js && node dist/test/refresh-queue-finalization.test.js && node dist/test/publish-summary.test.js && node dist/test/threads-refresh.test.js && node dist/test/linkedin-refresh.test.js && node dist/test/instagram-image-timeout.test.js && node dist/test/cost-control.test.js && node dist/test/recovery-scheduler.test.js && node dist/test/social-connector.test.js && node dist/test/meta-publication-boundary.test.js && node dist/test/cloudflare-health.test.js",
+    "test": "npm run build && node dist/test/security-hardening.test.js && node dist/test/source-ssrf.test.js && node dist/test/browser-collector-ingest.test.js && node dist/test/slot-scheduler.test.js && node dist/test/daily-inventory-planner.test.js && node dist/test/refresh-queue-finalization.test.js && node dist/test/publish-summary.test.js && node dist/test/threads-refresh.test.js && node dist/test/linkedin-refresh.test.js && node dist/test/instagram-image-timeout.test.js && node dist/test/cost-control.test.js && node dist/test/recovery-scheduler.test.js && node dist/test/social-connector.test.js && node dist/test/meta-publication-boundary.test.js && node dist/test/cloudflare-health.test.js && node dist/test/exclusive-run-gate.test.js",
     "smoke:dist": "node dist/src/cli.js status",
     "ci": "npm run typecheck && npm test && npm run smoke:dist",
     "dev": "tsx src/agent.ts",

src/cloudflare-worker.ts

@@ -1,3 +1,5 @@
+import { createExclusiveRunGate } from './exclusive-run-gate';
+
 interface WorkerVersionMetadata {
   id: string;
   tag?: string;
@@ -52,6 +54,7 @@ interface ExecutionContext {
 }
 
 const SCHEMA_CONTRACT = 'pre-publication-ledger-v1';
+const scheduledTickGate = createExclusiveRunGate();
 
 function canonicalGitSha(versionTag: string | undefined): string | null {
   const tag = String(versionTag || '').trim();
@@ -100,6 +103,7 @@ function healthPayload(env: Env): Record<string, unknown> {
       appliedSchema: 'unverified',
     },
     publicationCapabilities: publicationCapabilities(),
+    executionGate: scheduledTickGate.snapshot(),
   };
 }
 
@@ -113,7 +117,7 @@ function applyCloudflareEnv(env: Env): void {
   }
 }
 
-async function runScheduledTick(env: Env): Promise<Response> {
+async function executeScheduledTick(env: Env): Promise<Response> {
   applyCloudflareEnv(env);
 
   const [{ processPendingSupabaseJobs, runSupabaseAutomationScheduler }, logger] = await Promise.all([
@@ -130,6 +134,10 @@ async function runScheduledTick(env: Env): Promise<Response> {
   return Response.json({ ok: true, schedulerStats, stats });
 }
 
+function runScheduledTick(env: Env): Promise<Response> {
+  return scheduledTickGate.run(() => executeScheduledTick(env));
+}
+
 export default {
   async scheduled(_controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise<void> {
     ctx.waitUntil(runScheduledTick(env));

src/exclusive-run-gate.ts

@@ -0,0 +1,41 @@
+export interface ExclusiveRunGateSnapshot {
+  active: number;
+  waiting: number;
+}
+
+export interface ExclusiveRunGate {
+  run<T>(task: () => Promise<T>): Promise<T>;
+  snapshot(): ExclusiveRunGateSnapshot;
+}
+
+export function createExclusiveRunGate(): ExclusiveRunGate {
+  let tail: Promise<void> = Promise.resolve();
+  let active = 0;
+  let waiting = 0;
+
+  return {
+    async run<T>(task: () => Promise<T>): Promise<T> {
+      let release!: () => void;
+      const previous = tail;
+      tail = new Promise<void>(resolve => {
+        release = resolve;
+      });
+      waiting++;
+
+      try {
+        await previous.catch(() => undefined);
+        waiting--;
+        active++;
+        return await task();
+      } finally {
+        if (active > 0) active--;
+        else if (waiting > 0) waiting--;
+        release();
+      }
+    },
+
+    snapshot(): ExclusiveRunGateSnapshot {
+      return { active, waiting };
+    },
+  };
+}

test/exclusive-run-gate.test.ts

@@ -0,0 +1,88 @@
+import assert from 'node:assert/strict';
+
+import { createExclusiveRunGate } from '../src/exclusive-run-gate';
+
+async function test(name: string, fn: () => Promise<void>): Promise<void> {
+  try {
+    await fn();
+    console.log(`ok - ${name}`);
+  } catch (error) {
+    console.error(`not ok - ${name}`);
+    throw error;
+  }
+}
+
+async function main(): Promise<void> {
+  await test('serializes deliberately interleaved async work', async () => {
+    const gate = createExclusiveRunGate();
+    const events: string[] = [];
+    let active = 0;
+    let maxActive = 0;
+    let releaseFirst!: () => void;
+    let firstStarted!: () => void;
+
+    const firstStartedPromise = new Promise<void>(resolve => {
+      firstStarted = resolve;
+    });
+    const firstReleasePromise = new Promise<void>(resolve => {
+      releaseFirst = resolve;
+    });
+
+    const first = gate.run(async () => {
+      active++;
+      maxActive = Math.max(maxActive, active);
+      events.push('tenant-a:start');
+      firstStarted();
+      await firstReleasePromise;
+      events.push('tenant-a:end');
+      active--;
+      return 'tenant-a';
+    });
+
+    await firstStartedPromise;
+
+    const second = gate.run(async () => {
+      active++;
+      maxActive = Math.max(maxActive, active);
+      events.push('tenant-b:start');
+      events.push('tenant-b:end');
+      active--;
+      return 'tenant-b';
+    });
+
+    await Promise.resolve();
+    assert.deepEqual(gate.snapshot(), { active: 1, waiting: 1 });
+    assert.deepEqual(events, ['tenant-a:start']);
+
+    releaseFirst();
+    assert.deepEqual(await Promise.all([first, second]), ['tenant-a', 'tenant-b']);
+    assert.deepEqual(events, [
+      'tenant-a:start',
+      'tenant-a:end',
+      'tenant-b:start',
+      'tenant-b:end',
+    ]);
+    assert.equal(maxActive, 1);
+    assert.deepEqual(gate.snapshot(), { active: 0, waiting: 0 });
+  });
+
+  await test('a failed run does not poison the next queued run', async () => {
+    const gate = createExclusiveRunGate();
+
+    await assert.rejects(
+      gate.run(async () => {
+        throw new Error('expected failure');
+      }),
+      /expected failure/
+    );
+
+    const result = await gate.run(async () => 'next-run-completed');
+    assert.equal(result, 'next-run-completed');
+    assert.deepEqual(gate.snapshot(), { active: 0, waiting: 0 });
+  });
+}
+
+main().catch(error => {
+  console.error(error);
+  process.exitCode = 1;
+});