diff --git a/docs/reliability-repair-progress.md b/docs/reliability-repair-progress.md new file mode 100644 index 0000000..dba6308 --- /dev/null +++ b/docs/reliability-repair-progress.md @@ -0,0 +1,33 @@ +# Reliability repair progress + +This file records only implementation status that is evidenced by repository changes and CI. It does not claim production deployment or provider readiness. + +## Sequence 1: release identity and complete CI gate + +Status: implemented on `codex/reliability-foundation-1` and proven by upstream pull-request CI on head `a73d1d9202f5013a944b9a2320ebb22be7caf4b1`. + +Evidence: + +- `npm run ci` gates typecheck, the complete repository test suite, and the compiled runtime smoke check. +- the deploy workflow runs that same gate before Wrangler deployment. +- Cloudflare version metadata and the deployment Git SHA are exposed separately from provider readiness. +- hosted Threads and Instagram publication remain explicitly unavailable; Facebook remains paused; LinkedIn compatibility remains unverified; X remains tenant-scoped. +- no production deployment is claimed by this change. + +## D03 containment: overlapping tenant runtime state + +Status: containment implemented on `codex/tenant-runtime-containment`; immutable provider/client context refactor still required before D03 can be marked complete. + +The Cloudflare scheduled and authenticated `/tick` job-drain entry points now share one per-isolate exclusive run gate. This prevents two overlapping SaaS drains in the same Worker isolate from entering `withTenantRuntime` concurrently. Separate Worker isolates do not share process globals. + +This is deliberately a containment layer, not the target architecture. The remaining D03 work is to remove tenant-specific mutation of shared `config` and shared token-persistence callbacks and pass immutable tenant/connection context to provider and generation clients. + +Acceptance evidence required before this containment is considered proven: + +- deliberately interleaved async runs never overlap inside the gate; +- a failed run does not poison the next queued run; +- the complete CI gate passes on the exact branch SHA. + +## Next bounded repair + +After containment is green, replace shared tenant runtime mutation with immutable request-scoped/provider-scoped context. Do not re-enable Meta publication as part of that refactor. diff --git a/package.json b/package.json index 0385c45..32c794c 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "tsx scripts/build.ts", "typecheck": "tsc --noEmit --project tsconfig.json", - "test": "npm run build && node dist/test/security-hardening.test.js && node dist/test/source-ssrf.test.js && node dist/test/browser-collector-ingest.test.js && node dist/test/slot-scheduler.test.js && node dist/test/daily-inventory-planner.test.js && node dist/test/refresh-queue-finalization.test.js && node dist/test/publish-summary.test.js && node dist/test/threads-refresh.test.js && node dist/test/linkedin-refresh.test.js && node dist/test/instagram-image-timeout.test.js && node dist/test/cost-control.test.js && node dist/test/recovery-scheduler.test.js && node dist/test/social-connector.test.js && node dist/test/meta-publication-boundary.test.js && node dist/test/cloudflare-health.test.js", + "test": "npm run build && node dist/test/security-hardening.test.js && node dist/test/source-ssrf.test.js && node dist/test/browser-collector-ingest.test.js && node dist/test/slot-scheduler.test.js && node dist/test/daily-inventory-planner.test.js && node dist/test/refresh-queue-finalization.test.js && node dist/test/publish-summary.test.js && node dist/test/threads-refresh.test.js && node dist/test/linkedin-refresh.test.js && node dist/test/instagram-image-timeout.test.js && node dist/test/cost-control.test.js && node dist/test/recovery-scheduler.test.js && node dist/test/social-connector.test.js && node dist/test/meta-publication-boundary.test.js && node dist/test/cloudflare-health.test.js && node dist/test/exclusive-run-gate.test.js", "smoke:dist": "node dist/src/cli.js status", "ci": "npm run typecheck && npm test && npm run smoke:dist", "dev": "tsx src/agent.ts", diff --git a/src/cloudflare-worker.ts b/src/cloudflare-worker.ts index ad7bce4..6437123 100644 --- a/src/cloudflare-worker.ts +++ b/src/cloudflare-worker.ts @@ -1,3 +1,5 @@ +import { createExclusiveRunGate } from './exclusive-run-gate'; + interface WorkerVersionMetadata { id: string; tag?: string; @@ -52,6 +54,7 @@ interface ExecutionContext { } const SCHEMA_CONTRACT = 'pre-publication-ledger-v1'; +const scheduledTickGate = createExclusiveRunGate(); function canonicalGitSha(versionTag: string | undefined): string | null { const tag = String(versionTag || '').trim(); @@ -100,6 +103,7 @@ function healthPayload(env: Env): Record { appliedSchema: 'unverified', }, publicationCapabilities: publicationCapabilities(), + executionGate: scheduledTickGate.snapshot(), }; } @@ -113,7 +117,7 @@ function applyCloudflareEnv(env: Env): void { } } -async function runScheduledTick(env: Env): Promise { +async function executeScheduledTick(env: Env): Promise { applyCloudflareEnv(env); const [{ processPendingSupabaseJobs, runSupabaseAutomationScheduler }, logger] = await Promise.all([ @@ -130,6 +134,10 @@ async function runScheduledTick(env: Env): Promise { return Response.json({ ok: true, schedulerStats, stats }); } +function runScheduledTick(env: Env): Promise { + return scheduledTickGate.run(() => executeScheduledTick(env)); +} + export default { async scheduled(_controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise { ctx.waitUntil(runScheduledTick(env)); diff --git a/src/exclusive-run-gate.ts b/src/exclusive-run-gate.ts new file mode 100644 index 0000000..3f216fa --- /dev/null +++ b/src/exclusive-run-gate.ts @@ -0,0 +1,41 @@ +export interface ExclusiveRunGateSnapshot { + active: number; + waiting: number; +} + +export interface ExclusiveRunGate { + run(task: () => Promise): Promise; + snapshot(): ExclusiveRunGateSnapshot; +} + +export function createExclusiveRunGate(): ExclusiveRunGate { + let tail: Promise = Promise.resolve(); + let active = 0; + let waiting = 0; + + return { + async run(task: () => Promise): Promise { + let release!: () => void; + const previous = tail; + tail = new Promise(resolve => { + release = resolve; + }); + waiting++; + + try { + await previous.catch(() => undefined); + waiting--; + active++; + return await task(); + } finally { + if (active > 0) active--; + else if (waiting > 0) waiting--; + release(); + } + }, + + snapshot(): ExclusiveRunGateSnapshot { + return { active, waiting }; + }, + }; +} diff --git a/test/exclusive-run-gate.test.ts b/test/exclusive-run-gate.test.ts new file mode 100644 index 0000000..e3b7cc9 --- /dev/null +++ b/test/exclusive-run-gate.test.ts @@ -0,0 +1,88 @@ +import assert from 'node:assert/strict'; + +import { createExclusiveRunGate } from '../src/exclusive-run-gate'; + +async function test(name: string, fn: () => Promise): Promise { + try { + await fn(); + console.log(`ok - ${name}`); + } catch (error) { + console.error(`not ok - ${name}`); + throw error; + } +} + +async function main(): Promise { + await test('serializes deliberately interleaved async work', async () => { + const gate = createExclusiveRunGate(); + const events: string[] = []; + let active = 0; + let maxActive = 0; + let releaseFirst!: () => void; + let firstStarted!: () => void; + + const firstStartedPromise = new Promise(resolve => { + firstStarted = resolve; + }); + const firstReleasePromise = new Promise(resolve => { + releaseFirst = resolve; + }); + + const first = gate.run(async () => { + active++; + maxActive = Math.max(maxActive, active); + events.push('tenant-a:start'); + firstStarted(); + await firstReleasePromise; + events.push('tenant-a:end'); + active--; + return 'tenant-a'; + }); + + await firstStartedPromise; + + const second = gate.run(async () => { + active++; + maxActive = Math.max(maxActive, active); + events.push('tenant-b:start'); + events.push('tenant-b:end'); + active--; + return 'tenant-b'; + }); + + await Promise.resolve(); + assert.deepEqual(gate.snapshot(), { active: 1, waiting: 1 }); + assert.deepEqual(events, ['tenant-a:start']); + + releaseFirst(); + assert.deepEqual(await Promise.all([first, second]), ['tenant-a', 'tenant-b']); + assert.deepEqual(events, [ + 'tenant-a:start', + 'tenant-a:end', + 'tenant-b:start', + 'tenant-b:end', + ]); + assert.equal(maxActive, 1); + assert.deepEqual(gate.snapshot(), { active: 0, waiting: 0 }); + }); + + await test('a failed run does not poison the next queued run', async () => { + const gate = createExclusiveRunGate(); + + await assert.rejects( + gate.run(async () => { + throw new Error('expected failure'); + }), + /expected failure/ + ); + + const result = await gate.run(async () => 'next-run-completed'); + assert.equal(result, 'next-run-completed'); + assert.deepEqual(gate.snapshot(), { active: 0, waiting: 0 }); + }); +} + +main().catch(error => { + console.error(error); + process.exitCode = 1; +});