diff --git a/src/lib/lab-activation.ts b/src/lib/lab-activation.ts index dc37f85af1..18298b7fe2 100644 --- a/src/lib/lab-activation.ts +++ b/src/lib/lab-activation.ts @@ -29,6 +29,7 @@ import { labAutomationPolicyPath } from "../lab/paths"; import type { OcxConfig } from "../types"; import { LabAutomationError } from "../lab/automation/types"; import { registerLabPassiveRouteLinker } from "./lab-passive-linker-registration"; +import { registerCurrentServerResourceCleanup } from "./server-resource-ownership"; import { setCompatibilityEvidenceProvider } from "../routing/compatibility/provider-slot"; import { labCompatibilityEvidenceProvider } from "../routing/compatibility/lab-evidence-provider"; import { @@ -37,8 +38,18 @@ import { } from "../lab/automation/orchestrator"; import { createProductionLabRouteExecutor } from "./lab-live-route-production"; +interface LabRuntimeBinding { + release(): void; +} + +interface LabActivationRecord { + staticDetach: Array<() => void>; + runtime: LabRuntimeBinding | null; + seenRuntimeConfigs: WeakSet; +} + /** Activation records keyed by configDir, so one process can own several configs. */ -const activated = new Map void>>(); +const activated = new Map(); const activationKey = (configDir?: string): string => configDir ?? ""; @@ -83,60 +94,110 @@ export function labActivationRequired(config: OcxConfig, configDir?: string): bo return labAutomationEnabledOnDisk(configDir); } +function startAutomationIfEnabled(configDir?: string): void { + if (!labAutomationEnabledOnDisk(configDir)) return; + try { + startLabAutomationScheduler(configDir); + } catch (err) { + // Neither a malformed automation file nor a busy state lock may take the proxy down + // at startup. Lab automation stays off for this run; routing, evidence, and every + // other subsystem keep working. + // + // The two causes get different messages because they need different actions, and a + // lock-contention failure reported as "invalid config" sends the operator to fix a + // file that is fine. Contention can also stall startup by up to the 5s lock wait. + const code = err instanceof LabAutomationError ? err.code : null; + if (code === "state_lock_busy" || code === "state_lock_failed") { + console.warn( + "[lab] Lab automation did not start: another process holds the automation state lock." + + " Automation stays off for this run and will be retried on the next start.", + ); + } else { + console.warn( + "[lab] Lab automation is disabled for this run because its configuration could not be" + + " loaded:", + err instanceof Error ? err.message : err, + ); + } + } +} + +function installLabAutomationRuntime( + record: LabActivationRecord, + config: OcxConfig, + configDir?: string, +): void { + const previous = record.runtime; + const routeExecutor = createProductionLabRouteExecutor({ configDir, loadConfig: () => config }); + const releaseDispatchDeps = setLabAutomationDispatchDeps({ + configDir, + loadConfig: () => config, + routeExecutor, + }); + + let released = false; + let detachOwnerCleanup = () => {}; + const binding: LabRuntimeBinding = { + release() { + if (released) return; + released = true; + detachOwnerCleanup(); + releaseDispatchDeps(); + if (record.runtime === binding) record.runtime = null; + }, + }; + // `setLabAutomationDispatchDeps` already registers its own owner cleanup. This second + // receipt only keeps the activation record in sync with that owner-scoped lifetime, so a + // later same-process server can see that the static Lab slots survived but CL-08 authority + // did not. The release is idempotent, so cleanup order does not matter. + detachOwnerCleanup = registerCurrentServerResourceCleanup(binding.release); + + // Install the successor before releasing the predecessor. The dispatcher token check then + // makes the predecessor release a no-op for the successor scheduler/authority. + record.runtime = binding; + record.seenRuntimeConfigs.add(config); + previous?.release(); +} + /** - * Register Lab into the core slots. Idempotent per configDir and safe to call again after - * a routing profile is created at runtime. + * Register Lab into the core slots. Static activation is idempotent per configDir. The + * server-owned CL-08 runtime binding is refreshed when its prior owner ended or when a new + * server instance arrives with a config object that has not owned this activation before. */ export function activateLab(config: OcxConfig, configDir?: string): void { const key = activationKey(configDir); - // INVARIANT: activation is all-or-nothing and reason-independent. Every slot is - // registered here regardless of WHY activation was required, which is what makes this - // key safe as configDir alone -- an automation-only activation still installs the - // compatibility provider a later profile needs. If any registration ever becomes - // conditional on the activation reason, this key must include that reason, or the early - // return will silently skip it forever. - if (activated.has(key)) return; - - const detach: Array<() => void> = []; - detach.push(registerLabPassiveRouteLinker(configDir)); - detach.push(setCompatibilityEvidenceProvider(labCompatibilityEvidenceProvider)); + const existing = activated.get(key); + if (existing) { + // A released predecessor leaves the static Lab slots resident but removes dispatch + // authority and its scheduler. A live successor uses a fresh config object. Rebind in + // either case, but never let an older already-seen server steal authority back from a + // newer successor merely because it receives another management request. + if (existing.runtime === null || !existing.seenRuntimeConfigs.has(config)) { + installLabAutomationRuntime(existing, config, configDir); + startAutomationIfEnabled(configDir); + } + return; + } - const routeExecutor = createProductionLabRouteExecutor({ configDir, loadConfig: () => config }); - detach.push(setLabAutomationDispatchDeps({ configDir, loadConfig: () => config, routeExecutor })); + // INVARIANT: static activation is all-or-nothing and reason-independent. Every static + // slot is registered here regardless of WHY activation was required, so automation-only + // activation still installs the compatibility provider a later profile needs. + const record: LabActivationRecord = { + staticDetach: [], + runtime: null, + seenRuntimeConfigs: new WeakSet(), + }; + record.staticDetach.push(registerLabPassiveRouteLinker(configDir)); + record.staticDetach.push(setCompatibilityEvidenceProvider(labCompatibilityEvidenceProvider)); + installLabAutomationRuntime(record, config, configDir); // Record the activation BEFORE the scheduler start. startLabAutomationScheduler runs the // full automation normalizer, which throws on any field violation, and this call sits on // the startup path of every install that has a routing profile. Storing the record first // means a throw cannot orphan the detach receipts and leave slots registered with no - // activation record -- which would let a later activateLab register them a second time. - activated.set(key, detach); - - if (labAutomationEnabledOnDisk(configDir)) { - try { - startLabAutomationScheduler(configDir); - } catch (err) { - // Neither a malformed automation file nor a busy state lock may take the proxy down - // at startup. Lab automation stays off for this run; routing, evidence, and every - // other subsystem keep working. - // - // The two causes get different messages because they need different actions, and a - // lock-contention failure reported as "invalid config" sends the operator to fix a - // file that is fine. Contention can also stall startup by up to the 5s lock wait. - const code = err instanceof LabAutomationError ? err.code : null; - if (code === "state_lock_busy" || code === "state_lock_failed") { - console.warn( - "[lab] Lab automation did not start: another process holds the automation state lock." - + " Automation stays off for this run and will be retried on the next start.", - ); - } else { - console.warn( - "[lab] Lab automation is disabled for this run because its configuration could not be" - + " loaded:", - err instanceof Error ? err.message : err, - ); - } - } - } + // activation record, which would let a later activateLab register them a second time. + activated.set(key, record); + startAutomationIfEnabled(configDir); } /** True when this configDir has been activated. */ @@ -152,9 +213,10 @@ export function isLabActivated(configDir?: string): boolean { * users who never opted in. */ export function resetLabActivationForTests(): void { - for (const [key, detach] of [...activated]) { + for (const [key, record] of [...activated]) { activated.delete(key); - for (const release of [...detach].reverse()) { + try { record.runtime?.release(); } catch { /* teardown is best-effort */ } + for (const release of [...record.staticDetach].reverse()) { try { release(); } catch { /* teardown is best-effort */ } } } diff --git a/tests/lab-activation.test.ts b/tests/lab-activation.test.ts index ef3c6fed4f..d056af9c46 100644 --- a/tests/lab-activation.test.ts +++ b/tests/lab-activation.test.ts @@ -13,8 +13,13 @@ import { resolveCompatibilityEvidenceProvider, resetCompatibilityEvidenceProviderForTests, } from "../src/routing/compatibility/provider-slot"; +import { defaultLabAutomationPolicyV1 } from "../src/lab/automation/policy"; import { isLabAutomationSchedulerRunning, stopLabAutomationScheduler } from "../src/lab/automation/orchestrator"; import { runOptionalShutdownHooks, resetOptionalShutdownHooksForTests } from "../src/lib/optional-shutdown-hooks"; +import { + acquireServerResourceOwner, + resetServerResourceOwnershipForTests, +} from "../src/lib/server-resource-ownership"; import { hasPassiveRouteLinker, resetPassiveRouteLinkerForTests } from "../src/server/passive-route-linker"; import type { OcxConfig } from "../src/types"; @@ -23,7 +28,23 @@ function scratch(): string { mkdirSync(join(dir, "lab"), { recursive: true }); return dir; } -const withProfile = { providers: {}, routingProfiles: { p: { candidates: [] } } } as unknown as OcxConfig; + +function writeEnabledAutomationConfig(dir: string): void { + writeFileSync(join(dir, "lab", "automation-config.json"), JSON.stringify({ + schemaVersion: 1, + policy: { + ...defaultLabAutomationPolicyV1(), + enabled: true, + }, + routes: { schemaVersion: 1, routes: [] }, + })); +} + +function profileConfig(): OcxConfig { + return { providers: {}, routingProfiles: { p: { candidates: [] } } } as unknown as OcxConfig; +} + +const withProfile = profileConfig(); const bare = { providers: {} } as unknown as OcxConfig; describe("lab activation gate", () => { @@ -31,6 +52,7 @@ describe("lab activation gate", () => { // otherwise leak into the bare-install assertion below. beforeEach(() => { resetLabActivationForTests(); + resetServerResourceOwnershipForTests(); resetCompatibilityEvidenceProviderForTests(); resetPassiveRouteLinkerForTests(); }); @@ -79,6 +101,40 @@ describe("lab activation gate", () => { expect(isLabActivated(dir)).toBe(true); }); + test("same-root successor keeps automation after predecessor owner release", () => { + const dir = scratch(); + writeEnabledAutomationConfig(dir); + const firstOwner = acquireServerResourceOwner(); + activateLab(profileConfig(), dir); + expect(isLabAutomationSchedulerRunning(dir)).toBe(true); + + const secondOwner = acquireServerResourceOwner(); + activateLab(profileConfig(), dir); + firstOwner.release(); + expect(isLabAutomationSchedulerRunning(dir)).toBe(true); + + secondOwner.release(); + expect(isLabAutomationSchedulerRunning(dir)).toBe(false); + }); + + test("same-process restart reacquires automation after its prior owner ended", () => { + const dir = scratch(); + writeEnabledAutomationConfig(dir); + const config = profileConfig(); + + const firstOwner = acquireServerResourceOwner(); + activateLab(config, dir); + expect(isLabAutomationSchedulerRunning(dir)).toBe(true); + firstOwner.release(); + expect(isLabAutomationSchedulerRunning(dir)).toBe(false); + + const secondOwner = acquireServerResourceOwner(); + activateLab(config, dir); + expect(isLabAutomationSchedulerRunning(dir)).toBe(true); + secondOwner.release(); + expect(isLabAutomationSchedulerRunning(dir)).toBe(false); + }); + // Ordering trap: automation-only activation must not permanently satisfy a later // profile-driven one. Safe today only because activation is all-or-nothing; this test // fails the moment a registration becomes conditional on the activation reason. @@ -100,6 +156,7 @@ describe("automation detection reads the current authority", () => { // otherwise leak into the bare-install assertion below. beforeEach(() => { resetLabActivationForTests(); + resetServerResourceOwnershipForTests(); resetCompatibilityEvidenceProviderForTests(); resetPassiveRouteLinkerForTests(); }); @@ -139,7 +196,11 @@ describe("automation detection reads the current authority", () => { }); describe("failed scheduler start leaves nothing dangling", () => { - beforeEach(() => { resetLabActivationForTests(); resetOptionalShutdownHooksForTests(); }); + beforeEach(() => { + resetLabActivationForTests(); + resetServerResourceOwnershipForTests(); + resetOptionalShutdownHooksForTests(); + }); // startLabAutomationScheduler registers its shutdown hook BEFORE it can throw, so a // failed start leaves a hook with no timer behind it. That must be harmless: no running