fix(lab): rebind CL-08 runtime ownership across server replacement

src/lib/lab-activation.ts

@@ -29,6 +29,7 @@ import { labAutomationPolicyPath } from "../lab/paths";
 import type { OcxConfig } from "../types";
 import { LabAutomationError } from "../lab/automation/types";
 import { registerLabPassiveRouteLinker } from "./lab-passive-linker-registration";
+import { registerCurrentServerResourceCleanup } from "./server-resource-ownership";
 import { setCompatibilityEvidenceProvider } from "../routing/compatibility/provider-slot";
 import { labCompatibilityEvidenceProvider } from "../routing/compatibility/lab-evidence-provider";
 import {
@@ -37,8 +38,18 @@ import {
 } from "../lab/automation/orchestrator";
 import { createProductionLabRouteExecutor } from "./lab-live-route-production";
 
+interface LabRuntimeBinding {
+  release(): void;
+}
+
+interface LabActivationRecord {
+  staticDetach: Array<() => void>;
+  runtime: LabRuntimeBinding | null;
+  seenRuntimeConfigs: WeakSet<OcxConfig>;
+}
+
 /** Activation records keyed by configDir, so one process can own several configs. */
-const activated = new Map<string, Array<() => void>>();
+const activated = new Map<string, LabActivationRecord>();
 
 const activationKey = (configDir?: string): string => configDir ?? "";
 
@@ -83,60 +94,110 @@ export function labActivationRequired(config: OcxConfig, configDir?: string): bo
   return labAutomationEnabledOnDisk(configDir);
 }
 
+function startAutomationIfEnabled(configDir?: string): void {
+  if (!labAutomationEnabledOnDisk(configDir)) return;
+  try {
+    startLabAutomationScheduler(configDir);
+  } catch (err) {
+    // Neither a malformed automation file nor a busy state lock may take the proxy down
+    // at startup. Lab automation stays off for this run; routing, evidence, and every
+    // other subsystem keep working.
+    //
+    // The two causes get different messages because they need different actions, and a
+    // lock-contention failure reported as "invalid config" sends the operator to fix a
+    // file that is fine. Contention can also stall startup by up to the 5s lock wait.
+    const code = err instanceof LabAutomationError ? err.code : null;
+    if (code === "state_lock_busy" || code === "state_lock_failed") {
+      console.warn(
+        "[lab] Lab automation did not start: another process holds the automation state lock."
+        + " Automation stays off for this run and will be retried on the next start.",
+      );
+    } else {
+      console.warn(
+        "[lab] Lab automation is disabled for this run because its configuration could not be"
+        + " loaded:",
+        err instanceof Error ? err.message : err,
+      );
+    }
+  }
+}
+
+function installLabAutomationRuntime(
+  record: LabActivationRecord,
+  config: OcxConfig,
+  configDir?: string,
+): void {
+  const previous = record.runtime;
+  const routeExecutor = createProductionLabRouteExecutor({ configDir, loadConfig: () => config });
+  const releaseDispatchDeps = setLabAutomationDispatchDeps({
+    configDir,
+    loadConfig: () => config,
+    routeExecutor,
+  });
+
+  let released = false;
+  let detachOwnerCleanup = () => {};
+  const binding: LabRuntimeBinding = {
+    release() {
+      if (released) return;
+      released = true;
+      detachOwnerCleanup();
+      releaseDispatchDeps();
+      if (record.runtime === binding) record.runtime = null;
+    },
+  };
+  // `setLabAutomationDispatchDeps` already registers its own owner cleanup. This second
+  // receipt only keeps the activation record in sync with that owner-scoped lifetime, so a
+  // later same-process server can see that the static Lab slots survived but CL-08 authority
+  // did not. The release is idempotent, so cleanup order does not matter.
+  detachOwnerCleanup = registerCurrentServerResourceCleanup(binding.release);
+
+  // Install the successor before releasing the predecessor. The dispatcher token check then
+  // makes the predecessor release a no-op for the successor scheduler/authority.
+  record.runtime = binding;
+  record.seenRuntimeConfigs.add(config);
+  previous?.release();
+}
+
 /**
- * Register Lab into the core slots. Idempotent per configDir and safe to call again after
- * a routing profile is created at runtime.
+ * Register Lab into the core slots. Static activation is idempotent per configDir. The
+ * server-owned CL-08 runtime binding is refreshed when its prior owner ended or when a new
+ * server instance arrives with a config object that has not owned this activation before.
  */
 export function activateLab(config: OcxConfig, configDir?: string): void {
   const key = activationKey(configDir);
-  // INVARIANT: activation is all-or-nothing and reason-independent. Every slot is
-  // registered here regardless of WHY activation was required, which is what makes this
-  // key safe as configDir alone -- an automation-only activation still installs the
-  // compatibility provider a later profile needs. If any registration ever becomes
-  // conditional on the activation reason, this key must include that reason, or the early
-  // return will silently skip it forever.
-  if (activated.has(key)) return;
-
-  const detach: Array<() => void> = [];
-  detach.push(registerLabPassiveRouteLinker(configDir));
-  detach.push(setCompatibilityEvidenceProvider(labCompatibilityEvidenceProvider));
+  const existing = activated.get(key);
+  if (existing) {
+    // A released predecessor leaves the static Lab slots resident but removes dispatch
+    // authority and its scheduler. A live successor uses a fresh config object. Rebind in
+    // either case, but never let an older already-seen server steal authority back from a
+    // newer successor merely because it receives another management request.
+    if (existing.runtime === null || !existing.seenRuntimeConfigs.has(config)) {
+      installLabAutomationRuntime(existing, config, configDir);
+      startAutomationIfEnabled(configDir);
+    }
+    return;
+  }
 
-  const routeExecutor = createProductionLabRouteExecutor({ configDir, loadConfig: () => config });
-  detach.push(setLabAutomationDispatchDeps({ configDir, loadConfig: () => config, routeExecutor }));
+  // INVARIANT: static activation is all-or-nothing and reason-independent. Every static
+  // slot is registered here regardless of WHY activation was required, so automation-only
+  // activation still installs the compatibility provider a later profile needs.
+  const record: LabActivationRecord = {
+    staticDetach: [],
+    runtime: null,
+    seenRuntimeConfigs: new WeakSet<OcxConfig>(),
+  };
+  record.staticDetach.push(registerLabPassiveRouteLinker(configDir));
+  record.staticDetach.push(setCompatibilityEvidenceProvider(labCompatibilityEvidenceProvider));
+  installLabAutomationRuntime(record, config, configDir);
 
   // Record the activation BEFORE the scheduler start. startLabAutomationScheduler runs the
   // full automation normalizer, which throws on any field violation, and this call sits on
   // the startup path of every install that has a routing profile. Storing the record first
   // means a throw cannot orphan the detach receipts and leave slots registered with no
-  // activation record -- which would let a later activateLab register them a second time.
-  activated.set(key, detach);
-
-  if (labAutomationEnabledOnDisk(configDir)) {
-    try {
-      startLabAutomationScheduler(configDir);
-    } catch (err) {
-      // Neither a malformed automation file nor a busy state lock may take the proxy down
-      // at startup. Lab automation stays off for this run; routing, evidence, and every
-      // other subsystem keep working.
-      //
-      // The two causes get different messages because they need different actions, and a
-      // lock-contention failure reported as "invalid config" sends the operator to fix a
-      // file that is fine. Contention can also stall startup by up to the 5s lock wait.
-      const code = err instanceof LabAutomationError ? err.code : null;
-      if (code === "state_lock_busy" || code === "state_lock_failed") {
-        console.warn(
-          "[lab] Lab automation did not start: another process holds the automation state lock."
-          + " Automation stays off for this run and will be retried on the next start.",
-        );
-      } else {
-        console.warn(
-          "[lab] Lab automation is disabled for this run because its configuration could not be"
-          + " loaded:",
-          err instanceof Error ? err.message : err,
-        );
-      }
-    }
-  }
+  // activation record, which would let a later activateLab register them a second time.
+  activated.set(key, record);
+  startAutomationIfEnabled(configDir);
 }
 
 /** True when this configDir has been activated. */
@@ -152,9 +213,10 @@ export function isLabActivated(configDir?: string): boolean {
  * users who never opted in.
  */
 export function resetLabActivationForTests(): void {
-  for (const [key, detach] of [...activated]) {
+  for (const [key, record] of [...activated]) {
     activated.delete(key);
-    for (const release of [...detach].reverse()) {
+    try { record.runtime?.release(); } catch { /* teardown is best-effort */ }
+    for (const release of [...record.staticDetach].reverse()) {
       try { release(); } catch { /* teardown is best-effort */ }
     }
   }

tests/lab-activation.test.ts

@@ -13,8 +13,13 @@ import {
   resolveCompatibilityEvidenceProvider,
   resetCompatibilityEvidenceProviderForTests,
 } from "../src/routing/compatibility/provider-slot";
+import { defaultLabAutomationPolicyV1 } from "../src/lab/automation/policy";
 import { isLabAutomationSchedulerRunning, stopLabAutomationScheduler } from "../src/lab/automation/orchestrator";
 import { runOptionalShutdownHooks, resetOptionalShutdownHooksForTests } from "../src/lib/optional-shutdown-hooks";
+import {
+  acquireServerResourceOwner,
+  resetServerResourceOwnershipForTests,
+} from "../src/lib/server-resource-ownership";
 import { hasPassiveRouteLinker, resetPassiveRouteLinkerForTests } from "../src/server/passive-route-linker";
 import type { OcxConfig } from "../src/types";
 
@@ -23,14 +28,31 @@ function scratch(): string {
   mkdirSync(join(dir, "lab"), { recursive: true });
   return dir;
 }
-const withProfile = { providers: {}, routingProfiles: { p: { candidates: [] } } } as unknown as OcxConfig;
+
+function writeEnabledAutomationConfig(dir: string): void {
+  writeFileSync(join(dir, "lab", "automation-config.json"), JSON.stringify({
+    schemaVersion: 1,
+    policy: {
+      ...defaultLabAutomationPolicyV1(),
+      enabled: true,
+    },
+    routes: { schemaVersion: 1, routes: [] },
+  }));
+}
+
+function profileConfig(): OcxConfig {
+  return { providers: {}, routingProfiles: { p: { candidates: [] } } } as unknown as OcxConfig;
+}
+
+const withProfile = profileConfig();
 const bare = { providers: {} } as unknown as OcxConfig;
 
 describe("lab activation gate", () => {
   // Slots are process-global, so a sibling test file that registered one directly would
   // otherwise leak into the bare-install assertion below.
   beforeEach(() => {
     resetLabActivationForTests();
+    resetServerResourceOwnershipForTests();
     resetCompatibilityEvidenceProviderForTests();
     resetPassiveRouteLinkerForTests();
   });
@@ -79,6 +101,40 @@ describe("lab activation gate", () => {
     expect(isLabActivated(dir)).toBe(true);
   });
 
+  test("same-root successor keeps automation after predecessor owner release", () => {
+    const dir = scratch();
+    writeEnabledAutomationConfig(dir);
+    const firstOwner = acquireServerResourceOwner();
+    activateLab(profileConfig(), dir);
+    expect(isLabAutomationSchedulerRunning(dir)).toBe(true);
+
+    const secondOwner = acquireServerResourceOwner();
+    activateLab(profileConfig(), dir);
+    firstOwner.release();
+    expect(isLabAutomationSchedulerRunning(dir)).toBe(true);
+
+    secondOwner.release();
+    expect(isLabAutomationSchedulerRunning(dir)).toBe(false);
+  });
+
+  test("same-process restart reacquires automation after its prior owner ended", () => {
+    const dir = scratch();
+    writeEnabledAutomationConfig(dir);
+    const config = profileConfig();
+
+    const firstOwner = acquireServerResourceOwner();
+    activateLab(config, dir);
+    expect(isLabAutomationSchedulerRunning(dir)).toBe(true);
+    firstOwner.release();
+    expect(isLabAutomationSchedulerRunning(dir)).toBe(false);
+
+    const secondOwner = acquireServerResourceOwner();
+    activateLab(config, dir);
+    expect(isLabAutomationSchedulerRunning(dir)).toBe(true);
+    secondOwner.release();
+    expect(isLabAutomationSchedulerRunning(dir)).toBe(false);
+  });
+
   // Ordering trap: automation-only activation must not permanently satisfy a later
   // profile-driven one. Safe today only because activation is all-or-nothing; this test
   // fails the moment a registration becomes conditional on the activation reason.
@@ -100,6 +156,7 @@ describe("automation detection reads the current authority", () => {
   // otherwise leak into the bare-install assertion below.
   beforeEach(() => {
     resetLabActivationForTests();
+    resetServerResourceOwnershipForTests();
     resetCompatibilityEvidenceProviderForTests();
     resetPassiveRouteLinkerForTests();
   });
@@ -139,7 +196,11 @@ describe("automation detection reads the current authority", () => {
 });
 
 describe("failed scheduler start leaves nothing dangling", () => {
-  beforeEach(() => { resetLabActivationForTests(); resetOptionalShutdownHooksForTests(); });
+  beforeEach(() => {
+    resetLabActivationForTests();
+    resetServerResourceOwnershipForTests();
+    resetOptionalShutdownHooksForTests();
+  });
 
   // startLabAutomationScheduler registers its shutdown hook BEFORE it can throw, so a
   // failed start leaves a hook with no timer behind it. That must be harmless: no running