fix(keeper): persist budget circuit-breaker halt across restarts

src/lib/budget.ts

@@ -25,6 +25,7 @@
  * concurrent canSpend() / recordTx() callers.
  */
 
+import fs from "node:fs";
 import { createLogger } from "@percolatorct/shared";
 
 const logger = createLogger("keeper:budget");
@@ -111,6 +112,16 @@ export interface KeeperBudgetDeps {
   /** Fired when a latched halt is cleared (resume()). Lets callers reset a
    *  halted gauge without coupling this class to the metrics layer. */
   onResume?: () => void;
+  /**
+   * BUG-106: when set, a latched halt is persisted to this local file (and
+   * restored from it at construction) so a process crash/restart does not
+   * silently resume spending into whatever condition caused the halt.
+   * Unset by default -- every existing/test instance is unaffected unless it
+   * opts in explicitly. Covers process-level restarts that reuse the same
+   * filesystem (the common automatic-restart case); a full container
+   * recreate on an ephemeral filesystem is not covered.
+   */
+  haltStatePath?: string;
 }
 
 interface SpendEvent {
@@ -203,6 +214,7 @@ export class KeeperBudget {
   private _isHalted = false;
   private _haltKind: HaltKind | undefined;
   private _haltReason: string | undefined;
+  private readonly _haltStatePath: string | undefined;
 
   // M12: realized-cost reconciliation telemetry.
   private _realizedCostDriftLamports = 0;
@@ -214,6 +226,76 @@ export class KeeperBudget {
     this._now = deps.now ?? (() => Date.now());
     this._onHalt = deps.onHalt;
     this._onResume = deps.onResume;
+    this._haltStatePath = deps.haltStatePath;
+    this._restoreHaltState();
+  }
+
+  /**
+   * BUG-106: restore a latched halt persisted by a previous process, if any.
+   * A halt that was never resumed before a restart must keep the keeper
+   * halted -- silently resuming would defeat the entire point of a
+   * manual-resume-only breaker. Fires onHalt again so paging/metrics react
+   * exactly as if the halt had just occurred.
+   */
+  private _restoreHaltState(): void {
+    if (!this._haltStatePath) return;
+    let raw: string;
+    try {
+      raw = fs.readFileSync(this._haltStatePath, "utf8");
+    } catch {
+      return; // no persisted halt — start clean
+    }
+    try {
+      const parsed = JSON.parse(raw) as { kind?: HaltKind; reason?: string };
+      if (!parsed.kind || !parsed.reason) return;
+      this._isHalted = true;
+      this._haltKind = parsed.kind;
+      this._haltReason = parsed.reason;
+      logger.error(
+        "Restored a latched budget halt from a previous run — keeper is starting " +
+          "halted. This halt was never resume()d before the process restarted; " +
+          "investigate the original cause before calling resume().",
+        { kind: parsed.kind, reason: parsed.reason, path: this._haltStatePath },
+      );
+      try {
+        this._onHalt?.(parsed.kind, parsed.reason);
+      } catch (err) {
+        logger.warn("onHalt hook threw while restoring a persisted halt — ignoring", {
+          error: err instanceof Error ? err.message : String(err),
+        });
+      }
+    } catch (err) {
+      logger.warn("Failed to parse persisted budget halt state — starting clean", {
+        path: this._haltStatePath,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+
+  private _persistHaltState(): void {
+    if (!this._haltStatePath) return;
+    try {
+      fs.writeFileSync(
+        this._haltStatePath,
+        JSON.stringify({ kind: this._haltKind, reason: this._haltReason, haltedAt: this._now() }),
+        "utf8",
+      );
+    } catch (err) {
+      logger.warn("Failed to persist budget halt state — a restart will not remember this halt", {
+        path: this._haltStatePath,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+
+  private _clearHaltStateFile(): void {
+    if (!this._haltStatePath) return;
+    try {
+      fs.unlinkSync(this._haltStatePath);
+    } catch {
+      // Nothing to remove, or removal failed — not critical; the next halt
+      // (if any) overwrites it, and a missing file is treated as no-halt.
+    }
   }
 
   /**
@@ -538,6 +620,7 @@ export class KeeperBudget {
     this._isHalted = false;
     this._haltKind = undefined;
     this._haltReason = undefined;
+    this._clearHaltStateFile();
     try {
       this._onResume?.();
     } catch (err) {
@@ -560,6 +643,7 @@ export class KeeperBudget {
     this._isHalted = true;
     this._haltKind = kind;
     this._haltReason = reason;
+    this._persistHaltState();
     logger.error("Keeper budget halted — refusing further sends until manual resume()", {
       kind,
       reason,

src/lib/keeper-send.ts

@@ -80,6 +80,13 @@ function getCuEstimator(): CuEstimator {
 export const sharedBudget = new KeeperBudget(
   {},
   {
+    // BUG-106: persist a latched halt to disk so a crash/restart (the common
+    // case — same container/filesystem, e.g. an uncaught exception or an
+    // orchestrator-driven OOM restart) keeps the keeper halted instead of
+    // silently resuming spend into whatever condition tripped the breaker.
+    // A full container recreate on an ephemeral filesystem is not covered;
+    // operators wanting that can point this at a mounted persistent volume.
+    haltStatePath: process.env.KEEPER_BUDGET_HALT_STATE_PATH ?? "/tmp/keeper-budget-halt.json",
     onHalt: (kind, reason) => {
       budgetHalted.set(1);
       logger.error("KeeperBudget circuit-breaker halted — refusing all sends until resume", {

tests/lib/budget.test.ts

@@ -1,4 +1,7 @@
 import { describe, it, expect, vi } from "vitest";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
 import { KeeperBudget, type TxResult } from "../../src/lib/budget.js";
 
 function makeClock(start = 1_700_000_000_000) {
@@ -748,3 +751,82 @@ describe("KeeperBudget — M12 adjustForRealizedCost", () => {
     });
   });
 });
+
+// BUG-106: a latched halt was purely in-memory -- a process crash/restart
+// silently resumed spending into whatever condition tripped the breaker,
+// with no record that it was ever halted. haltStatePath is opt-in (undefined
+// by default) so every test above, and every consumer that doesn't pass it,
+// is completely unaffected.
+describe("KeeperBudget — BUG-106: halt-state persistence", () => {
+  function tempHaltPath(): string {
+    return path.join(os.tmpdir(), `keeper-budget-halt-test-${Math.random().toString(36).slice(2)}.json`);
+  }
+
+  it("does not touch the filesystem when haltStatePath is not set", () => {
+    const clock = makeClock();
+    const b = new KeeperBudget(TIGHT_CONFIG, { now: clock.now });
+    b.haltManually("cordon");
+    expect(b.isHalted()).toBe(true);
+    // No haltStatePath was given — nothing to assert on disk, just confirms
+    // the existing in-memory-only behavior is unchanged when opted out.
+  });
+
+  it("persists a halt to disk and restores it in a fresh instance constructed with the same path", () => {
+    const haltStatePath = tempHaltPath();
+    try {
+      const clock = makeClock();
+      const b1 = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath });
+      b1.haltManually("cordoning for deploy");
+      expect(fs.existsSync(haltStatePath)).toBe(true);
+
+      // Simulates a process restart: a brand-new instance, same path.
+      const onHalt = vi.fn();
+      const b2 = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath, onHalt });
+      expect(b2.isHalted()).toBe(true);
+      expect(b2.haltKind).toBe("operator");
+      expect(b2.canSpend(1, "crank")).toBe(false);
+      // The restore re-fires onHalt so paging/metrics react as if it just happened.
+      expect(onHalt).toHaveBeenCalledWith("operator", "cordoning for deploy");
+    } finally {
+      fs.rmSync(haltStatePath, { force: true });
+    }
+  });
+
+  it("clears the persisted file on resume — a later restart starts clean", () => {
+    const haltStatePath = tempHaltPath();
+    try {
+      const clock = makeClock();
+      const b1 = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath });
+      b1.haltManually("cordon");
+      expect(fs.existsSync(haltStatePath)).toBe(true);
+      b1.resume("op");
+      expect(fs.existsSync(haltStatePath)).toBe(false);
+
+      const b2 = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath });
+      expect(b2.isHalted()).toBe(false);
+      expect(b2.canSpend(1, "crank")).toBe(true);
+    } finally {
+      fs.rmSync(haltStatePath, { force: true });
+    }
+  });
+
+  it("starts clean when the configured path has no file yet", () => {
+    const haltStatePath = tempHaltPath(); // never written
+    const clock = makeClock();
+    const b = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath });
+    expect(b.isHalted()).toBe(false);
+  });
+
+  it("starts clean and logs rather than throwing when the persisted file is malformed", () => {
+    const haltStatePath = tempHaltPath();
+    try {
+      fs.writeFileSync(haltStatePath, "{not valid json", "utf8");
+      const clock = makeClock();
+      expect(() => new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath })).not.toThrow();
+      const b = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath });
+      expect(b.isHalted()).toBe(false);
+    } finally {
+      fs.rmSync(haltStatePath, { force: true });
+    }
+  });
+});