diff --git a/src/lib/budget.ts b/src/lib/budget.ts index febce7c..88d494c 100644 --- a/src/lib/budget.ts +++ b/src/lib/budget.ts @@ -25,6 +25,7 @@ * concurrent canSpend() / recordTx() callers. */ +import fs from "node:fs"; import { createLogger } from "@percolatorct/shared"; const logger = createLogger("keeper:budget"); @@ -111,6 +112,16 @@ export interface KeeperBudgetDeps { /** Fired when a latched halt is cleared (resume()). Lets callers reset a * halted gauge without coupling this class to the metrics layer. */ onResume?: () => void; + /** + * BUG-106: when set, a latched halt is persisted to this local file (and + * restored from it at construction) so a process crash/restart does not + * silently resume spending into whatever condition caused the halt. + * Unset by default -- every existing/test instance is unaffected unless it + * opts in explicitly. Covers process-level restarts that reuse the same + * filesystem (the common automatic-restart case); a full container + * recreate on an ephemeral filesystem is not covered. + */ + haltStatePath?: string; } interface SpendEvent { @@ -203,6 +214,7 @@ export class KeeperBudget { private _isHalted = false; private _haltKind: HaltKind | undefined; private _haltReason: string | undefined; + private readonly _haltStatePath: string | undefined; // M12: realized-cost reconciliation telemetry. private _realizedCostDriftLamports = 0; @@ -214,6 +226,76 @@ export class KeeperBudget { this._now = deps.now ?? (() => Date.now()); this._onHalt = deps.onHalt; this._onResume = deps.onResume; + this._haltStatePath = deps.haltStatePath; + this._restoreHaltState(); + } + + /** + * BUG-106: restore a latched halt persisted by a previous process, if any. + * A halt that was never resumed before a restart must keep the keeper + * halted -- silently resuming would defeat the entire point of a + * manual-resume-only breaker. Fires onHalt again so paging/metrics react + * exactly as if the halt had just occurred. + */ + private _restoreHaltState(): void { + if (!this._haltStatePath) return; + let raw: string; + try { + raw = fs.readFileSync(this._haltStatePath, "utf8"); + } catch { + return; // no persisted halt — start clean + } + try { + const parsed = JSON.parse(raw) as { kind?: HaltKind; reason?: string }; + if (!parsed.kind || !parsed.reason) return; + this._isHalted = true; + this._haltKind = parsed.kind; + this._haltReason = parsed.reason; + logger.error( + "Restored a latched budget halt from a previous run — keeper is starting " + + "halted. This halt was never resume()d before the process restarted; " + + "investigate the original cause before calling resume().", + { kind: parsed.kind, reason: parsed.reason, path: this._haltStatePath }, + ); + try { + this._onHalt?.(parsed.kind, parsed.reason); + } catch (err) { + logger.warn("onHalt hook threw while restoring a persisted halt — ignoring", { + error: err instanceof Error ? err.message : String(err), + }); + } + } catch (err) { + logger.warn("Failed to parse persisted budget halt state — starting clean", { + path: this._haltStatePath, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + private _persistHaltState(): void { + if (!this._haltStatePath) return; + try { + fs.writeFileSync( + this._haltStatePath, + JSON.stringify({ kind: this._haltKind, reason: this._haltReason, haltedAt: this._now() }), + "utf8", + ); + } catch (err) { + logger.warn("Failed to persist budget halt state — a restart will not remember this halt", { + path: this._haltStatePath, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + private _clearHaltStateFile(): void { + if (!this._haltStatePath) return; + try { + fs.unlinkSync(this._haltStatePath); + } catch { + // Nothing to remove, or removal failed — not critical; the next halt + // (if any) overwrites it, and a missing file is treated as no-halt. + } } /** @@ -538,6 +620,7 @@ export class KeeperBudget { this._isHalted = false; this._haltKind = undefined; this._haltReason = undefined; + this._clearHaltStateFile(); try { this._onResume?.(); } catch (err) { @@ -560,6 +643,7 @@ export class KeeperBudget { this._isHalted = true; this._haltKind = kind; this._haltReason = reason; + this._persistHaltState(); logger.error("Keeper budget halted — refusing further sends until manual resume()", { kind, reason, diff --git a/src/lib/keeper-send.ts b/src/lib/keeper-send.ts index 64af809..fee70c7 100644 --- a/src/lib/keeper-send.ts +++ b/src/lib/keeper-send.ts @@ -80,6 +80,13 @@ function getCuEstimator(): CuEstimator { export const sharedBudget = new KeeperBudget( {}, { + // BUG-106: persist a latched halt to disk so a crash/restart (the common + // case — same container/filesystem, e.g. an uncaught exception or an + // orchestrator-driven OOM restart) keeps the keeper halted instead of + // silently resuming spend into whatever condition tripped the breaker. + // A full container recreate on an ephemeral filesystem is not covered; + // operators wanting that can point this at a mounted persistent volume. + haltStatePath: process.env.KEEPER_BUDGET_HALT_STATE_PATH ?? "/tmp/keeper-budget-halt.json", onHalt: (kind, reason) => { budgetHalted.set(1); logger.error("KeeperBudget circuit-breaker halted — refusing all sends until resume", { diff --git a/tests/lib/budget.test.ts b/tests/lib/budget.test.ts index 11a2203..c0001bf 100644 --- a/tests/lib/budget.test.ts +++ b/tests/lib/budget.test.ts @@ -1,4 +1,7 @@ import { describe, it, expect, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { KeeperBudget, type TxResult } from "../../src/lib/budget.js"; function makeClock(start = 1_700_000_000_000) { @@ -748,3 +751,82 @@ describe("KeeperBudget — M12 adjustForRealizedCost", () => { }); }); }); + +// BUG-106: a latched halt was purely in-memory -- a process crash/restart +// silently resumed spending into whatever condition tripped the breaker, +// with no record that it was ever halted. haltStatePath is opt-in (undefined +// by default) so every test above, and every consumer that doesn't pass it, +// is completely unaffected. +describe("KeeperBudget — BUG-106: halt-state persistence", () => { + function tempHaltPath(): string { + return path.join(os.tmpdir(), `keeper-budget-halt-test-${Math.random().toString(36).slice(2)}.json`); + } + + it("does not touch the filesystem when haltStatePath is not set", () => { + const clock = makeClock(); + const b = new KeeperBudget(TIGHT_CONFIG, { now: clock.now }); + b.haltManually("cordon"); + expect(b.isHalted()).toBe(true); + // No haltStatePath was given — nothing to assert on disk, just confirms + // the existing in-memory-only behavior is unchanged when opted out. + }); + + it("persists a halt to disk and restores it in a fresh instance constructed with the same path", () => { + const haltStatePath = tempHaltPath(); + try { + const clock = makeClock(); + const b1 = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath }); + b1.haltManually("cordoning for deploy"); + expect(fs.existsSync(haltStatePath)).toBe(true); + + // Simulates a process restart: a brand-new instance, same path. + const onHalt = vi.fn(); + const b2 = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath, onHalt }); + expect(b2.isHalted()).toBe(true); + expect(b2.haltKind).toBe("operator"); + expect(b2.canSpend(1, "crank")).toBe(false); + // The restore re-fires onHalt so paging/metrics react as if it just happened. + expect(onHalt).toHaveBeenCalledWith("operator", "cordoning for deploy"); + } finally { + fs.rmSync(haltStatePath, { force: true }); + } + }); + + it("clears the persisted file on resume — a later restart starts clean", () => { + const haltStatePath = tempHaltPath(); + try { + const clock = makeClock(); + const b1 = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath }); + b1.haltManually("cordon"); + expect(fs.existsSync(haltStatePath)).toBe(true); + b1.resume("op"); + expect(fs.existsSync(haltStatePath)).toBe(false); + + const b2 = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath }); + expect(b2.isHalted()).toBe(false); + expect(b2.canSpend(1, "crank")).toBe(true); + } finally { + fs.rmSync(haltStatePath, { force: true }); + } + }); + + it("starts clean when the configured path has no file yet", () => { + const haltStatePath = tempHaltPath(); // never written + const clock = makeClock(); + const b = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath }); + expect(b.isHalted()).toBe(false); + }); + + it("starts clean and logs rather than throwing when the persisted file is malformed", () => { + const haltStatePath = tempHaltPath(); + try { + fs.writeFileSync(haltStatePath, "{not valid json", "utf8"); + const clock = makeClock(); + expect(() => new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath })).not.toThrow(); + const b = new KeeperBudget(TIGHT_CONFIG, { now: clock.now, haltStatePath }); + expect(b.isHalted()).toBe(false); + } finally { + fs.rmSync(haltStatePath, { force: true }); + } + }); +});