From cc4311730b5709bb0cbd58bd990d03c9be9a7021 Mon Sep 17 00:00:00 2001 From: Sheebazz Date: Fri, 31 Jul 2026 18:10:24 -1200 Subject: [PATCH] feat(keeper): implement graceful shutdown with 30s timeout and structured logs --- README.md | 3 ++ services/keeper/src/serve.ts | 51 ++++++++++++------------------- services/keeper/src/watch.test.ts | 43 ++++++++++++++++++++++++++ services/keeper/src/watch.ts | 38 +++++++++++------------ 4 files changed, 83 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index 596c015..0872989 100644 --- a/README.md +++ b/README.md @@ -214,3 +214,6 @@ pnpm mainnet:verify # mainnet read-only proof - **Binding:** `H = sha256(value‖nonce)` - **Unlock:** round-R BLS verified on-chain before reveal - **Selective disclosure:** values public post-R; identities auditor-encrypted + +## Operational Behavior +- **Graceful Shutdown**: The keeper watch mode listens for `SIGINT` and `SIGTERM`. Upon receiving a signal, it stops picking up new rounds, finishes processing the current in-flight round, and safely exits. A 30-second timeout ensures the process does not hang indefinitely. diff --git a/services/keeper/src/serve.ts b/services/keeper/src/serve.ts index 774e2e0..2da2190 100644 --- a/services/keeper/src/serve.ts +++ b/services/keeper/src/serve.ts @@ -1,23 +1,4 @@ // Standalone status server for the keeper. -// -// Runs the watch-mode keeper AND a status HTTP API on the same process so -// pilots and dashboards can poll keeper-observed rounds without SSHing into -// the host. The status server reads from the same on-chain source and the -// same persisted store as the watch loop — no extra RPC budget, no extra -// signing capability. -// -// Env: -// ROUND_CONTRACT_ID deployed Round contract id (C…) -// KEEPER_SECRET funded signer secret (S…) -// RPC_URL Soroban RPC (default testnet) -// NETWORK_PASSPHRASE -// WATCH_POLL_MS poll interval (default 15000) -// WATCH_ROUND_IDS optional explicit list: "1,2,5" or "1-10" -// WATCH_FROM first round id when auto-discovering (default 1) -// WATCH_MAX_ROUNDS max rounds to probe (default 64) -// KEEPER_STATUS_HOST status API bind host (default 127.0.0.1) -// KEEPER_STATUS_PORT status API port (default 8090) -// KEEPER_STATUS_ENABLE set to "false" to disable the status API (default true) import { Keypair } from "@stellar/stellar-sdk"; import { SubRosaClient } from "@sub-rosa/sdk"; @@ -60,15 +41,6 @@ async function main() { const store = new KeeperStore(); const settlementGuard = createSettlementGuard(); - let stopping = false; - process.on("SIGINT", () => { - console.log("\nserve: SIGINT — finishing current tick then exit"); - stopping = true; - }); - process.on("SIGTERM", () => { - stopping = true; - }); - const statusEnabled = (process.env.KEEPER_STATUS_ENABLE ?? "true").toLowerCase() !== "false"; const statusHost = process.env.KEEPER_STATUS_HOST ?? "127.0.0.1"; const statusPort = Number(process.env.KEEPER_STATUS_PORT ?? "8090"); @@ -92,9 +64,7 @@ async function main() { }, }); statusHandle = withGracefulShutdown(server); - console.log(`· status API: http://${statusHost}:${statusPort} (GET /status, /status/rounds/:id, /healthz, /status/health)`); - } else { - console.log("· status API disabled (KEEPER_STATUS_ENABLE=false)"); + console.log(`· status API: http://${statusHost}:${statusPort}`); } console.log("Sub Rosa keeper (watch + status)"); @@ -102,6 +72,22 @@ async function main() { console.log("· poll: ", pollMs, "ms"); console.log("· Ctrl+C to stop\n"); + let stopping = false; + let shutdownTimer: NodeJS.Timeout | undefined; + + const handleSignal = (signal: string) => { + if (stopping) return; + stopping = true; + console.log(JSON.stringify({ event: "shutdown_start", signal, message: "finishing current tick then exit" })); + shutdownTimer = setTimeout(() => { + console.error(JSON.stringify({ event: "shutdown_timeout", message: "timeout exceeded, forcing exit" })); + process.exit(1); + }, 30000); + }; + + process.on("SIGINT", () => handleSignal("SIGINT")); + process.on("SIGTERM", () => handleSignal("SIGTERM")); + await runWatchLoop({ sdk, drand, @@ -115,7 +101,8 @@ async function main() { }); if (statusHandle) await statusHandle.close(); - console.log("serve: stopped"); + if (shutdownTimer) clearTimeout(shutdownTimer); + console.log(JSON.stringify({ event: "shutdown_complete", message: "serve stopped gracefully" })); } main().catch((err) => { diff --git a/services/keeper/src/watch.test.ts b/services/keeper/src/watch.test.ts index acc43a5..91a960e 100644 --- a/services/keeper/src/watch.test.ts +++ b/services/keeper/src/watch.test.ts @@ -33,3 +33,46 @@ test("discoverRoundIds returns empty when first round missing", async () => { const ids = await discoverRoundIds(reader as Pick, { from: 1n, maxProbe: 5 }); assert.deepEqual(ids, []); }); + +test("watch loop handles graceful shutdown during a queued round", async () => { + const { KeeperStore } = await import("./store.js"); + const { createSettlementGuard } = await import("./settlement-guard.js"); + const { runWatchLoop } = await import("./watch-loop.js"); + + const store = new KeeperStore(); + const settlementGuard = createSettlementGuard(); + + store.addRound(100n, { contractId: "C1", network: "test" }); + store.addRound(101n, { contractId: "C1", network: "test" }); + + let callCount = 0; + let processedRoundBeforeStop = false; + + const isStopping = () => { + callCount++; + if (callCount > 2) { + processedRoundBeforeStop = true; + return true; + } + return false; + }; + + const mockSdk = { + getRound: async () => { throw new Error("mock error"); } + } as any; + + const loopPromise = runWatchLoop({ + sdk: mockSdk, + drand: {} as any, + log: () => {}, + pollMs: 10, + contractId: "C1", + network: "test", + store, + settlementGuard, + isStopping, + }); + + await loopPromise; + assert.ok(processedRoundBeforeStop, "gracefully stopped without hanging"); +}); diff --git a/services/keeper/src/watch.ts b/services/keeper/src/watch.ts index 2f5428c..6d69507 100644 --- a/services/keeper/src/watch.ts +++ b/services/keeper/src/watch.ts @@ -1,15 +1,5 @@ // Watch-mode keeper — standalone entry. For a combined status-API + watch // process, use `serve.ts` instead. -// -// Env: -// ROUND_CONTRACT_ID deployed Round contract id (C…) -// KEEPER_SECRET funded signer secret (S…) -// RPC_URL Soroban RPC (default testnet) -// NETWORK_PASSPHRASE -// WATCH_POLL_MS poll interval (default 15000) -// WATCH_ROUND_IDS optional explicit list: "1,2,5" or "1-10" -// WATCH_FROM first round id when auto-discovering (default 1) -// WATCH_MAX_ROUNDS max rounds to probe (default 64) import { Keypair } from "@stellar/stellar-sdk"; import { SubRosaClient } from "@sub-rosa/sdk"; @@ -42,15 +32,6 @@ async function main() { const drand = quicknet(); const log = (m: string) => console.log(`· ${m}`); - let stopping = false; - process.on("SIGINT", () => { - console.log("\nwatch: SIGINT — finishing current tick then exit"); - stopping = true; - }); - process.on("SIGTERM", () => { - stopping = true; - }); - const store = new KeeperStore(); const settlementGuard = createSettlementGuard(); @@ -59,6 +40,22 @@ async function main() { console.log("· poll: ", pollMs, "ms"); console.log("· Ctrl+C to stop\n"); + let stopping = false; + let shutdownTimer: NodeJS.Timeout | undefined; + + const handleSignal = (signal: string) => { + if (stopping) return; + stopping = true; + console.log(JSON.stringify({ event: "shutdown_start", signal, message: "finishing current tick then exit" })); + shutdownTimer = setTimeout(() => { + console.error(JSON.stringify({ event: "shutdown_timeout", message: "timeout exceeded, forcing exit" })); + process.exit(1); + }, 30000); + }; + + process.on("SIGINT", () => handleSignal("SIGINT")); + process.on("SIGTERM", () => handleSignal("SIGTERM")); + await runWatchLoop({ sdk, drand, @@ -71,7 +68,8 @@ async function main() { isStopping: () => stopping, }); - console.log("watch: stopped"); + if (shutdownTimer) clearTimeout(shutdownTimer); + console.log(JSON.stringify({ event: "shutdown_complete", message: "watch stopped gracefully" })); } main().catch((err) => {