fix(keeper): wire health monitors to real outcomes instead of placeholder data

src/index.ts

@@ -1,7 +1,8 @@
 import "dotenv/config";
 import http from "node:http";
 import { timingSafeEqual } from "node:crypto";
-import { config, createLogger, initSentry, captureException, sendInfoAlert, sendCriticalAlert, sendWarningAlert, createServiceMonitors, getConnection, loadKeypair } from "@percolatorct/shared";
+import { config, createLogger, initSentry, captureException, sendInfoAlert, sendCriticalAlert, sendWarningAlert, getConnection, loadKeypair } from "@percolatorct/shared";
+import { monitors } from "./lib/service-monitors.js";
 import { OracleService } from "./services/oracle.js";
 import { CrankService } from "./services/crank.js";
 import { LiquidationService } from "./services/liquidation.js";
@@ -26,8 +27,8 @@ import { initSharedShadowHarness, sharedShadowHarness } from "./lib/shadow-harne
 import { sharedDecisionLog } from "./lib/decision-log.js";
 import { createLaserStreamAccountLoader } from "./lib/laserstream-entrypoint.js";
 
-// Monitoring — alerts to Discord on threshold breaches
-export const monitors = createServiceMonitors("Keeper");
+// Monitoring — alerts to Discord on threshold breaches (see src/lib/service-monitors.ts)
+export { monitors };
 
 // Initialize Sentry first
 initSentry("keeper");
@@ -157,6 +158,10 @@ const solBalanceCheckInterval = setInterval(async () => {
     _keeperSolBalanceLamports = lamports;
     const solBalance = lamports / 1e9;
     walletBalanceSol.set(solBalance);
+    // BUG-110: this periodic getBalance is a real, regular RPC round trip —
+    // record it so /health's monitors.rpc reflects actual connectivity
+    // instead of permanently-green placeholder data.
+    monitors.rpc.recordSuccess().catch(() => {});
 
     if (solBalance < SOL_BALANCE_WARN_THRESHOLD) {
       // Rate-limit alerts to once per 5 minutes to avoid Discord spam
@@ -180,6 +185,7 @@ const solBalanceCheckInterval = setInterval(async () => {
     logger.warn("Failed to fetch keeper SOL balance", {
       error: err instanceof Error ? err.message : String(err),
     });
+    monitors.rpc.recordFailure(err instanceof Error ? err.message : String(err)).catch(() => {});
   }
 }, 60_000);
 solBalanceCheckInterval.unref();

src/lib/service-monitors.ts

@@ -0,0 +1,11 @@
+import { createServiceMonitors } from "@percolatorct/shared";
+
+/**
+ * BUG-110: standard health monitors (rpc/scan/oracle/db), surfaced in
+ * /health's `monitors` sub-object. Factored out of index.ts so crank.ts and
+ * oracle.ts can record real outcomes without a circular import on index.ts.
+ * Each monitor is only as accurate as its wiring — see the recordSuccess/
+ * recordFailure call sites in index.ts (rpc), crank.ts (scan, db), and
+ * oracle.ts (oracle).
+ */
+export const monitors = createServiceMonitors("Keeper");

src/services/crank.ts

@@ -33,6 +33,7 @@ import {
   txLandTimeSeconds,
 } from "../lib/metrics.js";
 import type { AccountLoader } from "../lib/account-loader.js";
+import { monitors } from "../lib/service-monitors.js";
 import { keeperSend, sharedBudget } from "../lib/keeper-send.js";
 import { sharedTxQueue } from "../lib/tx-queue.js";
 import { parseV17RiskParams, V17_RISK_PARAMS_MIN_DATA_LEN } from "../lib/v17-risk.js";
@@ -1189,6 +1190,10 @@ export class CrankService {
         .in("slab_address", slabAddresses);
       if (error) {
         logger.warn("Supabase market metadata query error", { error: error.message });
+        // BUG-110: record so /health's monitors.db reflects real DB outcomes.
+        monitors.db.recordFailure(error.message).catch(() => {});
+      } else {
+        monitors.db.recordSuccess().catch(() => {});
       }
       if (data) {
         const base58Re = /^[1-9A-HJ-NP-Za-km-z]{32,44}$/;
@@ -1207,6 +1212,7 @@ export class CrankService {
       logger.warn("Failed to fetch market metadata from Supabase", {
         error: err instanceof Error ? err.message : String(err),
       });
+      monitors.db.recordFailure(err instanceof Error ? err.message : String(err)).catch(() => {});
     }
 
     const discoveredKeys = new Set<string>();
@@ -2065,8 +2071,12 @@ export class CrankService {
             });
           }
         }
+        // BUG-110: the cycle (discovery + crank pass) completed without
+        // throwing — record so /health's monitors.scan reflects real outcomes.
+        monitors.scan.recordSuccess().catch(() => {});
       } catch (err) {
         logger.error("Crank cycle failed", { error: err instanceof Error ? err.message : String(err), stack: err instanceof Error ? err.stack : undefined });
+        monitors.scan.recordFailure(err instanceof Error ? err.message : String(err)).catch(() => {});
       } finally {
         this._cycling = false;
         // H4: disarm the watchdog on natural recovery so a transient slow

src/services/oracle.ts

@@ -5,6 +5,7 @@ import {
 import { eventBus, createLogger, getErrorMessage, sendWarningAlert, sendCriticalAlert } from "@percolatorct/shared";
 import { isMainnet } from "../config/network.js";
 import { oraclePushCountTotal, oracleStalenessSeconds } from "../lib/metrics.js";
+import { monitors } from "../lib/service-monitors.js";
 
 const logger = createLogger("keeper:oracle");
 
@@ -184,8 +185,14 @@ export class OracleService {
       });
       clearTimeout(timeoutId);
 
-      if (!res.ok) return null;
-
+      // BUG-110: record real connectivity outcomes so /health's monitors.oracle
+      // reflects whether the external price feeds are actually reachable.
+      if (!res.ok) {
+        monitors.oracle.recordFailure(`DexScreener HTTP ${res.status}`).catch(() => {});
+        return null;
+      }
+      monitors.oracle.recordSuccess().catch(() => {});
+
       const json = (await res.json()) as DexScreenerResponse;
 
       // M7: Validate BEFORE caching — don't cache bad responses that would
@@ -218,6 +225,7 @@ export class OracleService {
         mint,
         error: err instanceof Error ? err.message : String(err),
       });
+      monitors.oracle.recordFailure(err instanceof Error ? err.message : String(err)).catch(() => {});
       return null;
     }
   }
@@ -251,8 +259,13 @@ export class OracleService {
       });
       clearTimeout(timeoutId);
 
-      if (!res.ok) return null;
-
+      // BUG-110: see fetchDexScreenerPrice — same connectivity signal for Jupiter.
+      if (!res.ok) {
+        monitors.oracle.recordFailure(`Jupiter HTTP ${res.status}`).catch(() => {});
+        return null;
+      }
+      monitors.oracle.recordSuccess().catch(() => {});
+
       const json = (await res.json()) as JupiterResponse;
       const priceStr = json.data?.[mint]?.price;
       if (!priceStr) return null;
@@ -268,6 +281,7 @@ export class OracleService {
         mint,
         error: err instanceof Error ? err.message : String(err),
       });
+      monitors.oracle.recordFailure(err instanceof Error ? err.message : String(err)).catch(() => {});
       return null;
     }
   }

tests/services/crank-error-code.poc.test.ts

@@ -37,6 +37,11 @@ vi.mock("@percolatorct/shared", () => ({
   sendWithRetryKeeper: vi.fn(),
   eventBus: { publish: vi.fn() },
   getSupabase: vi.fn(),
+  // BUG-110: src/lib/service-monitors.ts calls this at import time.
+  createServiceMonitors: vi.fn(() => {
+    const m = () => ({ recordSuccess: vi.fn(async () => {}), recordFailure: vi.fn(async () => {}), getErrorRate: vi.fn(() => 0), getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })) });
+    return { rpc: m(), scan: m(), oracle: m(), db: m() };
+  }),
 }));
 vi.mock("../../src/lib/keeper-send.js", async () => {
   const { KeeperBudget } = await vi.importActual<typeof import("../../src/lib/budget.js")>("../../src/lib/budget.js");

tests/services/crank-hyperp-detection.poc.test.ts

@@ -37,6 +37,11 @@ vi.mock("@percolatorct/shared", () => ({
   sendWithRetryKeeper: vi.fn(),
   eventBus: { publish: vi.fn() },
   getSupabase: vi.fn(),
+  // BUG-110: src/lib/service-monitors.ts calls this at import time.
+  createServiceMonitors: vi.fn(() => {
+    const m = () => ({ recordSuccess: vi.fn(async () => {}), recordFailure: vi.fn(async () => {}), getErrorRate: vi.fn(() => 0), getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })) });
+    return { rpc: m(), scan: m(), oracle: m(), db: m() };
+  }),
 }));
 vi.mock("../../src/lib/keeper-send.js", async () => {
   const { KeeperBudget } = await vi.importActual<typeof import("../../src/lib/budget.js")>("../../src/lib/budget.js");

tests/services/crank.b-fixes.test.ts

@@ -76,6 +76,11 @@ vi.mock("@percolatorct/shared", () => ({
     })),
   })),
   eventBus: { publish: vi.fn() },
+  // BUG-110: src/lib/service-monitors.ts calls this at import time.
+  createServiceMonitors: vi.fn(() => {
+    const m = () => ({ recordSuccess: vi.fn(async () => {}), recordFailure: vi.fn(async () => {}), getErrorRate: vi.fn(() => 0), getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })) });
+    return { rpc: m(), scan: m(), oracle: m(), db: m() };
+  }),
 }));
 
 // After the #119 merge, crank.ts routes sends through keeperSend (not shared.sendWithRetryKeeper

tests/services/crank.processBatched.test.ts

@@ -63,6 +63,11 @@ vi.mock("@percolatorct/shared", () => ({
     from: vi.fn(() => ({ select: vi.fn(() => ({ in: vi.fn(() => ({ data: [], error: null })) })) })),
   })),
   eventBus: { publish: vi.fn() },
+  // BUG-110: src/lib/service-monitors.ts calls this at import time.
+  createServiceMonitors: vi.fn(() => {
+    const m = () => ({ recordSuccess: vi.fn(async () => {}), recordFailure: vi.fn(async () => {}), getErrorRate: vi.fn(() => 0), getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })) });
+    return { rpc: m(), scan: m(), oracle: m(), db: m() };
+  }),
 }));
 
 import { processBatched } from "../../src/services/crank.js";

tests/services/crank.test.ts

@@ -55,50 +55,65 @@ vi.mock('../../src/lib/v17-risk.js', () => ({
   })),
 }));
 
-vi.mock('@percolatorct/shared', () => ({
-  config: {
-    crankIntervalMs: 30000,
-    crankInactiveIntervalMs: 120000,
-    discoveryIntervalMs: 300000,
-    allProgramIds: ['11111111111111111111111111111111', 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA'],
-    crankKeypair: 'mock-keypair-path',
-  },
-  createLogger: vi.fn(() => ({
-    info: vi.fn(),
-    warn: vi.fn(),
-    error: vi.fn(),
-    debug: vi.fn(),
-  })),
-  getConnection: vi.fn(() => ({
-    getAccountInfo: vi.fn(),
-    getSlot: vi.fn().mockResolvedValue(200),
-  })),
-  getFallbackConnection: vi.fn(() => ({
-    getProgramAccounts: vi.fn(),
-  })),
-  loadKeypair: vi.fn(() => ({
-    publicKey: {
-      toBase58: () => '11111111111111111111111111111111',
-      // Use string-based equality so foreign oracle authorities correctly return false.
-      equals: (other: any) => other?.toBase58?.() === '11111111111111111111111111111111',
+vi.mock('@percolatorct/shared', () => {
+  const makeMonitor = () => ({
+    recordSuccess: vi.fn(async () => {}),
+    recordFailure: vi.fn(async () => {}),
+    getErrorRate: vi.fn(() => 0),
+    getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })),
+  });
+  return {
+    config: {
+      crankIntervalMs: 30000,
+      crankInactiveIntervalMs: 120000,
+      discoveryIntervalMs: 300000,
+      allProgramIds: ['11111111111111111111111111111111', 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA'],
+      crankKeypair: 'mock-keypair-path',
     },
-    secretKey: new Uint8Array(64),
-  })),
-  sendWithRetry: vi.fn(async () => 'mock-signature-' + Date.now()),
-  sendWithRetryKeeper: vi.fn(async () => 'mock-keeper-sig-' + Date.now()),
-  rateLimitedCall: vi.fn((fn) => fn()),
-  sendCriticalAlert: vi.fn(),
-  getSupabase: vi.fn(() => ({
-    from: vi.fn(() => ({
-      select: vi.fn(() => ({
-        in: vi.fn(() => ({ data: [], error: null })),
+    createLogger: vi.fn(() => ({
+      info: vi.fn(),
+      warn: vi.fn(),
+      error: vi.fn(),
+      debug: vi.fn(),
+    })),
+    getConnection: vi.fn(() => ({
+      getAccountInfo: vi.fn(),
+      getSlot: vi.fn().mockResolvedValue(200),
+    })),
+    getFallbackConnection: vi.fn(() => ({
+      getProgramAccounts: vi.fn(),
+    })),
+    loadKeypair: vi.fn(() => ({
+      publicKey: {
+        toBase58: () => '11111111111111111111111111111111',
+        // Use string-based equality so foreign oracle authorities correctly return false.
+        equals: (other: any) => other?.toBase58?.() === '11111111111111111111111111111111',
+      },
+      secretKey: new Uint8Array(64),
+    })),
+    sendWithRetry: vi.fn(async () => 'mock-signature-' + Date.now()),
+    sendWithRetryKeeper: vi.fn(async () => 'mock-keeper-sig-' + Date.now()),
+    rateLimitedCall: vi.fn((fn) => fn()),
+    sendCriticalAlert: vi.fn(),
+    getSupabase: vi.fn(() => ({
+      from: vi.fn(() => ({
+        select: vi.fn(() => ({
+          in: vi.fn(() => ({ data: [], error: null })),
+        })),
       })),
     })),
-  })),
-  eventBus: {
-    publish: vi.fn(),
-  },
-}));
+    eventBus: {
+      publish: vi.fn(),
+    },
+    // BUG-110: src/lib/service-monitors.ts calls this at import time.
+    createServiceMonitors: vi.fn(() => ({
+      rpc: makeMonitor(),
+      scan: makeMonitor(),
+      oracle: makeMonitor(),
+      db: makeMonitor(),
+    })),
+  };
+});
 
 vi.mock('../../src/lib/keeper-send.js', async () => {
   const { KeeperBudget } = await vi.importActual<typeof import('../../src/lib/budget.js')>('../../src/lib/budget.js');
@@ -113,6 +128,7 @@ import { CrankService } from '../../src/services/crank.js';
 import * as core from '@percolatorct/sdk';
 import * as shared from '@percolatorct/shared';
 import * as keeperSendModule from '../../src/lib/keeper-send.js';
+import { monitors } from '../../src/lib/service-monitors.js';
 
 const MOCK_PORTFOLIO = new PublicKey('9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin');
 
@@ -183,6 +199,87 @@ describe('CrankService', () => {
       expect(crankService.getMarkets().size).toBe(1);
     });
 
+    // BUG-110: monitors.db was never wired to a real outcome -- /health's
+    // monitors.db sub-object was permanently-green placeholder data.
+    it('BUG-110: records monitors.db success on a clean Supabase query', async () => {
+      vi.mocked(core.discoverMarkets).mockResolvedValue([] as any);
+
+      await crankService.discover();
+
+      expect(monitors.db.recordSuccess).toHaveBeenCalled();
+      expect(monitors.db.recordFailure).not.toHaveBeenCalled();
+    });
+
+    it('BUG-110: records monitors.db failure on a Supabase query error', async () => {
+      vi.mocked(core.discoverMarkets).mockResolvedValue([] as any);
+      vi.mocked(shared.getSupabase).mockReturnValueOnce({
+        from: vi.fn(() => ({
+          select: vi.fn(() => ({
+            in: vi.fn(() => ({ data: null, error: { message: 'connection refused' } })),
+          })),
+        })),
+      } as any);
+
+      await crankService.discover();
+
+      expect(monitors.db.recordFailure).toHaveBeenCalledWith('connection refused');
+      expect(monitors.db.recordSuccess).not.toHaveBeenCalled();
+    });
+
+    it('BUG-110: records monitors.db failure when the Supabase call throws', async () => {
+      vi.mocked(core.discoverMarkets).mockResolvedValue([] as any);
+      vi.mocked(shared.getSupabase).mockImplementationOnce(() => {
+        throw new Error('supabase client init failed');
+      });
+
+      await crankService.discover();
+
+      expect(monitors.db.recordFailure).toHaveBeenCalledWith('supabase client init failed');
+      expect(monitors.db.recordSuccess).not.toHaveBeenCalled();
+    });
+
+  });
+
+  // BUG-110: monitors.scan was never wired to a real outcome -- /health's
+  // monitors.scan sub-object was permanently-green placeholder data.
+  describe('BUG-110: monitors.scan wiring in the periodic cycle', () => {
+    it('records success after a cycle completes without throwing', async () => {
+      const mockMarket = {
+        slabAddress: { toBase58: () => 'MarketMonScan111111111111111111111111111' },
+        programId: { toBase58: () => '11111111111111111111111111111111' },
+        config: {
+          collateralMint: { toBase58: () => 'Mint1111111111111111111111111111111111' },
+          oracleAuthority: { toBase58: () => 'Oracle11111111111111111111111111111111', equals: () => false },
+          indexFeedId: { toBytes: () => new Uint8Array(32) },
+        },
+        params: { maintenanceMarginBps: 500n, initialMarginBps: 1000n },
+        header: { admin: { toBase58: () => 'Admin111111111111111111111111111111111' } },
+      };
+      vi.mocked(core.discoverMarkets).mockResolvedValue([mockMarket] as any);
+      const svc = new CrankService(mockOracleService, 1_000);
+      // Populate markets + lastDiscoveryTime under REAL timers first, so the
+      // periodic tick below takes the "markets pre-loaded" path and sees
+      // needsDiscovery===false -- avoiding crank.ts's real inter-program
+      // discovery delay (a setTimeout) while fake timers are active, which
+      // would otherwise hang this test and leak fake timers into every test
+      // that runs after it in this file.
+      await svc.discover();
+      vi.mocked(core.discoverMarkets).mockClear();
+
+      vi.useFakeTimers();
+      try {
+        await svc.start();
+        await vi.advanceTimersByTimeAsync(1_100);
+        expect(monitors.scan.recordSuccess).toHaveBeenCalled();
+        expect(monitors.scan.recordFailure).not.toHaveBeenCalled();
+      } finally {
+        svc.stop();
+        vi.useRealTimers();
+      }
+    });
+  });
+
+  describe('discover (LaserStream)', () => {
     it('LaserStream fast-path retries v17 keeper portfolio provisioning when keeperPortfolio is null', async () => {
       const prevLaserStream = process.env.KEEPER_USE_LASERSTREAM;
       process.env.KEEPER_USE_LASERSTREAM = 'true';