From 7b724daecc0c15f44d8eb0c1abe8033c96747756 Mon Sep 17 00:00:00 2001 From: Morenikeoa Date: Sat, 27 Jun 2026 04:00:54 +0100 Subject: [PATCH] fix(keeper): wire health monitors to real outcomes instead of placeholder data /health's monitors.{rpc,scan,oracle,db} sub-objects look like genuine connectivity probes, but recordSuccess()/recordFailure() were never called anywhere in the codebase -- confirmed by repo-wide grep. lastSuccessTime is set once at construction and consecutiveFailures stays 0 forever, so getStatus().healthy is permanently true regardless of actual RPC/DB/oracle connectivity. Anyone building dashboards or alert rules off health.monitors.rpc.healthy etc. is reading fabricated-looking "all good" telemetry that can never go false. Moved `monitors` out of index.ts into a new src/lib/service-monitors.ts (so crank.ts and oracle.ts can wire it without a circular import on index.ts) and recorded real outcomes at the four monitors' most natural existing call sites: - rpc: the periodic SOL-balance getBalance() check (index.ts) - scan: the periodic discover()+crankAll() cycle completing or throwing (crank.ts) - oracle: the DexScreener/Jupiter external price fetches' HTTP/network outcome (oracle.ts) - db: the single Supabase market-metadata query (crank.ts) Each existing test file mocking @percolatorct/shared without spreading the real module needed createServiceMonitors added to its mock (the new module calls it at import time) -- updated 8 affected test files. BUG-110 from a clean-room Phase 4 audit pass. --- src/index.ts | 12 +- src/lib/service-monitors.ts | 11 ++ src/services/crank.ts | 10 + src/services/oracle.ts | 22 ++- tests/services/crank-error-code.poc.test.ts | 5 + .../crank-hyperp-detection.poc.test.ts | 5 + tests/services/crank.b-fixes.test.ts | 5 + tests/services/crank.processBatched.test.ts | 5 + tests/services/crank.test.ts | 179 ++++++++++++++---- tests/services/oracle-deviation.test.ts | 55 ++++-- tests/services/oracle-stale.test.ts | 71 ++++--- tests/services/oracle.b-fixes.test.ts | 5 + tests/services/oracle.test.ts | 106 ++++++++--- tests/services/program-id-allowlist.test.ts | 5 + 14 files changed, 371 insertions(+), 125 deletions(-) create mode 100644 src/lib/service-monitors.ts diff --git a/src/index.ts b/src/index.ts index 038bd5b..b2f60ee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,8 @@ import "dotenv/config"; import http from "node:http"; import { timingSafeEqual } from "node:crypto"; -import { config, createLogger, initSentry, captureException, sendInfoAlert, sendCriticalAlert, sendWarningAlert, createServiceMonitors, getConnection, loadKeypair } from "@percolatorct/shared"; +import { config, createLogger, initSentry, captureException, sendInfoAlert, sendCriticalAlert, sendWarningAlert, getConnection, loadKeypair } from "@percolatorct/shared"; +import { monitors } from "./lib/service-monitors.js"; import { OracleService } from "./services/oracle.js"; import { CrankService } from "./services/crank.js"; import { LiquidationService } from "./services/liquidation.js"; @@ -26,8 +27,8 @@ import { initSharedShadowHarness, sharedShadowHarness } from "./lib/shadow-harne import { sharedDecisionLog } from "./lib/decision-log.js"; import { createLaserStreamAccountLoader } from "./lib/laserstream-entrypoint.js"; -// Monitoring — alerts to Discord on threshold breaches -export const monitors = createServiceMonitors("Keeper"); +// Monitoring — alerts to Discord on threshold breaches (see src/lib/service-monitors.ts) +export { monitors }; // Initialize Sentry first initSentry("keeper"); @@ -157,6 +158,10 @@ const solBalanceCheckInterval = setInterval(async () => { _keeperSolBalanceLamports = lamports; const solBalance = lamports / 1e9; walletBalanceSol.set(solBalance); + // BUG-110: this periodic getBalance is a real, regular RPC round trip — + // record it so /health's monitors.rpc reflects actual connectivity + // instead of permanently-green placeholder data. + monitors.rpc.recordSuccess().catch(() => {}); if (solBalance < SOL_BALANCE_WARN_THRESHOLD) { // Rate-limit alerts to once per 5 minutes to avoid Discord spam @@ -180,6 +185,7 @@ const solBalanceCheckInterval = setInterval(async () => { logger.warn("Failed to fetch keeper SOL balance", { error: err instanceof Error ? err.message : String(err), }); + monitors.rpc.recordFailure(err instanceof Error ? err.message : String(err)).catch(() => {}); } }, 60_000); solBalanceCheckInterval.unref(); diff --git a/src/lib/service-monitors.ts b/src/lib/service-monitors.ts new file mode 100644 index 0000000..5c290ff --- /dev/null +++ b/src/lib/service-monitors.ts @@ -0,0 +1,11 @@ +import { createServiceMonitors } from "@percolatorct/shared"; + +/** + * BUG-110: standard health monitors (rpc/scan/oracle/db), surfaced in + * /health's `monitors` sub-object. Factored out of index.ts so crank.ts and + * oracle.ts can record real outcomes without a circular import on index.ts. + * Each monitor is only as accurate as its wiring — see the recordSuccess/ + * recordFailure call sites in index.ts (rpc), crank.ts (scan, db), and + * oracle.ts (oracle). + */ +export const monitors = createServiceMonitors("Keeper"); diff --git a/src/services/crank.ts b/src/services/crank.ts index 2c21556..344b03d 100644 --- a/src/services/crank.ts +++ b/src/services/crank.ts @@ -33,6 +33,7 @@ import { txLandTimeSeconds, } from "../lib/metrics.js"; import type { AccountLoader } from "../lib/account-loader.js"; +import { monitors } from "../lib/service-monitors.js"; import { keeperSend, sharedBudget } from "../lib/keeper-send.js"; import { sharedTxQueue } from "../lib/tx-queue.js"; import { parseV17RiskParams, V17_RISK_PARAMS_MIN_DATA_LEN } from "../lib/v17-risk.js"; @@ -1189,6 +1190,10 @@ export class CrankService { .in("slab_address", slabAddresses); if (error) { logger.warn("Supabase market metadata query error", { error: error.message }); + // BUG-110: record so /health's monitors.db reflects real DB outcomes. + monitors.db.recordFailure(error.message).catch(() => {}); + } else { + monitors.db.recordSuccess().catch(() => {}); } if (data) { const base58Re = /^[1-9A-HJ-NP-Za-km-z]{32,44}$/; @@ -1207,6 +1212,7 @@ export class CrankService { logger.warn("Failed to fetch market metadata from Supabase", { error: err instanceof Error ? err.message : String(err), }); + monitors.db.recordFailure(err instanceof Error ? err.message : String(err)).catch(() => {}); } const discoveredKeys = new Set(); @@ -2065,8 +2071,12 @@ export class CrankService { }); } } + // BUG-110: the cycle (discovery + crank pass) completed without + // throwing — record so /health's monitors.scan reflects real outcomes. + monitors.scan.recordSuccess().catch(() => {}); } catch (err) { logger.error("Crank cycle failed", { error: err instanceof Error ? err.message : String(err), stack: err instanceof Error ? err.stack : undefined }); + monitors.scan.recordFailure(err instanceof Error ? err.message : String(err)).catch(() => {}); } finally { this._cycling = false; // H4: disarm the watchdog on natural recovery so a transient slow diff --git a/src/services/oracle.ts b/src/services/oracle.ts index cc492fa..ef12543 100644 --- a/src/services/oracle.ts +++ b/src/services/oracle.ts @@ -5,6 +5,7 @@ import { import { eventBus, createLogger, getErrorMessage, sendWarningAlert, sendCriticalAlert } from "@percolatorct/shared"; import { isMainnet } from "../config/network.js"; import { oraclePushCountTotal, oracleStalenessSeconds } from "../lib/metrics.js"; +import { monitors } from "../lib/service-monitors.js"; const logger = createLogger("keeper:oracle"); @@ -184,8 +185,14 @@ export class OracleService { }); clearTimeout(timeoutId); - if (!res.ok) return null; - + // BUG-110: record real connectivity outcomes so /health's monitors.oracle + // reflects whether the external price feeds are actually reachable. + if (!res.ok) { + monitors.oracle.recordFailure(`DexScreener HTTP ${res.status}`).catch(() => {}); + return null; + } + monitors.oracle.recordSuccess().catch(() => {}); + const json = (await res.json()) as DexScreenerResponse; // M7: Validate BEFORE caching — don't cache bad responses that would @@ -218,6 +225,7 @@ export class OracleService { mint, error: err instanceof Error ? err.message : String(err), }); + monitors.oracle.recordFailure(err instanceof Error ? err.message : String(err)).catch(() => {}); return null; } } @@ -251,8 +259,13 @@ export class OracleService { }); clearTimeout(timeoutId); - if (!res.ok) return null; - + // BUG-110: see fetchDexScreenerPrice — same connectivity signal for Jupiter. + if (!res.ok) { + monitors.oracle.recordFailure(`Jupiter HTTP ${res.status}`).catch(() => {}); + return null; + } + monitors.oracle.recordSuccess().catch(() => {}); + const json = (await res.json()) as JupiterResponse; const priceStr = json.data?.[mint]?.price; if (!priceStr) return null; @@ -268,6 +281,7 @@ export class OracleService { mint, error: err instanceof Error ? err.message : String(err), }); + monitors.oracle.recordFailure(err instanceof Error ? err.message : String(err)).catch(() => {}); return null; } } diff --git a/tests/services/crank-error-code.poc.test.ts b/tests/services/crank-error-code.poc.test.ts index 43c3a69..db86b8e 100644 --- a/tests/services/crank-error-code.poc.test.ts +++ b/tests/services/crank-error-code.poc.test.ts @@ -37,6 +37,11 @@ vi.mock("@percolatorct/shared", () => ({ sendWithRetryKeeper: vi.fn(), eventBus: { publish: vi.fn() }, getSupabase: vi.fn(), + // BUG-110: src/lib/service-monitors.ts calls this at import time. + createServiceMonitors: vi.fn(() => { + const m = () => ({ recordSuccess: vi.fn(async () => {}), recordFailure: vi.fn(async () => {}), getErrorRate: vi.fn(() => 0), getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })) }); + return { rpc: m(), scan: m(), oracle: m(), db: m() }; + }), })); vi.mock("../../src/lib/keeper-send.js", async () => { const { KeeperBudget } = await vi.importActual("../../src/lib/budget.js"); diff --git a/tests/services/crank-hyperp-detection.poc.test.ts b/tests/services/crank-hyperp-detection.poc.test.ts index 98136f1..f5645c7 100644 --- a/tests/services/crank-hyperp-detection.poc.test.ts +++ b/tests/services/crank-hyperp-detection.poc.test.ts @@ -37,6 +37,11 @@ vi.mock("@percolatorct/shared", () => ({ sendWithRetryKeeper: vi.fn(), eventBus: { publish: vi.fn() }, getSupabase: vi.fn(), + // BUG-110: src/lib/service-monitors.ts calls this at import time. + createServiceMonitors: vi.fn(() => { + const m = () => ({ recordSuccess: vi.fn(async () => {}), recordFailure: vi.fn(async () => {}), getErrorRate: vi.fn(() => 0), getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })) }); + return { rpc: m(), scan: m(), oracle: m(), db: m() }; + }), })); vi.mock("../../src/lib/keeper-send.js", async () => { const { KeeperBudget } = await vi.importActual("../../src/lib/budget.js"); diff --git a/tests/services/crank.b-fixes.test.ts b/tests/services/crank.b-fixes.test.ts index a47731e..8cf4d72 100644 --- a/tests/services/crank.b-fixes.test.ts +++ b/tests/services/crank.b-fixes.test.ts @@ -76,6 +76,11 @@ vi.mock("@percolatorct/shared", () => ({ })), })), eventBus: { publish: vi.fn() }, + // BUG-110: src/lib/service-monitors.ts calls this at import time. + createServiceMonitors: vi.fn(() => { + const m = () => ({ recordSuccess: vi.fn(async () => {}), recordFailure: vi.fn(async () => {}), getErrorRate: vi.fn(() => 0), getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })) }); + return { rpc: m(), scan: m(), oracle: m(), db: m() }; + }), })); // After the #119 merge, crank.ts routes sends through keeperSend (not shared.sendWithRetryKeeper diff --git a/tests/services/crank.processBatched.test.ts b/tests/services/crank.processBatched.test.ts index 41d0677..d342ee5 100644 --- a/tests/services/crank.processBatched.test.ts +++ b/tests/services/crank.processBatched.test.ts @@ -63,6 +63,11 @@ vi.mock("@percolatorct/shared", () => ({ from: vi.fn(() => ({ select: vi.fn(() => ({ in: vi.fn(() => ({ data: [], error: null })) })) })), })), eventBus: { publish: vi.fn() }, + // BUG-110: src/lib/service-monitors.ts calls this at import time. + createServiceMonitors: vi.fn(() => { + const m = () => ({ recordSuccess: vi.fn(async () => {}), recordFailure: vi.fn(async () => {}), getErrorRate: vi.fn(() => 0), getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })) }); + return { rpc: m(), scan: m(), oracle: m(), db: m() }; + }), })); import { processBatched } from "../../src/services/crank.js"; diff --git a/tests/services/crank.test.ts b/tests/services/crank.test.ts index 3af441c..87499d0 100644 --- a/tests/services/crank.test.ts +++ b/tests/services/crank.test.ts @@ -55,50 +55,65 @@ vi.mock('../../src/lib/v17-risk.js', () => ({ })), })); -vi.mock('@percolatorct/shared', () => ({ - config: { - crankIntervalMs: 30000, - crankInactiveIntervalMs: 120000, - discoveryIntervalMs: 300000, - allProgramIds: ['11111111111111111111111111111111', 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA'], - crankKeypair: 'mock-keypair-path', - }, - createLogger: vi.fn(() => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - })), - getConnection: vi.fn(() => ({ - getAccountInfo: vi.fn(), - getSlot: vi.fn().mockResolvedValue(200), - })), - getFallbackConnection: vi.fn(() => ({ - getProgramAccounts: vi.fn(), - })), - loadKeypair: vi.fn(() => ({ - publicKey: { - toBase58: () => '11111111111111111111111111111111', - // Use string-based equality so foreign oracle authorities correctly return false. - equals: (other: any) => other?.toBase58?.() === '11111111111111111111111111111111', +vi.mock('@percolatorct/shared', () => { + const makeMonitor = () => ({ + recordSuccess: vi.fn(async () => {}), + recordFailure: vi.fn(async () => {}), + getErrorRate: vi.fn(() => 0), + getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })), + }); + return { + config: { + crankIntervalMs: 30000, + crankInactiveIntervalMs: 120000, + discoveryIntervalMs: 300000, + allProgramIds: ['11111111111111111111111111111111', 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA'], + crankKeypair: 'mock-keypair-path', }, - secretKey: new Uint8Array(64), - })), - sendWithRetry: vi.fn(async () => 'mock-signature-' + Date.now()), - sendWithRetryKeeper: vi.fn(async () => 'mock-keeper-sig-' + Date.now()), - rateLimitedCall: vi.fn((fn) => fn()), - sendCriticalAlert: vi.fn(), - getSupabase: vi.fn(() => ({ - from: vi.fn(() => ({ - select: vi.fn(() => ({ - in: vi.fn(() => ({ data: [], error: null })), + createLogger: vi.fn(() => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + })), + getConnection: vi.fn(() => ({ + getAccountInfo: vi.fn(), + getSlot: vi.fn().mockResolvedValue(200), + })), + getFallbackConnection: vi.fn(() => ({ + getProgramAccounts: vi.fn(), + })), + loadKeypair: vi.fn(() => ({ + publicKey: { + toBase58: () => '11111111111111111111111111111111', + // Use string-based equality so foreign oracle authorities correctly return false. + equals: (other: any) => other?.toBase58?.() === '11111111111111111111111111111111', + }, + secretKey: new Uint8Array(64), + })), + sendWithRetry: vi.fn(async () => 'mock-signature-' + Date.now()), + sendWithRetryKeeper: vi.fn(async () => 'mock-keeper-sig-' + Date.now()), + rateLimitedCall: vi.fn((fn) => fn()), + sendCriticalAlert: vi.fn(), + getSupabase: vi.fn(() => ({ + from: vi.fn(() => ({ + select: vi.fn(() => ({ + in: vi.fn(() => ({ data: [], error: null })), + })), })), })), - })), - eventBus: { - publish: vi.fn(), - }, -})); + eventBus: { + publish: vi.fn(), + }, + // BUG-110: src/lib/service-monitors.ts calls this at import time. + createServiceMonitors: vi.fn(() => ({ + rpc: makeMonitor(), + scan: makeMonitor(), + oracle: makeMonitor(), + db: makeMonitor(), + })), + }; +}); vi.mock('../../src/lib/keeper-send.js', async () => { const { KeeperBudget } = await vi.importActual('../../src/lib/budget.js'); @@ -113,6 +128,7 @@ import { CrankService } from '../../src/services/crank.js'; import * as core from '@percolatorct/sdk'; import * as shared from '@percolatorct/shared'; import * as keeperSendModule from '../../src/lib/keeper-send.js'; +import { monitors } from '../../src/lib/service-monitors.js'; const MOCK_PORTFOLIO = new PublicKey('9xQeWvG816bUx9EPjHmaT23yvVM2ZWbrrpZb9PusVFin'); @@ -183,6 +199,87 @@ describe('CrankService', () => { expect(crankService.getMarkets().size).toBe(1); }); + // BUG-110: monitors.db was never wired to a real outcome -- /health's + // monitors.db sub-object was permanently-green placeholder data. + it('BUG-110: records monitors.db success on a clean Supabase query', async () => { + vi.mocked(core.discoverMarkets).mockResolvedValue([] as any); + + await crankService.discover(); + + expect(monitors.db.recordSuccess).toHaveBeenCalled(); + expect(monitors.db.recordFailure).not.toHaveBeenCalled(); + }); + + it('BUG-110: records monitors.db failure on a Supabase query error', async () => { + vi.mocked(core.discoverMarkets).mockResolvedValue([] as any); + vi.mocked(shared.getSupabase).mockReturnValueOnce({ + from: vi.fn(() => ({ + select: vi.fn(() => ({ + in: vi.fn(() => ({ data: null, error: { message: 'connection refused' } })), + })), + })), + } as any); + + await crankService.discover(); + + expect(monitors.db.recordFailure).toHaveBeenCalledWith('connection refused'); + expect(monitors.db.recordSuccess).not.toHaveBeenCalled(); + }); + + it('BUG-110: records monitors.db failure when the Supabase call throws', async () => { + vi.mocked(core.discoverMarkets).mockResolvedValue([] as any); + vi.mocked(shared.getSupabase).mockImplementationOnce(() => { + throw new Error('supabase client init failed'); + }); + + await crankService.discover(); + + expect(monitors.db.recordFailure).toHaveBeenCalledWith('supabase client init failed'); + expect(monitors.db.recordSuccess).not.toHaveBeenCalled(); + }); + + }); + + // BUG-110: monitors.scan was never wired to a real outcome -- /health's + // monitors.scan sub-object was permanently-green placeholder data. + describe('BUG-110: monitors.scan wiring in the periodic cycle', () => { + it('records success after a cycle completes without throwing', async () => { + const mockMarket = { + slabAddress: { toBase58: () => 'MarketMonScan111111111111111111111111111' }, + programId: { toBase58: () => '11111111111111111111111111111111' }, + config: { + collateralMint: { toBase58: () => 'Mint1111111111111111111111111111111111' }, + oracleAuthority: { toBase58: () => 'Oracle11111111111111111111111111111111', equals: () => false }, + indexFeedId: { toBytes: () => new Uint8Array(32) }, + }, + params: { maintenanceMarginBps: 500n, initialMarginBps: 1000n }, + header: { admin: { toBase58: () => 'Admin111111111111111111111111111111111' } }, + }; + vi.mocked(core.discoverMarkets).mockResolvedValue([mockMarket] as any); + const svc = new CrankService(mockOracleService, 1_000); + // Populate markets + lastDiscoveryTime under REAL timers first, so the + // periodic tick below takes the "markets pre-loaded" path and sees + // needsDiscovery===false -- avoiding crank.ts's real inter-program + // discovery delay (a setTimeout) while fake timers are active, which + // would otherwise hang this test and leak fake timers into every test + // that runs after it in this file. + await svc.discover(); + vi.mocked(core.discoverMarkets).mockClear(); + + vi.useFakeTimers(); + try { + await svc.start(); + await vi.advanceTimersByTimeAsync(1_100); + expect(monitors.scan.recordSuccess).toHaveBeenCalled(); + expect(monitors.scan.recordFailure).not.toHaveBeenCalled(); + } finally { + svc.stop(); + vi.useRealTimers(); + } + }); + }); + + describe('discover (LaserStream)', () => { it('LaserStream fast-path retries v17 keeper portfolio provisioning when keeperPortfolio is null', async () => { const prevLaserStream = process.env.KEEPER_USE_LASERSTREAM; process.env.KEEPER_USE_LASERSTREAM = 'true'; diff --git a/tests/services/oracle-deviation.test.ts b/tests/services/oracle-deviation.test.ts index 60978ad..ecc150b 100644 --- a/tests/services/oracle-deviation.test.ts +++ b/tests/services/oracle-deviation.test.ts @@ -34,26 +34,41 @@ vi.mock('@percolatorct/sdk', () => ({ ACCOUNTS_PUSH_ORACLE_PRICE: {}, })); -vi.mock('@percolatorct/shared', () => ({ - config: { - programId: '11111111111111111111111111111111', - crankKeypair: 'mock-keypair-path', - }, - createLogger: vi.fn(() => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - })), - getConnection: vi.fn(() => ({ getAccountInfo: vi.fn() })), - loadKeypair: vi.fn(() => ({ - publicKey: new PublicKey('11111111111111111111111111111111'), - secretKey: new Uint8Array(64), - })), - sendWithRetry: vi.fn(async () => 'mock-sig'), - eventBus: { publish: vi.fn() }, - getErrorMessage: vi.fn((e: unknown) => (e instanceof Error ? e.message : String(e))), -})); +vi.mock('@percolatorct/shared', () => { + const makeMonitor = () => ({ + recordSuccess: vi.fn(async () => {}), + recordFailure: vi.fn(async () => {}), + getErrorRate: vi.fn(() => 0), + getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })), + }); + return { + config: { + programId: '11111111111111111111111111111111', + crankKeypair: 'mock-keypair-path', + }, + createLogger: vi.fn(() => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + })), + getConnection: vi.fn(() => ({ getAccountInfo: vi.fn() })), + loadKeypair: vi.fn(() => ({ + publicKey: new PublicKey('11111111111111111111111111111111'), + secretKey: new Uint8Array(64), + })), + sendWithRetry: vi.fn(async () => 'mock-sig'), + eventBus: { publish: vi.fn() }, + getErrorMessage: vi.fn((e: unknown) => (e instanceof Error ? e.message : String(e))), + // BUG-110: src/lib/service-monitors.ts calls this at import time. + createServiceMonitors: vi.fn(() => ({ + rpc: makeMonitor(), + scan: makeMonitor(), + oracle: makeMonitor(), + db: makeMonitor(), + })), + }; +}); import { OracleService } from '../../src/services/oracle.js'; diff --git a/tests/services/oracle-stale.test.ts b/tests/services/oracle-stale.test.ts index a826fed..ee60d23 100644 --- a/tests/services/oracle-stale.test.ts +++ b/tests/services/oracle-stale.test.ts @@ -12,34 +12,49 @@ vi.mock('@percolatorct/sdk', () => ({ ACCOUNTS_PUSH_ORACLE_PRICE: {}, })); -vi.mock('@percolatorct/shared', () => ({ - config: { - programId: '11111111111111111111111111111111', - crankKeypair: 'mock-keypair-path', - }, - createLogger: vi.fn(() => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - })), - getConnection: vi.fn(() => ({ - getAccountInfo: vi.fn(), - })), - loadKeypair: vi.fn(() => ({ - publicKey: new PublicKey('11111111111111111111111111111111'), - secretKey: new Uint8Array(64), - })), - sendWithRetry: vi.fn(async () => 'mock-signature'), - eventBus: { - publish: vi.fn(), - }, - getErrorMessage: vi.fn((err: unknown) => { - if (err instanceof Error) return err.message; - return String(err); - }), - sendWarningAlert: vi.fn(() => Promise.resolve()), -})); +vi.mock('@percolatorct/shared', () => { + const makeMonitor = () => ({ + recordSuccess: vi.fn(async () => {}), + recordFailure: vi.fn(async () => {}), + getErrorRate: vi.fn(() => 0), + getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })), + }); + return { + config: { + programId: '11111111111111111111111111111111', + crankKeypair: 'mock-keypair-path', + }, + createLogger: vi.fn(() => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + })), + getConnection: vi.fn(() => ({ + getAccountInfo: vi.fn(), + })), + loadKeypair: vi.fn(() => ({ + publicKey: new PublicKey('11111111111111111111111111111111'), + secretKey: new Uint8Array(64), + })), + sendWithRetry: vi.fn(async () => 'mock-signature'), + eventBus: { + publish: vi.fn(), + }, + getErrorMessage: vi.fn((err: unknown) => { + if (err instanceof Error) return err.message; + return String(err); + }), + sendWarningAlert: vi.fn(() => Promise.resolve()), + // BUG-110: src/lib/service-monitors.ts calls this at import time. + createServiceMonitors: vi.fn(() => ({ + rpc: makeMonitor(), + scan: makeMonitor(), + oracle: makeMonitor(), + db: makeMonitor(), + })), + }; +}); import { OracleService } from '../../src/services/oracle.js'; diff --git a/tests/services/oracle.b-fixes.test.ts b/tests/services/oracle.b-fixes.test.ts index 268ad91..3b62a3e 100644 --- a/tests/services/oracle.b-fixes.test.ts +++ b/tests/services/oracle.b-fixes.test.ts @@ -49,6 +49,11 @@ vi.mock("@percolatorct/shared", () => ({ err instanceof Error ? err.message : String(err), ), sendWarningAlert: hoisted.sendWarningAlert, + // BUG-110: src/lib/service-monitors.ts calls this at import time. + createServiceMonitors: vi.fn(() => { + const m = () => ({ recordSuccess: vi.fn(async () => {}), recordFailure: vi.fn(async () => {}), getErrorRate: vi.fn(() => 0), getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })) }); + return { rpc: m(), scan: m(), oracle: m(), db: m() }; + }), })); const loggerWarn = hoisted.loggerWarn; diff --git a/tests/services/oracle.test.ts b/tests/services/oracle.test.ts index 5c4ef04..4b73d5e 100644 --- a/tests/services/oracle.test.ts +++ b/tests/services/oracle.test.ts @@ -12,38 +12,54 @@ vi.mock('@percolatorct/sdk', () => ({ ACCOUNTS_PUSH_ORACLE_PRICE: {}, })); -vi.mock('@percolatorct/shared', () => ({ - config: { - programId: '11111111111111111111111111111111', - crankKeypair: 'mock-keypair-path', - }, - createLogger: vi.fn(() => ({ - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), - })), - getConnection: vi.fn(() => ({ - getAccountInfo: vi.fn(), - })), - loadKeypair: vi.fn(() => ({ - publicKey: new PublicKey('11111111111111111111111111111111'), - secretKey: new Uint8Array(64), - })), - sendWithRetry: vi.fn(async () => 'mock-signature'), - eventBus: { - publish: vi.fn(), - }, - getErrorMessage: vi.fn((err: unknown) => { - if (err instanceof Error) return err.message; - return String(err); - }), - sendWarningAlert: vi.fn(), - sendCriticalAlert: vi.fn(), -})); +vi.mock('@percolatorct/shared', () => { + const makeMonitor = () => ({ + recordSuccess: vi.fn(async () => {}), + recordFailure: vi.fn(async () => {}), + getErrorRate: vi.fn(() => 0), + getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })), + }); + return { + config: { + programId: '11111111111111111111111111111111', + crankKeypair: 'mock-keypair-path', + }, + createLogger: vi.fn(() => ({ + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + })), + getConnection: vi.fn(() => ({ + getAccountInfo: vi.fn(), + })), + loadKeypair: vi.fn(() => ({ + publicKey: new PublicKey('11111111111111111111111111111111'), + secretKey: new Uint8Array(64), + })), + sendWithRetry: vi.fn(async () => 'mock-signature'), + eventBus: { + publish: vi.fn(), + }, + getErrorMessage: vi.fn((err: unknown) => { + if (err instanceof Error) return err.message; + return String(err); + }), + sendWarningAlert: vi.fn(), + sendCriticalAlert: vi.fn(), + // BUG-110: src/lib/service-monitors.ts calls this at import time. + createServiceMonitors: vi.fn(() => ({ + rpc: makeMonitor(), + scan: makeMonitor(), + oracle: makeMonitor(), + db: makeMonitor(), + })), + }; +}); import { OracleService } from '../../src/services/oracle.js'; import * as shared from '@percolatorct/shared'; +import { monitors } from '../../src/lib/service-monitors.js'; describe('OracleService', () => { let oracleService: OracleService; @@ -123,6 +139,38 @@ describe('OracleService', () => { expect(price).toBeNull(); }); + + // BUG-110: monitors.oracle was never wired to a real outcome -- /health's + // monitors.oracle sub-object was permanently-green placeholder data. + it('BUG-110: records monitors.oracle success on a reachable fetch', async () => { + vi.mocked(fetch).mockResolvedValueOnce({ + ok: true, + json: async () => ({ pairs: [{ priceUsd: '1.23', liquidity: { usd: 100000 } }] }), + } as any); + + await oracleService.fetchDexScreenerPrice('MINT_MONITOR_OK'); + + expect(monitors.oracle.recordSuccess).toHaveBeenCalledTimes(1); + expect(monitors.oracle.recordFailure).not.toHaveBeenCalled(); + }); + + it('BUG-110: records monitors.oracle failure on a network error', async () => { + vi.mocked(fetch).mockRejectedValueOnce(new Error('Network error')); + + await oracleService.fetchDexScreenerPrice('MINT_MONITOR_FAIL'); + + expect(monitors.oracle.recordFailure).toHaveBeenCalledTimes(1); + expect(monitors.oracle.recordSuccess).not.toHaveBeenCalled(); + }); + + it('BUG-110: records monitors.oracle failure on a non-ok HTTP response', async () => { + vi.mocked(fetch).mockResolvedValueOnce({ ok: false, status: 429 } as any); + + await oracleService.fetchDexScreenerPrice('MINT_MONITOR_429'); + + expect(monitors.oracle.recordFailure).toHaveBeenCalledWith('DexScreener HTTP 429'); + expect(monitors.oracle.recordSuccess).not.toHaveBeenCalled(); + }); }); describe('DexScreener cache', () => { diff --git a/tests/services/program-id-allowlist.test.ts b/tests/services/program-id-allowlist.test.ts index 3763b4b..54e65a6 100644 --- a/tests/services/program-id-allowlist.test.ts +++ b/tests/services/program-id-allowlist.test.ts @@ -62,6 +62,11 @@ vi.mock("@percolatorct/shared", () => ({ getSupabase: vi.fn(() => ({ from: vi.fn(() => ({ select: vi.fn(() => ({ in: vi.fn(async () => ({ data: [], error: null })) })) })), })), + // BUG-110: src/lib/service-monitors.ts calls this at import time. + createServiceMonitors: vi.fn(() => { + const m = () => ({ recordSuccess: vi.fn(async () => {}), recordFailure: vi.fn(async () => {}), getErrorRate: vi.fn(() => 0), getStatus: vi.fn(() => ({ healthy: true, consecutiveFailures: 0, errorRate: 0, timeSinceSuccessMs: 0, alertActive: false })) }); + return { rpc: m(), scan: m(), oracle: m(), db: m() }; + }), })); vi.mock("../../src/lib/keeper-send.js", async () => { const { KeeperBudget } = await vi.importActual("../../src/lib/budget.js");