From 7be092b158ad4e3776bcb174aad81f4c746e21c3 Mon Sep 17 00:00:00 2001 From: Shifat7 Date: Sat, 22 Aug 2026 18:12:39 +1000 Subject: [PATCH] feat: opt-in response cache keyed by canonical request fingerprint Store completed successful responses by sha256(canonical JSON body) in the private state directory and replay identical un-keyed requests locally until the TTL expires (bounded by maxEntries). Errors, cancellations, and indeterminate dispatches are never stored; explicit Idempotency-Key flows take precedence and skip the cache entirely. Disabled by default: enableResponseCache must be explicitly true. Replay is marked with X-Response-Cache-Replayed and a sanitized gateway log event. --- README.md | 14 +++++ src/bridge.mjs | 28 ++++++++- src/config.mjs | 74 +++++++++++++++++++++- test/bridge.test.mjs | 118 +++++++++++++++++++++++++++++++++++ test/response-cache.test.mjs | 56 +++++++++++++++++ 5 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 test/response-cache.test.mjs diff --git a/README.md b/README.md index e214825..7584ff0 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,20 @@ Budget state is reserved durably before provider dispatch, committed immediately These controls do not replace the Hyperagent agent-level budget. Configure each relay agent with low effort and a hard per-run USD cap before production use. +### Request fingerprint cache + +Codex retries identical requests after local hiccups without an `Idempotency-Key`; each retry previously created another Hyperagent thread and burned credits. The optional fingerprint cache stores completed successful responses by `sha256(canonical JSON body)` in the private state directory and replays them locally (header `X-Response-Cache-Replayed: true`) until the TTL expires. + +```json +{ + "enableResponseCache": true, + "responseCacheTtlMs": 1800000, + "responseCacheMaxEntries": 128 +} +``` + +Off by default because it changes replay semantics for un-keyed requests. Only completed successes are cached — never errors, cancellations, or in-progress outcomes — and an explicit `Idempotency-Key` always wins over the cache. + ## Security - The HTTP bridge binds only to `127.0.0.1` and requires a random local bearer token on every model and Responses request. diff --git a/src/bridge.mjs b/src/bridge.mjs index e8b0ac8..7de6489 100644 --- a/src/bridge.mjs +++ b/src/bridge.mjs @@ -12,6 +12,8 @@ import { reconcileIdempotency, releaseDailyRequestBudget, reserveDailyRequestBudget, + lookupResponseCache, + storeResponseCache, updateIdempotency, VERSION } from './config.mjs'; @@ -179,13 +181,16 @@ function etagFor(value) { } export class BridgeServer { - constructor(config, { clientFactory, auditWriter, logWriter, budgetGuard, budgetManager, idempotencyManager } = {}) { + constructor(config, { clientFactory, auditWriter, logWriter, cacheManager, budgetGuard, budgetManager, idempotencyManager } = {}) { this.config = config; this.server = null; this.agentCache = { at: 0, agents: [] }; this.clientFactory = clientFactory || (() => new HyperagentClient(this.config)); this.auditWriter = auditWriter || appendAudit; this.logWriter = logWriter || appendGatewayLog; + this.cacheManager = config.enableResponseCache === true + ? (cacheManager || { lookup: lookupResponseCache, store: storeResponseCache }) + : null; this.budgetManager = budgetManager || (budgetGuard ? { reserve: async configValue => ({ id: null, ...await budgetGuard(configValue) }), @@ -251,11 +256,12 @@ export class BridgeServer { }, { etag: tag, 'cache-control': 'private, max-age=60' }); } - renderCompleted(response, result, { replayed = false, createdSent = false } = {}) { + renderCompleted(response, result, { replayed = false, cacheReplayed = false, createdSent = false } = {}) { const headers = { 'x-hyperagent-thread-id': result.threadId, 'x-usage-source': 'unavailable', - ...(replayed ? { 'x-idempotency-replayed': 'true' } : {}) + ...(replayed ? { 'x-idempotency-replayed': 'true' } : {}), + ...(cacheReplayed ? { 'x-response-cache-replayed': 'true' } : {}) }; if (result.streaming) { if (!response.headersSent) { @@ -289,6 +295,19 @@ export class BridgeServer { if (!body.model) throw Object.assign(new Error('The model field is required.'), { status: 400, code: 'model_required' }); const keyHash = idempotencyKey(request); const fingerprint = requestFingerprint(body); + if (!keyHash && this.cacheManager) { + let cached = null; + try { + cached = await this.cacheManager.lookup(fingerprint, this.config); + } catch { + cached = null; + } + if (cached?.result) { + await this.safeLog({ event: 'response_cache_replayed', requestId: serverRequestId, originalRequestId: cached.result.requestId }); + this.renderCompleted(response, cached.result, { cacheReplayed: true }); + return; + } + } const abort = new AbortController(); const cancel = () => { if (!abort.signal.aborted) abort.abort(abortError()); @@ -443,6 +462,9 @@ export class BridgeServer { result: completed }, this.config); } + if (!keyHash && this.cacheManager) { + await this.cacheManager.store(fingerprint, completed, this.config).catch(() => {}); + } this.renderCompleted(response, completed, { createdSent: streaming }); } catch (error) { const code = errorCode(error); diff --git a/src/config.mjs b/src/config.mjs index 249c4e9..1d43b29 100644 --- a/src/config.mjs +++ b/src/config.mjs @@ -66,6 +66,14 @@ function idempotencyLockPath() { return join(stateDir(), 'idempotency.lock'); } +export function responseCachePath() { + return join(stateDir(), 'response-cache.json'); +} + +function responseCacheLockPath() { + return join(stateDir(), 'response-cache.lock'); +} + export const DEFAULT_CONFIG = Object.freeze({ configVersion: CONFIG_SCHEMA_VERSION, mcpUrl: DEFAULT_MCP_URL, @@ -99,7 +107,10 @@ export const DEFAULT_CONFIG = Object.freeze({ maxConversationTurns: 8, maxForwardedTools: 32, maxPromptChars: 70000, - blockMultiAgentTools: true + blockMultiAgentTools: true, + enableResponseCache: false, + responseCacheTtlMs: 30 * 60 * 1000, + responseCacheMaxEntries: 128 }); export async function ensureStateDir() { @@ -562,3 +573,64 @@ export function reconcileIdempotency(config) { idempotencyQueue = run.catch(() => {}); return run; } + + +let responseCacheQueue = Promise.resolve(); + +function withResponseCacheLock(callback) { + return withFileLock(responseCacheLockPath(), 'response_cache', callback); +} + +function normalizeResponseCache(current) { + return { + version: 1, + entries: current?.entries && typeof current.entries === 'object' && !Array.isArray(current.entries) + ? current.entries + : {} + }; +} + +function responseCacheOptions(config = {}) { + return { + ttlMs: Math.max(1000, Number(config.responseCacheTtlMs) || DEFAULT_CONFIG.responseCacheTtlMs), + maxEntries: Math.max(1, Math.min(10_000, Number(config.responseCacheMaxEntries) || DEFAULT_CONFIG.responseCacheMaxEntries)) + }; +} + +function pruneResponseCache(state, { ttlMs, now }) { + for (const [key, entry] of Object.entries(state.entries)) { + const at = Number(entry?.completedAt) || 0; + if (!at || now - at > ttlMs) delete state.entries[key]; + } +} + +export function lookupResponseCache(fingerprint, config = {}) { + const run = responseCacheQueue.then(() => withResponseCacheLock(async () => { + const { ttlMs, now } = { ...responseCacheOptions(config), now: Date.now() }; + const state = normalizeResponseCache(await readJson(responseCachePath(), { version: 1, entries: {} })); + pruneResponseCache(state, { ttlMs, now }); + const entry = state.entries[fingerprint]; + await atomicWriteJson(responseCachePath(), state, 0o600); + return entry ? structuredClone(entry) : null; + })); + responseCacheQueue = run.catch(() => null); + return run; +} + +export function storeResponseCache(fingerprint, result, config = {}, { completedAt = Date.now() } = {}) { + const run = responseCacheQueue.then(() => withResponseCacheLock(async () => { + const options = responseCacheOptions(config); + const state = normalizeResponseCache(await readJson(responseCachePath(), { version: 1, entries: {} })); + pruneResponseCache(state, { ttlMs: options.ttlMs, now: Date.now() }); + state.entries[fingerprint] = { result: structuredClone(result), completedAt }; + const entries = Object.entries(state.entries).sort((a, b) => (Number(a[1]?.completedAt) || 0) - (Number(b[1]?.completedAt) || 0)); + while (entries.length > options.maxEntries) { + const [oldest] = entries.shift(); + delete state.entries[oldest]; + } + await atomicWriteJson(responseCachePath(), state, 0o600); + return Object.keys(state.entries).length; + })); + responseCacheQueue = run.catch(() => {}); + return run; +} diff --git a/test/bridge.test.mjs b/test/bridge.test.mjs index e823817..abff3f1 100644 --- a/test/bridge.test.mjs +++ b/test/bridge.test.mjs @@ -645,3 +645,121 @@ test('disconnect while awaiting an idempotency claim cannot reserve or dispatch' await bridge.close(); } }); + +test('identical requests without an idempotency key replay from the fingerprint cache', async () => { + const home = await mkdtemp(join(tmpdir(), 'hacb-bridge-cache-')); + const previous = process.env.HACB_HOME; + process.env.HACB_HOME = home; + try { + let created = 0; + const bridge = new BridgeServer({ + bridgeHost: '127.0.0.1', bridgePort: 0, aliases: {}, exposeAllAgents: true, + localApiToken: 'test-local-token-12345678901234567890', + maxRequestsPerDay: 20, + enableResponseCache: true, + responseCacheTtlMs: 600000 + }, { + clientFactory: () => ({ + async listAgents() { return [agent]; }, + async createThread() { created += 1; return 'thread_cache_once'; }, + async waitForThread() { return { text: '{"type":"final","text":"cached"}' }; }, + async close() {} + }), + auditWriter: async () => {}, logWriter: async () => {}, idempotencyManager: createMemoryIdempotencyManager() + }); + await bridge.start(); + try { + const base = `http://127.0.0.1:${bridge.server.address().port}`; + const body = JSON.stringify({ model: 'hyperagent/sol-coder', input: 'same body', stream: false }); + const first = await fetch(`${base}/v1/responses`, { method: 'POST', headers: { ...AUTH, 'content-type': 'application/json' }, body }); + assert.equal(first.status, 200); + const second = await fetch(`${base}/v1/responses`, { method: 'POST', headers: { ...AUTH, 'content-type': 'application/json' }, body }); + assert.equal(second.status, 200); + assert.equal(second.headers.get('x-response-cache-replayed'), 'true'); + assert.equal((await second.json()).id, (await first.json()).id); + assert.equal(created, 1); + + const changed = await fetch(`${base}/v1/responses`, { + method: 'POST', + headers: { ...AUTH, 'content-type': 'application/json' }, + body: JSON.stringify({ model: 'hyperagent/sol-coder', input: 'different body', stream: false }) + }); + assert.equal(changed.headers.get('x-response-cache-replayed'), null); + assert.equal(created, 2); + } finally { + await bridge.close(); + } + } finally { + await rm(home, { recursive: true, force: true }); + if (previous === undefined) delete process.env.HACB_HOME; + else process.env.HACB_HOME = previous; + } +}); + +test('failed dispatches are never cached and an explicit Idempotency-Key still wins', async () => { + const home = await mkdtemp(join(tmpdir(), 'hacb-bridge-cache2-')); + const previous = process.env.HACB_HOME; + process.env.HACB_HOME = home; + try { + let attempts = 0; + const bridge = new BridgeServer({ + bridgeHost: '127.0.0.1', bridgePort: 0, aliases: {}, exposeAllAgents: true, + localApiToken: 'test-local-token-12345678901234567890', + maxRequestsPerDay: 20, + enableResponseCache: true + }, { + clientFactory: () => ({ + async listAgents() { return [agent]; }, + async createThread() { + attempts += 1; + throw Object.assign(new Error('upstream down'), { dispatchState: 'not_dispatched' }); + }, + async close() {} + }), + auditWriter: async () => {}, logWriter: async () => {}, idempotencyManager: createMemoryIdempotencyManager() + }); + await bridge.start(); + try { + const base = `http://127.0.0.1:${bridge.server.address().port}`; + const body = JSON.stringify({ model: 'hyperagent/sol-coder', input: 'will fail', stream: false }); + const response = () => fetch(`${base}/v1/responses`, { method: 'POST', headers: { ...AUTH, 'content-type': 'application/json' }, body }); + assert.equal((await response()).status, 500); + assert.equal((await response()).status, 500); + assert.equal(attempts, 2); + } finally { + await bridge.close(); + } + + const keyed = new BridgeServer({ + bridgeHost: '127.0.0.1', bridgePort: 0, aliases: {}, exposeAllAgents: true, + localApiToken: 'test-local-token-12345678901234567890', + maxRequestsPerDay: 20, + enableResponseCache: true + }, { + clientFactory: () => ({ + async listAgents() { return [agent]; }, + async createThread() { attempts += 1; return `thread_keyed_${attempts}`; }, + async waitForThread() { return { text: '{"type":"final","text":"keyed"}' }; }, + async close() {} + }), + auditWriter: async () => {}, logWriter: async () => {}, idempotencyManager: createMemoryIdempotencyManager() + }); + await keyed.start(); + try { + const base = `http://127.0.0.1:${keyed.server.address().port}`; + const headers = { ...AUTH, 'content-type': 'application/json', 'idempotency-key': 'explicit-key-wins' }; + const body = JSON.stringify({ model: 'hyperagent/sol-coder', input: 'keyed replay', stream: false }); + const first = await fetch(`${base}/v1/responses`, { method: 'POST', headers, body }); + const second = await fetch(`${base}/v1/responses`, { method: 'POST', headers, body }); + assert.equal(first.status, 200); + assert.equal(second.headers.get('x-idempotency-replayed'), 'true'); + assert.equal(second.headers.get('x-response-cache-replayed'), null); + } finally { + await keyed.close(); + } + } finally { + await rm(home, { recursive: true, force: true }); + if (previous === undefined) delete process.env.HACB_HOME; + else process.env.HACB_HOME = previous; + } +}); diff --git a/test/response-cache.test.mjs b/test/response-cache.test.mjs new file mode 100644 index 0000000..7e13a2b --- /dev/null +++ b/test/response-cache.test.mjs @@ -0,0 +1,56 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { lookupResponseCache, storeResponseCache } from '../src/config.mjs'; + +const RESULT = { output: { type: 'final', text: 'once' }, model: 'hyperagent/sol-coder' }; + +async function withCacheHome(fn) { + const home = await mkdtemp(join(tmpdir(), 'hacb-response-cache-')); + const previous = process.env.HACB_HOME; + process.env.HACB_HOME = home; + try { + await fn(home); + } finally { + if (previous === undefined) delete process.env.HACB_HOME; + else process.env.HACB_HOME = previous; + await rm(home, { recursive: true, force: true }); + } +} + +test('stored fingerprints replay the exact completed result', async () => { + await withCacheHome(async () => { + const config = { enableResponseCache: true, responseCacheTtlMs: 60000, responseCacheMaxEntries: 8 }; + assert.equal(await lookupResponseCache('fp_missing', config), null); + await storeResponseCache('fp_a', RESULT, config); + const hit = await lookupResponseCache('fp_a', config); + assert.deepEqual(hit.result, RESULT); + assert.ok(hit.completedAt <= Date.now()); + }); +}); + +test('expired entries miss and are pruned from the cache file', async () => { + await withCacheHome(async () => { + const config = { enableResponseCache: true, responseCacheTtlMs: 1000, responseCacheMaxEntries: 8 }; + await storeResponseCache('fp_old', RESULT, config, { completedAt: Date.now() - 5000 }); + assert.equal(await lookupResponseCache('fp_old', config), null); + await lookupResponseCache('fp_trigger_prune', config); + assert.equal(await lookupResponseCache('fp_old', config), null); + await storeResponseCache('fp_new', RESULT, config, { completedAt: Date.now() }); + assert.deepEqual((await lookupResponseCache('fp_new', config)).result, RESULT); + }); +}); + +test('the cache evicts oldest entries beyond maxEntries', async () => { + await withCacheHome(async () => { + const config = { enableResponseCache: true, responseCacheTtlMs: 600000, responseCacheMaxEntries: 2 }; + await storeResponseCache('fp_1', { n: 1 }, config, { completedAt: Date.now() - 4000 }); + await storeResponseCache('fp_2', { n: 2 }, config, { completedAt: Date.now() - 2000 }); + await storeResponseCache('fp_3', { n: 3 }, config, { completedAt: Date.now() }); + assert.equal(await lookupResponseCache('fp_1', config), null); + assert.deepEqual((await lookupResponseCache('fp_2', config)).result, { n: 2 }); + assert.deepEqual((await lookupResponseCache('fp_3', config)).result, { n: 3 }); + }); +});