From a1fb74dc24f069dde29342da78c7bdb792a6003d Mon Sep 17 00:00:00 2001 From: Amp Date: Wed, 16 Sep 2026 13:46:06 +0000 Subject: [PATCH 1/5] fix(openai): count compatible cache reads Amp-Thread-ID: https://ampcode.com/threads/T-01a0aa02-b6de-7763-97b8-1650ea4b46da --- .../__tests__/openai-usage-tracking.spec.ts | 48 +++++++++++++++++++ src/api/providers/openai.ts | 4 +- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/api/providers/__tests__/openai-usage-tracking.spec.ts b/src/api/providers/__tests__/openai-usage-tracking.spec.ts index e9c2f5e4fd..151defd3ac 100644 --- a/src/api/providers/__tests__/openai-usage-tracking.spec.ts +++ b/src/api/providers/__tests__/openai-usage-tracking.spec.ts @@ -181,6 +181,31 @@ describe("OpenAiHandler with usage tracking fix", () => { }) }) + it("should report OpenAI-compatible cached prompt tokens", async () => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + choices: [{ delta: { content: "Cached response" }, index: 0 }], + usage: { + prompt_tokens: 5_053, + completion_tokens: 16, + total_tokens: 5_069, + prompt_tokens_details: { cached_tokens: 4_864 }, + }, + }, + ]), + ) + + const chunks = await collectStream(handler.createMessage(systemPrompt, messages)) + + expect(chunks).toContainEqual({ + type: "usage", + inputTokens: 5_053, + outputTokens: 16, + cacheReadTokens: 4_864, + }) + }) + it("should handle case where no usage is provided", async () => { // Override the mock for this specific test mockCreate.mockImplementationOnce(async (options) => { @@ -212,4 +237,27 @@ describe("OpenAiHandler with usage tracking fix", () => { expect(usageChunks).toHaveLength(0) }) }) + + it("should report cached prompt tokens from a non-streaming response", async () => { + const nonStreamingHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled: false }) + mockCreate.mockImplementationOnce(async () => ({ + id: "test-completion", + choices: [{ message: { role: "assistant", content: "Cached response" } }], + usage: { + prompt_tokens: 4_621, + completion_tokens: 16, + total_tokens: 4_637, + prompt_tokens_details: { cached_tokens: 4_608 }, + }, + })) + + const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", [])) + + expect(chunks).toContainEqual({ + type: "usage", + inputTokens: 4_621, + outputTokens: 16, + cacheReadTokens: 4_608, + }) + }) }) diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index 04b12f233d..ae7405b346 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -282,12 +282,14 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl } protected processUsageMetrics(usage: any, _modelInfo?: ModelInfo): ApiStreamUsageChunk { + const cacheReadTokens = usage?.cache_read_input_tokens ?? usage?.prompt_tokens_details?.cached_tokens + return { type: "usage", inputTokens: usage?.prompt_tokens || 0, outputTokens: usage?.completion_tokens || 0, cacheWriteTokens: usage?.cache_creation_input_tokens || undefined, - cacheReadTokens: usage?.cache_read_input_tokens || undefined, + cacheReadTokens: cacheReadTokens || undefined, } } From b173f6357a3a79a37e7c282c549a51ab1ee787d6 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Fri, 18 Sep 2026 15:26:59 +0200 Subject: [PATCH 2/5] fix(api): validate compatible cache usage --- .../__tests__/openai-usage-tracking.spec.ts | 74 +++++++++++++++++++ src/api/providers/openai.ts | 33 ++++++--- src/eslint-suppressions.json | 2 +- 3 files changed, 99 insertions(+), 10 deletions(-) diff --git a/src/api/providers/__tests__/openai-usage-tracking.spec.ts b/src/api/providers/__tests__/openai-usage-tracking.spec.ts index 151defd3ac..b1f21b304d 100644 --- a/src/api/providers/__tests__/openai-usage-tracking.spec.ts +++ b/src/api/providers/__tests__/openai-usage-tracking.spec.ts @@ -260,4 +260,78 @@ describe("OpenAiHandler with usage tracking fix", () => { cacheReadTokens: 4_608, }) }) + + it("reports cached prompt tokens for a streaming O3 response", async () => { + const o3Handler = new OpenAiHandler({ ...mockOptions, openAiModelId: "o3-mini" }) + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + choices: [{ delta: { content: "Cached response" }, index: 0 }], + usage: { + prompt_tokens: 5_053, + completion_tokens: 16, + total_tokens: 5_069, + prompt_tokens_details: { cached_tokens: 4_864 }, + }, + }, + ]), + ) + + const chunks = await collectStream(o3Handler.createMessage("system prompt", [])) + + expect(chunks).toContainEqual({ + type: "usage", + inputTokens: 5_053, + outputTokens: 16, + cacheReadTokens: 4_864, + }) + }) + + it.each([ + ["string", "10"], + ["object", { tokens: 10 }], + ["negative", -1], + ["non-finite", Number.POSITIVE_INFINITY], + ["greater than prompt tokens", 101], + ])("ignores invalid %s cached prompt tokens in streaming responses", async (_name, cachedTokens) => { + mockCreate.mockImplementationOnce(async () => + asyncStreamFrom([ + { + choices: [{ delta: { content: "Response" }, index: 0 }], + usage: { + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: cachedTokens }, + }, + }, + ]), + ) + + const chunks = await collectStream(handler.createMessage("system prompt", [])) + + expect(chunks).toContainEqual({ type: "usage", inputTokens: 100, outputTokens: 5 }) + }) + + it.each([ + ["string", "10"], + ["object", { tokens: 10 }], + ["negative", -1], + ["non-finite", Number.POSITIVE_INFINITY], + ["greater than prompt tokens", 101], + ])("ignores invalid %s cached prompt tokens in non-streaming responses", async (_name, cachedTokens) => { + const nonStreamingHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled: false }) + mockCreate.mockImplementationOnce(async () => ({ + id: "test-completion", + choices: [{ message: { role: "assistant", content: "Response" } }], + usage: { + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: cachedTokens }, + }, + })) + + const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", [])) + + expect(chunks).toContainEqual({ type: "usage", inputTokens: 100, outputTokens: 5 }) + }) }) diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index d1f098e63d..a07ef121bf 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -28,6 +28,17 @@ import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata, Complete import { handleOpenAIError } from "./utils/error-handler" import { extractReasoningFromDelta } from "./utils/extract-reasoning" +type OpenAiUsage = + | { + prompt_tokens?: number + completion_tokens?: number + cache_creation_input_tokens?: number + cache_read_input_tokens?: unknown + prompt_tokens_details?: { cached_tokens?: unknown } | null + } + | null + | undefined + // TODO: Rename this to OpenAICompatibleHandler. Also, I think the // `OpenAINativeHandler` can subclass from this, since it's obviously // compatible with the OpenAI API. We can also rename it to `OpenAIHandler`. @@ -281,15 +292,23 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl } } - protected processUsageMetrics(usage: any, _modelInfo?: ModelInfo): ApiStreamUsageChunk { - const cacheReadTokens = usage?.cache_read_input_tokens ?? usage?.prompt_tokens_details?.cached_tokens + protected processUsageMetrics(usage: OpenAiUsage, _modelInfo?: ModelInfo): ApiStreamUsageChunk { + const inputTokens = usage?.prompt_tokens || 0 + const reportedCacheReadTokens = usage?.cache_read_input_tokens ?? usage?.prompt_tokens_details?.cached_tokens + const cacheReadTokens = + typeof reportedCacheReadTokens === "number" && + Number.isFinite(reportedCacheReadTokens) && + reportedCacheReadTokens >= 0 && + reportedCacheReadTokens <= inputTokens + ? reportedCacheReadTokens || undefined + : undefined return { type: "usage", - inputTokens: usage?.prompt_tokens || 0, + inputTokens, outputTokens: usage?.completion_tokens || 0, cacheWriteTokens: usage?.cache_creation_input_tokens || undefined, - cacheReadTokens: cacheReadTokens || undefined, + cacheReadTokens, } } @@ -473,11 +492,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl } if (chunk.usage) { - yield { - type: "usage", - inputTokens: chunk.usage.prompt_tokens || 0, - outputTokens: chunk.usage.completion_tokens || 0, - } + yield this.processUsageMetrics(chunk.usage) } } } diff --git a/src/eslint-suppressions.json b/src/eslint-suppressions.json index 93741e9174..bcb5947288 100644 --- a/src/eslint-suppressions.json +++ b/src/eslint-suppressions.json @@ -401,7 +401,7 @@ }, "api/providers/openai.ts": { "@typescript-eslint/no-explicit-any": { - "count": 3 + "count": 2 } }, "api/providers/openrouter.ts": { From ef29ef32f4f474de39e578bb5a8a1edc8688c492 Mon Sep 17 00:00:00 2001 From: Amp Date: Mon, 21 Sep 2026 18:16:30 +0000 Subject: [PATCH 3/5] fix: reject fractional OpenAI cache reads and cover precedence Amp-Thread-ID: https://ampcode.com/threads/T-01a0c52c-51a6-73ae-bafb-2fb2bc840b2b --- .../__tests__/openai-usage-tracking.spec.ts | 59 +++++++++++++++++++ src/api/providers/openai.ts | 1 + 2 files changed, 60 insertions(+) diff --git a/src/api/providers/__tests__/openai-usage-tracking.spec.ts b/src/api/providers/__tests__/openai-usage-tracking.spec.ts index b1f21b304d..22ab2daf97 100644 --- a/src/api/providers/__tests__/openai-usage-tracking.spec.ts +++ b/src/api/providers/__tests__/openai-usage-tracking.spec.ts @@ -291,6 +291,7 @@ describe("OpenAiHandler with usage tracking fix", () => { ["string", "10"], ["object", { tokens: 10 }], ["negative", -1], + ["fractional", 10.5], ["non-finite", Number.POSITIVE_INFINITY], ["greater than prompt tokens", 101], ])("ignores invalid %s cached prompt tokens in streaming responses", async (_name, cachedTokens) => { @@ -316,6 +317,7 @@ describe("OpenAiHandler with usage tracking fix", () => { ["string", "10"], ["object", { tokens: 10 }], ["negative", -1], + ["fractional", 10.5], ["non-finite", Number.POSITIVE_INFINITY], ["greater than prompt tokens", 101], ])("ignores invalid %s cached prompt tokens in non-streaming responses", async (_name, cachedTokens) => { @@ -334,4 +336,61 @@ describe("OpenAiHandler with usage tracking fix", () => { expect(chunks).toContainEqual({ type: "usage", inputTokens: 100, outputTokens: 5 }) }) + + describe.each([true, false])("cache accounting with streaming=%s", (openAiStreamingEnabled) => { + it.each([ + ["authoritative count", 23, 23], + ["authoritative zero", 0, undefined], + ["fully cached prompt", 100, 100], + ["null falls back", null, 71], + ["absent falls back", undefined, 71], + ["negative", -1, undefined], + ["fractional", 10.5, undefined], + ["infinite", Infinity, undefined], + ["NaN", NaN, undefined], + ["string", "23", undefined], + ["object", { tokens: 23 }, undefined], + ["exceeds input", 101, undefined], + ])("respects %s without substituting a conflicting fallback", async (_name, reported, expected) => { + const cacheHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled }) + const usage = { + prompt_tokens: 100, + completion_tokens: 5, + cache_creation_input_tokens: 7, + cache_read_input_tokens: reported, + prompt_tokens_details: { cached_tokens: 71 }, + } + mockCreate.mockResolvedValueOnce( + openAiStreamingEnabled + ? asyncStreamFrom([{ choices: [], usage }]) + : { choices: [{ message: { content: "Response" } }], usage }, + ) + + const chunks = await collectStream(cacheHandler.createMessage("system prompt", [])) + + expect(chunks.filter((chunk) => chunk.type === "usage")).toEqual([ + { + type: "usage", + inputTokens: 100, + outputTokens: 5, + cacheWriteTokens: 7, + cacheReadTokens: expected, + }, + ]) + }) + }) + + it.each([null, undefined, {}, { prompt_tokens_details: null }])( + "defaults missing non-streaming usage counters to zero: %j", + async (usage) => { + const nonStreamingHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled: false }) + mockCreate.mockResolvedValueOnce({ choices: [{ message: { content: "Response" } }], usage }) + + const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", [])) + + expect(chunks.filter((chunk) => chunk.type === "usage")).toEqual([ + { type: "usage", inputTokens: 0, outputTokens: 0 }, + ]) + }, + ) }) diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index a07ef121bf..eb04e04312 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -298,6 +298,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl const cacheReadTokens = typeof reportedCacheReadTokens === "number" && Number.isFinite(reportedCacheReadTokens) && + Number.isInteger(reportedCacheReadTokens) && reportedCacheReadTokens >= 0 && reportedCacheReadTokens <= inputTokens ? reportedCacheReadTokens || undefined From f6eb03b64d031496bc5784a20082c747ac076078 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Mon, 5 Oct 2026 17:53:56 +0200 Subject: [PATCH 4/5] fix(openai): preserve proxy cache counters and cover nonstreaming o3 --- .../__tests__/openai-usage-tracking.spec.ts | 49 +++++++++++-------- src/api/providers/openai.ts | 4 +- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/api/providers/__tests__/openai-usage-tracking.spec.ts b/src/api/providers/__tests__/openai-usage-tracking.spec.ts index 22ab2daf97..6ac5f1a526 100644 --- a/src/api/providers/__tests__/openai-usage-tracking.spec.ts +++ b/src/api/providers/__tests__/openai-usage-tracking.spec.ts @@ -238,28 +238,35 @@ describe("OpenAiHandler with usage tracking fix", () => { }) }) - it("should report cached prompt tokens from a non-streaming response", async () => { - const nonStreamingHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled: false }) - mockCreate.mockImplementationOnce(async () => ({ - id: "test-completion", - choices: [{ message: { role: "assistant", content: "Cached response" } }], - usage: { - prompt_tokens: 4_621, - completion_tokens: 16, - total_tokens: 4_637, - prompt_tokens_details: { cached_tokens: 4_608 }, - }, - })) + it.each(["gpt-4", "o3-mini"])( + "reports cached prompt tokens from a non-streaming %s response", + async (openAiModelId) => { + const nonStreamingHandler = new OpenAiHandler({ + ...mockOptions, + openAiModelId, + openAiStreamingEnabled: false, + }) + mockCreate.mockImplementationOnce(async () => ({ + id: "test-completion", + choices: [{ message: { role: "assistant", content: "Cached response" } }], + usage: { + prompt_tokens: 4_621, + completion_tokens: 16, + total_tokens: 4_637, + prompt_tokens_details: { cached_tokens: 4_608 }, + }, + })) - const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", [])) + const chunks = await collectStream(nonStreamingHandler.createMessage("system prompt", [])) - expect(chunks).toContainEqual({ - type: "usage", - inputTokens: 4_621, - outputTokens: 16, - cacheReadTokens: 4_608, - }) - }) + expect(chunks).toContainEqual({ + type: "usage", + inputTokens: 4_621, + outputTokens: 16, + cacheReadTokens: 4_608, + }) + }, + ) it("reports cached prompt tokens for a streaming O3 response", async () => { const o3Handler = new OpenAiHandler({ ...mockOptions, openAiModelId: "o3-mini" }) @@ -350,7 +357,7 @@ describe("OpenAiHandler with usage tracking fix", () => { ["NaN", NaN, undefined], ["string", "23", undefined], ["object", { tokens: 23 }, undefined], - ["exceeds input", 101, undefined], + ["proxy cache reads exceed uncached input", 101, 101], ])("respects %s without substituting a conflicting fallback", async (_name, reported, expected) => { const cacheHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled }) const usage = { diff --git a/src/api/providers/openai.ts b/src/api/providers/openai.ts index eb04e04312..6fa55de3b3 100644 --- a/src/api/providers/openai.ts +++ b/src/api/providers/openai.ts @@ -300,7 +300,9 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl Number.isFinite(reportedCacheReadTokens) && Number.isInteger(reportedCacheReadTokens) && reportedCacheReadTokens >= 0 && - reportedCacheReadTokens <= inputTokens + // Proxy-specific counters may exclude cache reads from prompt_tokens. + // Only OpenAI's nested cached_tokens must be a subset of prompt_tokens. + (usage?.cache_read_input_tokens != null || reportedCacheReadTokens <= inputTokens) ? reportedCacheReadTokens || undefined : undefined From 0fbcc962460d8de6de500d93f963bd1c27d3959e Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Sun, 25 Oct 2026 11:51:33 +0100 Subject: [PATCH 5/5] test(openai): pin the cached_tokens upper bound when it equals prompt_tokens Co-Authored-By: Claude Sonnet 5.5 --- .../__tests__/openai-usage-tracking.spec.ts | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/api/providers/__tests__/openai-usage-tracking.spec.ts b/src/api/providers/__tests__/openai-usage-tracking.spec.ts index 6ac5f1a526..0441270498 100644 --- a/src/api/providers/__tests__/openai-usage-tracking.spec.ts +++ b/src/api/providers/__tests__/openai-usage-tracking.spec.ts @@ -385,6 +385,34 @@ describe("OpenAiHandler with usage tracking fix", () => { }, ]) }) + + // Only OpenAI's nested counter must be a subset of prompt_tokens; this pins the inclusive boundary. + it.each([ + ["below the prompt size", 99, 99], + ["equal to the prompt size", 100, 100], + ["above the prompt size", 101, undefined], + ])( + "accepts a nested cached_tokens count %s with no cache_read_input_tokens", + async (_name, cached, expected) => { + const cacheHandler = new OpenAiHandler({ ...mockOptions, openAiStreamingEnabled }) + const usage = { + prompt_tokens: 100, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: cached }, + } + mockCreate.mockResolvedValueOnce( + openAiStreamingEnabled + ? asyncStreamFrom([{ choices: [], usage }]) + : { choices: [{ message: { content: "Response" } }], usage }, + ) + + const chunks = await collectStream(cacheHandler.createMessage("system prompt", [])) + + expect(chunks.filter((chunk) => chunk.type === "usage")).toEqual([ + { type: "usage", inputTokens: 100, outputTokens: 5, cacheReadTokens: expected }, + ]) + }, + ) }) it.each([null, undefined, {}, { prompt_tokens_details: null }])(