Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 54 additions & 2 deletions packages/ai/src/providers/openai-completions-usage-parity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,57 @@ const context = {

type UsageScenario = {
name: string;
usage: Record<string, unknown>;
usage?: Record<string, unknown>;
trailingUsage?: Record<string, unknown>;
expectedUsage: Record<string, unknown>;
expectedProvenance?: "partial" | "unmeasured";
expectedCost?: number;
inChoice?: boolean;
};

const scenarios: UsageScenario[] = [
{
name: "a completed stream without a usage chunk",
expectedUsage: { totalTokens: 0 },
expectedProvenance: "unmeasured",
},
{
name: "provider total in a trailing usage-only chunk",
trailingUsage: { total_tokens: 42 },
expectedUsage: { totalTokens: 42 },
},
{
name: "provider aggregate above the component subtotal",
usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 17 },
expectedUsage: { input: 10, output: 5, totalTokens: 17 },
},
{
name: "partial token usage with billed cost",
usage: { prompt_tokens: 12, cost: 0.025 },
expectedUsage: {
input: 12,
totalTokens: 12,
cost: { total: 0.025, totalOrigin: "provider-billed" },
},
expectedProvenance: "partial",
},
{
name: "explicit measured zero",
trailingUsage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
expectedUsage: { totalTokens: 0 },
},
{
name: "complete usage followed by an empty usage object",
usage: { prompt_tokens: 20, completion_tokens: 10, total_tokens: 30 },
trailingUsage: {},
expectedUsage: { input: 20, output: 10, totalTokens: 30 },
},
{
name: "complete usage followed by billed cost only",
usage: { prompt_tokens: 20, completion_tokens: 10, total_tokens: 30 },
trailingUsage: { cost: 0.025 },
expectedUsage: { totalTokens: 30, cost: { total: 0.025, totalOrigin: "provider-billed" } },
},
{
name: "documented reasoning tokens and cache buckets",
usage: {
Expand Down Expand Up @@ -131,7 +175,10 @@ function installUsageChunk(scenario: UsageScenario): void {
choices: [choice],
...(scenario.inChoice ? {} : { usage: scenario.usage }),
} as ChatCompletionChunk;
const body = `data: ${JSON.stringify(chunk)}\n\ndata: [DONE]\n\n`;
const chunks = scenario.trailingUsage
? [chunk, { ...chunk, choices: [], usage: scenario.trailingUsage }]
: [chunk];
const body = `${chunks.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("")}data: [DONE]\n\n`;
configureAiTransportHost({
buildModelFetch: () => async () =>
new Response(body, {
Expand Down Expand Up @@ -187,6 +234,11 @@ describe.each([

expect(result.stopReason).toBe("stop");
expect(result.usage).toMatchObject(expectedUsage);
if (scenario.expectedProvenance) {
expect(result).toHaveProperty("openclawUsageProvenance", scenario.expectedProvenance);
} else {
expect(result).not.toHaveProperty("openclawUsageProvenance");
}
if (scenario.expectedCost !== undefined) {
expect(result.usage.cost.total).toBeCloseTo(scenario.expectedCost, 10);
}
Expand Down
14 changes: 9 additions & 5 deletions packages/ai/src/providers/openai-completions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@ import {
resolveOpenAICompletionsCompat,
type ResolvedOpenAICompletionsCompat,
} from "../transports/openai-completions-compat.js";
import {
applyOpenAICompletionsUsage,
type OpenAICompletionsUsageTarget,
} from "../transports/openai-completions-usage.js";
import { resolveOpenAIReasoningEffortMap } from "../transports/openai-reasoning-compat.js";
import {
isOpenAICompletionsThinkingEnabled,
parseOpenAICompletionsUsage,
readOpenAICompletionsContentDeltas,
} from "../transports/openai-transport-shared.js";
import { transportAbortError } from "../transports/transport-stream-shared.js";
Expand Down Expand Up @@ -144,12 +147,13 @@ export const streamOpenAICompletions: StreamFunction<
const stream = new AssistantMessageEventStream();

void (async () => {
const output: AssistantMessage = {
const output: AssistantMessage & OpenAICompletionsUsageTarget = {
role: "assistant",
content: [],
api: model.api,
provider: model.provider,
model: model.id,
openclawUsageProvenance: "unmeasured",
usage: {
input: 0,
output: 0,
Expand Down Expand Up @@ -406,7 +410,7 @@ export const streamOpenAICompletions: StreamFunction<
output.responseModel ||= chunk.model;
}
if (chunk.usage) {
output.usage = parseOpenAICompletionsUsage(chunk.usage, model, {
applyOpenAICompletionsUsage(output, chunk.usage, model, {
includeReasoningTokens: false,
});
}
Expand All @@ -419,10 +423,10 @@ export const streamOpenAICompletions: StreamFunction<
// Fallback: some providers (e.g., Moonshot) return usage
// in choice.usage instead of the standard chunk.usage
const choiceUsage = (
choice as typeof choice & { usage?: Parameters<typeof parseOpenAICompletionsUsage>[0] }
choice as typeof choice & { usage?: Parameters<typeof applyOpenAICompletionsUsage>[1] }
).usage;
if (!chunk.usage && choiceUsage) {
output.usage = parseOpenAICompletionsUsage(choiceUsage, model, {
applyOpenAICompletionsUsage(output, choiceUsage, model, {
includeReasoningTokens: false,
});
}
Expand Down
11 changes: 8 additions & 3 deletions packages/ai/src/transports/openai-completions-transport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@ import {
flattenCompletionMessagesToStringContent,
stripCompletionMessagesToRoleContent,
} from "./openai-completions-string-content.js";
import {
applyOpenAICompletionsUsage,
type OpenAICompletionsUsageTarget,
} from "./openai-completions-usage.js";
import type { OpenAIModelInput } from "./openai-model-input-internal.js";
import {
assertCodeModeResponsesToolSurface,
Expand Down Expand Up @@ -265,12 +269,13 @@ export function createOpenAICompletionsTransportStreamFn(): StreamFn {
const eventStream = createAssistantMessageEventStream();
const stream = eventStream as unknown as { push(event: unknown): void; end(): void };
void (async () => {
const output: MutableAssistantOutput = {
const output: MutableAssistantOutput & OpenAICompletionsUsageTarget = {
role: "assistant" as const,
content: [],
api: model.api,
provider: model.provider,
model: model.id,
openclawUsageProvenance: "unmeasured",
usage: {
input: 0,
output: 0,
Expand Down Expand Up @@ -661,7 +666,7 @@ async function processOpenAICompletionsStream(
output.responseId ||= chunk.id;
let hasReasoningUsageActivity = false;
if (chunk.usage) {
output.usage = parseOpenAICompletionsUsage(chunk.usage, model);
applyOpenAICompletionsUsage(output, chunk.usage, model);
hasReasoningUsageActivity = hasOpenAICompletionsReasoningUsageActivity(chunk.usage);
}
const choice = Array.isArray(chunk.choices) ? chunk.choices[0] : undefined;
Expand All @@ -672,7 +677,7 @@ async function processOpenAICompletionsStream(
}
const choiceUsage = (choice as unknown as { usage?: ChatCompletionChunk["usage"] }).usage;
if (!chunk.usage && choiceUsage) {
output.usage = parseOpenAICompletionsUsage(choiceUsage, model);
applyOpenAICompletionsUsage(output, choiceUsage, model);
hasReasoningUsageActivity = hasOpenAICompletionsReasoningUsageActivity(choiceUsage);
}
if (choice.finish_reason) {
Expand Down
100 changes: 100 additions & 0 deletions packages/ai/src/transports/openai-completions-usage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
import type { Model, Usage } from "@openclaw/llm-core";
import { parseStrictNonNegativeInteger } from "@openclaw/normalization-core/number-coercion";
import type { ChatCompletionChunk } from "openai/resources/chat/completions.js";
import { applyProviderReportedUsageCost, calculateCost } from "../model-utils.js";

export type OpenAICompletionsUsageTarget = {
usage: Usage & { reasoningTokens?: number };
openclawUsageProvenance?: "unmeasured" | "partial";
};

type RawOpenAICompletionsUsage = NonNullable<ChatCompletionChunk["usage"]> & {
cost?: unknown;
prompt_cache_hit_tokens?: number;
};

type OpenAICompletionsUsageOptions = { includeReasoningTokens?: boolean };

function readOpenAICompletionsUsage(
rawUsage: RawOpenAICompletionsUsage,
model: Model,
options?: OpenAICompletionsUsageOptions,
): OpenAICompletionsUsageTarget {
const prompt = parseStrictNonNegativeInteger(rawUsage.prompt_tokens);
const completion = parseStrictNonNegativeInteger(rawUsage.completion_tokens);
const providerTotal = parseStrictNonNegativeInteger(rawUsage.total_tokens);
const cached =
parseStrictNonNegativeInteger(rawUsage.prompt_tokens_details?.cached_tokens) ??
parseStrictNonNegativeInteger(rawUsage.prompt_cache_hit_tokens);
const cacheWritten = parseStrictNonNegativeInteger(
rawUsage.prompt_tokens_details?.cache_write_tokens,
);
const cacheRead = cached ?? 0;
const cacheWrite = cacheWritten ?? 0;
const input = Math.max(0, (prompt ?? 0) - cacheRead - cacheWrite);
const output = completion ?? 0;
const componentTotal = input + output + cacheRead + cacheWrite;
const hasCompleteComponents = prompt !== undefined && completion !== undefined;
// Compatible providers can report an aggregate independently of its components.
// Reject totals smaller than the recorded subtotal, including cache overflow.
const measuredAggregate =
providerTotal !== undefined && providerTotal >= componentTotal ? providerTotal : undefined;
const reasoningTokens = rawUsage.completion_tokens_details?.reasoning_tokens;
const usage: OpenAICompletionsUsageTarget["usage"] = {
input,
output,
cacheRead,
cacheWrite,
// Managed transport exposes reasoning telemetry; the shipped package Usage shape does not.
...(options?.includeReasoningTokens !== false &&
typeof reasoningTokens === "number" &&
Number.isFinite(reasoningTokens)
? { reasoningTokens }
: {}),
totalTokens: measuredAggregate ?? componentTotal,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
calculateCost(model, usage);
applyProviderReportedUsageCost(usage, rawUsage.cost);
if (hasCompleteComponents || measuredAggregate !== undefined) {
return { usage };
}
const hasComponents = [prompt, completion, cached, cacheWritten].some(
(count) => count !== undefined,
);
return { usage, openclawUsageProvenance: hasComponents ? "partial" : "unmeasured" };
}

export function applyOpenAICompletionsUsage(
target: OpenAICompletionsUsageTarget,
rawUsage: RawOpenAICompletionsUsage,
model: Model,
options?: OpenAICompletionsUsageOptions,
): void {
const sample = readOpenAICompletionsUsage(rawUsage, model, options);
if (
sample.openclawUsageProvenance &&
(target.openclawUsageProvenance === undefined ||
(sample.openclawUsageProvenance === "unmeasured" &&
target.openclawUsageProvenance === "partial"))
) {
if (sample.usage.cost.totalOrigin === "provider-billed") {
applyProviderReportedUsageCost(target.usage, sample.usage.cost.total);
}
return;
}
target.usage = sample.usage;
if (sample.openclawUsageProvenance) {
target.openclawUsageProvenance = sample.openclawUsageProvenance;
} else {
delete target.openclawUsageProvenance;
}
}

export function parseOpenAICompletionsUsage(
rawUsage: RawOpenAICompletionsUsage,
model: Model,
options?: OpenAICompletionsUsageOptions,
): OpenAICompletionsUsageTarget["usage"] {
return readOpenAICompletionsUsage(rawUsage, model, options).usage;
}
37 changes: 2 additions & 35 deletions packages/ai/src/transports/openai-transport-shared.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import type { Api, Model, OpenAICompletionsCompat, Usage } from "@openclaw/llm-core";
import type { ChatCompletionChunk } from "openai/resources/chat/completions.js";
import { getAiTransportHost } from "../host.js";
import { applyProviderReportedUsageCost, calculateCost } from "../model-utils.js";
import type { BaseOpenAIStreamOptions } from "../provider-options.js";
/** Shared options, usage shape, cache identity, ordering, and stream scheduling for OpenAI APIs. */
import { clampOpenAIPromptCacheKey } from "../providers/openai-prompt-cache.js";
Expand All @@ -25,6 +23,8 @@ export const log = {
},
};

export { parseOpenAICompletionsUsage } from "./openai-completions-usage.js";

export type { OpenAICompletionsOptions } from "../provider-options.js";

export type OpenAICompletionsContentDelta =
Expand Down Expand Up @@ -68,39 +68,6 @@ export type MutableAssistantOutput = {
errorBody?: string;
};

export function parseOpenAICompletionsUsage(
rawUsage: NonNullable<ChatCompletionChunk["usage"]> & {
cost?: unknown;
prompt_cache_hit_tokens?: number;
},
model: Model,
options?: { includeReasoningTokens?: boolean },
): MutableAssistantOutput["usage"] {
const cacheRead =
rawUsage.prompt_tokens_details?.cached_tokens ?? rawUsage.prompt_cache_hit_tokens ?? 0;
const cacheWrite = rawUsage.prompt_tokens_details?.cache_write_tokens || 0;
const input = Math.max(0, (rawUsage.prompt_tokens || 0) - cacheRead - cacheWrite);
const output = rawUsage.completion_tokens || 0;
const reasoningTokens = rawUsage.completion_tokens_details?.reasoning_tokens;
const usage: MutableAssistantOutput["usage"] = {
input,
output,
cacheRead,
cacheWrite,
// Managed transport exposes reasoning telemetry; the shipped package Usage shape does not.
...(options?.includeReasoningTokens !== false &&
typeof reasoningTokens === "number" &&
Number.isFinite(reasoningTokens)
? { reasoningTokens }
: {}),
totalTokens: input + output + cacheRead + cacheWrite,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
};
calculateCost(model, usage);
applyProviderReportedUsageCost(usage, rawUsage.cost);
return usage;
}

type ModelStreamCooperativeScheduler = {
afterEvent: () => Promise<void>;
};
Expand Down
5 changes: 5 additions & 0 deletions src/infra/session-cost-usage-accounting.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ describe("session usage accounting conservation", () => {
{
provider: "example-provider",
model: "unmeasured",
usage: { ...placeholder, cost: { total: 0.025, totalOrigin: "provider-billed" } },
openclawUsageProvenance: "unmeasured",
},
{
Expand Down Expand Up @@ -209,6 +210,10 @@ describe("session usage accounting conservation", () => {
missingTokenEntries: 1,
});
}
expect(first.modelUsage?.find((entry) => entry.model === "unmeasured")).toMatchObject({
count: 1,
totals: { totalCost: 0.025, missingTokenEntries: 1 },
});
expect(
first.dailyModelUsage?.find((entry) => entry.model === "measured-cli"),
).toMatchObject({
Expand Down
6 changes: 5 additions & 1 deletion src/infra/session-cost-usage-pricing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,11 @@ const parseTranscriptEntry = (entry: Record<string, unknown>): ParsedTranscriptE
[usage?.total, usage?.input, usage?.output, usage?.cacheRead, usage?.cacheWrite].every(
(value) => value === undefined || value === 0,
)));
const costBreakdown = partial || unmeasured ? undefined : extractCostBreakdown(usageRaw);
const recordedCost = extractCostBreakdown(usageRaw);
const costBreakdown =
(partial || unmeasured) && recordedCost?.totalOrigin !== "provider-billed"
? undefined
: recordedCost;
const stopReason = typeof message.stopReason === "string" ? message.stopReason : undefined;
const durationMs = asFiniteNumber(message.durationMs ?? entry.durationMs);

Expand Down