From 1b97734ebac7c96c86035b671c989af7c821c957 Mon Sep 17 00:00:00 2001 From: devSejung Date: Thu, 5 Nov 2026 17:53:38 +0900 Subject: [PATCH] fix: report measured AI usage and stale snapshots accurately --- docs/platformclaw/ai-usage-report-api.md | 6 ++ .../ai-usage-report-api.openapi.json | 4 +- .../event-projector-assistant-message.ts | 51 ++++++++---- .../event-projector-tool-transcript.ts | 1 + .../event-projector.commentary.test.ts | 8 ++ .../event-projector.reasoning.test.ts | 6 ++ .../codex/src/app-server/event-projector.ts | 6 +- .../app-server/event-projector.usage.test.ts | 25 ++++++ .../src/app-server/transcript-mirror.test.ts | 24 ++++-- .../codex/src/app-server/transcript-mirror.ts | 17 +++- src/agents/cli-output.test.ts | 35 +++++++- src/agents/cli-output.ts | 25 +++--- src/agents/cli-runner.reliability.test.ts | 2 + src/agents/cli-runner.ts | 33 ++++---- .../assistant-transcript-repair.test.ts | 10 +++ .../command/assistant-transcript-repair.ts | 18 +---- .../command/attempt-execution.cli.test.ts | 38 +++++++++ src/agents/command/attempt-execution.ts | 42 +--------- src/agents/stream-message-shared.ts | 32 ++++++++ .../usage.sessions-usage-cache.test.ts | 71 +++++++++++++--- src/gateway/server-methods/usage.ts | 25 +++++- .../session-cost-usage-accounting.test.ts | 80 +++++++++++++++++-- src/infra/session-cost-usage-aggregation.ts | 4 +- src/infra/session-cost-usage-pricing.ts | 46 ++++++++--- src/infra/session-cost-usage.test.ts | 2 +- src/infra/session-cost-usage.types.ts | 1 + 26 files changed, 470 insertions(+), 142 deletions(-) diff --git a/docs/platformclaw/ai-usage-report-api.md b/docs/platformclaw/ai-usage-report-api.md index b861a0171cf..7931e4d6f20 100644 --- a/docs/platformclaw/ai-usage-report-api.md +++ b/docs/platformclaw/ai-usage-report-api.md @@ -246,6 +246,12 @@ Without a provider total, both input and output counts must be recorded; absent cache buckets follow the existing normalized zero-default convention. A missing input or output count makes the affected total unknown, even when another component is positive. +Transcript messages written only for display or delivery do not count as usage +records. A recorded model reply without token measurements still contributes to +`call_count`, but its token total is null. Older ACP and CLI transcript writers +used zero placeholders; when the record cannot establish a measured zero, the +report treats those placeholders as unknown. Rebuilding the usage cache cannot +recover token counts that were never recorded. This is an activity metric, not an invoice reconciliation API. The initial scope is owned personal agents, including historical activity of diff --git a/docs/platformclaw/ai-usage-report-api.openapi.json b/docs/platformclaw/ai-usage-report-api.openapi.json index be1428779f0..a01f0d79e02 100644 --- a/docs/platformclaw/ai-usage-report-api.openapi.json +++ b/docs/platformclaw/ai-usage-report-api.openapi.json @@ -1163,12 +1163,12 @@ "call_count_basis": { "type": "string", "const": "usage_records", - "description": "Count persisted model usage records, not provider HTTP attempts." + "description": "Count persisted model usage records, including recorded replies with unknown token measurements. Excludes transcript-only display and delivery records." }, "token_usage_basis": { "type": "string", "const": "provider_total_or_components", - "description": "Provider total when recorded, otherwise normalized nonduplicated input/output/cache components. Without a total, both input and output must be recorded; absent cache buckets follow normalized zero defaults. Missing input or output makes the total unknown." + "description": "Provider total when measured, otherwise normalized nonduplicated input/output/cache components. Without a total, both input and output must be recorded; absent cache buckets follow normalized zero defaults. Unmeasured usage and legacy zero placeholders produce null token totals." }, "coverage": { "type": "string", diff --git a/extensions/codex/src/app-server/event-projector-assistant-message.ts b/extensions/codex/src/app-server/event-projector-assistant-message.ts index 3415eeeeaf0..4e67b88fcad 100644 --- a/extensions/codex/src/app-server/event-projector-assistant-message.ts +++ b/extensions/codex/src/app-server/event-projector-assistant-message.ts @@ -18,6 +18,11 @@ export type AssistantMessageOptions = { | undefined; aborted: boolean; promptError: unknown; + usageUnmeasured?: boolean; +}; + +export type CodexUsageProvenance = { + openclawUsageProvenance?: "transcript-only" | "unmeasured" | "partial"; }; const ZERO_USAGE: Usage = { @@ -39,23 +44,31 @@ export function createAssistantMessage( params: EmbeddedRunAttemptParams, text: string, options: AssistantMessageOptions, -): AssistantMessage { +): AssistantMessage & CodexUsageProvenance { const attribution = resolveCodexLocalRuntimeAttribution(params); - const usage: Usage = options.tokenUsage + const { tokenUsage } = options; + const hasMeasuredTotal = + tokenUsage?.total !== undefined || + (tokenUsage?.input !== undefined && tokenUsage.output !== undefined); + const hasMeasuredComponents = [ + tokenUsage?.input, + tokenUsage?.output, + tokenUsage?.cacheRead, + tokenUsage?.cacheWrite, + ].some((value) => value !== undefined); + const usage: Usage = tokenUsage ? { - input: options.tokenUsage.input ?? 0, - output: options.tokenUsage.output ?? 0, - cacheRead: options.tokenUsage.cacheRead ?? 0, - cacheWrite: options.tokenUsage.cacheWrite ?? 0, - ...(options.tokenUsage.contextUsage - ? { contextUsage: options.tokenUsage.contextUsage } - : {}), + input: tokenUsage.input ?? 0, + output: tokenUsage.output ?? 0, + cacheRead: tokenUsage.cacheRead ?? 0, + cacheWrite: tokenUsage.cacheWrite ?? 0, + ...(tokenUsage.contextUsage ? { contextUsage: tokenUsage.contextUsage } : {}), totalTokens: - options.tokenUsage.total ?? - (options.tokenUsage.input ?? 0) + - (options.tokenUsage.output ?? 0) + - (options.tokenUsage.cacheRead ?? 0) + - (options.tokenUsage.cacheWrite ?? 0), + tokenUsage.total ?? + (tokenUsage.input ?? 0) + + (tokenUsage.output ?? 0) + + (tokenUsage.cacheRead ?? 0) + + (tokenUsage.cacheWrite ?? 0), cost: ZERO_USAGE.cost, } : ZERO_USAGE; @@ -66,6 +79,12 @@ export function createAssistantMessage( provider: attribution.provider, model: params.modelId, usage, + ...(options.usageUnmeasured || !hasMeasuredTotal + ? { + openclawUsageProvenance: + !options.usageUnmeasured && hasMeasuredComponents ? "partial" : "unmeasured", + } + : {}), stopReason: options.aborted ? "aborted" : options.promptError ? "error" : "stop", errorMessage: options.promptError ? formatErrorMessage(options.promptError) : undefined, timestamp: Date.now(), @@ -86,6 +105,7 @@ export function createAssistantCommentaryMessage( provider: attribution.provider, model: params.modelId, usage: ZERO_USAGE, + openclawUsageProvenance: "transcript-only", stopReason: "stop", timestamp, // Keep this unphased: gateway history hides commentary-phase assistant rows. @@ -102,7 +122,7 @@ export function createAssistantMirrorMessage( params: EmbeddedRunAttemptParams, title: string, text: string, -): AssistantMessage { +): AssistantMessage & CodexUsageProvenance { const attribution = resolveCodexLocalRuntimeAttribution(params); return { role: "assistant", @@ -111,6 +131,7 @@ export function createAssistantMirrorMessage( provider: attribution.provider, model: params.modelId, usage: ZERO_USAGE, + openclawUsageProvenance: "transcript-only", stopReason: "stop", timestamp: Date.now(), }; diff --git a/extensions/codex/src/app-server/event-projector-tool-transcript.ts b/extensions/codex/src/app-server/event-projector-tool-transcript.ts index 0f390c90950..b38dec59028 100644 --- a/extensions/codex/src/app-server/event-projector-tool-transcript.ts +++ b/extensions/codex/src/app-server/event-projector-tool-transcript.ts @@ -594,6 +594,7 @@ export class CodexToolTranscriptProjection { provider: attribution.provider, model: this.params.modelId, usage: ZERO_USAGE, + openclawUsageProvenance: "transcript-only", stopReason: "toolUse", timestamp: this.nextTranscriptTimestamp(), } as unknown as AgentMessage; diff --git a/extensions/codex/src/app-server/event-projector.commentary.test.ts b/extensions/codex/src/app-server/event-projector.commentary.test.ts index 8a493017f79..25d79234731 100644 --- a/extensions/codex/src/app-server/event-projector.commentary.test.ts +++ b/extensions/codex/src/app-server/event-projector.commentary.test.ts @@ -277,6 +277,7 @@ describe("CodexAppServerEventProjector commentary projection", () => { expect(commentary).toMatchObject({ role: "assistant", content: [{ type: "text", text: "Checking the app-server stream" }], + openclawUsageProvenance: "transcript-only", openclawStreamFallback: { replacementText: "Checking the app-server stream", source: "segment", @@ -373,6 +374,13 @@ describe("CodexAppServerEventProjector commentary projection", () => { `${TURN_ID}:tool:call-search:result`, `${TURN_ID}:commentary:msg-after-tool`, ]); + expect(result.messagesSnapshot).toContainEqual( + expect.objectContaining({ + role: "assistant", + openclawUsageProvenance: "transcript-only", + __openclaw: expect.objectContaining({ mirrorIdentity: `${TURN_ID}:tool:call-search:call` }), + }), + ); }); it("does not double-deliver a commentary note echoed on the raw response lane", async () => { diff --git a/extensions/codex/src/app-server/event-projector.reasoning.test.ts b/extensions/codex/src/app-server/event-projector.reasoning.test.ts index 901da4477d2..6460d2a6ce2 100644 --- a/extensions/codex/src/app-server/event-projector.reasoning.test.ts +++ b/extensions/codex/src/app-server/event-projector.reasoning.test.ts @@ -186,6 +186,12 @@ describe("CodexAppServerEventProjector reasoning and guardian projection", () => ]); expect(JSON.stringify(result.messagesSnapshot[1])).toContain("Codex reasoning"); expect(JSON.stringify(result.messagesSnapshot[2])).toContain("Codex plan"); + expect(result.messagesSnapshot[1]).toMatchObject({ + openclawUsageProvenance: "transcript-only", + }); + expect(result.messagesSnapshot[2]).toMatchObject({ + openclawUsageProvenance: "transcript-only", + }); expect(JSON.stringify(result.messagesSnapshot[2])).toContain("next"); expect(JSON.stringify(result.messagesSnapshot[2])).toContain("[in_progress] patch"); expect(result.compactionCount).toBe(1); diff --git a/extensions/codex/src/app-server/event-projector.ts b/extensions/codex/src/app-server/event-projector.ts index d72d32ab638..aa46897cdfd 100644 --- a/extensions/codex/src/app-server/event-projector.ts +++ b/extensions/codex/src/app-server/event-projector.ts @@ -368,6 +368,9 @@ export class CodexAppServerEventProjector { } const assistantMessageOptions = { tokenUsage: projectedUsage, + // Thread snapshots can replay an earlier response. Keep them for live + // context, but require current-response measurements for accounting. + usageUnmeasured: this.aborted || this.responseCompletions.usage === undefined, aborted: this.aborted, promptError: this.promptError, }; @@ -415,7 +418,6 @@ export class CodexAppServerEventProjector { promptError, turnCompleted: Boolean(this.completedTurn), }); - const toolMetas = this.toolProgressProjection.toolMetas; const hadPotentialSideEffects = toolTelemetry.didSendViaMessagingTool || Boolean(toolTelemetry.successfulCronAdds || toolTelemetry.acceptedSessionSpawns?.length) || @@ -436,7 +438,7 @@ export class CodexAppServerEventProjector { : {}), messagesSnapshot, assistantTexts, - toolMetas, + toolMetas: this.toolProgressProjection.toolMetas, lastAssistant, currentAttemptAssistant, ...(this.toolProgressProjection.lastToolError diff --git a/extensions/codex/src/app-server/event-projector.usage.test.ts b/extensions/codex/src/app-server/event-projector.usage.test.ts index 023998b4faa..cd9d29f1d56 100644 --- a/extensions/codex/src/app-server/event-projector.usage.test.ts +++ b/extensions/codex/src/app-server/event-projector.usage.test.ts @@ -95,6 +95,28 @@ describe("CodexAppServerEventProjector usage projection", () => { expect(projector.buildResult(buildEmptyToolTelemetry()).modelIterations).toBe(2); }); + it("preserves a measured zero-token response", async () => { + const projector = await createProjector(); + + await projector.handleNotification(agentMessageDelta("done")); + await projector.handleNotification( + forCurrentTurn("rawResponse/completed", { + responseId: "response-1", + usage: { + totalTokens: 0, + inputTokens: 0, + cachedInputTokens: 0, + outputTokens: 0, + reasoningOutputTokens: 0, + }, + }), + ); + + const result = projector.buildResult(buildEmptyToolTelemetry()); + expect(result.lastAssistant).toMatchObject({ usage: { totalTokens: 0 } }); + expect(result.lastAssistant).not.toHaveProperty("openclawUsageProvenance"); + }); + it("keeps cumulative-only thread usage unknown", async () => { const projector = await createProjector(); @@ -130,6 +152,7 @@ describe("CodexAppServerEventProjector usage projection", () => { total: 12, }); expect(result.lastAssistant?.usage.contextUsage).toEqual({ state: "unavailable" }); + expect(result.lastAssistant).toMatchObject({ openclawUsageProvenance: "unmeasured" }); }); it.each([ @@ -168,6 +191,7 @@ describe("CodexAppServerEventProjector usage projection", () => { expect(result.assistantTexts).toEqual(["done"]); expect(result.attemptUsage).toBeUndefined(); expect(result.lastAssistant?.usage.contextUsage).toBeUndefined(); + expect(result.lastAssistant).toMatchObject({ openclawUsageProvenance: "unmeasured" }); }); it("clears prior response usage when the final response omits usage", async () => { @@ -207,6 +231,7 @@ describe("CodexAppServerEventProjector usage projection", () => { expectUsageFields(result.attemptUsage, { input: 3, output: 7, cacheRead: 2, total: 12 }); expect(result.attemptUsage?.contextUsage).toEqual({ state: "unavailable" }); expect(result.lastAssistant?.usage.contextUsage).toEqual({ state: "unavailable" }); + expect(result.lastAssistant).toMatchObject({ openclawUsageProvenance: "unmeasured" }); }); it.each(["failed", "interrupted"])( diff --git a/extensions/codex/src/app-server/transcript-mirror.test.ts b/extensions/codex/src/app-server/transcript-mirror.test.ts index a1865479568..221d2195a1b 100644 --- a/extensions/codex/src/app-server/transcript-mirror.test.ts +++ b/extensions/codex/src/app-server/transcript-mirror.test.ts @@ -521,6 +521,7 @@ describe("importCodexThreadHistoryToTranscript", () => { provider: "source-provider", model: "native-history", usage: { totalTokens: 0 }, + openclawUsageProvenance: "transcript-only", stopReason: "stop", timestamp: expect.any(Number), }, @@ -1287,7 +1288,7 @@ describe("mirrorCodexAppServerTranscript", () => { } as unknown as Parameters[0]["params"], result: { messagesSnapshot: [assistantMessage], - } as Parameters[0]["result"], + }, notifyUserMessagePersisted: () => undefined, cwd: root, threadId: "thread-1", @@ -1328,7 +1329,7 @@ describe("mirrorCodexAppServerTranscript", () => { } as unknown as Parameters[0]["params"], result: { messagesSnapshot: [currentMessage], - } as Parameters[0]["result"], + }, agentId: target.agentId, sessionKey: target.sessionKey, notifyUserMessagePersisted: () => undefined, @@ -1357,10 +1358,13 @@ describe("mirrorCodexAppServerTranscript", () => { ); const target = await createSqliteMirrorTarget("openclaw-codex-mirror-attested-hook-"); const sourceMessage = attachCodexMirrorIdentity( - makeAgentAssistantMessage({ - content: [{ type: "text", text: "sensitive answer" }], - timestamp: Date.now(), - }), + { + ...makeAgentAssistantMessage({ + content: [{ type: "text", text: "sensitive answer" }], + timestamp: Date.now(), + }), + openclawUsageProvenance: "unmeasured", + }, "turn-1:assistant", ); @@ -1373,7 +1377,7 @@ describe("mirrorCodexAppServerTranscript", () => { } as unknown as Parameters[0]["params"], result: { messagesSnapshot: [sourceMessage], - } as Parameters[0]["result"], + }, agentId: target.agentId, sessionKey: target.sessionKey, notifyUserMessagePersisted: () => undefined, @@ -1387,7 +1391,11 @@ describe("mirrorCodexAppServerTranscript", () => { "codex-app-server:thread-1:turn-1:assistant", ); expect(mirrorOutcome.mirroredMessages).toMatchObject([ - { role: "assistant", content: [{ type: "text", text: "[redacted by hook]" }] }, + { + role: "assistant", + content: [{ type: "text", text: "[redacted by hook]" }], + openclawUsageProvenance: "unmeasured", + }, ]); expect(JSON.stringify(mirrorOutcome.mirroredMessages)).not.toContain("sensitive answer"); }); diff --git a/extensions/codex/src/app-server/transcript-mirror.ts b/extensions/codex/src/app-server/transcript-mirror.ts index 3968ee96d5b..0caa7ed192e 100644 --- a/extensions/codex/src/app-server/transcript-mirror.ts +++ b/extensions/codex/src/app-server/transcript-mirror.ts @@ -17,6 +17,7 @@ import { } from "openclaw/plugin-sdk/session-transcript-runtime"; import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime"; import type { EmbeddedRunAttemptResult } from "./attempt-terminal.js"; +import type { CodexUsageProvenance } from "./event-projector-assistant-message.js"; import type { CodexThread, JsonValue } from "./protocol.js"; import { attachCodexMirrorAttestation, @@ -223,6 +224,7 @@ function projectCodexThreadHistory(params: { CODEX_HISTORY_ASSISTANT_PROVIDER, model: CODEX_HISTORY_ASSISTANT_MODEL, usage: CODEX_HISTORY_ZERO_USAGE, + openclawUsageProvenance: "transcript-only", stopReason: turn.status === "interrupted" ? "aborted" @@ -233,7 +235,7 @@ function projectCodexThreadHistory(params: { ? { errorMessage: turn.error.message } : {}), timestamp, - } satisfies AssistantMessage, + } satisfies AssistantMessage & CodexUsageProvenance, identity, ) : attachCodexMirrorIdentity({ role, content: text, timestamp } as AgentMessage, identity); @@ -351,7 +353,7 @@ async function mirrorBestEffort(params: { params: EmbeddedRunAttemptParams; agentId?: string; notifyUserMessagePersisted: (message: Extract) => void; - result: EmbeddedRunAttemptResult; + result: Pick; sessionKey?: string; cwd: string; threadId: string; @@ -645,6 +647,17 @@ async function mirror(params: { // identity so retries cannot turn a stale idempotency hit into evidence. messageToAppend = attachCodexMirrorIdentity(messageToAppend, mirrorIdentity); } + const usageProvenance = (message as MirroredAgentMessage & CodexUsageProvenance) + .openclawUsageProvenance; + if (usageProvenance) { + // A message-write hook may replace the payload, but the Codex producer + // owns whether the row represents measured usage or a projection. + const withUsageProvenance = { + ...messageToAppend, + openclawUsageProvenance: usageProvenance, + }; + messageToAppend = withUsageProvenance; + } messageToAppend = projectAgentHarnessTranscriptMessageForDisplay({ hidden: (message as { display?: boolean }).display === false, message: messageToAppend, diff --git a/src/agents/cli-output.test.ts b/src/agents/cli-output.test.ts index fb0a0cbd8e4..d82cc758bfc 100644 --- a/src/agents/cli-output.test.ts +++ b/src/agents/cli-output.test.ts @@ -47,6 +47,39 @@ const OPENAI_COMPATIBLE_CLI_USAGE_CASES = [ }, normalized: { input: 15, output: 7, cacheRead: 4, cacheWrite: undefined, total: 26 }, }, + { + name: "explicit zero aggregate with missing components", + raw: { total_tokens: 0 }, + normalized: { + input: undefined, + output: undefined, + cacheRead: undefined, + cacheWrite: undefined, + total: 0, + }, + }, + { + name: "explicit zero components with optional zero cache buckets", + raw: { input_tokens: 0, output_tokens: 0, cache_read_input_tokens: 0, cacheWrite: 0 }, + normalized: { + input: 0, + output: 0, + cacheRead: undefined, + cacheWrite: undefined, + total: undefined, + }, + }, + { + name: "explicit zero input with missing output", + raw: { input_tokens: 0 }, + normalized: { + input: 0, + output: undefined, + cacheRead: undefined, + cacheWrite: undefined, + total: undefined, + }, + }, { name: "flat Codex cached input is included in input_tokens", raw: { @@ -1054,7 +1087,7 @@ describe("parseCliJsonl", () => { sessionId: "session-456", usage: { input: 18, - output: undefined, + output: 0, cacheRead: undefined, cacheWrite: undefined, total: undefined, diff --git a/src/agents/cli-output.ts b/src/agents/cli-output.ts index 4d2868e9c13..42718d0df90 100644 --- a/src/agents/cli-output.ts +++ b/src/agents/cli-output.ts @@ -294,7 +294,7 @@ function toCliUsage(raw: Record): CliUsage | undefined { return typeof nested[field] === "number" && nested[field] > 0 ? nested[field] : undefined; }; const pick = (key: string) => - typeof raw[key] === "number" && raw[key] > 0 ? raw[key] : undefined; + typeof raw[key] === "number" && raw[key] >= 0 ? raw[key] : undefined; // Chat Completions calls these prompt/completion tokens; preserve existing CLI-field precedence. const totalInput = pick("input_tokens") ?? pick("inputTokens") ?? pick("prompt_tokens") ?? pick("promptTokens"); @@ -305,19 +305,20 @@ function toCliUsage(raw: Record): CliUsage | undefined { pick("completionTokens"); const nestedCached = readNestedCached("input_tokens_details") ?? readNestedCached("prompt_tokens_details"); + // Zero counts establish input/output/total coverage. Cache buckets are optional. const cacheRead = - pick("cache_read_input_tokens") ?? - pick("cached_input_tokens") ?? - pick("cacheRead") ?? - pick("cached") ?? + pick("cache_read_input_tokens") || + pick("cached_input_tokens") || + pick("cacheRead") || + pick("cached") || nestedCached; const nestedCacheWrite = readNestedCached("input_tokens_details", "cache_write_tokens") ?? readNestedCached("prompt_tokens_details", "cache_write_tokens"); const cacheWrite = - pick("cache_creation_input_tokens") ?? - pick("cache_write_input_tokens") ?? - pick("cacheWrite") ?? + pick("cache_creation_input_tokens") || + pick("cache_write_input_tokens") || + pick("cacheWrite") || nestedCacheWrite; const input = pick("input") ?? @@ -330,7 +331,13 @@ function toCliUsage(raw: Record): CliUsage | undefined { ? Math.max(0, totalInput - (cacheRead ?? 0) - (cacheWrite ?? 0)) : totalInput); const total = pick("total_tokens") ?? pick("total"); - if (!input && !output && !cacheRead && !cacheWrite && !total) { + if ( + input === undefined && + output === undefined && + cacheRead === undefined && + cacheWrite === undefined && + total === undefined + ) { return undefined; } return { input, output, cacheRead, cacheWrite, total }; diff --git a/src/agents/cli-runner.reliability.test.ts b/src/agents/cli-runner.reliability.test.ts index f672636ed4e..c9758064a79 100644 --- a/src/agents/cli-runner.reliability.test.ts +++ b/src/agents/cli-runner.reliability.test.ts @@ -3468,6 +3468,8 @@ describe("runCliAgent reliability", () => { provider: "codex-cli", model: "gpt-5.4", idempotencyKey: "cli-assistant:run-persist-cli", + usage: expect.objectContaining({ totalTokens: 0 }), + openclawUsageProvenance: "unmeasured", }), ); expect( diff --git a/src/agents/cli-runner.ts b/src/agents/cli-runner.ts index 1afb7a6ed3a..b32b22298ff 100644 --- a/src/agents/cli-runner.ts +++ b/src/agents/cli-runner.ts @@ -81,7 +81,10 @@ import { } from "./harness/lifecycle-hook-helpers.js"; import type { AgentMessage } from "./runtime/index.js"; import { SessionManager } from "./sessions/session-manager.js"; -import { buildAssistantMessage, buildUsageWithNoCost } from "./stream-message-shared.js"; +import { + buildAssistantMessage, + buildTranscriptUsageWithProvenance, +} from "./stream-message-shared.js"; const log = createSubsystemLogger("agents/cli-runner"); @@ -363,6 +366,7 @@ async function persistCliAssistantTranscript(params: { return true; } try { + const { usage, ...usageProvenance } = buildTranscriptUsageWithProvenance(params.usage); const result = await appendExactAssistantMessageToSessionTranscript({ sessionKey: runParams.sessionKey, agentId: runParams.agentId, @@ -371,22 +375,19 @@ async function persistCliAssistantTranscript(params: { idempotencyKey: `cli-assistant:${runParams.runId}`, config: runParams.config, beforeMessageWrite: runAgentHarnessBeforeMessageWriteHook, - message: buildAssistantMessage({ - model: { - api: "cli", - provider: runParams.provider, - id: params.modelId, - }, - content: [{ type: "text", text: params.text }], - stopReason: "stop", - usage: buildUsageWithNoCost({ - input: params.usage?.input, - output: params.usage?.output, - cacheRead: params.usage?.cacheRead, - cacheWrite: params.usage?.cacheWrite, - totalTokens: params.usage?.total, + message: { + ...buildAssistantMessage({ + model: { + api: "cli", + provider: runParams.provider, + id: params.modelId, + }, + content: [{ type: "text", text: params.text }], + stopReason: "stop", + usage, }), - }), + ...usageProvenance, + }, }); if (!result.ok) { log.warn(`CLI assistant transcript persistence skipped: ${result.reason}`); diff --git a/src/agents/command/assistant-transcript-repair.test.ts b/src/agents/command/assistant-transcript-repair.test.ts index ca53edbcb60..2df7953dfe0 100644 --- a/src/agents/command/assistant-transcript-repair.test.ts +++ b/src/agents/command/assistant-transcript-repair.test.ts @@ -418,6 +418,16 @@ describe("assistant transcript repair", () => { await agentCommand({ message: "user one", sessionId, sessionKey, cwd: state.workspaceDir }); state.runAgentAttemptMock.mockImplementationOnce(async (params) => { + expect( + await readSessionMessages({ agentId: "main", sessionId, storePath: requireStorePath() }), + ).toContainEqual( + expect.objectContaining({ + role: "assistant", + content: [{ type: "text", text: "assistant one" }], + usage: expect.objectContaining({ totalTokens: 0 }), + openclawUsageProvenance: "unmeasured", + }), + ); expect( await readMessageSequence({ agentId: "main", sessionId, storePath: requireStorePath() }), ).toEqual([ diff --git a/src/agents/command/assistant-transcript-repair.ts b/src/agents/command/assistant-transcript-repair.ts index 42df8a89407..359868ba42b 100644 --- a/src/agents/command/assistant-transcript-repair.ts +++ b/src/agents/command/assistant-transcript-repair.ts @@ -4,6 +4,7 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; import { runAgentHarnessBeforeMessageWriteHook } from "../harness/hook-helpers.js"; +import { buildTranscriptUsageWithProvenance } from "../stream-message-shared.js"; import { loadTranscriptAppendRuntime } from "./runtime-loaders.js"; import { persistSessionEntry } from "./session-helpers.js"; @@ -18,21 +19,6 @@ type AssistantTranscriptRepairContext = { config: OpenClawConfig; }; -const EMPTY_USAGE = { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - total: 0, - }, -} as const; - /** Records a final whose canonical transcript append failed. */ export async function persistAssistantTranscriptRepairRecord(params: { context: AssistantTranscriptRepairContext; @@ -117,7 +103,7 @@ export async function repairPendingAssistantTranscriptTurns(params: { api: "cli", provider: item.provider ?? "cli", model: item.model ?? "default", - usage: EMPTY_USAGE, + ...buildTranscriptUsageWithProvenance(), stopReason: "stop", timestamp: item.createdAt, }, diff --git a/src/agents/command/attempt-execution.cli.test.ts b/src/agents/command/attempt-execution.cli.test.ts index 779626ec45c..441027df5a0 100644 --- a/src/agents/command/attempt-execution.cli.test.ts +++ b/src/agents/command/attempt-execution.cli.test.ts @@ -2130,6 +2130,7 @@ describe("CLI attempt execution", () => { expect.objectContaining({ role: "assistant", content: [{ type: "text", text: "internal reply" }], + ...(runtime === "ACP" ? { openclawUsageProvenance: "unmeasured" } : {}), }), ); expect( @@ -2230,6 +2231,43 @@ describe("CLI attempt execution", () => { expect(sessionStore[sessionKey]?.updatedAt).toBe(persisted[sessionKey]?.updatedAt); }); + it.each([ + { usage: undefined, provenance: "unmeasured", totalTokens: 0 }, + { usage: { input: 12 }, provenance: "partial", totalTokens: 12 }, + { usage: { total: 0 }, provenance: "measured", totalTokens: 0 }, + { usage: { input: 0, output: 0 }, provenance: "measured", totalTokens: 0 }, + ])("persists CLI usage $usage with $provenance provenance", async (testCase) => { + const sessionKey = "agent:main:direct:cli-usage-provenance"; + const sessionId = "session-cli-usage-provenance"; + const result = makeCliResult("hello from cli"); + if (!result.meta.agentMeta) { + throw new Error("missing test CLI agent metadata"); + } + result.meta.agentMeta.usage = testCase.usage; + + await persistCliTurnTranscript({ + body: "persist this", + result, + sessionId, + sessionKey, + sessionEntry: { sessionId, updatedAt: Date.now() }, + storePath, + sessionAgentId: "main", + sessionCwd: tmpDir, + config: {}, + }); + + expect( + await readSessionMessages({ agentId: "main", sessionId, sessionKey, storePath }), + ).toContainEqual( + expect.objectContaining({ + role: "assistant", + usage: expect.objectContaining({ totalTokens: testCase.totalTokens }), + openclawUsageProvenance: testCase.provenance, + }), + ); + }); + it("mirrors only the CLI reply when the shared recorder already persisted the user turn", async () => { const sessionKey = "agent:main:direct:cli-recorder-owned-user"; const sessionFile = path.join(tmpDir, "session-cli-recorder-owned-user.jsonl"); diff --git a/src/agents/command/attempt-execution.ts b/src/agents/command/attempt-execution.ts index ff7e409d811..6971b81f45a 100644 --- a/src/agents/command/attempt-execution.ts +++ b/src/agents/command/attempt-execution.ts @@ -91,13 +91,14 @@ import { resolveSessionWriteLockOptions, resolveSessionWriteLockTargetKey, } from "../session-write-lock.js"; -import { buildUsageWithNoCost } from "../stream-message-shared.js"; +import { buildTranscriptUsageWithProvenance } from "../stream-message-shared.js"; import { isSubagentAnnounceCompletionHandoff, isTrustedSubagentCompletionHandoffForRun, } from "../subagent-announce-handoff.js"; import { isRuntimeToolAllowed, isToolAllowedByPolicies } from "../tool-policy-match.js"; import { DEFAULT_MAX_LIVE_TOOL_RESULT_CHARS } from "../tool-result-limits.js"; +import type { NormalizedUsage } from "../usage.js"; import { buildClaudeCliFallbackContextPrelude, claudeCliSessionTranscriptHasContent, @@ -141,32 +142,10 @@ function normalizeTranscriptMirrorText(value: string): string { return value.trim().replace(/\s+/gu, " "); } -const ACP_TRANSCRIPT_USAGE = { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - total: 0, - }, -} as const; function shouldSuppressEmbeddedLiveStreamOutput(params: { opts: AgentCommandOpts }): boolean { return params.opts.sessionEffects === "internal" && params.opts.deliver !== true; } -type TranscriptUsage = { - input?: number; - output?: number; - cacheRead?: number; - cacheWrite?: number; - total?: number; -}; - type PersistTextTurnTranscriptParams = { body: string; transcriptBody?: string; @@ -188,7 +167,7 @@ type PersistTextTurnTranscriptParams = { api: string; provider: string; model: string; - usage?: TranscriptUsage; + usage?: NormalizedUsage; }; }; @@ -285,19 +264,6 @@ function resolveHarnessAuthProfileSelection(params: { : { authProfileProvider: params.authProfileProvider }; } -function resolveTranscriptUsage(usage: PersistTextTurnTranscriptParams["assistant"]["usage"]) { - if (!usage) { - return ACP_TRANSCRIPT_USAGE; - } - return buildUsageWithNoCost({ - input: usage.input, - output: usage.output, - cacheRead: usage.cacheRead, - cacheWrite: usage.cacheWrite, - totalTokens: usage.total, - }); -} - async function persistTextTurnTranscript( params: PersistTextTurnTranscriptParams, ): Promise { @@ -338,7 +304,7 @@ async function persistTextTurnTranscript( api: params.assistant.api, provider: params.assistant.provider, model: params.assistant.model, - usage: resolveTranscriptUsage(params.assistant.usage), + ...buildTranscriptUsageWithProvenance(params.assistant.usage), stopReason: "stop", timestamp: Date.now(), }, diff --git a/src/agents/stream-message-shared.ts b/src/agents/stream-message-shared.ts index 4dc51b4e65a..58769cf4aed 100644 --- a/src/agents/stream-message-shared.ts +++ b/src/agents/stream-message-shared.ts @@ -3,7 +3,9 @@ * * Centralizes zero-cost usage records and assistant message construction for simple stream transports. */ +import { parseStrictNonNegativeInteger } from "@openclaw/normalization-core/number-coercion"; import type { AssistantMessage, StopReason, Usage } from "../llm/types.js"; +import type { NormalizedUsage } from "./usage.js"; type StreamModelDescriptor = { api: string; @@ -33,6 +35,36 @@ export function buildUsageWithNoCost(params: { }; } +export function buildTranscriptUsageWithProvenance(reported?: NormalizedUsage) { + const usage = buildUsageWithNoCost({ + input: reported?.input, + output: reported?.output, + cacheRead: reported?.cacheRead, + cacheWrite: reported?.cacheWrite, + totalTokens: reported?.total, + }); + // The transcript retains a full Usage shape for replay. Classify coverage + // before filling defaults so missing telemetry cannot masquerade as zero. + const measured = + parseStrictNonNegativeInteger(reported?.total) !== undefined || + (parseStrictNonNegativeInteger(reported?.input) !== undefined && + parseStrictNonNegativeInteger(reported?.output) !== undefined); + if (!measured) { + const hasComponents = [ + reported?.input, + reported?.output, + reported?.cacheRead, + reported?.cacheWrite, + ].some((value) => parseStrictNonNegativeInteger(value) !== undefined); + const openclawUsageProvenance = hasComponents ? "partial" : "unmeasured"; + return { usage, openclawUsageProvenance }; + } + // Older CLI transcripts used the same zero-shaped usage as a placeholder. + return usage.totalTokens === 0 + ? { usage, openclawUsageProvenance: "measured" as const } + : { usage }; +} + export function buildAssistantMessage(params: { model: StreamModelDescriptor; content: AssistantMessage["content"]; diff --git a/src/gateway/server-methods/usage.sessions-usage-cache.test.ts b/src/gateway/server-methods/usage.sessions-usage-cache.test.ts index 05b503fc25e..b0b2d3b15ab 100644 --- a/src/gateway/server-methods/usage.sessions-usage-cache.test.ts +++ b/src/gateway/server-methods/usage.sessions-usage-cache.test.ts @@ -28,7 +28,7 @@ vi.mock("../../infra/session-cost-usage.js", async () => { }; }); -import { testApi, usageHandlers } from "./usage.js"; +import { testApi, usageHandlers, type SessionsUsageResult } from "./usage.js"; const config = { agents: { list: [{ id: "main", default: true }, { id: "opus" }] }, @@ -338,7 +338,10 @@ describe("sessions.usage result cache", () => { }, }); now = 31_000; - await runSessionsUsage(baseParams); + expect(await runSessionsUsage(baseParams)).toMatchObject({ + totals: { totalTokens: 10 }, + cacheStatus: { status: "stale" }, + }); await vi.waitFor(() => { expect( Array.from(testApi.sessionsUsageCache.values()).every((entry) => !entry.inFlight), @@ -377,7 +380,13 @@ describe("sessions.usage result cache", () => { await aggregationStarted; await Promise.resolve(); expect(returned).toBe(true); - expect((await next).totals.totalTokens).toBe(10); + const retained = await next; + expect(retained).toMatchObject({ + totals: { totalTokens: 10 }, + cacheStatus: { status: "stale", cachedFiles: 1, pendingFiles: 0, staleFiles: 0 }, + }); + expect(await runSessionsUsage(baseParams)).toEqual(retained); + expect(mocks.loadSessionCostSummariesFromCache).toHaveBeenCalledTimes(3); release(); await vi.waitFor(() => { expect( @@ -387,7 +396,7 @@ describe("sessions.usage result cache", () => { }); it("serves stale data while one 30s refresh replaces it", async () => { - const first = await runSessionsUsage(baseParams); + const first = (await runSessionsUsage(baseParams)) as SessionsUsageResult; mocks.loadSessionCostSummariesFromCache.mockImplementationOnce( async (params: { sessions: unknown[] }) => ({ summaries: params.sessions.map(() => sessionSummary(20)), @@ -402,17 +411,61 @@ describe("sessions.usage result cache", () => { now = 31_000; const stale = await runSessionsUsage(baseParams); - expect(JSON.stringify(stale)).toBe(JSON.stringify(first)); + expect(stale).toEqual({ ...first, cacheStatus: { ...first.cacheStatus, status: "stale" } }); + expect(first.cacheStatus?.status).toBe("fresh"); await vi.waitFor(() => { expect(mocks.loadSessionCostSummariesFromCache).toHaveBeenCalledTimes(2); }); await vi.waitFor(async () => { - const refreshed = (await runSessionsUsage(baseParams)) as { - totals: { totalTokens: number }; - }; - expect(refreshed.totals.totalTokens).toBe(20); + expect(await runSessionsUsage(baseParams)).toMatchObject({ + totals: { totalTokens: 20 }, + cacheStatus: { status: "fresh" }, + }); + }); + expect(mocks.loadSessionCostSummariesFromCache).toHaveBeenCalledTimes(2); + }); + + it("marks a retained empty result stale while rediscovery fails", async () => { + mocks.discoverAllSessions.mockResolvedValueOnce([]); + const empty = (await runSessionsUsage(baseParams)) as SessionsUsageResult; + expect(empty.cacheStatus).toBeUndefined(); + expect(empty.totals.totalTokens).toBe(0); + expect(mocks.loadSessionCostSummariesFromCache).not.toHaveBeenCalled(); + + now = 31_000; + mocks.discoverAllSessions.mockRejectedValueOnce(new Error("discovery failed")); + expect(await runSessionsUsage(baseParams)).toEqual({ + ...empty, + cacheStatus: { status: "stale", cachedFiles: 0, pendingFiles: 0, staleFiles: 0 }, + }); + await vi.waitFor(() => { + expect( + Array.from(testApi.sessionsUsageCache.values()).every((entry) => !entry.inFlight), + ).toBe(true); + }); + expect(empty.cacheStatus).toBeUndefined(); + }); + + it("marks a fallback partial snapshot stale when the awaited refresh fails", async () => { + mocks.discoverAllSessions.mockResolvedValueOnce([ + { sessionId: "s-a", sessionFile: "/tmp/s-a.jsonl", mtime: 100 }, + { sessionId: "s-b", sessionFile: "/tmp/s-b.jsonl", mtime: 200 }, + ]); + mocks.loadSessionCostSummariesFromCache + .mockResolvedValueOnce({ + summaries: [sessionSummary(10), null], + cacheStatus: { status: "partial", cachedFiles: 1, pendingFiles: 1, staleFiles: 1 }, + }) + .mockRejectedValueOnce(new Error("refresh failed")); + const first = (await runSessionsUsage(baseParams)) as SessionsUsageResult; + expect(first.cacheStatus?.status).toBe("partial"); + + expect(await runSessionsUsage(baseParams)).toEqual({ + ...first, + cacheStatus: { status: "stale", cachedFiles: 1, pendingFiles: 1, staleFiles: 1 }, }); + expect(first.cacheStatus?.status).toBe("partial"); expect(mocks.loadSessionCostSummariesFromCache).toHaveBeenCalledTimes(2); }); }); diff --git a/src/gateway/server-methods/usage.ts b/src/gateway/server-methods/usage.ts index 2baa8b9b9ca..23cea35407b 100644 --- a/src/gateway/server-methods/usage.ts +++ b/src/gateway/server-methods/usage.ts @@ -198,8 +198,9 @@ async function loadUsageResultCached(params: { configRef?: object; load: () => Promise; isComplete?: (value: T) => boolean; + projectStale?: (value: T) => T; }): Promise { - const { cache, cacheKey, configRef } = params; + const { cache, cacheKey, configRef, projectStale } = params; const candidate = cache.get(cacheKey); const cached = configRef === undefined || candidate?.configRef === configRef ? candidate : undefined; @@ -207,7 +208,9 @@ async function loadUsageResultCached(params: { return cached.value; } if (cached?.inFlight) { - return cached.value && cached.updatedAt ? cached.value : await cached.inFlight; + return cached.value && cached.updatedAt + ? (projectStale?.(cached.value) ?? cached.value) + : await cached.inFlight; } const entry: UsageCacheEntry = cached ?? { ...(configRef && { configRef }) }; @@ -229,7 +232,7 @@ async function loadUsageResultCached(params: { }) .catch((error: unknown) => { if (entry.value) { - return entry.value; + return projectStale?.(entry.value) ?? entry.value; } throw error; }) @@ -242,7 +245,9 @@ async function loadUsageResultCached(params: { entry.inFlight = inFlight; setUsageCache(cache, cacheKey, entry); - return entry.value && entry.updatedAt ? entry.value : await inFlight; + return entry.value && entry.updatedAt + ? (projectStale?.(entry.value) ?? entry.value) + : await inFlight; } function usageDayBucketCacheKey(dayBucket: UsageDailyBucket | undefined): string { @@ -297,6 +302,18 @@ async function loadSessionsUsageResultCached( load: params.load, // Incomplete lower-cache snapshots must not acquire the outer freshness TTL. isComplete: (result) => !result.cacheStatus || result.cacheStatus.status === "fresh", + // A retained snapshot can outlive its lower-cache freshness. Project that + // fact onto the response without changing the snapshot or its counters. + projectStale: (result) => ({ + ...result, + cacheStatus: { + cachedFiles: 0, + pendingFiles: 0, + staleFiles: 0, + ...result.cacheStatus, + status: "stale", + }, + }), }); } diff --git a/src/infra/session-cost-usage-accounting.test.ts b/src/infra/session-cost-usage-accounting.test.ts index 7042c79352c..b66ea449cd5 100644 --- a/src/infra/session-cost-usage-accounting.test.ts +++ b/src/infra/session-cost-usage-accounting.test.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { expectDefined } from "@openclaw/normalization-core"; import { describe, expect, it } from "vitest"; +import { buildTranscriptUsageWithProvenance } from "../agents/stream-message-shared.js"; import { persistSessionTranscriptTurn, upsertSessionEntry, @@ -26,6 +27,37 @@ describe("session usage accounting conservation", () => { const target = { agentId, sessionId, sessionKey, storePath }; const september = Date.parse("2026-09-30T14:59:59.000Z"); const october = september + 1_000; + const placeholder = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0 }; + const provenanceCases = [ + { provider: "openclaw", model: "gateway-injected", usage: placeholder }, + { provider: "openclaw", model: "delivery-mirror", usage: placeholder }, + { provider: "openclaw", model: "acp-runtime", usage: placeholder }, + { api: "cli", provider: "example-provider", model: "old-cli", usage: placeholder }, + { + api: "cli", + provider: "example-provider", + model: "measured-cli", + usage: placeholder, + openclawUsageProvenance: "measured", + }, + { + provider: "example-provider", + model: "unmeasured", + openclawUsageProvenance: "unmeasured", + }, + { + api: "cli", + provider: "example-provider", + model: "partial-cli", + ...buildTranscriptUsageWithProvenance({ input: 12 }), + }, + { + provider: "example-provider", + model: "display-copy", + usage: { totalTokens: 10 }, + openclawUsageProvenance: "transcript-only", + }, + ]; const componentCases = [ { model: "input-only", usage: { input: 10 }, tokens: 10, missingTokenEntries: 1 }, { model: "output-only", usage: { output: 5 }, tokens: 5, missingTokenEntries: 1 }, @@ -117,6 +149,14 @@ describe("session usage accounting conservation", () => { usage: entry.usage, }, })), + ...provenanceCases.map((entry, index) => ({ + message: { + role: "assistant", + content: "Usage provenance fixture", + timestamp: october + 10 + index, + ...entry, + }, + })), ], touchSessionEntry: false, }); @@ -125,13 +165,13 @@ describe("session usage accounting conservation", () => { sessionId, sessionTarget: target, startMs: september - 1, - endMs: october + 10, + endMs: october + 20, dayBucket: { mode: "time-zone", timeZone: "Asia/Seoul" } as const, }; const first = expectDefined(await loadSessionCostSummary(query), "usage summary"); - expect(first.totalTokens).toBe(155); - expect(first.missingTokenEntries).toBe(4); - expect(first.modelUsage).toHaveLength(8); + expect(first.totalTokens).toBe(167); + expect(first.missingTokenEntries).toBe(8); + expect(first.modelUsage).toHaveLength(13); expect(first.modelUsage?.find((entry) => entry.provider === undefined)).toMatchObject({ count: 1, totals: { totalTokens: 12 }, @@ -158,7 +198,31 @@ describe("session usage accounting conservation", () => { expect(first.modelUsage?.reduce((sum, model) => sum + model.totals.totalTokens, 0)).toBe( first.totalTokens, ); - expect(first.dailyModelUsage?.reduce((sum, model) => sum + model.count, 0)).toBe(11); + expect(first.dailyModelUsage?.reduce((sum, model) => sum + model.count, 0)).toBe(16); + for (const model of ["gateway-injected", "delivery-mirror", "display-copy"]) { + expect(first.modelUsage?.some((entry) => entry.model === model)).toBe(false); + } + for (const model of ["acp-runtime", "old-cli", "unmeasured"]) { + expect(first.dailyModelUsage?.find((entry) => entry.model === model)).toMatchObject({ + tokens: 0, + count: 1, + missingTokenEntries: 1, + }); + } + expect( + first.dailyModelUsage?.find((entry) => entry.model === "measured-cli"), + ).toMatchObject({ + tokens: 0, + count: 1, + }); + expect( + first.dailyModelUsage?.find((entry) => entry.model === "measured-cli") + ?.missingTokenEntries, + ).toBeUndefined(); + expect(first.modelUsage?.find((entry) => entry.model === "partial-cli")).toMatchObject({ + count: 1, + totals: { input: 12, totalTokens: 12, missingTokenEntries: 1 }, + }); for (const entry of componentCases) { const daily = expectDefined( first.dailyModelUsage?.find((model) => model.model === entry.model), @@ -170,7 +234,7 @@ describe("session usage accounting conservation", () => { const row = expectDefined(readSessionCostUsageRollupRows(agentId)[0], "persisted rollup"); const oldRollup = JSON.parse(row.valueJson); - oldRollup.version = 2; + oldRollup.version = 3; for (const key of Object.keys(oldRollup.rollup.buckets)) { oldRollup.rollup.buckets[key].models = []; } @@ -185,12 +249,12 @@ describe("session usage accounting conservation", () => { ).toBe(true); const rebuilt = expectDefined(await loadSessionCostSummary(query), "rebuilt usage summary"); expect(rebuilt.dailyModelUsage).toEqual(first.dailyModelUsage); - expect(rebuilt.missingTokenEntries).toBe(4); + expect(rebuilt.missingTokenEntries).toBe(8); const rebuiltRow = expectDefined( readSessionCostUsageRollupRows(agentId)[0], "rebuilt rollup", ); - expect(JSON.parse(rebuiltRow.valueJson).version).toBe(3); + expect(JSON.parse(rebuiltRow.valueJson).version).toBe(4); }); } finally { await fs.rm(root, { recursive: true, force: true }); diff --git a/src/infra/session-cost-usage-aggregation.ts b/src/infra/session-cost-usage-aggregation.ts index 658dac0d1b5..9e43b10884e 100644 --- a/src/infra/session-cost-usage-aggregation.ts +++ b/src/infra/session-cost-usage-aggregation.ts @@ -46,7 +46,7 @@ import type { CostUsageTotals, ParsedTranscriptEntry } from "./session-cost-usag // Cache data is rebuildable. Semantic changes get a new version; old rows are // ignored and rebuilt instead of normalized through a runtime compatibility path. -const USAGE_COST_ROLLUP_VERSION = 3; +const USAGE_COST_ROLLUP_VERSION = 4; const USAGE_COST_FILE_ANCHOR_BYTES = 4096; type UsageCostJsonlCheckpoint = { @@ -311,7 +311,7 @@ function appendParsedEntryToRollup( let usageTotals: CostUsageTotals | undefined; if (entry.usage) { usageTotals = emptyTotals(); - applyUsageTotals(usageTotals, entry.usage); + applyUsageTotals(usageTotals, entry.usage, entry.usageUnmeasured); if (entry.costBreakdown?.total !== undefined) { applyCostBreakdown(usageTotals, entry.costBreakdown); } else { diff --git a/src/infra/session-cost-usage-pricing.ts b/src/infra/session-cost-usage-pricing.ts index 487f03e8584..20a04d6e9eb 100644 --- a/src/infra/session-cost-usage-pricing.ts +++ b/src/infra/session-cost-usage-pricing.ts @@ -3,6 +3,7 @@ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coe import type { NormalizedUsage, UsageLike } from "../agents/usage.js"; import { normalizeUsage } from "../agents/usage.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { isTranscriptOnlyOpenClawAssistantMessage } from "../shared/transcript-only-openclaw-assistant.js"; import { countToolResults, extractToolCallNames } from "../utils/transcript-tools.js"; import { estimateUsageCost, resolveModelCostConfig } from "../utils/usage-format.js"; import type { @@ -71,10 +72,6 @@ const parseTranscriptEntry = (entry: Record): ParsedTranscriptE return null; } - const usageRaw = - (message.usage as UsageLike | undefined) ?? (entry.usage as UsageLike | undefined); - const usage = usageRaw ? (normalizeUsage(usageRaw) ?? undefined) : undefined; - const provider = (typeof message.provider === "string" ? message.provider : undefined) ?? (typeof entry.provider === "string" ? entry.provider : undefined); @@ -82,7 +79,28 @@ const parseTranscriptEntry = (entry: Record): ParsedTranscriptE (typeof message.model === "string" ? message.model : undefined) ?? (typeof entry.model === "string" ? entry.model : undefined); - const costBreakdown = extractCostBreakdown(usageRaw); + const transcriptOnly = + role === "assistant" && + (message.openclawUsageProvenance === "transcript-only" || + isTranscriptOnlyOpenClawAssistantMessage(message)); + const usageRaw = transcriptOnly + ? undefined + : ((message.usage as UsageLike | undefined) ?? (entry.usage as UsageLike | undefined)); + const usage = normalizeUsage(usageRaw); + const partial = + !transcriptOnly && role === "assistant" && message.openclawUsageProvenance === "partial"; + // Historical ACP/CLI transcript writers used all-zero placeholders when no + // token measurements arrived. Rebuild rollups without treating them as measured. + const unmeasured = + role === "assistant" && + !transcriptOnly && + (message.openclawUsageProvenance === "unmeasured" || + (message.openclawUsageProvenance !== "measured" && + (message.api === "cli" || (provider === "openclaw" && model === "acp-runtime")) && + [usage?.total, usage?.input, usage?.output, usage?.cacheRead, usage?.cacheWrite].every( + (value) => value === undefined || value === 0, + ))); + const costBreakdown = partial || unmeasured ? undefined : extractCostBreakdown(usageRaw); const stopReason = typeof message.stopReason === "string" ? message.stopReason : undefined; const durationMs = asFiniteNumber(message.durationMs ?? entry.durationMs); @@ -91,7 +109,10 @@ const parseTranscriptEntry = (entry: Record): ParsedTranscriptE role, timestamp: parseTimestamp(entry), durationMs, - usage, + // Partial measurements can contribute their observed components, but their + // padded transcript total does not establish complete coverage. + usage: partial ? { ...usage, total: undefined } : unmeasured ? {} : usage, + usageUnmeasured: partial || unmeasured || undefined, costTotal: costBreakdown?.total, costBreakdown, provider, @@ -123,7 +144,11 @@ export const computeUsageTokenTotals = (usage: NormalizedUsage) => { }; }; -export const applyUsageTotals = (totals: CostUsageTotals, usage: NormalizedUsage): void => { +export const applyUsageTotals = ( + totals: CostUsageTotals, + usage: NormalizedUsage, + unmeasured = false, +): void => { const usageTotals = computeUsageTokenTotals(usage); totals.input += usageTotals.input; totals.output += usageTotals.output; @@ -133,7 +158,10 @@ export const applyUsageTotals = (totals: CostUsageTotals, usage: NormalizedUsage // Without an aggregate, both input and output must be measured. An absent // component is unknown, unlike an explicitly recorded zero; cache buckets // remain optional under the normalized usage contract. - if (usage.total === undefined && (usage.input === undefined || usage.output === undefined)) { + if ( + unmeasured || + (usage.total === undefined && (usage.input === undefined || usage.output === undefined)) + ) { totals.missingTokenEntries = (totals.missingTokenEntries ?? 0) + 1; } }; @@ -237,7 +265,7 @@ export function parseUsageCostTranscriptEntry( resolveCost: UsageCostResolver, ): ParsedTranscriptEntry | null { const entry = parseTranscriptEntry(parsed); - if (!entry?.usage) { + if (!entry?.usage || entry.usageUnmeasured) { return entry; } const cost = resolveCost({ provider: entry.provider, model: entry.model }); diff --git a/src/infra/session-cost-usage.test.ts b/src/infra/session-cost-usage.test.ts index 134dcc88e61..d0acb2c02aa 100644 --- a/src/infra/session-cost-usage.test.ts +++ b/src/infra/session-cost-usage.test.ts @@ -1195,7 +1195,7 @@ describe("session cost usage", () => { version: number; rollup: { untimestamped: { totals: { totalTokens: number } } }; }; - expect(appendedRollup.version).toBe(3); + expect(appendedRollup.version).toBe(4); expect(appendedRollup.rollup.untimestamped.totals.totalTokens).toBe(1_000); const allTime = await loadSessionCostSummariesFromCache({ diff --git a/src/infra/session-cost-usage.types.ts b/src/infra/session-cost-usage.types.ts index ba6aee61619..108eefa8ea5 100644 --- a/src/infra/session-cost-usage.types.ts +++ b/src/infra/session-cost-usage.types.ts @@ -14,6 +14,7 @@ export type ParsedTranscriptEntry = { timestamp?: Date; durationMs?: number; usage?: NormalizedUsage; + usageUnmeasured?: true; costTotal?: number; costBreakdown?: CostBreakdown; provider?: string;