Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/platformclaw/ai-usage-report-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,12 @@ Without a provider total, both input and output counts must be recorded; absent
cache buckets follow the existing normalized zero-default convention. A missing
input or output count makes the affected total unknown, even when another
component is positive.
Transcript messages written only for display or delivery do not count as usage
records. A recorded model reply without token measurements still contributes to
`call_count`, but its token total is null. Older ACP and CLI transcript writers
used zero placeholders; when the record cannot establish a measured zero, the
report treats those placeholders as unknown. Rebuilding the usage cache cannot
recover token counts that were never recorded.
This is an activity metric, not an invoice reconciliation API.

The initial scope is owned personal agents, including historical activity of
Expand Down
4 changes: 2 additions & 2 deletions docs/platformclaw/ai-usage-report-api.openapi.json
Original file line number Diff line number Diff line change
Expand Up @@ -1163,12 +1163,12 @@
"call_count_basis": {
"type": "string",
"const": "usage_records",
"description": "Count persisted model usage records, not provider HTTP attempts."
"description": "Count persisted model usage records, including recorded replies with unknown token measurements. Excludes transcript-only display and delivery records."
},
"token_usage_basis": {
"type": "string",
"const": "provider_total_or_components",
"description": "Provider total when recorded, otherwise normalized nonduplicated input/output/cache components. Without a total, both input and output must be recorded; absent cache buckets follow normalized zero defaults. Missing input or output makes the total unknown."
"description": "Provider total when measured, otherwise normalized nonduplicated input/output/cache components. Without a total, both input and output must be recorded; absent cache buckets follow normalized zero defaults. Unmeasured usage and legacy zero placeholders produce null token totals."
},
"coverage": {
"type": "string",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ export type AssistantMessageOptions = {
| undefined;
aborted: boolean;
promptError: unknown;
usageUnmeasured?: boolean;
};

export type CodexUsageProvenance = {
openclawUsageProvenance?: "transcript-only" | "unmeasured" | "partial";
};

const ZERO_USAGE: Usage = {
Expand All @@ -39,23 +44,31 @@ export function createAssistantMessage(
params: EmbeddedRunAttemptParams,
text: string,
options: AssistantMessageOptions,
): AssistantMessage {
): AssistantMessage & CodexUsageProvenance {
const attribution = resolveCodexLocalRuntimeAttribution(params);
const usage: Usage = options.tokenUsage
const { tokenUsage } = options;
const hasMeasuredTotal =
tokenUsage?.total !== undefined ||
(tokenUsage?.input !== undefined && tokenUsage.output !== undefined);
const hasMeasuredComponents = [
tokenUsage?.input,
tokenUsage?.output,
tokenUsage?.cacheRead,
tokenUsage?.cacheWrite,
].some((value) => value !== undefined);
const usage: Usage = tokenUsage
? {
input: options.tokenUsage.input ?? 0,
output: options.tokenUsage.output ?? 0,
cacheRead: options.tokenUsage.cacheRead ?? 0,
cacheWrite: options.tokenUsage.cacheWrite ?? 0,
...(options.tokenUsage.contextUsage
? { contextUsage: options.tokenUsage.contextUsage }
: {}),
input: tokenUsage.input ?? 0,
output: tokenUsage.output ?? 0,
cacheRead: tokenUsage.cacheRead ?? 0,
cacheWrite: tokenUsage.cacheWrite ?? 0,
...(tokenUsage.contextUsage ? { contextUsage: tokenUsage.contextUsage } : {}),
totalTokens:
options.tokenUsage.total ??
(options.tokenUsage.input ?? 0) +
(options.tokenUsage.output ?? 0) +
(options.tokenUsage.cacheRead ?? 0) +
(options.tokenUsage.cacheWrite ?? 0),
tokenUsage.total ??
(tokenUsage.input ?? 0) +
(tokenUsage.output ?? 0) +
(tokenUsage.cacheRead ?? 0) +
(tokenUsage.cacheWrite ?? 0),
cost: ZERO_USAGE.cost,
}
: ZERO_USAGE;
Expand All @@ -66,6 +79,12 @@ export function createAssistantMessage(
provider: attribution.provider,
model: params.modelId,
usage,
...(options.usageUnmeasured || !hasMeasuredTotal
? {
openclawUsageProvenance:
!options.usageUnmeasured && hasMeasuredComponents ? "partial" : "unmeasured",
}
: {}),
stopReason: options.aborted ? "aborted" : options.promptError ? "error" : "stop",
errorMessage: options.promptError ? formatErrorMessage(options.promptError) : undefined,
timestamp: Date.now(),
Expand All @@ -86,6 +105,7 @@ export function createAssistantCommentaryMessage(
provider: attribution.provider,
model: params.modelId,
usage: ZERO_USAGE,
openclawUsageProvenance: "transcript-only",
stopReason: "stop",
timestamp,
// Keep this unphased: gateway history hides commentary-phase assistant rows.
Expand All @@ -102,7 +122,7 @@ export function createAssistantMirrorMessage(
params: EmbeddedRunAttemptParams,
title: string,
text: string,
): AssistantMessage {
): AssistantMessage & CodexUsageProvenance {
const attribution = resolveCodexLocalRuntimeAttribution(params);
return {
role: "assistant",
Expand All @@ -111,6 +131,7 @@ export function createAssistantMirrorMessage(
provider: attribution.provider,
model: params.modelId,
usage: ZERO_USAGE,
openclawUsageProvenance: "transcript-only",
stopReason: "stop",
timestamp: Date.now(),
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,7 @@ export class CodexToolTranscriptProjection {
provider: attribution.provider,
model: this.params.modelId,
usage: ZERO_USAGE,
openclawUsageProvenance: "transcript-only",
stopReason: "toolUse",
timestamp: this.nextTranscriptTimestamp(),
} as unknown as AgentMessage;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,7 @@ describe("CodexAppServerEventProjector commentary projection", () => {
expect(commentary).toMatchObject({
role: "assistant",
content: [{ type: "text", text: "Checking the app-server stream" }],
openclawUsageProvenance: "transcript-only",
openclawStreamFallback: {
replacementText: "Checking the app-server stream",
source: "segment",
Expand Down Expand Up @@ -373,6 +374,13 @@ describe("CodexAppServerEventProjector commentary projection", () => {
`${TURN_ID}:tool:call-search:result`,
`${TURN_ID}:commentary:msg-after-tool`,
]);
expect(result.messagesSnapshot).toContainEqual(
expect.objectContaining({
role: "assistant",
openclawUsageProvenance: "transcript-only",
__openclaw: expect.objectContaining({ mirrorIdentity: `${TURN_ID}:tool:call-search:call` }),
}),
);
});

it("does not double-deliver a commentary note echoed on the raw response lane", async () => {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,12 @@ describe("CodexAppServerEventProjector reasoning and guardian projection", () =>
]);
expect(JSON.stringify(result.messagesSnapshot[1])).toContain("Codex reasoning");
expect(JSON.stringify(result.messagesSnapshot[2])).toContain("Codex plan");
expect(result.messagesSnapshot[1]).toMatchObject({
openclawUsageProvenance: "transcript-only",
});
expect(result.messagesSnapshot[2]).toMatchObject({
openclawUsageProvenance: "transcript-only",
});
expect(JSON.stringify(result.messagesSnapshot[2])).toContain("next");
expect(JSON.stringify(result.messagesSnapshot[2])).toContain("[in_progress] patch");
expect(result.compactionCount).toBe(1);
Expand Down
6 changes: 4 additions & 2 deletions extensions/codex/src/app-server/event-projector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -368,6 +368,9 @@ export class CodexAppServerEventProjector {
}
const assistantMessageOptions = {
tokenUsage: projectedUsage,
// Thread snapshots can replay an earlier response. Keep them for live
// context, but require current-response measurements for accounting.
usageUnmeasured: this.aborted || this.responseCompletions.usage === undefined,
aborted: this.aborted,
promptError: this.promptError,
};
Expand Down Expand Up @@ -415,7 +418,6 @@ export class CodexAppServerEventProjector {
promptError,
turnCompleted: Boolean(this.completedTurn),
});
const toolMetas = this.toolProgressProjection.toolMetas;
const hadPotentialSideEffects =
toolTelemetry.didSendViaMessagingTool ||
Boolean(toolTelemetry.successfulCronAdds || toolTelemetry.acceptedSessionSpawns?.length) ||
Expand All @@ -436,7 +438,7 @@ export class CodexAppServerEventProjector {
: {}),
messagesSnapshot,
assistantTexts,
toolMetas,
toolMetas: this.toolProgressProjection.toolMetas,
lastAssistant,
currentAttemptAssistant,
...(this.toolProgressProjection.lastToolError
Expand Down
25 changes: 25 additions & 0 deletions extensions/codex/src/app-server/event-projector.usage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,28 @@ describe("CodexAppServerEventProjector usage projection", () => {
expect(projector.buildResult(buildEmptyToolTelemetry()).modelIterations).toBe(2);
});

it("preserves a measured zero-token response", async () => {
const projector = await createProjector();

await projector.handleNotification(agentMessageDelta("done"));
await projector.handleNotification(
forCurrentTurn("rawResponse/completed", {
responseId: "response-1",
usage: {
totalTokens: 0,
inputTokens: 0,
cachedInputTokens: 0,
outputTokens: 0,
reasoningOutputTokens: 0,
},
}),
);

const result = projector.buildResult(buildEmptyToolTelemetry());
expect(result.lastAssistant).toMatchObject({ usage: { totalTokens: 0 } });
expect(result.lastAssistant).not.toHaveProperty("openclawUsageProvenance");
});

it("keeps cumulative-only thread usage unknown", async () => {
const projector = await createProjector();

Expand Down Expand Up @@ -130,6 +152,7 @@ describe("CodexAppServerEventProjector usage projection", () => {
total: 12,
});
expect(result.lastAssistant?.usage.contextUsage).toEqual({ state: "unavailable" });
expect(result.lastAssistant).toMatchObject({ openclawUsageProvenance: "unmeasured" });
});

it.each([
Expand Down Expand Up @@ -168,6 +191,7 @@ describe("CodexAppServerEventProjector usage projection", () => {
expect(result.assistantTexts).toEqual(["done"]);
expect(result.attemptUsage).toBeUndefined();
expect(result.lastAssistant?.usage.contextUsage).toBeUndefined();
expect(result.lastAssistant).toMatchObject({ openclawUsageProvenance: "unmeasured" });
});

it("clears prior response usage when the final response omits usage", async () => {
Expand Down Expand Up @@ -207,6 +231,7 @@ describe("CodexAppServerEventProjector usage projection", () => {
expectUsageFields(result.attemptUsage, { input: 3, output: 7, cacheRead: 2, total: 12 });
expect(result.attemptUsage?.contextUsage).toEqual({ state: "unavailable" });
expect(result.lastAssistant?.usage.contextUsage).toEqual({ state: "unavailable" });
expect(result.lastAssistant).toMatchObject({ openclawUsageProvenance: "unmeasured" });
});

it.each(["failed", "interrupted"])(
Expand Down
24 changes: 16 additions & 8 deletions extensions/codex/src/app-server/transcript-mirror.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,7 @@ describe("importCodexThreadHistoryToTranscript", () => {
provider: "source-provider",
model: "native-history",
usage: { totalTokens: 0 },
openclawUsageProvenance: "transcript-only",
stopReason: "stop",
timestamp: expect.any(Number),
},
Expand Down Expand Up @@ -1287,7 +1288,7 @@ describe("mirrorCodexAppServerTranscript", () => {
} as unknown as Parameters<typeof mirrorTranscriptBestEffort>[0]["params"],
result: {
messagesSnapshot: [assistantMessage],
} as Parameters<typeof mirrorTranscriptBestEffort>[0]["result"],
},
notifyUserMessagePersisted: () => undefined,
cwd: root,
threadId: "thread-1",
Expand Down Expand Up @@ -1328,7 +1329,7 @@ describe("mirrorCodexAppServerTranscript", () => {
} as unknown as Parameters<typeof mirrorTranscriptBestEffort>[0]["params"],
result: {
messagesSnapshot: [currentMessage],
} as Parameters<typeof mirrorTranscriptBestEffort>[0]["result"],
},
agentId: target.agentId,
sessionKey: target.sessionKey,
notifyUserMessagePersisted: () => undefined,
Expand Down Expand Up @@ -1357,10 +1358,13 @@ describe("mirrorCodexAppServerTranscript", () => {
);
const target = await createSqliteMirrorTarget("openclaw-codex-mirror-attested-hook-");
const sourceMessage = attachCodexMirrorIdentity(
makeAgentAssistantMessage({
content: [{ type: "text", text: "sensitive answer" }],
timestamp: Date.now(),
}),
{
...makeAgentAssistantMessage({
content: [{ type: "text", text: "sensitive answer" }],
timestamp: Date.now(),
}),
openclawUsageProvenance: "unmeasured",
},
"turn-1:assistant",
);

Expand All @@ -1373,7 +1377,7 @@ describe("mirrorCodexAppServerTranscript", () => {
} as unknown as Parameters<typeof mirrorTranscriptBestEffort>[0]["params"],
result: {
messagesSnapshot: [sourceMessage],
} as Parameters<typeof mirrorTranscriptBestEffort>[0]["result"],
},
agentId: target.agentId,
sessionKey: target.sessionKey,
notifyUserMessagePersisted: () => undefined,
Expand All @@ -1387,7 +1391,11 @@ describe("mirrorCodexAppServerTranscript", () => {
"codex-app-server:thread-1:turn-1:assistant",
);
expect(mirrorOutcome.mirroredMessages).toMatchObject([
{ role: "assistant", content: [{ type: "text", text: "[redacted by hook]" }] },
{
role: "assistant",
content: [{ type: "text", text: "[redacted by hook]" }],
openclawUsageProvenance: "unmeasured",
},
]);
expect(JSON.stringify(mirrorOutcome.mirroredMessages)).not.toContain("sensitive answer");
});
Expand Down
17 changes: 15 additions & 2 deletions extensions/codex/src/app-server/transcript-mirror.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
} from "openclaw/plugin-sdk/session-transcript-runtime";
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
import type { EmbeddedRunAttemptResult } from "./attempt-terminal.js";
import type { CodexUsageProvenance } from "./event-projector-assistant-message.js";
import type { CodexThread, JsonValue } from "./protocol.js";
import {
attachCodexMirrorAttestation,
Expand Down Expand Up @@ -223,6 +224,7 @@ function projectCodexThreadHistory(params: {
CODEX_HISTORY_ASSISTANT_PROVIDER,
model: CODEX_HISTORY_ASSISTANT_MODEL,
usage: CODEX_HISTORY_ZERO_USAGE,
openclawUsageProvenance: "transcript-only",
stopReason:
turn.status === "interrupted"
? "aborted"
Expand All @@ -233,7 +235,7 @@ function projectCodexThreadHistory(params: {
? { errorMessage: turn.error.message }
: {}),
timestamp,
} satisfies AssistantMessage,
} satisfies AssistantMessage & CodexUsageProvenance,
identity,
)
: attachCodexMirrorIdentity({ role, content: text, timestamp } as AgentMessage, identity);
Expand Down Expand Up @@ -351,7 +353,7 @@ async function mirrorBestEffort(params: {
params: EmbeddedRunAttemptParams;
agentId?: string;
notifyUserMessagePersisted: (message: Extract<AgentMessage, { role: "user" }>) => void;
result: EmbeddedRunAttemptResult;
result: Pick<EmbeddedRunAttemptResult, "messagesSnapshot">;
sessionKey?: string;
cwd: string;
threadId: string;
Expand Down Expand Up @@ -645,6 +647,17 @@ async function mirror(params: {
// identity so retries cannot turn a stale idempotency hit into evidence.
messageToAppend = attachCodexMirrorIdentity(messageToAppend, mirrorIdentity);
}
const usageProvenance = (message as MirroredAgentMessage & CodexUsageProvenance)
.openclawUsageProvenance;
if (usageProvenance) {
// A message-write hook may replace the payload, but the Codex producer
// owns whether the row represents measured usage or a projection.
const withUsageProvenance = {
...messageToAppend,
openclawUsageProvenance: usageProvenance,
};
messageToAppend = withUsageProvenance;
}
messageToAppend = projectAgentHarnessTranscriptMessageForDisplay({
hidden: (message as { display?: boolean }).display === false,
message: messageToAppend,
Expand Down
Loading