From 1de68e975d72b463121b3ecfda43b12f1b07dec4 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 30 Sep 2026 12:07:31 -0400 Subject: [PATCH 1/7] Add ByTool cost breakdown types and regenerate manifests --- ...tprimitives.authzed.com_agentsessions.yaml | 43 +++++++++++++++++-- pkg/apis/v1alpha1/agentsession_types.go | 43 ++++++++++++++++--- pkg/apis/v1alpha1/toolcostbucket_test.go | 24 +++++++++++ pkg/apis/v1alpha1/zz_generated.deepcopy.go | 20 +++++++++ pkg/platform/manifests/install.yaml | 43 +++++++++++++++++-- pkg/platform/pipeline/types.go | 21 +++++++++ site/content/docs/crd-agentsession.mdx | 8 +++- 7 files changed, 185 insertions(+), 17 deletions(-) create mode 100644 pkg/apis/v1alpha1/toolcostbucket_test.go diff --git a/config/crds/agentprimitives.authzed.com_agentsessions.yaml b/config/crds/agentprimitives.authzed.com_agentsessions.yaml index 00984ca9..3d8fd368 100644 --- a/config/crds/agentprimitives.authzed.com_agentsessions.yaml +++ b/config/crds/agentprimitives.authzed.com_agentsessions.yaml @@ -2012,8 +2012,11 @@ spec: properties: amountMicroUSD: description: |- - AmountMicroUSD is the session total in micro-USD (1e-6 USD); 0 and - meaningless when PricingKnown is false. + AmountMicroUSD is the session total in micro-USD (1e-6 USD): the sum of + every priced component (ByModel + ByTool). When PricingKnown is false, at + least one served MODEL had no rate, so this is a LOWER BOUND on spend (the + components that could be priced), not the full session cost — it is no + longer necessarily 0, because provider-reported tool cost is always priced. format: int64 type: integer asOf: @@ -2065,6 +2068,37 @@ spec: type: object type: array x-kubernetes-list-type: atomic + byTool: + description: |- + ByTool breaks out the cost of inner interactive toolkits (e.g. a `claude` + Claude Code sub-run's own provider-reported spend) a session drove. These + are ADDED to AmountMicroUSD alongside ByModel — a session's total is model + spend plus tool spend. Empty when no interactive toolkit reported a cost. + Tool cost is always provider-reported, never table-priced, so a bucket's + PricingKnown is always true. + items: + description: ToolCostBucket is one interactive toolkit's slice + of a session's cost. + properties: + amountMicroUSD: + description: |- + AmountMicroUSD is this tool's provider-reported cost, summed across its + invocations, in micro-USD (1e-6 USD). + format: int64 + type: integer + pricingKnown: + description: |- + PricingKnown is true when AmountMicroUSD is a real cost. Tool cost is + always provider-reported, so this is true whenever a cost was reported. + type: boolean + tool: + description: Tool is the outer tool name (e.g. "claude-oauth"). + type: string + required: + - tool + type: object + type: array + x-kubernetes-list-type: atomic currency: description: Currency is the ISO code the amount is denominated in ("USD"). @@ -2074,8 +2108,9 @@ spec: type: string pricingKnown: description: |- - PricingKnown is false when the model had no price; the amount is then 0 - and must not be shown as a real cost. + PricingKnown is false when a served model had no price. The amount is then + a lower bound (priced components only), not the full cost. Tool buckets are + always priced, so this tracks model-pricing coverage specifically. type: boolean type: object failureReason: diff --git a/pkg/apis/v1alpha1/agentsession_types.go b/pkg/apis/v1alpha1/agentsession_types.go index 0191a853..a323a3b5 100644 --- a/pkg/apis/v1alpha1/agentsession_types.go +++ b/pkg/apis/v1alpha1/agentsession_types.go @@ -964,12 +964,16 @@ type AgentSessionProgress struct { // EstimatedSessionCost is a best-effort USD estimate of a session's LLM spend, // stamped at SessionEnd from cumulative token usage × the provider's per-model -// pricing. Money is stored as integer micro-USD (1e-6 USD) to avoid float drift. -// PricingKnown=false means the model had no price; AmountMicroUSD is then 0 and -// must not be shown as a real cost. +// pricing PLUS any interactive toolkit's own provider-reported cost (ByTool). +// Money is stored as integer micro-USD (1e-6 USD) to avoid float drift. +// PricingKnown=false means a served model had no price, so AmountMicroUSD is a +// lower bound (the priced components only), not the full cost. type EstimatedSessionCost struct { - // AmountMicroUSD is the session total in micro-USD (1e-6 USD); 0 and - // meaningless when PricingKnown is false. + // AmountMicroUSD is the session total in micro-USD (1e-6 USD): the sum of + // every priced component (ByModel + ByTool). When PricingKnown is false, at + // least one served MODEL had no rate, so this is a LOWER BOUND on spend (the + // components that could be priced), not the full session cost — it is no + // longer necessarily 0, because provider-reported tool cost is always priced. // +optional AmountMicroUSD int64 `json:"amountMicroUSD,omitempty"` // Currency is the ISO code the amount is denominated in ("USD"). @@ -978,8 +982,9 @@ type EstimatedSessionCost struct { // Model is the configured model the session ran under. // +optional Model string `json:"model,omitempty"` - // PricingKnown is false when the model had no price; the amount is then 0 - // and must not be shown as a real cost. + // PricingKnown is false when a served model had no price. The amount is then + // a lower bound (priced components only), not the full cost. Tool buckets are + // always priced, so this tracks model-pricing coverage specifically. // +optional PricingKnown bool `json:"pricingKnown,omitempty"` // AsOf is when the estimate was computed. @@ -994,6 +999,16 @@ type EstimatedSessionCost struct { // +optional // +listType=atomic ByModel []ModelCostBucket `json:"byModel,omitempty"` + + // ByTool breaks out the cost of inner interactive toolkits (e.g. a `claude` + // Claude Code sub-run's own provider-reported spend) a session drove. These + // are ADDED to AmountMicroUSD alongside ByModel — a session's total is model + // spend plus tool spend. Empty when no interactive toolkit reported a cost. + // Tool cost is always provider-reported, never table-priced, so a bucket's + // PricingKnown is always true. + // +optional + // +listType=atomic + ByTool []ToolCostBucket `json:"byTool,omitempty"` } // ModelCostBucket is one served model's slice of a session's cost estimate. @@ -1020,6 +1035,20 @@ type ModelCostBucket struct { PricingKnown bool `json:"pricingKnown,omitempty"` } +// ToolCostBucket is one interactive toolkit's slice of a session's cost. +type ToolCostBucket struct { + // Tool is the outer tool name (e.g. "claude-oauth"). + Tool string `json:"tool"` + // AmountMicroUSD is this tool's provider-reported cost, summed across its + // invocations, in micro-USD (1e-6 USD). + // +optional + AmountMicroUSD int64 `json:"amountMicroUSD,omitempty"` + // PricingKnown is true when AmountMicroUSD is a real cost. Tool cost is + // always provider-reported, so this is true whenever a cost was reported. + // +optional + PricingKnown bool `json:"pricingKnown,omitempty"` +} + // ParentExchange is a delegated child's record of asking the agent that // delegated to it a question, and of whether that question is still // outstanding. diff --git a/pkg/apis/v1alpha1/toolcostbucket_test.go b/pkg/apis/v1alpha1/toolcostbucket_test.go new file mode 100644 index 00000000..b899caff --- /dev/null +++ b/pkg/apis/v1alpha1/toolcostbucket_test.go @@ -0,0 +1,24 @@ +package v1alpha1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEstimatedSessionCost_DeepCopy_ByTool(t *testing.T) { + in := &EstimatedSessionCost{ + AmountMicroUSD: 19_860_000, + PricingKnown: true, + ByModel: []ModelCostBucket{{Model: "anthropic/claude-sonnet-5", AmountMicroUSD: 840_000, PricingKnown: true}}, + ByTool: []ToolCostBucket{{Tool: "claude-oauth", AmountMicroUSD: 19_020_000, PricingKnown: true}}, + } + out := in.DeepCopy() + require.Len(t, out.ByTool, 1) + assert.Equal(t, "claude-oauth", out.ByTool[0].Tool) + assert.Equal(t, int64(19_020_000), out.ByTool[0].AmountMicroUSD) + // Deep, not shallow: mutating the copy must not touch the original. + out.ByTool[0].AmountMicroUSD = 0 + assert.Equal(t, int64(19_020_000), in.ByTool[0].AmountMicroUSD) +} diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index dcb66ad0..030752be 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -3016,6 +3016,11 @@ func (in *EstimatedSessionCost) DeepCopyInto(out *EstimatedSessionCost) { *out = make([]ModelCostBucket, len(*in)) copy(*out, *in) } + if in.ByTool != nil { + in, out := &in.ByTool, &out.ByTool + *out = make([]ToolCostBucket, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EstimatedSessionCost. @@ -8026,6 +8031,21 @@ func (in *ToolCallsAuthz) DeepCopy() *ToolCallsAuthz { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ToolCostBucket) DeepCopyInto(out *ToolCostBucket) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ToolCostBucket. +func (in *ToolCostBucket) DeepCopy() *ToolCostBucket { + if in == nil { + return nil + } + out := new(ToolCostBucket) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ToolGuardCeiling) DeepCopyInto(out *ToolGuardCeiling) { *out = *in diff --git a/pkg/platform/manifests/install.yaml b/pkg/platform/manifests/install.yaml index c0e362f8..b0a10980 100644 --- a/pkg/platform/manifests/install.yaml +++ b/pkg/platform/manifests/install.yaml @@ -6572,8 +6572,11 @@ spec: properties: amountMicroUSD: description: |- - AmountMicroUSD is the session total in micro-USD (1e-6 USD); 0 and - meaningless when PricingKnown is false. + AmountMicroUSD is the session total in micro-USD (1e-6 USD): the sum of + every priced component (ByModel + ByTool). When PricingKnown is false, at + least one served MODEL had no rate, so this is a LOWER BOUND on spend (the + components that could be priced), not the full session cost — it is no + longer necessarily 0, because provider-reported tool cost is always priced. format: int64 type: integer asOf: @@ -6625,6 +6628,37 @@ spec: type: object type: array x-kubernetes-list-type: atomic + byTool: + description: |- + ByTool breaks out the cost of inner interactive toolkits (e.g. a `claude` + Claude Code sub-run's own provider-reported spend) a session drove. These + are ADDED to AmountMicroUSD alongside ByModel — a session's total is model + spend plus tool spend. Empty when no interactive toolkit reported a cost. + Tool cost is always provider-reported, never table-priced, so a bucket's + PricingKnown is always true. + items: + description: ToolCostBucket is one interactive toolkit's slice + of a session's cost. + properties: + amountMicroUSD: + description: |- + AmountMicroUSD is this tool's provider-reported cost, summed across its + invocations, in micro-USD (1e-6 USD). + format: int64 + type: integer + pricingKnown: + description: |- + PricingKnown is true when AmountMicroUSD is a real cost. Tool cost is + always provider-reported, so this is true whenever a cost was reported. + type: boolean + tool: + description: Tool is the outer tool name (e.g. "claude-oauth"). + type: string + required: + - tool + type: object + type: array + x-kubernetes-list-type: atomic currency: description: Currency is the ISO code the amount is denominated in ("USD"). @@ -6634,8 +6668,9 @@ spec: type: string pricingKnown: description: |- - PricingKnown is false when the model had no price; the amount is then 0 - and must not be shown as a real cost. + PricingKnown is false when a served model had no price. The amount is then + a lower bound (priced components only), not the full cost. Tool buckets are + always priced, so this tracks model-pricing coverage specifically. type: boolean type: object failureReason: diff --git a/pkg/platform/pipeline/types.go b/pkg/platform/pipeline/types.go index 6a4125aa..8ef17201 100644 --- a/pkg/platform/pipeline/types.go +++ b/pkg/platform/pipeline/types.go @@ -106,6 +106,13 @@ type SessionEndInfo struct { // Model above. Empty means "no per-model breakdown available" — reporters // must NOT read that as zero usage. ByModel []ModelUsage + + // ByTool is the per-interactive-toolkit provider-reported cost accumulated + // across this session (e.g. an inner `claude` Claude Code sub-run's own + // billed cost), keyed by outer tool name in first-encountered order. Empty + // means no interactive toolkit reported a terminal cost — reporters must + // NOT read that as zero spend elsewhere. + ByTool []ToolUsage } // ModelUsage is one served model's cumulative usage for a session, keyed by its @@ -134,6 +141,20 @@ type ModelUsage struct { CostReported bool } +// ToolUsage is one interactive toolkit's cumulative provider-reported cost for +// a session, keyed by the outer tool name. Distinct from ModelUsage: a toolkit +// reports a single billed dollar figure (not token counts we re-price), so this +// carries only the summed cost and whether any invocation reported one. +type ToolUsage struct { + // Tool is the outer tool name (e.g. "claude-oauth"). + Tool string + // CostMicroUSD is the provider-reported cost summed across this tool's + // invocations, in micro-USD (1e-6 USD). + CostMicroUSD int64 + // CostReported is true when at least one invocation reported a cost. + CostReported bool +} + // MetaagentInfo is the payload for the four metaagent control-plane points // (MetaagentReceived/Extract/Decide/Apply). It carries ONLY generic primitives // — pkg/platform/pipeline imports no domain types, so the AgentClass envelope, the diff --git a/site/content/docs/crd-agentsession.mdx b/site/content/docs/crd-agentsession.mdx index a15a5b51..d7c7cafe 100644 --- a/site/content/docs/crd-agentsession.mdx +++ b/site/content/docs/crd-agentsession.mdx @@ -271,7 +271,7 @@ Status is controller-owned (observed state). | `status.effectiveSettings.toolGuard.namespace.rules[].rateLimit.maxCallsPerTurn` | `integer (int32)` | MaxCallsPerTurn caps calls within a single agent turn; 0 is unlimited. _(min 1)_ | | `status.effectiveSettings.toolGuard.namespace.rules[].rateLimit.window` | `string` | Window is the sliding-window span for MaxCalls; both must be set together. | | `status.estimatedCost` | `object` | EstimatedCost is the best-effort end-of-session USD cost estimate. Runner-owned; written at SessionEnd when reportSessionCost is on. | -| `status.estimatedCost.amountMicroUSD` | `integer (int64)` | AmountMicroUSD is the session total in micro-USD (1e-6 USD); 0 and meaningless when PricingKnown is false. | +| `status.estimatedCost.amountMicroUSD` | `integer (int64)` | AmountMicroUSD is the session total in micro-USD (1e-6 USD): the sum of every priced component (ByModel + ByTool). When PricingKnown is false, at least one served MODEL had no rate, so this is a LOWER BOUND on spend (the components that could be priced), not the full session cost — it is no longer necessarily 0, because provider-reported tool cost is always priced. | | `status.estimatedCost.asOf` | `string (date-time)` | AsOf is when the estimate was computed. | | `status.estimatedCost.byModel` | `[]object` | ByModel breaks the total down per actually-served model, so a session that fell back or auto-routed across models shows where the spend went instead of one blended total under the configured model. Empty when the runner reported no per-model usage; AmountMicroUSD and PricingKnown are then the sole source of truth. | | `status.estimatedCost.byModel[].amountMicroUSD` | `integer (int64)` | AmountMicroUSD is this bucket's share of the session cost, in micro-USD (1e-6 USD): the provider-reported cost when available, else tokens × the resolved per-model pricing table rate. | @@ -279,9 +279,13 @@ Status is controller-owned (observed state). | `status.estimatedCost.byModel[].model` \* | `string` | Model is the served-model display id verbatim (provider/model, or a provider-prefixed served model under routing) — never re-parsed here. | | `status.estimatedCost.byModel[].outputTokens` | `integer (int64)` | OutputTokens is this model's share of completion tokens. | | `status.estimatedCost.byModel[].pricingKnown` | `boolean` | PricingKnown is true when AmountMicroUSD is a real cost — either the provider reported it directly, or the pricing table had a rate for this model. False means AmountMicroUSD is 0 and must not be shown as real spend (same no-fabrication contract as EstimatedSessionCost). | +| `status.estimatedCost.byTool` | `[]object` | ByTool breaks out the cost of inner interactive toolkits (e.g. a `claude` Claude Code sub-run's own provider-reported spend) a session drove. These are ADDED to AmountMicroUSD alongside ByModel — a session's total is model spend plus tool spend. Empty when no interactive toolkit reported a cost. Tool cost is always provider-reported, never table-priced, so a bucket's PricingKnown is always true. | +| `status.estimatedCost.byTool[].amountMicroUSD` | `integer (int64)` | AmountMicroUSD is this tool's provider-reported cost, summed across its invocations, in micro-USD (1e-6 USD). | +| `status.estimatedCost.byTool[].pricingKnown` | `boolean` | PricingKnown is true when AmountMicroUSD is a real cost. Tool cost is always provider-reported, so this is true whenever a cost was reported. | +| `status.estimatedCost.byTool[].tool` \* | `string` | Tool is the outer tool name (e.g. "claude-oauth"). | | `status.estimatedCost.currency` | `string` | Currency is the ISO code the amount is denominated in ("USD"). | | `status.estimatedCost.model` | `string` | Model is the configured model the session ran under. | -| `status.estimatedCost.pricingKnown` | `boolean` | PricingKnown is false when the model had no price; the amount is then 0 and must not be shown as a real cost. | +| `status.estimatedCost.pricingKnown` | `boolean` | PricingKnown is false when a served model had no price. The amount is then a lower bound (priced components only), not the full cost. Tool buckets are always priced, so this tracks model-pricing coverage specifically. | | `status.failureReason` | `string` | FailureReason is the human-readable reason phase became Failed. | | `status.finishedAt` | `string (date-time)` | FinishedAt is when the session reached a terminal phase. | | `status.identityChoiceParkedAt` | `string (date-time)` | IdentityChoiceParkedAt is when the session entered AwaitingIdentityChoice. The operator measures IdentityChoiceTimeout from here. | From 47a6c6c58827e85976c3ca2b5dea3f43f9fbfce5 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 30 Sep 2026 12:08:56 -0400 Subject: [PATCH 2/7] Accumulate interactive toolkit cost per tool on the Loop --- pkg/agent/runner/loop_deps.go | 6 ++++ pkg/agent/runner/loop_usage.go | 55 ++++++++++++++++++++++++++++++ pkg/agent/runner/toolusage_test.go | 53 ++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+) create mode 100644 pkg/agent/runner/toolusage_test.go diff --git a/pkg/agent/runner/loop_deps.go b/pkg/agent/runner/loop_deps.go index 4702077a..d0bd358f 100644 --- a/pkg/agent/runner/loop_deps.go +++ b/pkg/agent/runner/loop_deps.go @@ -504,6 +504,12 @@ type Loop struct { // accumulate into the existing bucket instead of appending a duplicate. usageByModel []modelUsageBucket usageByModelIdx map[string]int + // usageByTool is the per-interactive-toolkit running cost accumulation, in + // first-encountered order, keyed by outer tool name in usageByToolIdx. Also + // guarded by usageMu (see above) — addToolCost fires from the same terminal + // bookkeeping path as addUsage/addModelUsage. + usageByTool []toolUsageBucket + usageByToolIdx map[string]int // Engine is the runner's single authz dependency. Production wiring comes from // internal/cmd/runner/main.go; tests inject a fake or leave nil (falling back to the diff --git a/pkg/agent/runner/loop_usage.go b/pkg/agent/runner/loop_usage.go index f6a913be..8b56b22b 100644 --- a/pkg/agent/runner/loop_usage.go +++ b/pkg/agent/runner/loop_usage.go @@ -120,6 +120,61 @@ func (l *Loop) usageByModelSnapshot() []pipeline.ModelUsage { return out } +// toolUsageBucket is one interactive toolkit's running cost accumulation across +// its invocations this session, keyed by outer tool name. +type toolUsageBucket struct { + tool string + costMicroUSD int64 + costReported bool +} + +// addToolCost folds one interactive toolkit result event's provider-reported +// cost into the bucket for outerTool. Shares usageMu with addUsage/addModelUsage +// (see usageMu's doc). ok is the toolkit's success flag; v1 accumulates cost +// regardless — a run the provider billed cost money whether or not it succeeded, +// and an unbilled run reports 0 anyway. +func (l *Loop) addToolCost(outerTool string, costUSD float64, ok bool) { + l.usageMu.Lock() + defer l.usageMu.Unlock() + if l.usageByToolIdx == nil { + l.usageByToolIdx = make(map[string]int) + } + i, seen := l.usageByToolIdx[outerTool] + if !seen { + i = len(l.usageByTool) + l.usageByToolIdx[outerTool] = i + l.usageByTool = append(l.usageByTool, toolUsageBucket{tool: outerTool}) + } + b := &l.usageByTool[i] + if costUSD < 0 { + // A negative provider-reported tool cost would reduce the session total. + // Treat it as unreported rather than netting it against real charges — + // same guard as addModelUsage. + slog.Default().Info("negative reported tool cost; ignoring", + "tool", outerTool, "costUSD", costUSD) + return + } + // Round once at the USD->micro-USD boundary so accumulation is pure int64. + b.costMicroUSD += int64(math.Round(costUSD * 1e6)) + b.costReported = true + _ = ok +} + +// usageByToolSnapshot returns the Loop's per-tool cost as pipeline.ToolUsage in +// first-encountered order (nil when unused). Safe for concurrent use. +func (l *Loop) usageByToolSnapshot() []pipeline.ToolUsage { + l.usageMu.Lock() + defer l.usageMu.Unlock() + if len(l.usageByTool) == 0 { + return nil + } + out := make([]pipeline.ToolUsage, len(l.usageByTool)) + for i, b := range l.usageByTool { + out[i] = pipeline.ToolUsage{Tool: b.tool, CostMicroUSD: b.costMicroUSD, CostReported: b.costReported} + } + return out +} + // StampEstimatedCost persists the cost estimate via the StatusPatcher. nil-safe: // a Loop without a Status (tests/kubectl-driven) silently no-ops. func (l *Loop) StampEstimatedCost(ctx context.Context, c spiceboxv1alpha1.EstimatedSessionCost) error { diff --git a/pkg/agent/runner/toolusage_test.go b/pkg/agent/runner/toolusage_test.go new file mode 100644 index 00000000..4bd7817a --- /dev/null +++ b/pkg/agent/runner/toolusage_test.go @@ -0,0 +1,53 @@ +package runner + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoopAddToolCost_AccumulatesPerTool(t *testing.T) { + var l Loop + // codebot's shape: one tool ("claude-oauth") invoked many times -> ONE + // summed bucket; a second tool -> a distinct bucket, first-seen order. + l.addToolCost("claude-oauth", 5.43, true) + l.addToolCost("claude-oauth", 1.31, true) + l.addToolCost("aider", 2.10, true) + + got := l.usageByToolSnapshot() + require.Len(t, got, 2, "two distinct tools -> two buckets in first-encountered order") + + assert.Equal(t, "claude-oauth", got[0].Tool) + assert.True(t, got[0].CostReported) + assert.Equal(t, int64(6_740_000), got[0].CostMicroUSD, "5.43+1.31 USD -> 6,740,000 micro-USD") + + assert.Equal(t, "aider", got[1].Tool) + assert.Equal(t, int64(2_100_000), got[1].CostMicroUSD) +} + +func TestLoopAddToolCost_NegativeCostIgnored(t *testing.T) { + var l Loop + l.addToolCost("claude-oauth", -1.0, true) + + got := l.usageByToolSnapshot() + require.Len(t, got, 1, "the tool is still recorded") + assert.False(t, got[0].CostReported, "negative cost -> treated as unreported") + assert.Equal(t, int64(0), got[0].CostMicroUSD, "no negative amount is ever added") +} + +func TestLoopAddToolCost_ZeroCostUnbilled(t *testing.T) { + // An unbilled / credential-halt run reports cost 0; it adds 0 and does not + // mark the bucket as having a reported (billed) cost. + var l Loop + l.addToolCost("claude-oauth", 0, false) + + got := l.usageByToolSnapshot() + require.Len(t, got, 1) + assert.Equal(t, int64(0), got[0].CostMicroUSD) +} + +func TestLoopUsageByToolSnapshot_EmptyWhenUnused(t *testing.T) { + var l Loop + assert.Nil(t, l.usageByToolSnapshot(), "no tool runs -> nil, not a slice of zero buckets") +} From fed25caa24d4aec23c4eb18123db348e9ef8ec57 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 30 Sep 2026 12:12:10 -0400 Subject: [PATCH 3/7] Feed interactive toolkit cost into SessionEnd via ByTool --- .../runner/interactive_event_publish_test.go | 34 +++++++++++++++++-- internal/cmd/runner/main.go | 12 ++++++- internal/cmd/runner/main_test.go | 2 +- pkg/agent/runner/loop_failure.go | 1 + pkg/agent/runner/loop_usage.go | 8 +++-- pkg/agent/runner/modelusage_test.go | 15 ++++++++ pkg/agent/runner/toolusage_test.go | 10 +++--- 7 files changed, 70 insertions(+), 12 deletions(-) diff --git a/internal/cmd/runner/interactive_event_publish_test.go b/internal/cmd/runner/interactive_event_publish_test.go index 543928fc..8fe536f7 100644 --- a/internal/cmd/runner/interactive_event_publish_test.go +++ b/internal/cmd/runner/interactive_event_publish_test.go @@ -38,7 +38,7 @@ func TestBuildToolSessionEventPublisher_PublishesKindToolSessionEvent(t *testing mem := memory.NewLocal(inmem.NewBackend()) scope := memory.Scope{Kind: "session", ID: "default/agent-1"} onEvent := buildToolSessionEventPublisher( - context.Background(), pub, mem, scope, "off", "default", "agent-1", nil) + context.Background(), pub, mem, scope, "off", "default", "agent-1", nil, nil) onEvent("tc-abc", "", "", toolkitstream.Event{Type: toolkitstream.EventTextDelta, Text: "hi"}) mu.Lock() @@ -60,7 +60,7 @@ func TestBuildToolSessionEventPublisher_CarriesReasonAndOuterTool(t *testing.T) return nil } emit := buildToolSessionEventPublisher(context.Background(), pub, - nil /*mem*/, memory.Scope{} /*scope*/, spiceboxv1alpha1.ToolSessionLogOff /*no persist*/, "ns", "sess", nil) + nil /*mem*/, memory.Scope{} /*scope*/, spiceboxv1alpha1.ToolSessionLogOff /*no persist*/, "ns", "sess", nil, nil) emit("tc-1", "Have claude write the README", "claude", toolkitstream.Event{Type: toolkitstream.EventTextDelta, Text: "hi"}) @@ -91,7 +91,7 @@ func TestBuildToolSessionEventPublisher_AllFieldsRoundTrip(t *testing.T) { mem := memory.NewLocal(inmem.NewBackend()) scope := memory.Scope{Kind: "session", ID: "default/agent-1"} onEvent := buildToolSessionEventPublisher( - context.Background(), pub, mem, scope, "off", "default", "agent-1", nil) + context.Background(), pub, mem, scope, "off", "default", "agent-1", nil, nil) onEvent("tc-1", "", "", toolkitstream.Event{ Type: toolkitstream.EventToolUseStop, @@ -110,3 +110,31 @@ func TestBuildToolSessionEventPublisher_AllFieldsRoundTrip(t *testing.T) { assert.False(t, pl.OK) assert.Equal(t, "exit code 1", pl.Summary) } + +func TestBuildToolSessionEventPublisher_InvokesOnResultForResultEvent(t *testing.T) { + type call struct { + tool string + cost float64 + ok bool + } + var got []call + onResult := func(tool string, cost float64, ok bool) { + got = append(got, call{tool, cost, ok}) + } + pub := func(_ context.Context, _ string, _ []byte) error { return nil } + + emit := buildToolSessionEventPublisher(context.Background(), pub, + nil, memory.Scope{}, spiceboxv1alpha1.ToolSessionLogOff, "ns", "sess", nil, onResult) + + // A non-result event must NOT trigger onResult. + emit("tc-1", "reason", "claude-oauth", + toolkitstream.Event{Type: toolkitstream.EventTextDelta, Text: "hi"}) + // A result event MUST trigger it once, carrying tool/cost/ok. + emit("tc-1", "reason", "claude-oauth", + toolkitstream.Event{Type: toolkitstream.EventResult, OK: true, CostUSD: 5.43}) + + require.Len(t, got, 1, "onResult fires only on EventResult") + assert.Equal(t, "claude-oauth", got[0].tool) + assert.Equal(t, 5.43, got[0].cost) + assert.True(t, got[0].ok) +} diff --git a/internal/cmd/runner/main.go b/internal/cmd/runner/main.go index 0d1968e8..eb956d13 100644 --- a/internal/cmd/runner/main.go +++ b/internal/cmd/runner/main.go @@ -2984,7 +2984,8 @@ func run(cfg *config) error { } }, OnEvent: buildToolSessionEventPublisher( - rootCtx, pub, memSigned, scope, class.Spec.ToolSessionLog, hooksNS, hooksName, envSigner), + rootCtx, pub, memSigned, scope, class.Spec.ToolSessionLog, hooksNS, hooksName, envSigner, + loop.AddToolCost), Register: toolSessionReg.register, } } @@ -3241,6 +3242,7 @@ func buildToolSessionEventPublisher( logMode string, ns, name string, signer *channelevents.EnvelopeSigner, + onResult func(outerTool string, costUSD float64, ok bool), ) func(toolCallRef, reason, outerTool string, ev toolkitstream.Event) { return func(toolCallRef, reason, outerTool string, ev toolkitstream.Event) { // NATS -> channelsd -> Slack — unchanged, always runs. @@ -3287,6 +3289,14 @@ func buildToolSessionEventPublisher( "toolCallRef", toolCallRef, "eventType", string(ev.Type)) } + + // Fold this interactive toolkit's own provider-reported cost into the + // session total. Fires on the terminal result event only, and regardless + // of the ToolSessionLog persist gate above — accumulation must not depend + // on logging being on. + if ev.Type == toolkitstream.EventResult && onResult != nil { + onResult(outerTool, ev.CostUSD, ev.OK) + } } } diff --git a/internal/cmd/runner/main_test.go b/internal/cmd/runner/main_test.go index ab392ce4..1a03ab39 100644 --- a/internal/cmd/runner/main_test.go +++ b/internal/cmd/runner/main_test.go @@ -181,7 +181,7 @@ func TestBuildToolSessionEventPublisher_Persist(t *testing.T) { noopPub := func(context.Context, string, []byte) error { return nil } onEvent := buildToolSessionEventPublisher( - ctx, noopPub, mem, scope, tc.mode, "default", "sess1", nil) + ctx, noopPub, mem, scope, tc.mode, "default", "sess1", nil, nil) onEvent("ref-1", "", "", toolkitstream.Event{ Type: toolkitstream.EventToolUseStart, ToolName: "Edit", ToolID: "t1", }) diff --git a/pkg/agent/runner/loop_failure.go b/pkg/agent/runner/loop_failure.go index 5725f2ef..c2713bf7 100644 --- a/pkg/agent/runner/loop_failure.go +++ b/pkg/agent/runner/loop_failure.go @@ -111,6 +111,7 @@ func (l *Loop) fireSessionEnd(ctx context.Context, reason string) { CacheCreationTokens: snap.CacheCreationTokens, CacheReadTokens: snap.CacheReadTokens, ByModel: l.usageByModelSnapshot(), + ByTool: l.usageByToolSnapshot(), }, }, host); err != nil { slog.Default().Info("fireSessionEnd: SessionEnd executor errored (best-effort)", diff --git a/pkg/agent/runner/loop_usage.go b/pkg/agent/runner/loop_usage.go index 8b56b22b..f75438f2 100644 --- a/pkg/agent/runner/loop_usage.go +++ b/pkg/agent/runner/loop_usage.go @@ -128,12 +128,16 @@ type toolUsageBucket struct { costReported bool } -// addToolCost folds one interactive toolkit result event's provider-reported +// AddToolCost folds one interactive toolkit result event's provider-reported // cost into the bucket for outerTool. Shares usageMu with addUsage/addModelUsage // (see usageMu's doc). ok is the toolkit's success flag; v1 accumulates cost // regardless — a run the provider billed cost money whether or not it succeeded, // and an unbilled run reports 0 anyway. -func (l *Loop) addToolCost(outerTool string, costUSD float64, ok bool) { +// +// Exported (unlike addModelUsage) because it is wired from internal/cmd/runner's +// interactive-tool result callback, which lives in a different package; the +// runner's turn loop feeds addModelUsage internally, so that one stays private. +func (l *Loop) AddToolCost(outerTool string, costUSD float64, ok bool) { l.usageMu.Lock() defer l.usageMu.Unlock() if l.usageByToolIdx == nil { diff --git a/pkg/agent/runner/modelusage_test.go b/pkg/agent/runner/modelusage_test.go index 2cf3138d..85e36397 100644 --- a/pkg/agent/runner/modelusage_test.go +++ b/pkg/agent/runner/modelusage_test.go @@ -89,3 +89,18 @@ func TestFireSessionEnd_PopulatesByModel(t *testing.T) { assert.True(t, b.CostReported) assert.Equal(t, int64(50_000), b.ReportedCostMicroUSD) } + +func TestFireSessionEnd_PopulatesByTool(t *testing.T) { + cr := &endCapturingRunner{} + l := &Loop{Model: "claude-sonnet-5"} + l.pipelineExec = cr + l.pipelineOnce.Do(func() {}) + + l.AddToolCost("claude-oauth", 5.43, true) + l.fireSessionEnd(context.Background(), "completed") + + require.NotNil(t, cr.in.End) + require.Len(t, cr.in.End.ByTool, 1) + assert.Equal(t, "claude-oauth", cr.in.End.ByTool[0].Tool) + assert.Equal(t, int64(5_430_000), cr.in.End.ByTool[0].CostMicroUSD) +} diff --git a/pkg/agent/runner/toolusage_test.go b/pkg/agent/runner/toolusage_test.go index 4bd7817a..7673b515 100644 --- a/pkg/agent/runner/toolusage_test.go +++ b/pkg/agent/runner/toolusage_test.go @@ -11,9 +11,9 @@ func TestLoopAddToolCost_AccumulatesPerTool(t *testing.T) { var l Loop // codebot's shape: one tool ("claude-oauth") invoked many times -> ONE // summed bucket; a second tool -> a distinct bucket, first-seen order. - l.addToolCost("claude-oauth", 5.43, true) - l.addToolCost("claude-oauth", 1.31, true) - l.addToolCost("aider", 2.10, true) + l.AddToolCost("claude-oauth", 5.43, true) + l.AddToolCost("claude-oauth", 1.31, true) + l.AddToolCost("aider", 2.10, true) got := l.usageByToolSnapshot() require.Len(t, got, 2, "two distinct tools -> two buckets in first-encountered order") @@ -28,7 +28,7 @@ func TestLoopAddToolCost_AccumulatesPerTool(t *testing.T) { func TestLoopAddToolCost_NegativeCostIgnored(t *testing.T) { var l Loop - l.addToolCost("claude-oauth", -1.0, true) + l.AddToolCost("claude-oauth", -1.0, true) got := l.usageByToolSnapshot() require.Len(t, got, 1, "the tool is still recorded") @@ -40,7 +40,7 @@ func TestLoopAddToolCost_ZeroCostUnbilled(t *testing.T) { // An unbilled / credential-halt run reports cost 0; it adds 0 and does not // mark the bucket as having a reported (billed) cost. var l Loop - l.addToolCost("claude-oauth", 0, false) + l.AddToolCost("claude-oauth", 0, false) got := l.usageByToolSnapshot() require.Len(t, got, 1) From ec3dfecfdb93917cc1432009c958d3b13c116768 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 30 Sep 2026 12:13:30 -0400 Subject: [PATCH 4/7] Include interactive toolkit cost in the session total and notice --- pkg/agent/postsession/cost/cost.go | 54 ++++++++++++++++++++----- pkg/agent/postsession/cost/cost_test.go | 43 ++++++++++++++++++++ 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/pkg/agent/postsession/cost/cost.go b/pkg/agent/postsession/cost/cost.go index 1c6ff3f5..339f2068 100644 --- a/pkg/agent/postsession/cost/cost.go +++ b/pkg/agent/postsession/cost/cost.go @@ -75,13 +75,21 @@ func (r *Reporter) Eval(ctx context.Context, in pipeline.Input) pipeline.Decisio micro = microUSD(e, price) } + // Interactive toolkit cost (e.g. an inner `claude` sub-run's own billed + // spend) is provider-reported, so it is always "known" and is ADDED to the + // model total. When a served model is unpriced (known=false, micro=0), the + // grand total is still the priced tool component — a lower bound, per the + // EstimatedSessionCost.AmountMicroUSD contract. + toolBuckets, toolTotal := buildToolBuckets(e.ByTool) + cost := v1.EstimatedSessionCost{ - AmountMicroUSD: micro, + AmountMicroUSD: micro + toolTotal, Currency: currency, Model: e.Model, PricingKnown: known, AsOf: r.d.Now(), ByModel: buckets, + ByTool: toolBuckets, } if known && cost.Currency == "" { cost.Currency = "USD" @@ -165,6 +173,22 @@ func (r *Reporter) buildCostBuckets(usages []pipeline.ModelUsage) (buckets []v1. return buckets, total, allKnown, currency } +// buildToolBuckets maps each interactive-toolkit usage bucket to a priced +// ToolCostBucket. Tool cost is always provider-reported (already micro-USD and +// "known"), so there is no table lookup here — this is a pure projection plus a +// sum. Returns the buckets in caller order and their total. +func buildToolBuckets(usages []pipeline.ToolUsage) (buckets []v1.ToolCostBucket, total int64) { + if len(usages) == 0 { + return nil, 0 + } + buckets = make([]v1.ToolCostBucket, len(usages)) + for i, u := range usages { + buckets[i] = v1.ToolCostBucket{Tool: u.Tool, AmountMicroUSD: u.CostMicroUSD, PricingKnown: u.CostReported} + total += u.CostMicroUSD + } + return buckets, total +} + func costNotice(e *pipeline.SessionEndInfo, c v1.EstimatedSessionCost, known bool) *notice.Notice { if !known { return notice.New(categories.SessionCost, notice.Args{ @@ -177,15 +201,27 @@ func costNotice(e *pipeline.SessionEndInfo, c v1.EstimatedSessionCost, known boo Audience: channelevents.InteractionAudience{Scope: channelevents.AudienceParticipants}, }) } + fields := []channelevents.InteractionField{ + {Label: "Tokens", Value: fmt.Sprintf("%s in / %s out", + formatTokens(e.InputTokens), formatTokens(e.OutputTokens))}, + {Label: "Cache", Value: fmt.Sprintf("%s read / %s write", + formatTokens(e.CacheReadTokens), formatTokens(e.CacheCreationTokens))}, + {Label: "Model", Value: e.Model}, + } + // Break out inner interactive-toolkit spend (e.g. a passthrough `claude` + // sub-run) so the total's provenance is visible, not folded silently. + var toolTotal int64 + for _, b := range c.ByTool { + toolTotal += b.AmountMicroUSD + } + if toolTotal > 0 { + fields = append(fields, channelevents.InteractionField{ + Label: "Sub-agent tools", Value: formatUSD(toolTotal), + }) + } return notice.New(categories.SessionCost, notice.Args{ - Lead: fmt.Sprintf("This session cost ~%s", formatUSD(c.AmountMicroUSD)), - Fields: []channelevents.InteractionField{ - {Label: "Tokens", Value: fmt.Sprintf("%s in / %s out", - formatTokens(e.InputTokens), formatTokens(e.OutputTokens))}, - {Label: "Cache", Value: fmt.Sprintf("%s read / %s write", - formatTokens(e.CacheReadTokens), formatTokens(e.CacheCreationTokens))}, - {Label: "Model", Value: e.Model}, - }, + Lead: fmt.Sprintf("This session cost ~%s", formatUSD(c.AmountMicroUSD)), + Fields: fields, Audience: channelevents.InteractionAudience{Scope: channelevents.AudienceParticipants}, }) } diff --git a/pkg/agent/postsession/cost/cost_test.go b/pkg/agent/postsession/cost/cost_test.go index 81173827..35d62099 100644 --- a/pkg/agent/postsession/cost/cost_test.go +++ b/pkg/agent/postsession/cost/cost_test.go @@ -163,6 +163,49 @@ func TestEval_ByModel_ReportedPreferredOverEstimated(t *testing.T) { assert.True(t, stamped.PricingKnown, "every bucket priced -> session PricingKnown") } +func TestEval_ByTool_AddsToGrandTotalAndItemizes(t *testing.T) { + r, stamped := newWithCapturePricer(multiModelPricer) + in := endInput("completed", 0, 0, 0, 0) + in.End.ByModel = []pipeline.ModelUsage{ + {Display: "anthropic/claude-3.5-sonnet", Model: "anthropic/claude-3.5-sonnet", + InputTokens: 1_000_000, OutputTokens: 0}, // 1M in × $3 = $3.00 -> 3_000_000 micro + } + in.End.ByTool = []pipeline.ToolUsage{{Tool: "claude-oauth", CostMicroUSD: 19_020_000, CostReported: true}} + r.Eval(context.Background(), in) + + require.Len(t, stamped.ByTool, 1) + assert.Equal(t, "claude-oauth", stamped.ByTool[0].Tool) + assert.Equal(t, int64(19_020_000), stamped.ByTool[0].AmountMicroUSD) + assert.True(t, stamped.ByTool[0].PricingKnown) + assert.Equal(t, int64(3_000_000+19_020_000), stamped.AmountMicroUSD, "grand total = model + tool") + assert.True(t, stamped.PricingKnown, "model priced + tool always priced") +} + +func TestEval_NoTool_IdenticalToToday(t *testing.T) { + r, stamped := newWithCapturePricer(multiModelPricer) + in := endInput("completed", 0, 0, 0, 0) + in.End.ByModel = []pipeline.ModelUsage{ + {Display: "anthropic/claude-3.5-sonnet", Model: "anthropic/claude-3.5-sonnet", InputTokens: 1_000_000}, + } + r.Eval(context.Background(), in) + + assert.Nil(t, stamped.ByTool, "no tool -> no byTool") + assert.Equal(t, int64(3_000_000), stamped.AmountMicroUSD, "unchanged from today") +} + +func TestEval_UnknownModel_KnownTool_LowerBound(t *testing.T) { + r, stamped := newWithCapturePricer(multiModelPricer) // "mystery-model" unpriced + in := endInput("completed", 0, 0, 0, 0) + in.End.ByModel = []pipeline.ModelUsage{ + {Display: "openrouter/mystery/model", Model: "mystery-model", InputTokens: 1_000_000}, + } + in.End.ByTool = []pipeline.ToolUsage{{Tool: "claude-oauth", CostMicroUSD: 19_020_000, CostReported: true}} + r.Eval(context.Background(), in) + + assert.False(t, stamped.PricingKnown, "an unpriced model makes the total a lower bound") + assert.Equal(t, int64(19_020_000), stamped.AmountMicroUSD, "nonzero: the priced (tool) component") +} + func TestEval_ByModel_UnknownPriceBucket_ZeroAndUnknown(t *testing.T) { r, stamped := newWithCapturePricer(multiModelPricer) in := endInput("completed", 0, 0, 0, 0) From 495ee1925aecc93be6d840990208278fe1c239b3 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 30 Sep 2026 12:15:43 -0400 Subject: [PATCH 5/7] Surface interactive toolkit cost in admind budget and overview --- pkg/web/admind/aggregator.go | 6 +++++ pkg/web/admind/budget.go | 12 +++++++++- pkg/web/admind/budget_internal_test.go | 16 +++++++++++++ pkg/web/admind/budget_test.go | 32 ++++++++++++++++++++++++++ pkg/web/admind/handlers.go | 2 +- 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 pkg/web/admind/budget_internal_test.go diff --git a/pkg/web/admind/aggregator.go b/pkg/web/admind/aggregator.go index 96de875c..2184b5d9 100644 --- a/pkg/web/admind/aggregator.go +++ b/pkg/web/admind/aggregator.go @@ -54,6 +54,10 @@ type SessionState struct { // yet, in which case the Overview/Budget aggregators attribute this // session's InputTokens/OutputTokens/Model as a single blended row. ByModel []spiceboxv1alpha1.ModelCostBucket `json:"byModel,omitempty"` + // ByTool is the session's per-interactive-toolkit cost breakdown + // (status.estimatedCost.byTool). Added to the token-repriced estimate in the + // budget/overview panels so admind totals include inner sub-agent spend. + ByTool []spiceboxv1alpha1.ToolCostBucket `json:"byTool,omitempty"` // BundleSessions mirrors AgentSession.status.bundleSessions — one entry per // tool bundle this session resolved to a SpiceboxSession, carried straight // through from the watched CRD (no new fetch). A bundle's sandbox backend @@ -362,8 +366,10 @@ func (a *Aggregator) UpsertSession(s *spiceboxv1alpha1.AgentSession) { // later Snapshot()/sortedModels-style sort-in-place elsewhere on the // SAME backing array would race the informer's own reads of it. st.ByModel = append([]spiceboxv1alpha1.ModelCostBucket(nil), ec.ByModel...) + st.ByTool = append([]spiceboxv1alpha1.ToolCostBucket(nil), ec.ByTool...) } else { st.ByModel = nil + st.ByTool = nil } // Defensive copy for the same reason as ByModel above: s.Status.BundleSessions // backs onto the informer cache's object. diff --git a/pkg/web/admind/budget.go b/pkg/web/admind/budget.go index f670229a..81352853 100644 --- a/pkg/web/admind/budget.go +++ b/pkg/web/admind/budget.go @@ -215,7 +215,7 @@ func (a *Admind) handleBudget(w http.ResponseWriter, r *http.Request) { // so it must be stripped of its provider prefix before pricing. bs := budgetSession{ State: s, - Est: prices.Estimate(bareModel(s.Model), s.InputTokens, s.OutputTokens), + Est: prices.Estimate(bareModel(s.Model), s.InputTokens, s.OutputTokens) + toolCostUSD(s.ByTool), Starters: starters, } for _, d := range budgetDimensions { @@ -259,6 +259,16 @@ func bucketCostUSD(b spiceboxv1alpha1.ModelCostBucket) cost.USD { return cost.USD(float64(b.AmountMicroUSD) / 1e6) } +// toolCostUSD sums a session's provider-reported interactive-toolkit spend. +// Nil/empty -> 0. Tool cost is always priced, so this never yields NaN. +func toolCostUSD(buckets []spiceboxv1alpha1.ToolCostBucket) cost.USD { + var micro int64 + for _, b := range buckets { + micro += b.AmountMicroUSD + } + return cost.USD(float64(micro) / 1e6) +} + // sortedBudgetRows flattens the accumulator into a non-nil slice ordered by // estimated cost desc, key asc as the tie-break — a stable dashboard order and // a JSON [] (never null) when empty. diff --git a/pkg/web/admind/budget_internal_test.go b/pkg/web/admind/budget_internal_test.go new file mode 100644 index 00000000..f231ada4 --- /dev/null +++ b/pkg/web/admind/budget_internal_test.go @@ -0,0 +1,16 @@ +package admind + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + spiceboxv1alpha1 "github.com/authzed/openagentprimitives/pkg/apis/v1alpha1" +) + +func TestToolCostUSD_NilByToolContributesZero(t *testing.T) { + assert.Equal(t, 0.0, float64(toolCostUSD(nil)), "nil/empty -> 0") + assert.InDelta(t, 19.02, float64(toolCostUSD([]spiceboxv1alpha1.ToolCostBucket{ + {Tool: "claude-oauth", AmountMicroUSD: 19_020_000, PricingKnown: true}, + })), 1e-9) +} diff --git a/pkg/web/admind/budget_test.go b/pkg/web/admind/budget_test.go index abb4ce2a..9742b43b 100644 --- a/pkg/web/admind/budget_test.go +++ b/pkg/web/admind/budget_test.go @@ -422,3 +422,35 @@ func TestAdmindBudget_EmptyIsNonNilSlices(t *testing.T) { assert.Contains(t, body, "\""+axis+"\":[]", axis+" must be an empty array, not null") } } + +func TestAdmindBudget_ByToolAddsToEstimate(t *testing.T) { + s := &spiceboxv1alpha1.AgentSession{} + s.Namespace, s.Name = "default", "s-codebot" + s.Spec.Class = "codebot" + s.Status.Phase = "Completed" + s.Status.StartedAt = &metav1.Time{Time: time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC)} + s.Status.EffectiveSettings = &spiceboxv1alpha1.EffectiveSettings{ + Model: spiceboxv1alpha1.ModelConfig{Provider: "anthropic", Name: "claude-opus-4-8"}, + } + // Zero tokens -> $0 model estimate, isolating the tool-cost contribution. + s.Status.Progress = &spiceboxv1alpha1.AgentSessionProgress{InputTokens: 0, OutputTokens: 0} + s.Status.EstimatedCost = &spiceboxv1alpha1.EstimatedSessionCost{ + ByTool: []spiceboxv1alpha1.ToolCostBucket{ + {Tool: "claude-oauth", AmountMicroUSD: 19_020_000, PricingKnown: true}, + }, + } + + k8s := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(s).Build() + a := newTestAdmind(t, k8s) + a.Aggregator().UpsertSession(s) + + w := do(t, a.Handler(), http.MethodGet, "/admin/v1/budget", "test-token", "user:YWRtaW4", "") + require.Equal(t, http.StatusOK, w.Code) + var bd admind.BudgetBreakdown + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &bd)) + + sess := rowsByKey(bd.BySession) + require.Contains(t, sess, "default/s-codebot") + assert.InDelta(t, 19.02, float64(sess["default/s-codebot"].EstimatedCostUSD), 1e-9, + "$0 model (0 tokens) + $19.02 tool cost") +} diff --git a/pkg/web/admind/handlers.go b/pkg/web/admind/handlers.go index c7c4cfaa..4f612ec8 100644 --- a/pkg/web/admind/handlers.go +++ b/pkg/web/admind/handlers.go @@ -670,7 +670,7 @@ func (a *Admind) enrichSessionRows(r *http.Request, rows []audit.EntityRow) { rows[i].OutputTokens = st.OutputTokens // st.Model is the uniform "/" display id; strip the // provider prefix before the bare-keyed price lookup. - rows[i].EstimatedCostUSD = prices.Estimate(bareModel(st.Model), st.InputTokens, st.OutputTokens) + rows[i].EstimatedCostUSD = prices.Estimate(bareModel(st.Model), st.InputTokens, st.OutputTokens) + toolCostUSD(st.ByTool) } if m, ok := meta[key]; ok { rows[i].StartedBy = m.StartedBy From abf64a96fa43782dbeb44c14d5f43d76b48f0a8f Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Wed, 30 Sep 2026 13:10:22 -0400 Subject: [PATCH 6/7] Include interactive toolkit cost in admind Overview spend panel --- pkg/web/admind/handler_test.go | 32 ++++++++++++++++++++++++++++++++ pkg/web/admind/handlers.go | 9 +++++++++ 2 files changed, 41 insertions(+) diff --git a/pkg/web/admind/handler_test.go b/pkg/web/admind/handler_test.go index 221248af..ee74761b 100644 --- a/pkg/web/admind/handler_test.go +++ b/pkg/web/admind/handler_test.go @@ -226,6 +226,38 @@ func TestAdmindAuditEndpoints(t *testing.T) { assert.Equal(t, http.StatusBadRequest, w.Code) } +func TestAdmindOverview_IncludesToolCost(t *testing.T) { + s := &spiceboxv1alpha1.AgentSession{} + s.Namespace, s.Name = "default", "s-codebot" + s.Spec.Class = "codebot" + s.Status.Phase = "Completed" + s.Status.EffectiveSettings = &spiceboxv1alpha1.EffectiveSettings{ + Model: spiceboxv1alpha1.ModelConfig{Provider: "anthropic", Name: "claude-opus-4-8"}, + } + // Zero tokens -> $0 model estimate, isolating the tool-cost contribution. + s.Status.Progress = &spiceboxv1alpha1.AgentSessionProgress{InputTokens: 0, OutputTokens: 0} + s.Status.EstimatedCost = &spiceboxv1alpha1.EstimatedSessionCost{ + ByTool: []spiceboxv1alpha1.ToolCostBucket{ + {Tool: "claude-oauth", AmountMicroUSD: 19_020_000, PricingKnown: true}, + }, + } + + k8s := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(s).Build() + a := newTestAdmind(t, k8s) + a.Aggregator().UpsertSession(s) + + w := do(t, a.Handler(), http.MethodGet, "/admin/v1/overview", "test-token", "user:YWRtaW4", "") + require.Equal(t, http.StatusOK, w.Code) + var ov struct { + Budget struct { + EstimatedCostUSD float64 `json:"estimatedCostUSD"` + } `json:"budget"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &ov)) + assert.InDelta(t, 19.02, ov.Budget.EstimatedCostUSD, 1e-9, + "Overview headline spend must include inner tool cost ($0 model + $19.02 tool)") +} + func TestAdmindOverviewAndHealth(t *testing.T) { sess := sessionCR("default", "s1", "support-bot", "Running") k8s := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(sess).Build() diff --git a/pkg/web/admind/handlers.go b/pkg/web/admind/handlers.go index 4f612ec8..ecfcd3c3 100644 --- a/pkg/web/admind/handlers.go +++ b/pkg/web/admind/handlers.go @@ -537,6 +537,15 @@ func (a *Admind) handleOverview(w http.ResponseWriter, r *http.Request) { estCost += prices.Estimate(bareModel(m.Model), m.InputTokens, m.OutputTokens) tokensSpent += m.InputTokens + m.OutputTokens } + // Inner interactive-toolkit spend (e.g. a passthrough `claude` sub-run) is + // not token-based, so it never appears in ov.ByModel. Add it over the SAME + // session scope that produced ov.ByModel — the overview engine's own live + // input is liveSessionSnapshot(a.agg), so a.agg.Snapshot() is exactly that + // set (no double-count). Without this the headline spend understates a + // sub-agent session by the full tool cost (see the codebot case). + for _, s := range a.agg.Snapshot() { + estCost += toolCostUSD(s.ByTool) + } writeJSON(w, http.StatusOK, overviewResponse{ Overview: ov, Budget: budgetInfo{TokensSpent: tokensSpent, EstimatedCostUSD: estCost, Estimated: true}, From bd065be4bc02ca4a8318b2de8ed465d7fc715689 Mon Sep 17 00:00:00 2001 From: Joseph Schorr Date: Tue, 13 Oct 2026 13:22:57 -0400 Subject: [PATCH 7/7] Address PR #13 review: per-pod cost accumulation + subscription exclusion Interactive-toolkit cost accounting fixes from code review: - AddToolCost carries a costReported presence bit (was an unused ok flag), so an absent total_cost_usd no longer asserts a known $0. - Negative tool costs are kept as "seen" but flagged Anomalous and excluded from the total. - byModel budget axis gives tool spend its own rows so its grand total matches the other axes (no dropped/hidden tool cost). - Session-cost notice shows a lower bound instead of "unavailable" when the model is unpriced but tool spend exists. - Overview headline sums tool cost inside the engine over the same cached snapshot as byModel (no TTL-window skew). - Session cost ACCUMULATES across pods: status.estimatedCost.perPod is the per-pod source of truth; each pod upserts its own entry and the aggregate (amount / byModel / byTool / pricingKnown) is recomputed as sum / merge / AND. Replaces the monotonic keep-higher guard. - Subscription-equivalent sub-run cost (new toolkit costTreatment field) is recorded and shown (flagged subscriptionEquivalent, separate notice line) but excluded from session spend so it cannot inflate the figure or trip alerts. CRDs, deepcopy, and install.yaml regenerated. --- ...tprimitives.authzed.com_agentsessions.yaml | 187 ++++++++++++++-- ...imitives.authzed.com_spiceboxtoolkits.yaml | 12 ++ .../runner/interactive_event_publish_test.go | 28 ++- internal/cmd/runner/main.go | 6 +- pkg/agent/postsession/cost/cost.go | 145 +++++++++---- pkg/agent/postsession/cost/cost_test.go | 47 ++++- pkg/agent/runner/loop_usage.go | 55 +++-- pkg/agent/runner/modelusage_test.go | 2 +- pkg/agent/runner/status.go | 152 +++++++++++-- pkg/agent/runner/status_test.go | 102 +++++++-- pkg/agent/runner/toolusage_test.go | 55 ++++- pkg/agent/tool/sandbox/interactive_tool.go | 8 + pkg/agent/tool/sandbox/sandbox_tool.go | 7 + pkg/apis/v1alpha1/agentsession_types.go | 99 +++++++-- pkg/apis/v1alpha1/spiceboxtoolkit_types.go | 10 + pkg/apis/v1alpha1/zz_generated.deepcopy.go | 33 +++ pkg/platform/manifests/install.yaml | 199 ++++++++++++++++-- pkg/platform/pipeline/types.go | 7 + pkg/tools/toolkitstream/claude/parser.go | 9 +- pkg/tools/toolkitstream/contract.go | 12 ++ pkg/tools/toolspec/toolkit/types.go | 19 ++ pkg/web/admind/admind.go | 1 + pkg/web/admind/budget.go | 64 ++++-- pkg/web/admind/budget_internal_test.go | 10 + pkg/web/admind/budget_test.go | 57 +++++ pkg/web/admind/handlers.go | 15 +- pkg/web/admind/overview/aggregate.go | 12 ++ pkg/web/admind/overview/aggregate_test.go | 5 +- 28 files changed, 1154 insertions(+), 204 deletions(-) diff --git a/config/crds/agentprimitives.authzed.com_agentsessions.yaml b/config/crds/agentprimitives.authzed.com_agentsessions.yaml index 3d8fd368..d0891e50 100644 --- a/config/crds/agentprimitives.authzed.com_agentsessions.yaml +++ b/config/crds/agentprimitives.authzed.com_agentsessions.yaml @@ -2012,11 +2012,12 @@ spec: properties: amountMicroUSD: description: |- - AmountMicroUSD is the session total in micro-USD (1e-6 USD): the sum of - every priced component (ByModel + ByTool). When PricingKnown is false, at - least one served MODEL had no rate, so this is a LOWER BOUND on spend (the - components that could be priced), not the full session cost — it is no - longer necessarily 0, because provider-reported tool cost is always priced. + AmountMicroUSD is the SESSION total in micro-USD (1e-6 USD), accumulated + across every pod that served the session: the sum of PerPod amounts. When + PricingKnown is false, at least one served MODEL (in some pod) had no rate, + so this is a LOWER BOUND on spend (the components that could be priced), + not the full session cost — it is no longer necessarily 0, because + provider-reported tool cost is always priced. format: int64 type: integer asOf: @@ -2025,10 +2026,11 @@ spec: type: string byModel: description: |- - ByModel breaks the total down per actually-served model, so a session - that fell back or auto-routed across models shows where the spend went - instead of one blended total under the configured model. Empty when the - runner reported no per-model usage; AmountMicroUSD and PricingKnown are + ByModel breaks the total down per actually-served model, MERGED across + every pod (summed per model display id), so a session that fell back or + auto-routed across models — or ran across a reap/wake — shows where the + spend went instead of one blended total under the configured model. Empty + when no pod reported per-model usage; AmountMicroUSD and PricingKnown are then the sole source of truth. items: description: ModelCostBucket is one served model's slice of @@ -2071,11 +2073,10 @@ spec: byTool: description: |- ByTool breaks out the cost of inner interactive toolkits (e.g. a `claude` - Claude Code sub-run's own provider-reported spend) a session drove. These - are ADDED to AmountMicroUSD alongside ByModel — a session's total is model - spend plus tool spend. Empty when no interactive toolkit reported a cost. - Tool cost is always provider-reported, never table-priced, so a bucket's - PricingKnown is always true. + Claude Code sub-run's own provider-reported spend) a session drove, MERGED + across every pod (summed per tool). These are ADDED to AmountMicroUSD + alongside ByModel — a session's total is model spend plus tool spend. Empty + when no interactive toolkit reported a cost. items: description: ToolCostBucket is one interactive toolkit's slice of a session's cost. @@ -2083,13 +2084,31 @@ spec: amountMicroUSD: description: |- AmountMicroUSD is this tool's provider-reported cost, summed across its - invocations, in micro-USD (1e-6 USD). + invocations, in micro-USD (1e-6 USD). Negative reports are excluded from + the sum (see Anomalous). format: int64 type: integer + anomalous: + description: |- + Anomalous is true when a negative (nonsensical) provider cost was reported + for this tool. The tool is still recorded as seen, but the negative value + is excluded from AmountMicroUSD and the session total — it signals "a cost + report here does not make sense" rather than silently dropping it. + type: boolean pricingKnown: description: |- - PricingKnown is true when AmountMicroUSD is a real cost. Tool cost is - always provider-reported, so this is true whenever a cost was reported. + PricingKnown is true when AmountMicroUSD is a real cost — i.e. at least one + non-negative cost tally was reported. False when none was reported (the + amount is 0 and must not be shown as real spend) or every report was + anomalous. + type: boolean + subscriptionEquivalent: + description: |- + SubscriptionEquivalent is true when this tool ran under a flat-rate + subscription (toolkit CostTreatment=subscription-equivalent), so its + AmountMicroUSD is an API-equivalent estimate, not money charged. The + bucket is recorded and shown, but its amount is EXCLUDED from the session + total / admind spend so it cannot inflate the figure or trip cost alerts. type: boolean tool: description: Tool is the outer tool name (e.g. "claude-oauth"). @@ -2106,11 +2125,139 @@ spec: model: description: Model is the configured model the session ran under. type: string + perPod: + description: |- + PerPod is the per-pod cost SOURCE OF TRUTH: one entry per pod that served + the session (keyed by pod name), each carrying that pod's own total and + breakdown. The aggregate fields above are DERIVED from this list — the + total is the sum of PerPod amounts, ByModel/ByTool are merged by key, and + PricingKnown is the AND. Each pod upserts only its own entry, so cost + ACCUMULATES across a reap/wake instead of a later pod's single-pod total + replacing an earlier one. Empty only for a session no pod has costed yet. + items: + description: |- + PerPodCost is one serving pod's slice of a session's cost — the per-pod + source of truth the EstimatedSessionCost aggregate is derived from. A pod + rewrites only its OWN entry (keyed by Pod), so the session total accumulates + across pods rather than a later pod clobbering an earlier one. + properties: + amountMicroUSD: + description: AmountMicroUSD is this pod's own total in micro-USD + (model + tool). + format: int64 + type: integer + asOf: + description: AsOf is when this pod last computed its estimate. + format: date-time + type: string + byModel: + description: ByModel is this pod's per-served-model breakdown. + items: + description: ModelCostBucket is one served model's slice + of a session's cost estimate. + properties: + amountMicroUSD: + description: |- + AmountMicroUSD is this bucket's share of the session cost, in + micro-USD (1e-6 USD): the provider-reported cost when available, + else tokens × the resolved per-model pricing table rate. + format: int64 + type: integer + inputTokens: + description: InputTokens is this model's share of + prompt tokens. + format: int64 + type: integer + model: + description: |- + Model is the served-model display id verbatim (provider/model, or a + provider-prefixed served model under routing) — never re-parsed here. + type: string + outputTokens: + description: OutputTokens is this model's share of + completion tokens. + format: int64 + type: integer + pricingKnown: + description: |- + PricingKnown is true when AmountMicroUSD is a real cost — either the + provider reported it directly, or the pricing table had a rate for + this model. False means AmountMicroUSD is 0 and must not be shown as + real spend (same no-fabrication contract as EstimatedSessionCost). + type: boolean + required: + - model + type: object + type: array + x-kubernetes-list-type: atomic + byTool: + description: ByTool is this pod's per-interactive-toolkit + breakdown. + items: + description: ToolCostBucket is one interactive toolkit's + slice of a session's cost. + properties: + amountMicroUSD: + description: |- + AmountMicroUSD is this tool's provider-reported cost, summed across its + invocations, in micro-USD (1e-6 USD). Negative reports are excluded from + the sum (see Anomalous). + format: int64 + type: integer + anomalous: + description: |- + Anomalous is true when a negative (nonsensical) provider cost was reported + for this tool. The tool is still recorded as seen, but the negative value + is excluded from AmountMicroUSD and the session total — it signals "a cost + report here does not make sense" rather than silently dropping it. + type: boolean + pricingKnown: + description: |- + PricingKnown is true when AmountMicroUSD is a real cost — i.e. at least one + non-negative cost tally was reported. False when none was reported (the + amount is 0 and must not be shown as real spend) or every report was + anomalous. + type: boolean + subscriptionEquivalent: + description: |- + SubscriptionEquivalent is true when this tool ran under a flat-rate + subscription (toolkit CostTreatment=subscription-equivalent), so its + AmountMicroUSD is an API-equivalent estimate, not money charged. The + bucket is recorded and shown, but its amount is EXCLUDED from the session + total / admind spend so it cannot inflate the figure or trip cost alerts. + type: boolean + tool: + description: Tool is the outer tool name (e.g. "claude-oauth"). + type: string + required: + - tool + type: object + type: array + x-kubernetes-list-type: atomic + currency: + description: Currency is the ISO code this pod's amount + is denominated in ("USD"). + type: string + pod: + description: Pod is the serving pod's name (its HOSTNAME); + the upsert key. + type: string + pricingKnown: + description: |- + PricingKnown is false when a model this pod served had no rate (this pod's + amount is then a lower bound). + type: boolean + required: + - pod + type: object + type: array + x-kubernetes-list-type: atomic pricingKnown: description: |- - PricingKnown is false when a served model had no price. The amount is then - a lower bound (priced components only), not the full cost. Tool buckets are - always priced, so this tracks model-pricing coverage specifically. + PricingKnown is false when a served model had no price in ANY contributing + pod (the AND across PerPod). The amount is then a lower bound (priced + components only), not the full cost. Tool buckets are always priced, so + this tracks model-pricing coverage specifically. type: boolean type: object failureReason: diff --git a/config/crds/agentprimitives.authzed.com_spiceboxtoolkits.yaml b/config/crds/agentprimitives.authzed.com_spiceboxtoolkits.yaml index d5968021..01175e89 100644 --- a/config/crds/agentprimitives.authzed.com_spiceboxtoolkits.yaml +++ b/config/crds/agentprimitives.authzed.com_spiceboxtoolkits.yaml @@ -58,6 +58,18 @@ spec: Conversion uses a JSON round-trip (ToToolkit), so this field set must stay byte-compatible with toolkit.Toolkit's JSON shape. properties: + costTreatment: + description: |- + CostTreatment declares how this toolkit's provider-reported cost counts + toward session spend. "metered" (default) is real billed money and counts; + "subscription-equivalent" is a flat-rate estimate (e.g. a Claude Max OAuth + run) that is recorded and shown but EXCLUDED from the session total, so it + cannot inflate spend or trip cost alerts. Mirrors + pkg/tools/toolspec/toolkit.Toolkit.CostTreatment. + enum: + - metered + - subscription-equivalent + type: string docs: description: Docs is prose about the CLI, surfaced to authors. type: string diff --git a/internal/cmd/runner/interactive_event_publish_test.go b/internal/cmd/runner/interactive_event_publish_test.go index 8fe536f7..3e108e57 100644 --- a/internal/cmd/runner/interactive_event_publish_test.go +++ b/internal/cmd/runner/interactive_event_publish_test.go @@ -113,13 +113,14 @@ func TestBuildToolSessionEventPublisher_AllFieldsRoundTrip(t *testing.T) { func TestBuildToolSessionEventPublisher_InvokesOnResultForResultEvent(t *testing.T) { type call struct { - tool string - cost float64 - ok bool + tool string + cost float64 + costReported bool + subscription bool } var got []call - onResult := func(tool string, cost float64, ok bool) { - got = append(got, call{tool, cost, ok}) + onResult := func(tool string, cost float64, costReported, subscription bool) { + got = append(got, call{tool, cost, costReported, subscription}) } pub := func(_ context.Context, _ string, _ []byte) error { return nil } @@ -129,12 +130,21 @@ func TestBuildToolSessionEventPublisher_InvokesOnResultForResultEvent(t *testing // A non-result event must NOT trigger onResult. emit("tc-1", "reason", "claude-oauth", toolkitstream.Event{Type: toolkitstream.EventTextDelta, Text: "hi"}) - // A result event MUST trigger it once, carrying tool/cost/ok. + // A result event that reported a cost tally MUST trigger onResult once, + // carrying tool/cost, costReported=true, and the subscription flag. emit("tc-1", "reason", "claude-oauth", - toolkitstream.Event{Type: toolkitstream.EventResult, OK: true, CostUSD: 5.43}) + toolkitstream.Event{Type: toolkitstream.EventResult, OK: true, CostUSD: 5.43, CostReported: true, SubscriptionEquivalent: true}) + // A result event that reported NO tally still fires, with costReported=false + // so the bucket is never asserted as a known $0. + emit("tc-2", "reason", "claude-oauth", + toolkitstream.Event{Type: toolkitstream.EventResult, OK: true, CostUSD: 0}) - require.Len(t, got, 1, "onResult fires only on EventResult") + require.Len(t, got, 2, "onResult fires once per EventResult") assert.Equal(t, "claude-oauth", got[0].tool) assert.Equal(t, 5.43, got[0].cost) - assert.True(t, got[0].ok) + assert.True(t, got[0].costReported, "a reported tally marks cost reported") + assert.True(t, got[0].subscription, "the event's subscription flag is passed through") + assert.Equal(t, float64(0), got[1].cost) + assert.False(t, got[1].costReported, "an absent tally is not reported") + assert.False(t, got[1].subscription) } diff --git a/internal/cmd/runner/main.go b/internal/cmd/runner/main.go index eb956d13..aad502f3 100644 --- a/internal/cmd/runner/main.go +++ b/internal/cmd/runner/main.go @@ -483,7 +483,7 @@ func run(cfg *config) error { cancel() }() - statusPatcher := runner.NewStatusPatcher(c, sessKey) + statusPatcher := runner.NewStatusPatcher(c, sessKey).WithPod(hostname) // failSession is this file's ONE terminal-failure path. Every boot-time // failure below goes through it rather than calling WriteFailed directly, @@ -3242,7 +3242,7 @@ func buildToolSessionEventPublisher( logMode string, ns, name string, signer *channelevents.EnvelopeSigner, - onResult func(outerTool string, costUSD float64, ok bool), + onResult func(outerTool string, costUSD float64, costReported, subscriptionEquivalent bool), ) func(toolCallRef, reason, outerTool string, ev toolkitstream.Event) { return func(toolCallRef, reason, outerTool string, ev toolkitstream.Event) { // NATS -> channelsd -> Slack — unchanged, always runs. @@ -3295,7 +3295,7 @@ func buildToolSessionEventPublisher( // of the ToolSessionLog persist gate above — accumulation must not depend // on logging being on. if ev.Type == toolkitstream.EventResult && onResult != nil { - onResult(outerTool, ev.CostUSD, ev.OK) + onResult(outerTool, ev.CostUSD, ev.CostReported, ev.SubscriptionEquivalent) } } } diff --git a/pkg/agent/postsession/cost/cost.go b/pkg/agent/postsession/cost/cost.go index 339f2068..e608124c 100644 --- a/pkg/agent/postsession/cost/cost.go +++ b/pkg/agent/postsession/cost/cost.go @@ -24,9 +24,11 @@ import ( type Deps struct { // Pricing returns per-model rates; ok=false ⇒ unknown model (no fabrication). Pricing func(model string) (llm.ModelPricing, bool) - // Stamp persists the estimate to status.estimatedCost (a Decision cannot - // write a CRD field, so the hook mutates via this injected handle). - Stamp func(ctx context.Context, c v1.EstimatedSessionCost) error + // Stamp folds the estimate into status.estimatedCost (a Decision cannot + // write a CRD field, so the hook mutates via this injected handle) and + // returns the resulting SESSION aggregate. Cost accumulates across pods, so + // the returned total may exceed this pod's own estimate. + Stamp func(ctx context.Context, c v1.EstimatedSessionCost) (v1.EstimatedSessionCost, error) // Now supplies AsOf; nil ⇒ metav1.Now. Now func() metav1.Time } @@ -76,14 +78,16 @@ func (r *Reporter) Eval(ctx context.Context, in pipeline.Input) pipeline.Decisio } // Interactive toolkit cost (e.g. an inner `claude` sub-run's own billed - // spend) is provider-reported, so it is always "known" and is ADDED to the - // model total. When a served model is unpriced (known=false, micro=0), the - // grand total is still the priced tool component — a lower bound, per the - // EstimatedSessionCost.AmountMicroUSD contract. - toolBuckets, toolTotal := buildToolBuckets(e.ByTool) + // spend) is provider-reported, so it is always "known" and the BILLED + // (metered) portion is ADDED to the model total. When a served model is + // unpriced (known=false, micro=0), the grand total is still the billed tool + // component — a lower bound, per the EstimatedSessionCost.AmountMicroUSD + // contract. Subscription-equivalent tool cost is recorded on its bucket but + // NOT added, so a flat-rate sub-run never inflates spend. + toolBuckets, billedToolTotal, _ := buildToolBuckets(e.ByTool) cost := v1.EstimatedSessionCost{ - AmountMicroUSD: micro + toolTotal, + AmountMicroUSD: micro + billedToolTotal, Currency: currency, Model: e.Model, PricingKnown: known, @@ -94,9 +98,13 @@ func (r *Reporter) Eval(ctx context.Context, in pipeline.Input) pipeline.Decisio if known && cost.Currency == "" { cost.Currency = "USD" } - // Stamp on EVERY path (cheap, gives admin UI a live running spend). + // Stamp on EVERY path (cheap, gives admin UI a live running spend). The + // returned aggregate is the SESSION total accumulated across pods; the notice + // shows it (not just this pod's estimate) so notice and status/admind agree. + sessionCost := cost if r.d.Stamp != nil { - if err := r.d.Stamp(ctx, cost); err != nil { + acc, err := r.d.Stamp(ctx, cost) + if err != nil { // no-silent-errors: log directly with structured context. Best-effort — // a failed stamp must not block teardown, and it must NOT suppress the // closing message (which does not depend on the stamp). We log here @@ -104,6 +112,8 @@ func (r *Reporter) Eval(ctx context.Context, in pipeline.Input) pipeline.Decisio // host has no case for a cost kind and would drop the err/model detail. slog.Default().Info("session_cost: stamp estimatedCost failed", "session", in.Session.String(), "model", e.Model, "err", err.Error()) + } else { + sessionCost = acc } } @@ -113,7 +123,7 @@ func (r *Reporter) Eval(ctx context.Context, in pipeline.Input) pipeline.Decisio } return pipeline.Decision{ Verdict: pipeline.Allow, - Notices: []pipeline.Notice{{Notice: costNotice(e, cost, known), ToRequester: false}}, + Notices: []pipeline.Notice{{Notice: costNotice(e, sessionCost, sessionCost.PricingKnown), ToRequester: false}}, } } @@ -175,29 +185,81 @@ func (r *Reporter) buildCostBuckets(usages []pipeline.ModelUsage) (buckets []v1. // buildToolBuckets maps each interactive-toolkit usage bucket to a priced // ToolCostBucket. Tool cost is always provider-reported (already micro-USD and -// "known"), so there is no table lookup here — this is a pure projection plus a -// sum. Returns the buckets in caller order and their total. -func buildToolBuckets(usages []pipeline.ToolUsage) (buckets []v1.ToolCostBucket, total int64) { +// "known"), so there is no table lookup here — a pure projection. Returns the +// buckets in caller order, the BILLED total (metered tools, which count toward +// session spend), and the SUBSCRIPTION-equivalent total (flat-rate estimates, +// recorded on their buckets but kept OUT of session spend). +func buildToolBuckets(usages []pipeline.ToolUsage) (buckets []v1.ToolCostBucket, billed, subscription int64) { if len(usages) == 0 { - return nil, 0 + return nil, 0, 0 } buckets = make([]v1.ToolCostBucket, len(usages)) for i, u := range usages { - buckets[i] = v1.ToolCostBucket{Tool: u.Tool, AmountMicroUSD: u.CostMicroUSD, PricingKnown: u.CostReported} - total += u.CostMicroUSD + buckets[i] = v1.ToolCostBucket{ + Tool: u.Tool, AmountMicroUSD: u.CostMicroUSD, PricingKnown: u.CostReported, + Anomalous: u.Anomalous, SubscriptionEquivalent: u.SubscriptionEquivalent, + } + if u.SubscriptionEquivalent { + subscription += u.CostMicroUSD + } else { + billed += u.CostMicroUSD + } } - return buckets, total + return buckets, billed, subscription } func costNotice(e *pipeline.SessionEndInfo, c v1.EstimatedSessionCost, known bool) *notice.Notice { + anomalous := anyToolAnomalous(c.ByTool) + // Split tool spend: billed (metered) counts toward the headline total; + // subscription-equivalent is shown on its own line, never in the total. + var billedTool, subscriptionTool int64 + for _, b := range c.ByTool { + if b.SubscriptionEquivalent { + subscriptionTool += b.AmountMicroUSD + } else { + billedTool += b.AmountMicroUSD + } + } + // extraFields are the breakdown lines common to both the priced and unpriced + // notices: billed sub-agent spend, subscription-equivalent spend (flagged as + // not billed), and the cost-anomaly note. + extraFields := func() []channelevents.InteractionField { + var fs []channelevents.InteractionField + if billedTool > 0 { + fs = append(fs, channelevents.InteractionField{Label: "Sub-agent tools", Value: formatUSD(billedTool)}) + } + if subscriptionTool > 0 { + fs = append(fs, channelevents.InteractionField{ + Label: "Sub-agent tools (subscription est., not billed)", Value: formatUSD(subscriptionTool)}) + } + if anomalous { + fs = append(fs, anomalyField()) + } + return fs + } + if !known { + fields := []channelevents.InteractionField{{ + Label: "Tokens", + Value: fmt.Sprintf("%s in / %s out", formatTokens(e.InputTokens), formatTokens(e.OutputTokens)), + }} + fields = append(fields, extraFields()...) + // Model pricing is unknown. If there is BILLED tool spend on record, + // surface it as a LOWER BOUND so the notice agrees with status/admind + // (c.AmountMicroUSD is exactly that billed total here) instead of saying + // "unavailable". Subscription-equivalent spend, shown above, is excluded. + if billedTool > 0 { + return notice.New(categories.SessionCost, notice.Args{ + Lead: fmt.Sprintf("This session cost at least ~%s", formatUSD(c.AmountMicroUSD)), + Body: fmt.Sprintf("Model pricing for %s is unavailable, so this covers sub-agent tool spend only.", e.Model), + Fields: fields, + Audience: channelevents.InteractionAudience{Scope: channelevents.AudienceParticipants}, + }) + } return notice.New(categories.SessionCost, notice.Args{ - Lead: "Session cost unavailable", - Body: fmt.Sprintf("No pricing is configured for %s.", e.Model), - Fields: []channelevents.InteractionField{{ - Label: "Tokens", - Value: fmt.Sprintf("%s in / %s out", formatTokens(e.InputTokens), formatTokens(e.OutputTokens)), - }}, + Lead: "Session cost unavailable", + Body: fmt.Sprintf("No pricing is configured for %s.", e.Model), + Fields: fields, Audience: channelevents.InteractionAudience{Scope: channelevents.AudienceParticipants}, }) } @@ -208,17 +270,7 @@ func costNotice(e *pipeline.SessionEndInfo, c v1.EstimatedSessionCost, known boo formatTokens(e.CacheReadTokens), formatTokens(e.CacheCreationTokens))}, {Label: "Model", Value: e.Model}, } - // Break out inner interactive-toolkit spend (e.g. a passthrough `claude` - // sub-run) so the total's provenance is visible, not folded silently. - var toolTotal int64 - for _, b := range c.ByTool { - toolTotal += b.AmountMicroUSD - } - if toolTotal > 0 { - fields = append(fields, channelevents.InteractionField{ - Label: "Sub-agent tools", Value: formatUSD(toolTotal), - }) - } + fields = append(fields, extraFields()...) return notice.New(categories.SessionCost, notice.Args{ Lead: fmt.Sprintf("This session cost ~%s", formatUSD(c.AmountMicroUSD)), Fields: fields, @@ -226,4 +278,25 @@ func costNotice(e *pipeline.SessionEndInfo, c v1.EstimatedSessionCost, known boo }) } +// anyToolAnomalous reports whether any tool bucket flagged a nonsensical +// (negative) provider cost report — see ToolCostBucket.Anomalous. +func anyToolAnomalous(buckets []v1.ToolCostBucket) bool { + for _, b := range buckets { + if b.Anomalous { + return true + } + } + return false +} + +// anomalyField is the cost-notice line shown when a sub-agent tool reported a +// nonsensical (negative) cost: the value is excluded from the total and flagged +// rather than silently dropped. +func anomalyField() channelevents.InteractionField { + return channelevents.InteractionField{ + Label: "Cost anomaly", + Value: "a sub-agent tool reported a negative cost; excluded from the total", + } +} + var _ pipeline.Hook = (*Reporter)(nil) diff --git a/pkg/agent/postsession/cost/cost_test.go b/pkg/agent/postsession/cost/cost_test.go index 35d62099..b37e490a 100644 --- a/pkg/agent/postsession/cost/cost_test.go +++ b/pkg/agent/postsession/cost/cost_test.go @@ -56,7 +56,10 @@ func newWithCapturePricer(pricer func(string) (llm.ModelPricing, bool)) (*Report var stamped v1.EstimatedSessionCost r := New(Deps{ Pricing: pricer, - Stamp: func(_ context.Context, c v1.EstimatedSessionCost) error { stamped = c; return nil }, + Stamp: func(_ context.Context, c v1.EstimatedSessionCost) (v1.EstimatedSessionCost, error) { + stamped = c + return c, nil + }, Now: func() metav1.Time { return metav1.Time{} }, }) return r, &stamped @@ -74,7 +77,9 @@ func TestEval_StampError_StillMessages(t *testing.T) { // completion/failure message (the message doesn't depend on the stamp). r := New(Deps{ Pricing: opusPricer, - Stamp: func(_ context.Context, _ v1.EstimatedSessionCost) error { return errors.New("boom") }, + Stamp: func(_ context.Context, _ v1.EstimatedSessionCost) (v1.EstimatedSessionCost, error) { + return v1.EstimatedSessionCost{}, errors.New("boom") + }, Now: func() metav1.Time { return metav1.Time{} }, }) dec := r.Eval(context.Background(), endInput("completed", 1_000_000, 0, 0, 0)) @@ -193,6 +198,33 @@ func TestEval_NoTool_IdenticalToToday(t *testing.T) { assert.Equal(t, int64(3_000_000), stamped.AmountMicroUSD, "unchanged from today") } +func TestEval_ByTool_SubscriptionEquivalentExcludedFromTotal(t *testing.T) { + r, stamped := newWithCapturePricer(opusPricer) + in := endInput("completed", 1_000_000, 0, 0, 0) // 1M in x $15 = $15 model + in.End.ByTool = []pipeline.ToolUsage{ + {Tool: "claude-oauth", CostMicroUSD: 5_000_000, CostReported: true, SubscriptionEquivalent: true}, + {Tool: "aider", CostMicroUSD: 2_000_000, CostReported: true}, + } + dec := r.Eval(context.Background(), in) + + require.Len(t, stamped.ByTool, 2, "both tools recorded, subscription included") + assert.Equal(t, int64(17_000_000), stamped.AmountMicroUSD, + "$15 model + $2 metered tool; the $5 subscription-equivalent tool is excluded") + + var sub *v1.ToolCostBucket + for i := range stamped.ByTool { + if stamped.ByTool[i].Tool == "claude-oauth" { + sub = &stamped.ByTool[i] + } + } + require.NotNil(t, sub, "subscription bucket still recorded") + assert.True(t, sub.SubscriptionEquivalent, "flagged") + assert.Equal(t, int64(5_000_000), sub.AmountMicroUSD, "amount kept on the bucket") + + require.Len(t, dec.Notices, 1) + assert.Contains(t, dec.Notices[0].Notice.Args().Lead, "$17.00", "headline excludes subscription cost") +} + func TestEval_UnknownModel_KnownTool_LowerBound(t *testing.T) { r, stamped := newWithCapturePricer(multiModelPricer) // "mystery-model" unpriced in := endInput("completed", 0, 0, 0, 0) @@ -200,10 +232,17 @@ func TestEval_UnknownModel_KnownTool_LowerBound(t *testing.T) { {Display: "openrouter/mystery/model", Model: "mystery-model", InputTokens: 1_000_000}, } in.End.ByTool = []pipeline.ToolUsage{{Tool: "claude-oauth", CostMicroUSD: 19_020_000, CostReported: true}} - r.Eval(context.Background(), in) + dec := r.Eval(context.Background(), in) assert.False(t, stamped.PricingKnown, "an unpriced model makes the total a lower bound") assert.Equal(t, int64(19_020_000), stamped.AmountMicroUSD, "nonzero: the priced (tool) component") + + // The notice must surface the stamped tool spend as a lower bound, not claim + // the cost is unavailable while $19.02 is on record. + require.Len(t, dec.Notices, 1) + lead := dec.Notices[0].Notice.Args().Lead + assert.NotContains(t, lead, "unavailable", "tool spend is on record; notice must not say unavailable") + assert.Contains(t, lead, "$19.02", "notice surfaces the priced (tool) lower bound") } func TestEval_ByModel_UnknownPriceBucket_ZeroAndUnknown(t *testing.T) { @@ -246,7 +285,7 @@ func TestEval_ByModel_OpenRouterAutoBucket_UnknownNotZero(t *testing.T) { } func TestEval_ByModel_NilPricingDep_NoPanic(t *testing.T) { - r := New(Deps{Stamp: func(_ context.Context, _ v1.EstimatedSessionCost) error { return nil }}) + r := New(Deps{Stamp: func(_ context.Context, c v1.EstimatedSessionCost) (v1.EstimatedSessionCost, error) { return c, nil }}) in := endInput("completed", 0, 0, 0, 0) in.End.ByModel = []pipeline.ModelUsage{{Display: "x/y", Model: "y", InputTokens: 1}} assert.NotPanics(t, func() { r.Eval(context.Background(), in) }) diff --git a/pkg/agent/runner/loop_usage.go b/pkg/agent/runner/loop_usage.go index f75438f2..43e1d12a 100644 --- a/pkg/agent/runner/loop_usage.go +++ b/pkg/agent/runner/loop_usage.go @@ -123,21 +123,32 @@ func (l *Loop) usageByModelSnapshot() []pipeline.ModelUsage { // toolUsageBucket is one interactive toolkit's running cost accumulation across // its invocations this session, keyed by outer tool name. type toolUsageBucket struct { - tool string - costMicroUSD int64 - costReported bool + tool string + costMicroUSD int64 + costReported bool + anomalous bool + subscriptionEquivalent bool } // AddToolCost folds one interactive toolkit result event's provider-reported // cost into the bucket for outerTool. Shares usageMu with addUsage/addModelUsage -// (see usageMu's doc). ok is the toolkit's success flag; v1 accumulates cost -// regardless — a run the provider billed cost money whether or not it succeeded, -// and an unbilled run reports 0 anyway. +// (see usageMu's doc). costReported is the presence bit from the toolkit's +// terminal event (toolkitstream.Event.CostReported): a run whose stream carried +// a total_cost_usd tally — even an explicit 0 — marks the bucket priced, while a +// run that reported no tally leaves it unpriced, so an absent cost is never +// asserted as a known $0. costUSD accumulates regardless of the inner run's +// success: a run the provider billed cost money whether or not it succeeded. A +// negative (nonsensical) cost is treated as anomalous: the bucket is kept and +// flagged, but the value is excluded from the total. subscriptionEquivalent (the +// toolkit's CostTreatment) marks the cost a flat-rate estimate rather than +// metered money — recorded on the bucket but excluded from session spend by the +// cost reporter; it is a property of the tool, so it is recorded even for a +// report that is otherwise ignored (negative). // // Exported (unlike addModelUsage) because it is wired from internal/cmd/runner's // interactive-tool result callback, which lives in a different package; the // runner's turn loop feeds addModelUsage internally, so that one stays private. -func (l *Loop) AddToolCost(outerTool string, costUSD float64, ok bool) { +func (l *Loop) AddToolCost(outerTool string, costUSD float64, costReported, subscriptionEquivalent bool) { l.usageMu.Lock() defer l.usageMu.Unlock() if l.usageByToolIdx == nil { @@ -150,18 +161,24 @@ func (l *Loop) AddToolCost(outerTool string, costUSD float64, ok bool) { l.usageByTool = append(l.usageByTool, toolUsageBucket{tool: outerTool}) } b := &l.usageByTool[i] + // A property of the tool, not of this call — record it before the negative + // guard so it survives a report that is otherwise dropped. + b.subscriptionEquivalent = b.subscriptionEquivalent || subscriptionEquivalent if costUSD < 0 { - // A negative provider-reported tool cost would reduce the session total. - // Treat it as unreported rather than netting it against real charges — - // same guard as addModelUsage. - slog.Default().Info("negative reported tool cost; ignoring", + // A negative provider-reported tool cost is nonsensical — it would reduce + // the session total. Keep the bucket (the tool DID run) but flag it + // anomalous and exclude the value, rather than netting it against real + // charges or silently dropping the report. + slog.Default().Info("negative reported tool cost; flagging anomalous, excluding from total", "tool", outerTool, "costUSD", costUSD) + b.anomalous = true return } // Round once at the USD->micro-USD boundary so accumulation is pure int64. b.costMicroUSD += int64(math.Round(costUSD * 1e6)) - b.costReported = true - _ = ok + if costReported { + b.costReported = true + } } // usageByToolSnapshot returns the Loop's per-tool cost as pipeline.ToolUsage in @@ -174,16 +191,18 @@ func (l *Loop) usageByToolSnapshot() []pipeline.ToolUsage { } out := make([]pipeline.ToolUsage, len(l.usageByTool)) for i, b := range l.usageByTool { - out[i] = pipeline.ToolUsage{Tool: b.tool, CostMicroUSD: b.costMicroUSD, CostReported: b.costReported} + out[i] = pipeline.ToolUsage{Tool: b.tool, CostMicroUSD: b.costMicroUSD, CostReported: b.costReported, Anomalous: b.anomalous, SubscriptionEquivalent: b.subscriptionEquivalent} } return out } -// StampEstimatedCost persists the cost estimate via the StatusPatcher. nil-safe: -// a Loop without a Status (tests/kubectl-driven) silently no-ops. -func (l *Loop) StampEstimatedCost(ctx context.Context, c spiceboxv1alpha1.EstimatedSessionCost) error { +// StampEstimatedCost folds this pod's cost estimate into the session total via +// the StatusPatcher and returns the resulting session aggregate (which +// accumulates across pods). nil-safe: a Loop without a Status +// (tests/kubectl-driven) returns c unchanged. +func (l *Loop) StampEstimatedCost(ctx context.Context, c spiceboxv1alpha1.EstimatedSessionCost) (spiceboxv1alpha1.EstimatedSessionCost, error) { if l.Status == nil { - return nil + return c, nil } return l.Status.PatchEstimatedCost(ctx, c) } diff --git a/pkg/agent/runner/modelusage_test.go b/pkg/agent/runner/modelusage_test.go index 85e36397..5ae3b185 100644 --- a/pkg/agent/runner/modelusage_test.go +++ b/pkg/agent/runner/modelusage_test.go @@ -96,7 +96,7 @@ func TestFireSessionEnd_PopulatesByTool(t *testing.T) { l.pipelineExec = cr l.pipelineOnce.Do(func() {}) - l.AddToolCost("claude-oauth", 5.43, true) + l.AddToolCost("claude-oauth", 5.43, true, false) l.fireSessionEnd(context.Background(), "completed") require.NotNil(t, cr.in.End) diff --git a/pkg/agent/runner/status.go b/pkg/agent/runner/status.go index 8dd23d07..bafde7b4 100644 --- a/pkg/agent/runner/status.go +++ b/pkg/agent/runner/status.go @@ -49,6 +49,11 @@ type StatusPatcher struct { // WithDirectReader when c reads through an informer cache. reader client.Reader + // pod is this serving pod's name (its HOSTNAME), the upsert key for + // PatchEstimatedCost's per-pod cost accumulation. Empty for in-process / + // test patchers, which collapse to a single "" entry. + pod string + // local, when true, makes mutate and Get short-circuit to nil. Used by // in-process callers (the gen-agent loop) that have no AgentSession CR // to patch. localResult captures the AgentResult passed to @@ -136,6 +141,14 @@ func (s *StatusPatcher) WithDirectReader(r client.Reader) *StatusPatcher { return s } +// WithPod sets this patcher's pod name, the upsert key PatchEstimatedCost uses +// to accumulate session cost across pods. Left empty, per-pod entries collapse +// to a single "" key (correct for a single-process in-process run). +func (s *StatusPatcher) WithPod(pod string) *StatusPatcher { + s.pod = pod + return s +} + const maxRetries = 5 // PatchProgress updates the live counters. Optimistic-lock retried. @@ -178,28 +191,131 @@ func (s *StatusPatcher) PatchToolGuard(ctx context.Context, tg *spiceboxv1alpha1 }) } -// PatchEstimatedCost stamps the end-of-session cost estimate. Optimistic-lock -// retried, same mutate path as PatchProgress. +// PatchEstimatedCost folds THIS pod's cost estimate into the session total and +// returns the resulting SESSION aggregate. Optimistic-lock retried, same mutate +// path as PatchProgress. // -// MONOTONIC: a lower amount never replaces a higher one. The cost hook stamps on -// every terminal path (including idle) from the loop's per-POD token counters, -// which restart at zero after an idle→reap→wake cycle; without the guard a woken -// pod that made no LLM calls of its own overwrites a real session cost with -// AmountMicroUSD: 0 and a nil breakdown. The whole prior record is kept (rather -// than max-ing the amount alone) so ByModel stays consistent with the amount it -// explains. +// ACCUMULATES across pods. status.estimatedCost.perPod is the per-pod source of +// truth; this pod upserts only its OWN entry (keyed by s.pod), then the +// aggregate (amount, byModel, byTool, pricingKnown) is recomputed as the sum / +// merge-by-key / AND over every entry. The cost hook stamps on every terminal +// path (including idle) from the loop's per-POD counters, which restart at zero +// after an idle→reap→wake cycle — but a woken pod now writes a $0 entry for +// ITSELF and leaves earlier pods' entries untouched, so the session total is +// preserved and grows rather than being clobbered. That is why this needs no +// "keep the higher total" guard: summation over immutable per-pod truth is +// inherently monotonic and double-count-free (a re-stamp overwrites the same +// pod's entry). // -// This bounds the damage without making the figure cumulative across pods: a -// woken pod that DOES spend still reports only its own spend. Full accumulation -// needs the loop to seed its usage counters from durable state at boot. -func (s *StatusPatcher) PatchEstimatedCost(ctx context.Context, c spiceboxv1alpha1.EstimatedSessionCost) error { - return s.mutate(ctx, func(sess *spiceboxv1alpha1.AgentSession) { - if prev := sess.Status.EstimatedCost; prev != nil && prev.AmountMicroUSD > c.AmountMicroUSD { - return +// Returns the recomputed aggregate so the cost reporter can show the SESSION +// total in the end-of-session notice (notice and status then agree). In local +// mode (no CR) the pod cost is returned unchanged — a single in-process run is +// its own and only pod. +func (s *StatusPatcher) PatchEstimatedCost(ctx context.Context, podCost spiceboxv1alpha1.EstimatedSessionCost) (spiceboxv1alpha1.EstimatedSessionCost, error) { + if s.local { + return podCost, nil + } + var agg spiceboxv1alpha1.EstimatedSessionCost + err := s.mutate(ctx, func(sess *spiceboxv1alpha1.AgentSession) { + ec := spiceboxv1alpha1.EstimatedSessionCost{} + if prev := sess.Status.EstimatedCost; prev != nil { + ec = *prev } - cc := c - sess.Status.EstimatedCost = &cc + ec.Model = podCost.Model + ec.PerPod = upsertPerPod(ec.PerPod, perPodFrom(s.pod, podCost)) + recomputeCostAggregate(&ec) + sess.Status.EstimatedCost = &ec + agg = ec }) + if err != nil { + return spiceboxv1alpha1.EstimatedSessionCost{}, err + } + return agg, nil +} + +// perPodFrom projects one pod's EstimatedSessionCost into its PerPodCost entry. +func perPodFrom(pod string, c spiceboxv1alpha1.EstimatedSessionCost) spiceboxv1alpha1.PerPodCost { + return spiceboxv1alpha1.PerPodCost{ + Pod: pod, + AmountMicroUSD: c.AmountMicroUSD, + Currency: c.Currency, + PricingKnown: c.PricingKnown, + AsOf: c.AsOf, + ByModel: c.ByModel, + ByTool: c.ByTool, + } +} + +// upsertPerPod returns a fresh slice with e replacing any same-Pod entry +// (preserving order) or appended when new — so a re-stamp by the same pod is +// idempotent and the caller never mutates the prior status slice in place. +func upsertPerPod(entries []spiceboxv1alpha1.PerPodCost, e spiceboxv1alpha1.PerPodCost) []spiceboxv1alpha1.PerPodCost { + out := make([]spiceboxv1alpha1.PerPodCost, 0, len(entries)+1) + replaced := false + for _, p := range entries { + if p.Pod == e.Pod { + out = append(out, e) + replaced = true + continue + } + out = append(out, p) + } + if !replaced { + out = append(out, e) + } + return out +} + +// recomputeCostAggregate derives c's top-level fields from its PerPod entries: +// amount = Σ, byModel/byTool merged by key, pricingKnown = AND, currency = first +// non-empty, asOf = latest. ByModel/ByTool are rebuilt into fresh slices so no +// per-pod entry's buckets are aliased or mutated. +func recomputeCostAggregate(c *spiceboxv1alpha1.EstimatedSessionCost) { + var total int64 + pricingKnown := len(c.PerPod) > 0 + var currency string + var asOf metav1.Time + modelIdx := map[string]int{} + toolIdx := map[string]int{} + var byModel []spiceboxv1alpha1.ModelCostBucket + var byTool []spiceboxv1alpha1.ToolCostBucket + for _, p := range c.PerPod { + total += p.AmountMicroUSD + pricingKnown = pricingKnown && p.PricingKnown + if currency == "" { + currency = p.Currency + } + if asOf.IsZero() || p.AsOf.Time.After(asOf.Time) { + asOf = p.AsOf + } + for _, b := range p.ByModel { + if i, ok := modelIdx[b.Model]; ok { + byModel[i].InputTokens += b.InputTokens + byModel[i].OutputTokens += b.OutputTokens + byModel[i].AmountMicroUSD += b.AmountMicroUSD + byModel[i].PricingKnown = byModel[i].PricingKnown && b.PricingKnown + } else { + modelIdx[b.Model] = len(byModel) + byModel = append(byModel, b) + } + } + for _, b := range p.ByTool { + if i, ok := toolIdx[b.Tool]; ok { + byTool[i].AmountMicroUSD += b.AmountMicroUSD + byTool[i].PricingKnown = byTool[i].PricingKnown && b.PricingKnown + byTool[i].Anomalous = byTool[i].Anomalous || b.Anomalous + } else { + toolIdx[b.Tool] = len(byTool) + byTool = append(byTool, b) + } + } + } + c.AmountMicroUSD = total + c.PricingKnown = pricingKnown + c.Currency = currency + c.AsOf = asOf + c.ByModel = byModel + c.ByTool = byTool } // WriteSucceeded writes the terminal success state. diff --git a/pkg/agent/runner/status_test.go b/pkg/agent/runner/status_test.go index 1b60d102..fefa8573 100644 --- a/pkg/agent/runner/status_test.go +++ b/pkg/agent/runner/status_test.go @@ -339,33 +339,93 @@ func TestTerminalWriteIsIdempotent(t *testing.T) { "FinishedAt was overwritten: was %v, now %v", firstFinished, out.Status.FinishedAt) } -// TestPatchEstimatedCost_NeverRegresses asserts the session cost estimate is -// monotonic. The SessionEnd cost hook stamps on EVERY terminal path — including -// idle — from per-pod token counters that reset on every wake, so a woken pod -// that reaches a terminal path having made no LLM calls used to overwrite a -// real number with zero. -func TestPatchEstimatedCost_NeverRegresses(t *testing.T) { +// TestPatchEstimatedCost_AccumulatesAcrossPods asserts the session cost is the +// SUM across pods, not the highest single-pod total: each pod upserts its own +// perPod entry and the aggregate is recomputed. The returned value is that +// aggregate. +func TestPatchEstimatedCost_AccumulatesAcrossPods(t *testing.T) { c, sess := newFakeClientWithSession(t) - sp := runner.NewStatusPatcher(c, client.ObjectKeyFromObject(sess)) + key := client.ObjectKeyFromObject(sess) + ctx := context.Background() + + podA := runner.NewStatusPatcher(c, key).WithPod("pod-a") + podB := runner.NewStatusPatcher(c, key).WithPod("pod-b") + + aggA, err := podA.PatchEstimatedCost(ctx, spiceboxv1alpha1.EstimatedSessionCost{ + AmountMicroUSD: 5_000_000, Currency: "USD", Model: "m1", PricingKnown: true, + ByModel: []spiceboxv1alpha1.ModelCostBucket{{Model: "m1", AmountMicroUSD: 5_000_000, PricingKnown: true}}, + }) + require.NoError(t, err, "pod-a stamp") + assert.Equal(t, int64(5_000_000), aggA.AmountMicroUSD, "first pod: aggregate = its own spend") + + aggB, err := podB.PatchEstimatedCost(ctx, spiceboxv1alpha1.EstimatedSessionCost{ + AmountMicroUSD: 3_000_000, Currency: "USD", Model: "m1", PricingKnown: true, + ByModel: []spiceboxv1alpha1.ModelCostBucket{{Model: "m1", AmountMicroUSD: 3_000_000, PricingKnown: true}}, + }) + require.NoError(t, err, "pod-b stamp") + assert.Equal(t, int64(8_000_000), aggB.AmountMicroUSD, "second pod: aggregate accumulates ($5 + $3)") + + var out spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(ctx, key, &out), "Get after stamps") + ec := out.Status.EstimatedCost + require.NotNil(t, ec) + assert.Equal(t, int64(8_000_000), ec.AmountMicroUSD, "session total = sum across pods") + assert.Len(t, ec.PerPod, 2, "one perPod entry per pod") + require.Len(t, ec.ByModel, 1, "same model merged into one aggregate bucket") + assert.Equal(t, int64(8_000_000), ec.ByModel[0].AmountMicroUSD, "merged model bucket = $5 + $3") +} + +// TestPatchEstimatedCost_WokenZeroPodPreservesTotal asserts a woken pod that +// made no LLM calls of its own writes a $0 entry for ITSELF and leaves earlier +// pods' spend intact — the clobber the old monotonic guard existed to prevent, +// now handled structurally by per-pod accumulation. +func TestPatchEstimatedCost_WokenZeroPodPreservesTotal(t *testing.T) { + c, sess := newFakeClientWithSession(t) + key := client.ObjectKeyFromObject(sess) + ctx := context.Background() + + _, err := runner.NewStatusPatcher(c, key).WithPod("pod-a").PatchEstimatedCost(ctx, + spiceboxv1alpha1.EstimatedSessionCost{ + AmountMicroUSD: 4_200_000, Currency: "USD", Model: "m1", PricingKnown: true, + ByModel: []spiceboxv1alpha1.ModelCostBucket{{Model: "m1", AmountMicroUSD: 4_200_000, PricingKnown: true}}, + }) + require.NoError(t, err, "pod-a stamp") + + agg, err := runner.NewStatusPatcher(c, key).WithPod("pod-b").PatchEstimatedCost(ctx, + spiceboxv1alpha1.EstimatedSessionCost{Currency: "USD", PricingKnown: false}) + require.NoError(t, err, "woken pod-b stamp") + assert.Equal(t, int64(4_200_000), agg.AmountMicroUSD, "a pod that spent nothing must not wipe recorded spend") + + var out spiceboxv1alpha1.AgentSession + require.NoError(t, c.Get(ctx, key, &out), "Get after stamps") + assert.Equal(t, int64(4_200_000), out.Status.EstimatedCost.AmountMicroUSD) + assert.Len(t, out.Status.EstimatedCost.PerPod, 2, "both pods recorded, even the zero one") + assert.Len(t, out.Status.EstimatedCost.ByModel, 1, "breakdown stays consistent with the amount") +} + +// TestPatchEstimatedCost_SamePodRestampIsIdempotent asserts a pod re-stamping +// (idle then completed) overwrites its OWN entry rather than adding twice. +func TestPatchEstimatedCost_SamePodRestampIsIdempotent(t *testing.T) { + c, sess := newFakeClientWithSession(t) + key := client.ObjectKeyFromObject(sess) ctx := context.Background() + sp := runner.NewStatusPatcher(c, key).WithPod("pod-a") - require.NoError(t, sp.PatchEstimatedCost(ctx, spiceboxv1alpha1.EstimatedSessionCost{ - AmountMicroUSD: 4_200_000, Currency: "USD", Model: "m1", PricingKnown: true, - ByModel: []spiceboxv1alpha1.ModelCostBucket{{Model: "m1", AmountMicroUSD: 4_200_000}}, - }), "first stamp") + _, err := sp.PatchEstimatedCost(ctx, spiceboxv1alpha1.EstimatedSessionCost{ + AmountMicroUSD: 2_000_000, Currency: "USD", Model: "m1", PricingKnown: true, + }) + require.NoError(t, err, "first (idle) stamp") - // A woken pod with no LLM calls of its own. - require.NoError(t, sp.PatchEstimatedCost(ctx, spiceboxv1alpha1.EstimatedSessionCost{ - Currency: "USD", PricingKnown: false, - }), "second stamp from a woken pod") + agg, err := sp.PatchEstimatedCost(ctx, spiceboxv1alpha1.EstimatedSessionCost{ + AmountMicroUSD: 5_000_000, Currency: "USD", Model: "m1", PricingKnown: true, + }) + require.NoError(t, err, "second (completed) stamp from the same pod") + assert.Equal(t, int64(5_000_000), agg.AmountMicroUSD, "same pod overwrites its entry, not added twice") var out spiceboxv1alpha1.AgentSession - require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(sess), &out), "Get after stamps") - require.NotNil(t, out.Status.EstimatedCost, "EstimatedCost must survive the second stamp") - assert.Equal(t, int64(4_200_000), out.Status.EstimatedCost.AmountMicroUSD, - "a pod that spent nothing must not wipe the session's recorded spend") - assert.Len(t, out.Status.EstimatedCost.ByModel, 1, - "the per-model breakdown must stay consistent with the amount it explains") + require.NoError(t, c.Get(ctx, key, &out), "Get after stamps") + assert.Equal(t, int64(5_000_000), out.Status.EstimatedCost.AmountMicroUSD) + assert.Len(t, out.Status.EstimatedCost.PerPod, 1, "one pod -> one entry, re-stamp overwrites") } // TestAppendRunnerNote_CapsOldestFirst asserts runnerNotes is bounded. Nothing diff --git a/pkg/agent/runner/toolusage_test.go b/pkg/agent/runner/toolusage_test.go index 7673b515..7790f70e 100644 --- a/pkg/agent/runner/toolusage_test.go +++ b/pkg/agent/runner/toolusage_test.go @@ -11,9 +11,9 @@ func TestLoopAddToolCost_AccumulatesPerTool(t *testing.T) { var l Loop // codebot's shape: one tool ("claude-oauth") invoked many times -> ONE // summed bucket; a second tool -> a distinct bucket, first-seen order. - l.AddToolCost("claude-oauth", 5.43, true) - l.AddToolCost("claude-oauth", 1.31, true) - l.AddToolCost("aider", 2.10, true) + l.AddToolCost("claude-oauth", 5.43, true, false) + l.AddToolCost("claude-oauth", 1.31, true, false) + l.AddToolCost("aider", 2.10, true, false) got := l.usageByToolSnapshot() require.Len(t, got, 2, "two distinct tools -> two buckets in first-encountered order") @@ -28,23 +28,62 @@ func TestLoopAddToolCost_AccumulatesPerTool(t *testing.T) { func TestLoopAddToolCost_NegativeCostIgnored(t *testing.T) { var l Loop - l.AddToolCost("claude-oauth", -1.0, true) + l.AddToolCost("claude-oauth", -1.0, true, false) got := l.usageByToolSnapshot() - require.Len(t, got, 1, "the tool is still recorded") + require.Len(t, got, 1, "the tool is still recorded as seen") assert.False(t, got[0].CostReported, "negative cost -> treated as unreported") assert.Equal(t, int64(0), got[0].CostMicroUSD, "no negative amount is ever added") + assert.True(t, got[0].Anomalous, "negative cost -> flagged as not making sense") } func TestLoopAddToolCost_ZeroCostUnbilled(t *testing.T) { - // An unbilled / credential-halt run reports cost 0; it adds 0 and does not - // mark the bucket as having a reported (billed) cost. + // A run whose terminal event carried NO cost tally (costReported=false) adds + // 0 and does NOT mark the bucket priced — an absent cost is never asserted as + // a known $0 (the absent-vs-zero distinction Outcome.Unbilled documents). var l Loop - l.AddToolCost("claude-oauth", 0, false) + l.AddToolCost("claude-oauth", 0, false, false) got := l.usageByToolSnapshot() require.Len(t, got, 1) assert.Equal(t, int64(0), got[0].CostMicroUSD) + assert.False(t, got[0].CostReported, "absent cost -> bucket is not priced") +} + +func TestLoopAddToolCost_ExplicitZeroReported(t *testing.T) { + // An explicit total_cost_usd:0 IS a reported tally: amount 0, but priced — + // distinct from an absent cost. + var l Loop + l.AddToolCost("claude-oauth", 0, true, false) + + got := l.usageByToolSnapshot() + require.Len(t, got, 1) + assert.Equal(t, int64(0), got[0].CostMicroUSD) + assert.True(t, got[0].CostReported, "explicit $0 is a known price, not 'unreported'") +} + +func TestLoopAddToolCost_SubscriptionEquivalentFlagged(t *testing.T) { + // A subscription-equivalent toolkit's cost is recorded on the bucket and + // flagged; the cost reporter / admind exclude it from spend on that flag. + var l Loop + l.AddToolCost("claude-oauth", 5.43, true, true) + + got := l.usageByToolSnapshot() + require.Len(t, got, 1) + assert.Equal(t, int64(5_430_000), got[0].CostMicroUSD, "amount is still recorded on the bucket") + assert.True(t, got[0].SubscriptionEquivalent, "flagged so downstream excludes it from spend") +} + +func TestLoopAddToolCost_SubscriptionFlagSurvivesNegativeReport(t *testing.T) { + // The subscription flag is a property of the tool, recorded before the + // negative-cost guard returns, so a negative-only report still carries it. + var l Loop + l.AddToolCost("claude-oauth", -1.0, true, true) + + got := l.usageByToolSnapshot() + require.Len(t, got, 1) + assert.True(t, got[0].SubscriptionEquivalent, "flag recorded even when the cost report is dropped") + assert.True(t, got[0].Anomalous) } func TestLoopUsageByToolSnapshot_EmptyWhenUnused(t *testing.T) { diff --git a/pkg/agent/tool/sandbox/interactive_tool.go b/pkg/agent/tool/sandbox/interactive_tool.go index 36a2cf20..5f6e8e20 100644 --- a/pkg/agent/tool/sandbox/interactive_tool.go +++ b/pkg/agent/tool/sandbox/interactive_tool.go @@ -249,6 +249,11 @@ func (s *SandboxTool) executeStreaming(ctx context.Context, raw json.RawMessage, // a future internal/cmd/runner log call can warn about the missing factory. } + // subEquiv stamps each result event with whether this toolkit's cost is a + // flat-rate subscription estimate (CostTreatment), so the runner can record + // but exclude it from session spend. A toolkit property, constant per run. + subEquiv := s.costSubscriptionEquivalent() + // The bridge's idle timer differs by mode. Interactive uses the resolved // idle window — a parked-but-live session is normal there. Stream is a // one-shot run to completion: it has no legitimate idle state, so the only @@ -308,6 +313,9 @@ func (s *SandboxTool) executeStreaming(ctx context.Context, raw json.RawMessage, return } for _, ev := range events { + if ev.Type == toolkitstream.EventResult { + ev.SubscriptionEquivalent = subEquiv + } hooks.OnEvent(tcName, a.Reason, s.opts.Suffix, ev) } }, diff --git a/pkg/agent/tool/sandbox/sandbox_tool.go b/pkg/agent/tool/sandbox/sandbox_tool.go index 1ecac005..f8c089b3 100644 --- a/pkg/agent/tool/sandbox/sandbox_tool.go +++ b/pkg/agent/tool/sandbox/sandbox_tool.go @@ -227,6 +227,13 @@ func (s *SandboxTool) Name() string { return s.opts.BundleName + "_" + s. func (s *SandboxTool) Kind() tool.Kind { return tool.KindSandbox } func (s *SandboxTool) Description() string { return s.opts.Description } +// costSubscriptionEquivalent reports whether this tool's toolkit declares its +// cost as a flat-rate subscription estimate (CostTreatment=subscription- +// equivalent), so the runner records but excludes it from session spend. +func (s *SandboxTool) costSubscriptionEquivalent() bool { + return s.opts.Toolkit != nil && s.opts.Toolkit.CostTreatment == toolkit.CostTreatmentSubscriptionEquivalent +} + // Origin implements tool.OriginTool. It returns "toolkit/" where // is the SpiceboxToolkit this tool belongs to — the revocation unit // used by toolguard for origin-level circuit breakers. Returns "" when the diff --git a/pkg/apis/v1alpha1/agentsession_types.go b/pkg/apis/v1alpha1/agentsession_types.go index a323a3b5..d7b00e00 100644 --- a/pkg/apis/v1alpha1/agentsession_types.go +++ b/pkg/apis/v1alpha1/agentsession_types.go @@ -969,11 +969,12 @@ type AgentSessionProgress struct { // PricingKnown=false means a served model had no price, so AmountMicroUSD is a // lower bound (the priced components only), not the full cost. type EstimatedSessionCost struct { - // AmountMicroUSD is the session total in micro-USD (1e-6 USD): the sum of - // every priced component (ByModel + ByTool). When PricingKnown is false, at - // least one served MODEL had no rate, so this is a LOWER BOUND on spend (the - // components that could be priced), not the full session cost — it is no - // longer necessarily 0, because provider-reported tool cost is always priced. + // AmountMicroUSD is the SESSION total in micro-USD (1e-6 USD), accumulated + // across every pod that served the session: the sum of PerPod amounts. When + // PricingKnown is false, at least one served MODEL (in some pod) had no rate, + // so this is a LOWER BOUND on spend (the components that could be priced), + // not the full session cost — it is no longer necessarily 0, because + // provider-reported tool cost is always priced. // +optional AmountMicroUSD int64 `json:"amountMicroUSD,omitempty"` // Currency is the ISO code the amount is denominated in ("USD"). @@ -982,30 +983,72 @@ type EstimatedSessionCost struct { // Model is the configured model the session ran under. // +optional Model string `json:"model,omitempty"` - // PricingKnown is false when a served model had no price. The amount is then - // a lower bound (priced components only), not the full cost. Tool buckets are - // always priced, so this tracks model-pricing coverage specifically. + // PricingKnown is false when a served model had no price in ANY contributing + // pod (the AND across PerPod). The amount is then a lower bound (priced + // components only), not the full cost. Tool buckets are always priced, so + // this tracks model-pricing coverage specifically. // +optional PricingKnown bool `json:"pricingKnown,omitempty"` // AsOf is when the estimate was computed. // +optional AsOf metav1.Time `json:"asOf,omitempty"` - // ByModel breaks the total down per actually-served model, so a session - // that fell back or auto-routed across models shows where the spend went - // instead of one blended total under the configured model. Empty when the - // runner reported no per-model usage; AmountMicroUSD and PricingKnown are + // ByModel breaks the total down per actually-served model, MERGED across + // every pod (summed per model display id), so a session that fell back or + // auto-routed across models — or ran across a reap/wake — shows where the + // spend went instead of one blended total under the configured model. Empty + // when no pod reported per-model usage; AmountMicroUSD and PricingKnown are // then the sole source of truth. // +optional // +listType=atomic ByModel []ModelCostBucket `json:"byModel,omitempty"` // ByTool breaks out the cost of inner interactive toolkits (e.g. a `claude` - // Claude Code sub-run's own provider-reported spend) a session drove. These - // are ADDED to AmountMicroUSD alongside ByModel — a session's total is model - // spend plus tool spend. Empty when no interactive toolkit reported a cost. - // Tool cost is always provider-reported, never table-priced, so a bucket's - // PricingKnown is always true. + // Claude Code sub-run's own provider-reported spend) a session drove, MERGED + // across every pod (summed per tool). These are ADDED to AmountMicroUSD + // alongside ByModel — a session's total is model spend plus tool spend. Empty + // when no interactive toolkit reported a cost. + // +optional + // +listType=atomic + ByTool []ToolCostBucket `json:"byTool,omitempty"` + + // PerPod is the per-pod cost SOURCE OF TRUTH: one entry per pod that served + // the session (keyed by pod name), each carrying that pod's own total and + // breakdown. The aggregate fields above are DERIVED from this list — the + // total is the sum of PerPod amounts, ByModel/ByTool are merged by key, and + // PricingKnown is the AND. Each pod upserts only its own entry, so cost + // ACCUMULATES across a reap/wake instead of a later pod's single-pod total + // replacing an earlier one. Empty only for a session no pod has costed yet. + // +optional + // +listType=atomic + PerPod []PerPodCost `json:"perPod,omitempty"` +} + +// PerPodCost is one serving pod's slice of a session's cost — the per-pod +// source of truth the EstimatedSessionCost aggregate is derived from. A pod +// rewrites only its OWN entry (keyed by Pod), so the session total accumulates +// across pods rather than a later pod clobbering an earlier one. +type PerPodCost struct { + // Pod is the serving pod's name (its HOSTNAME); the upsert key. + Pod string `json:"pod"` + // AmountMicroUSD is this pod's own total in micro-USD (model + tool). + // +optional + AmountMicroUSD int64 `json:"amountMicroUSD,omitempty"` + // Currency is the ISO code this pod's amount is denominated in ("USD"). + // +optional + Currency string `json:"currency,omitempty"` + // PricingKnown is false when a model this pod served had no rate (this pod's + // amount is then a lower bound). + // +optional + PricingKnown bool `json:"pricingKnown,omitempty"` + // AsOf is when this pod last computed its estimate. + // +optional + AsOf metav1.Time `json:"asOf,omitempty"` + // ByModel is this pod's per-served-model breakdown. + // +optional + // +listType=atomic + ByModel []ModelCostBucket `json:"byModel,omitempty"` + // ByTool is this pod's per-interactive-toolkit breakdown. // +optional // +listType=atomic ByTool []ToolCostBucket `json:"byTool,omitempty"` @@ -1040,13 +1083,29 @@ type ToolCostBucket struct { // Tool is the outer tool name (e.g. "claude-oauth"). Tool string `json:"tool"` // AmountMicroUSD is this tool's provider-reported cost, summed across its - // invocations, in micro-USD (1e-6 USD). + // invocations, in micro-USD (1e-6 USD). Negative reports are excluded from + // the sum (see Anomalous). // +optional AmountMicroUSD int64 `json:"amountMicroUSD,omitempty"` - // PricingKnown is true when AmountMicroUSD is a real cost. Tool cost is - // always provider-reported, so this is true whenever a cost was reported. + // PricingKnown is true when AmountMicroUSD is a real cost — i.e. at least one + // non-negative cost tally was reported. False when none was reported (the + // amount is 0 and must not be shown as real spend) or every report was + // anomalous. // +optional PricingKnown bool `json:"pricingKnown,omitempty"` + // Anomalous is true when a negative (nonsensical) provider cost was reported + // for this tool. The tool is still recorded as seen, but the negative value + // is excluded from AmountMicroUSD and the session total — it signals "a cost + // report here does not make sense" rather than silently dropping it. + // +optional + Anomalous bool `json:"anomalous,omitempty"` + // SubscriptionEquivalent is true when this tool ran under a flat-rate + // subscription (toolkit CostTreatment=subscription-equivalent), so its + // AmountMicroUSD is an API-equivalent estimate, not money charged. The + // bucket is recorded and shown, but its amount is EXCLUDED from the session + // total / admind spend so it cannot inflate the figure or trip cost alerts. + // +optional + SubscriptionEquivalent bool `json:"subscriptionEquivalent,omitempty"` } // ParentExchange is a delegated child's record of asking the agent that diff --git a/pkg/apis/v1alpha1/spiceboxtoolkit_types.go b/pkg/apis/v1alpha1/spiceboxtoolkit_types.go index 60887504..c8fb3293 100644 --- a/pkg/apis/v1alpha1/spiceboxtoolkit_types.go +++ b/pkg/apis/v1alpha1/spiceboxtoolkit_types.go @@ -99,6 +99,16 @@ type SpiceboxToolkitSpec struct { // +optional StreamFormat string `json:"streamFormat,omitempty"` + // CostTreatment declares how this toolkit's provider-reported cost counts + // toward session spend. "metered" (default) is real billed money and counts; + // "subscription-equivalent" is a flat-rate estimate (e.g. a Claude Max OAuth + // run) that is recorded and shown but EXCLUDED from the session total, so it + // cannot inflate spend or trip cost alerts. Mirrors + // pkg/tools/toolspec/toolkit.Toolkit.CostTreatment. + // +kubebuilder:validation:Enum=metered;subscription-equivalent + // +optional + CostTreatment string `json:"costTreatment,omitempty"` + // SiteURL is the toolkit-backed service's user-facing homepage. // identityd discovers a favicon from this URL for the credential // rows backed by this toolkit's sensitive env vars (e.g., a diff --git a/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/v1alpha1/zz_generated.deepcopy.go index 030752be..360f961f 100644 --- a/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -3021,6 +3021,13 @@ func (in *EstimatedSessionCost) DeepCopyInto(out *EstimatedSessionCost) { *out = make([]ToolCostBucket, len(*in)) copy(*out, *in) } + if in.PerPod != nil { + in, out := &in.PerPod, &out.PerPod + *out = make([]PerPodCost, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EstimatedSessionCost. @@ -4286,6 +4293,32 @@ func (in *PendingRestartSignature) DeepCopy() *PendingRestartSignature { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PerPodCost) DeepCopyInto(out *PerPodCost) { + *out = *in + in.AsOf.DeepCopyInto(&out.AsOf) + if in.ByModel != nil { + in, out := &in.ByModel, &out.ByModel + *out = make([]ModelCostBucket, len(*in)) + copy(*out, *in) + } + if in.ByTool != nil { + in, out := &in.ByTool, &out.ByTool + *out = make([]ToolCostBucket, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PerPodCost. +func (in *PerPodCost) DeepCopy() *PerPodCost { + if in == nil { + return nil + } + out := new(PerPodCost) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PermissionSurfaceEntry) DeepCopyInto(out *PermissionSurfaceEntry) { *out = *in diff --git a/pkg/platform/manifests/install.yaml b/pkg/platform/manifests/install.yaml index b0a10980..2d06755c 100644 --- a/pkg/platform/manifests/install.yaml +++ b/pkg/platform/manifests/install.yaml @@ -6572,11 +6572,12 @@ spec: properties: amountMicroUSD: description: |- - AmountMicroUSD is the session total in micro-USD (1e-6 USD): the sum of - every priced component (ByModel + ByTool). When PricingKnown is false, at - least one served MODEL had no rate, so this is a LOWER BOUND on spend (the - components that could be priced), not the full session cost — it is no - longer necessarily 0, because provider-reported tool cost is always priced. + AmountMicroUSD is the SESSION total in micro-USD (1e-6 USD), accumulated + across every pod that served the session: the sum of PerPod amounts. When + PricingKnown is false, at least one served MODEL (in some pod) had no rate, + so this is a LOWER BOUND on spend (the components that could be priced), + not the full session cost — it is no longer necessarily 0, because + provider-reported tool cost is always priced. format: int64 type: integer asOf: @@ -6585,10 +6586,11 @@ spec: type: string byModel: description: |- - ByModel breaks the total down per actually-served model, so a session - that fell back or auto-routed across models shows where the spend went - instead of one blended total under the configured model. Empty when the - runner reported no per-model usage; AmountMicroUSD and PricingKnown are + ByModel breaks the total down per actually-served model, MERGED across + every pod (summed per model display id), so a session that fell back or + auto-routed across models — or ran across a reap/wake — shows where the + spend went instead of one blended total under the configured model. Empty + when no pod reported per-model usage; AmountMicroUSD and PricingKnown are then the sole source of truth. items: description: ModelCostBucket is one served model's slice of @@ -6631,11 +6633,10 @@ spec: byTool: description: |- ByTool breaks out the cost of inner interactive toolkits (e.g. a `claude` - Claude Code sub-run's own provider-reported spend) a session drove. These - are ADDED to AmountMicroUSD alongside ByModel — a session's total is model - spend plus tool spend. Empty when no interactive toolkit reported a cost. - Tool cost is always provider-reported, never table-priced, so a bucket's - PricingKnown is always true. + Claude Code sub-run's own provider-reported spend) a session drove, MERGED + across every pod (summed per tool). These are ADDED to AmountMicroUSD + alongside ByModel — a session's total is model spend plus tool spend. Empty + when no interactive toolkit reported a cost. items: description: ToolCostBucket is one interactive toolkit's slice of a session's cost. @@ -6643,13 +6644,31 @@ spec: amountMicroUSD: description: |- AmountMicroUSD is this tool's provider-reported cost, summed across its - invocations, in micro-USD (1e-6 USD). + invocations, in micro-USD (1e-6 USD). Negative reports are excluded from + the sum (see Anomalous). format: int64 type: integer + anomalous: + description: |- + Anomalous is true when a negative (nonsensical) provider cost was reported + for this tool. The tool is still recorded as seen, but the negative value + is excluded from AmountMicroUSD and the session total — it signals "a cost + report here does not make sense" rather than silently dropping it. + type: boolean pricingKnown: description: |- - PricingKnown is true when AmountMicroUSD is a real cost. Tool cost is - always provider-reported, so this is true whenever a cost was reported. + PricingKnown is true when AmountMicroUSD is a real cost — i.e. at least one + non-negative cost tally was reported. False when none was reported (the + amount is 0 and must not be shown as real spend) or every report was + anomalous. + type: boolean + subscriptionEquivalent: + description: |- + SubscriptionEquivalent is true when this tool ran under a flat-rate + subscription (toolkit CostTreatment=subscription-equivalent), so its + AmountMicroUSD is an API-equivalent estimate, not money charged. The + bucket is recorded and shown, but its amount is EXCLUDED from the session + total / admind spend so it cannot inflate the figure or trip cost alerts. type: boolean tool: description: Tool is the outer tool name (e.g. "claude-oauth"). @@ -6666,11 +6685,139 @@ spec: model: description: Model is the configured model the session ran under. type: string + perPod: + description: |- + PerPod is the per-pod cost SOURCE OF TRUTH: one entry per pod that served + the session (keyed by pod name), each carrying that pod's own total and + breakdown. The aggregate fields above are DERIVED from this list — the + total is the sum of PerPod amounts, ByModel/ByTool are merged by key, and + PricingKnown is the AND. Each pod upserts only its own entry, so cost + ACCUMULATES across a reap/wake instead of a later pod's single-pod total + replacing an earlier one. Empty only for a session no pod has costed yet. + items: + description: |- + PerPodCost is one serving pod's slice of a session's cost — the per-pod + source of truth the EstimatedSessionCost aggregate is derived from. A pod + rewrites only its OWN entry (keyed by Pod), so the session total accumulates + across pods rather than a later pod clobbering an earlier one. + properties: + amountMicroUSD: + description: AmountMicroUSD is this pod's own total in micro-USD + (model + tool). + format: int64 + type: integer + asOf: + description: AsOf is when this pod last computed its estimate. + format: date-time + type: string + byModel: + description: ByModel is this pod's per-served-model breakdown. + items: + description: ModelCostBucket is one served model's slice + of a session's cost estimate. + properties: + amountMicroUSD: + description: |- + AmountMicroUSD is this bucket's share of the session cost, in + micro-USD (1e-6 USD): the provider-reported cost when available, + else tokens × the resolved per-model pricing table rate. + format: int64 + type: integer + inputTokens: + description: InputTokens is this model's share of + prompt tokens. + format: int64 + type: integer + model: + description: |- + Model is the served-model display id verbatim (provider/model, or a + provider-prefixed served model under routing) — never re-parsed here. + type: string + outputTokens: + description: OutputTokens is this model's share of + completion tokens. + format: int64 + type: integer + pricingKnown: + description: |- + PricingKnown is true when AmountMicroUSD is a real cost — either the + provider reported it directly, or the pricing table had a rate for + this model. False means AmountMicroUSD is 0 and must not be shown as + real spend (same no-fabrication contract as EstimatedSessionCost). + type: boolean + required: + - model + type: object + type: array + x-kubernetes-list-type: atomic + byTool: + description: ByTool is this pod's per-interactive-toolkit + breakdown. + items: + description: ToolCostBucket is one interactive toolkit's + slice of a session's cost. + properties: + amountMicroUSD: + description: |- + AmountMicroUSD is this tool's provider-reported cost, summed across its + invocations, in micro-USD (1e-6 USD). Negative reports are excluded from + the sum (see Anomalous). + format: int64 + type: integer + anomalous: + description: |- + Anomalous is true when a negative (nonsensical) provider cost was reported + for this tool. The tool is still recorded as seen, but the negative value + is excluded from AmountMicroUSD and the session total — it signals "a cost + report here does not make sense" rather than silently dropping it. + type: boolean + pricingKnown: + description: |- + PricingKnown is true when AmountMicroUSD is a real cost — i.e. at least one + non-negative cost tally was reported. False when none was reported (the + amount is 0 and must not be shown as real spend) or every report was + anomalous. + type: boolean + subscriptionEquivalent: + description: |- + SubscriptionEquivalent is true when this tool ran under a flat-rate + subscription (toolkit CostTreatment=subscription-equivalent), so its + AmountMicroUSD is an API-equivalent estimate, not money charged. The + bucket is recorded and shown, but its amount is EXCLUDED from the session + total / admind spend so it cannot inflate the figure or trip cost alerts. + type: boolean + tool: + description: Tool is the outer tool name (e.g. "claude-oauth"). + type: string + required: + - tool + type: object + type: array + x-kubernetes-list-type: atomic + currency: + description: Currency is the ISO code this pod's amount + is denominated in ("USD"). + type: string + pod: + description: Pod is the serving pod's name (its HOSTNAME); + the upsert key. + type: string + pricingKnown: + description: |- + PricingKnown is false when a model this pod served had no rate (this pod's + amount is then a lower bound). + type: boolean + required: + - pod + type: object + type: array + x-kubernetes-list-type: atomic pricingKnown: description: |- - PricingKnown is false when a served model had no price. The amount is then - a lower bound (priced components only), not the full cost. Tool buckets are - always priced, so this tracks model-pricing coverage specifically. + PricingKnown is false when a served model had no price in ANY contributing + pod (the AND across PerPod). The amount is then a lower bound (priced + components only), not the full cost. Tool buckets are always priced, so + this tracks model-pricing coverage specifically. type: boolean type: object failureReason: @@ -20012,6 +20159,18 @@ spec: Conversion uses a JSON round-trip (ToToolkit), so this field set must stay byte-compatible with toolkit.Toolkit's JSON shape. properties: + costTreatment: + description: |- + CostTreatment declares how this toolkit's provider-reported cost counts + toward session spend. "metered" (default) is real billed money and counts; + "subscription-equivalent" is a flat-rate estimate (e.g. a Claude Max OAuth + run) that is recorded and shown but EXCLUDED from the session total, so it + cannot inflate spend or trip cost alerts. Mirrors + pkg/tools/toolspec/toolkit.Toolkit.CostTreatment. + enum: + - metered + - subscription-equivalent + type: string docs: description: Docs is prose about the CLI, surfaced to authors. type: string diff --git a/pkg/platform/pipeline/types.go b/pkg/platform/pipeline/types.go index 8ef17201..245f9921 100644 --- a/pkg/platform/pipeline/types.go +++ b/pkg/platform/pipeline/types.go @@ -153,6 +153,13 @@ type ToolUsage struct { CostMicroUSD int64 // CostReported is true when at least one invocation reported a cost. CostReported bool + // Anomalous is true when a negative provider cost was observed for this + // tool; the value is excluded from CostMicroUSD and the bucket is flagged. + Anomalous bool + // SubscriptionEquivalent marks this tool's cost as a flat-rate subscription + // estimate (toolkit CostTreatment=subscription-equivalent): recorded and + // shown, but excluded from session spend. + SubscriptionEquivalent bool } // MetaagentInfo is the payload for the four metaagent control-plane points diff --git a/pkg/tools/toolkitstream/claude/parser.go b/pkg/tools/toolkitstream/claude/parser.go index d6117458..d7facc56 100644 --- a/pkg/tools/toolkitstream/claude/parser.go +++ b/pkg/tools/toolkitstream/claude/parser.go @@ -131,10 +131,11 @@ func (p *parser) decodeLine(line []byte) []toolkitstream.Event { Unbilled: ev.TotalCostUSD != nil && *ev.TotalCostUSD == 0, } return []toolkitstream.Event{{ - Type: toolkitstream.EventResult, - OK: ev.Subtype == "success", - DurationMs: ev.DurationMs, - CostUSD: ev.cost(), + Type: toolkitstream.EventResult, + OK: ev.Subtype == "success", + DurationMs: ev.DurationMs, + CostUSD: ev.cost(), + CostReported: ev.TotalCostUSD != nil, }} case "system": return nil diff --git a/pkg/tools/toolkitstream/contract.go b/pkg/tools/toolkitstream/contract.go index 39571acf..18135177 100644 --- a/pkg/tools/toolkitstream/contract.go +++ b/pkg/tools/toolkitstream/contract.go @@ -41,6 +41,18 @@ type Event struct { // EventResult only. DurationMs int64 CostUSD float64 + // CostReported is the presence bit for CostUSD on an EventResult: true when + // the toolkit's terminal event actually carried a cost tally (even an + // explicit 0), false when it reported none. CostUSD collapses absent to 0, + // so this is the only per-event signal that tells "metered $0" from "said + // nothing about cost" — see Outcome.Unbilled for why that matters. + CostReported bool + // SubscriptionEquivalent marks this result's cost as a flat-rate + // subscription estimate (e.g. a Claude Max OAuth run) rather than metered + // money. Stamped by the interactive tool from its toolkit's CostTreatment, + // not by the parser. Carried so the cost reporter can record it but keep it + // out of session spend. EventResult only. + SubscriptionEquivalent bool } // Outcome is the caller-facing result a Parser distills for the orchestrator diff --git a/pkg/tools/toolspec/toolkit/types.go b/pkg/tools/toolspec/toolkit/types.go index df077f7c..e3e4c0a9 100644 --- a/pkg/tools/toolspec/toolkit/types.go +++ b/pkg/tools/toolspec/toolkit/types.go @@ -10,6 +10,19 @@ const ( SubcommandModeInteractive = "interactive" ) +// Toolkit.CostTreatment values. "" is treated as metered. +const ( + // CostTreatmentMetered: the toolkit's reported total_cost_usd is money + // actually billed (an API key); it counts toward session spend. + CostTreatmentMetered = "metered" + // CostTreatmentSubscriptionEquivalent: the toolkit runs under a flat-rate + // subscription (e.g. a Claude Max/Pro OAuth token), so its reported cost is + // an API-EQUIVALENT estimate, not money charged. It is recorded and shown + // but excluded from session spend so it cannot inflate the figure or trip + // cost alerts. + CostTreatmentSubscriptionEquivalent = "subscription-equivalent" +) + // Toolkit is the closed-world description of one CLI tool. type Toolkit struct { // Name is the catalog key a spec names to reach this toolkit. @@ -57,6 +70,12 @@ type Toolkit struct { // publishing KindToolSessionEvent envelopes. Empty publishes raw // KindToolSessionDelta instead. Example value: "claude-stream-json". StreamFormat string `json:"streamFormat,omitempty"` + // CostTreatment declares how this toolkit's provider-reported cost counts + // toward session spend: "metered" (default, empty) is real billed money; + // "subscription-equivalent" is a flat-rate estimate (e.g. a Claude Max OAuth + // run) that is recorded and shown but kept OUT of the session total. See the + // CostTreatment* constants. + CostTreatment string `json:"costTreatment,omitempty"` // SiteURL mirrors SpiceboxToolkit.Spec.SiteURL for embedded // catalog entries. Set in toolkits/.yaml for built-in // toolkits whose homepage is known at compile time. identityd diff --git a/pkg/web/admind/admind.go b/pkg/web/admind/admind.go index f4e3aaab..1b8703b7 100644 --- a/pkg/web/admind/admind.go +++ b/pkg/web/admind/admind.go @@ -405,6 +405,7 @@ func liveSessionSnapshot(agg *Aggregator) func() []overview.LiveSession { OutputTokens: s.OutputTokens, PendingApprovals: s.PendingToolGrants + s.PendingLeakageApprovals + s.PendingContentInspectionApprovals, ByModel: buckets, + ToolCostMicroUSD: toolCostMicro(s.ByTool), }) } return out diff --git a/pkg/web/admind/budget.go b/pkg/web/admind/budget.go index 81352853..7fa72d04 100644 --- a/pkg/web/admind/budget.go +++ b/pkg/web/admind/budget.go @@ -54,10 +54,14 @@ const budgetUnknownKey = "(unknown)" type budgetSession struct { // State is the aggregator's live snapshot of the session. State SessionState - // Est is the whole-session cost estimate, computed once per session - // against its own model — see handleBudget for why pricing happens before - // grouping, not after. + // Est is the whole-session cost estimate (model + interactive-toolkit), + // computed once per session against its own model — see handleBudget for why + // pricing happens before grouping, not after. Est cost.USD + // ModelEst is the model-only portion of Est (no tool cost). byModel uses it + // for the no-buckets fallback row so tool spend lands on its own row rather + // than inflating the outer-model row. + ModelEst cost.USD // Starters maps "ns/name" to the session's started-by canonical subject. // nil means the lookup FAILED (not "no starters"), which is why the byUser // axis contributes nothing rather than a fabricated "(unknown)" row. @@ -93,17 +97,32 @@ var budgetDimensions = []budgetDimension{ // its spend to EACH bucket's own Model using that bucket's own // already-priced amount, so an OpenRouter session that routed or fell // back across models shows where the spend actually went instead of one - // blended total. Exactly one of the two branches runs per session, so - // tokens and spend are never double-counted. + // blended total; a session with no buckets contributes one row for its + // configured model. Either way the per-row cost is MODEL-ONLY (the + // bucket amount, or ModelEst) and interactive-toolkit spend — which is + // not token-based and belongs to no model — is emitted as its own row(s) + // keyed by tool. That keeps byModel's grand total equal to the other + // axes (which fold tool cost into Est) while showing where it went, + // instead of dropping it (bucket path) or hiding it inside the model row + // (no-bucket path). Name: "byModel", Contribute: func(s budgetSession, emit func(string, int64, int64, cost.USD)) { if len(s.State.ByModel) > 0 { for _, b := range s.State.ByModel { emit(budgetKeyOrUnknown(b.Model), b.InputTokens, b.OutputTokens, bucketCostUSD(b)) } - return + } else { + emit(budgetKeyOrUnknown(s.State.Model), s.State.InputTokens, s.State.OutputTokens, s.ModelEst) + } + for _, tb := range s.State.ByTool { + // Subscription-equivalent tool cost is excluded from spend, so it + // gets no spend row here — keeping byModel's grand total equal to + // the other axes (whose Est already excludes it). + if tb.SubscriptionEquivalent { + continue + } + emit(budgetKeyOrUnknown(tb.Tool), 0, 0, toolBucketCostUSD(tb)) } - emit(budgetKeyOrUnknown(s.State.Model), s.State.InputTokens, s.State.OutputTokens, s.Est) }, Assign: func(b *BudgetBreakdown, rows []BudgetRow) { b.ByModel = rows }, }, @@ -213,9 +232,11 @@ func (a *Admind) handleBudget(w http.ResponseWriter, r *http.Request) { // PriceMap.Estimate does an exact map-key lookup against BARE model ids // (the built-in tables and the catalog overrides are both keyed bare), // so it must be stripped of its provider prefix before pricing. + modelEst := prices.Estimate(bareModel(s.Model), s.InputTokens, s.OutputTokens) bs := budgetSession{ State: s, - Est: prices.Estimate(bareModel(s.Model), s.InputTokens, s.OutputTokens) + toolCostUSD(s.ByTool), + Est: modelEst + toolCostUSD(s.ByTool), + ModelEst: modelEst, Starters: starters, } for _, d := range budgetDimensions { @@ -259,14 +280,33 @@ func bucketCostUSD(b spiceboxv1alpha1.ModelCostBucket) cost.USD { return cost.USD(float64(b.AmountMicroUSD) / 1e6) } -// toolCostUSD sums a session's provider-reported interactive-toolkit spend. -// Nil/empty -> 0. Tool cost is always priced, so this never yields NaN. -func toolCostUSD(buckets []spiceboxv1alpha1.ToolCostBucket) cost.USD { +// toolCostMicro sums a session's BILLED (metered) interactive-toolkit spend in +// micro-USD. Nil/empty -> 0. Subscription-equivalent buckets are flat-rate +// estimates excluded from spend, so they are skipped — the single place that +// exclusion lands for every admind spend view (budget axes, per-session rows, +// and the Overview headline via liveSessionSnapshot). +func toolCostMicro(buckets []spiceboxv1alpha1.ToolCostBucket) int64 { var micro int64 for _, b := range buckets { + if b.SubscriptionEquivalent { + continue + } micro += b.AmountMicroUSD } - return cost.USD(float64(micro) / 1e6) + return micro +} + +// toolCostUSD is toolCostMicro in the admin-dashboard USD type. Tool cost is +// always priced, so this never yields NaN. +func toolCostUSD(buckets []spiceboxv1alpha1.ToolCostBucket) cost.USD { + return cost.USD(float64(toolCostMicro(buckets)) / 1e6) +} + +// toolBucketCostUSD converts one ToolCostBucket's micro-USD amount to the +// admin-dashboard USD type. Tool cost is always priced (an anomalous/negative +// report contributes 0), so unlike bucketCostUSD this never yields NaN. +func toolBucketCostUSD(b spiceboxv1alpha1.ToolCostBucket) cost.USD { + return cost.USD(float64(b.AmountMicroUSD) / 1e6) } // sortedBudgetRows flattens the accumulator into a non-nil slice ordered by diff --git a/pkg/web/admind/budget_internal_test.go b/pkg/web/admind/budget_internal_test.go index f231ada4..e9457428 100644 --- a/pkg/web/admind/budget_internal_test.go +++ b/pkg/web/admind/budget_internal_test.go @@ -14,3 +14,13 @@ func TestToolCostUSD_NilByToolContributesZero(t *testing.T) { {Tool: "claude-oauth", AmountMicroUSD: 19_020_000, PricingKnown: true}, })), 1e-9) } + +func TestToolCostUSD_ExcludesSubscriptionEquivalent(t *testing.T) { + // A subscription-equivalent bucket is a flat-rate estimate, excluded from + // spend; only the metered bucket counts. + got := toolCostUSD([]spiceboxv1alpha1.ToolCostBucket{ + {Tool: "aider", AmountMicroUSD: 2_000_000, PricingKnown: true}, + {Tool: "claude-oauth", AmountMicroUSD: 5_000_000, PricingKnown: true, SubscriptionEquivalent: true}, + }) + assert.InDelta(t, 2.00, float64(got), 1e-9, "only the $2 metered tool counts; the $5 subscription tool is excluded") +} diff --git a/pkg/web/admind/budget_test.go b/pkg/web/admind/budget_test.go index 9742b43b..4a78b71e 100644 --- a/pkg/web/admind/budget_test.go +++ b/pkg/web/admind/budget_test.go @@ -453,4 +453,61 @@ func TestAdmindBudget_ByToolAddsToEstimate(t *testing.T) { require.Contains(t, sess, "default/s-codebot") assert.InDelta(t, 19.02, float64(sess["default/s-codebot"].EstimatedCostUSD), 1e-9, "$0 model (0 tokens) + $19.02 tool cost") + + // byModel: tool spend gets its own row and the grand total matches the other + // axes — the fix for tool cost being hidden inside the model row (no-bucket + // path). + model := rowsByKey(bd.ByModel) + require.Contains(t, model, "claude-oauth", "tool spend gets its own byModel row") + assert.InDelta(t, 19.02, float64(model["claude-oauth"].EstimatedCostUSD), 1e-9) + var byModelTotal float64 + for _, r := range bd.ByModel { + byModelTotal += float64(r.EstimatedCostUSD) + } + assert.InDelta(t, 19.02, byModelTotal, 1e-9, "byModel total equals bySession") +} + +// TestAdmindBudget_ByModelWithBucketsAndTool proves tool spend keeps its own +// byModel row even when per-model buckets are present — the sub-case where the +// bucket path previously dropped tool cost entirely from byModel. +func TestAdmindBudget_ByModelWithBucketsAndTool(t *testing.T) { + s := &spiceboxv1alpha1.AgentSession{} + s.Namespace, s.Name = "default", "s-routed" + s.Spec.Class = "router-bot" + s.Status.Phase = "Completed" + s.Status.StartedAt = &metav1.Time{Time: time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC)} + s.Status.EffectiveSettings = &spiceboxv1alpha1.EffectiveSettings{ + Model: spiceboxv1alpha1.ModelConfig{Provider: "openrouter", Name: "auto"}, + } + s.Status.EstimatedCost = &spiceboxv1alpha1.EstimatedSessionCost{ + ByModel: []spiceboxv1alpha1.ModelCostBucket{ + {Model: "openrouter/anthropic/claude-3.5-sonnet", AmountMicroUSD: 1_000_000, PricingKnown: true}, + }, + ByTool: []spiceboxv1alpha1.ToolCostBucket{ + {Tool: "claude-oauth", AmountMicroUSD: 5_000_000, PricingKnown: true}, + }, + } + + k8s := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(s).Build() + a := newTestAdmind(t, k8s) + a.Aggregator().UpsertSession(s) + + w := do(t, a.Handler(), http.MethodGet, "/admin/v1/budget", "test-token", "user:YWRtaW4", "") + require.Equal(t, http.StatusOK, w.Code) + var bd admind.BudgetBreakdown + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &bd)) + + model := rowsByKey(bd.ByModel) + require.Contains(t, model, "openrouter/anthropic/claude-3.5-sonnet") + assert.InDelta(t, 1.0, float64(model["openrouter/anthropic/claude-3.5-sonnet"].EstimatedCostUSD), 1e-9) + require.Contains(t, model, "claude-oauth", "tool spend is NOT dropped when buckets are present") + assert.InDelta(t, 5.0, float64(model["claude-oauth"].EstimatedCostUSD), 1e-9) + + // byModel total is the $1 model bucket + $5 tool row, computed from the + // byModel rows alone — the tool cost is attributed, not dropped. + var byModelTotal float64 + for _, r := range bd.ByModel { + byModelTotal += float64(r.EstimatedCostUSD) + } + assert.InDelta(t, 6.0, byModelTotal, 1e-9) } diff --git a/pkg/web/admind/handlers.go b/pkg/web/admind/handlers.go index ecfcd3c3..9129b6a5 100644 --- a/pkg/web/admind/handlers.go +++ b/pkg/web/admind/handlers.go @@ -538,14 +538,13 @@ func (a *Admind) handleOverview(w http.ResponseWriter, r *http.Request) { tokensSpent += m.InputTokens + m.OutputTokens } // Inner interactive-toolkit spend (e.g. a passthrough `claude` sub-run) is - // not token-based, so it never appears in ov.ByModel. Add it over the SAME - // session scope that produced ov.ByModel — the overview engine's own live - // input is liveSessionSnapshot(a.agg), so a.agg.Snapshot() is exactly that - // set (no double-count). Without this the headline spend understates a - // sub-agent session by the full tool cost (see the codebot case). - for _, s := range a.agg.Snapshot() { - estCost += toolCostUSD(s.ByTool) - } + // not token-based, so it never appears in ov.ByModel. The overview engine + // sums it into ov.ToolCostMicroUSD over the SAME cached snapshot that + // produced ov.ByModel, so model and tool cost share one point in time — + // reading a.agg.Snapshot() live here would instead mix the ≤TTL-stale + // ov.ByModel with a newer tool total. Without this the headline spend + // understates a sub-agent session by the full tool cost (see the codebot case). + estCost += cost.USD(float64(ov.ToolCostMicroUSD) / 1e6) writeJSON(w, http.StatusOK, overviewResponse{ Overview: ov, Budget: budgetInfo{TokensSpent: tokensSpent, EstimatedCostUSD: estCost, Estimated: true}, diff --git a/pkg/web/admind/overview/aggregate.go b/pkg/web/admind/overview/aggregate.go index 71171eb4..31de0dba 100644 --- a/pkg/web/admind/overview/aggregate.go +++ b/pkg/web/admind/overview/aggregate.go @@ -61,6 +61,12 @@ type LiveSession struct { // under the single Model field above — EITHER the buckets OR the session // totals, never both, so nothing is double-counted. ByModel []ModelBucket + // ToolCostMicroUSD is the session's summed provider-reported interactive- + // toolkit cost (micro-USD). Carried here so the engine folds it from the + // SAME snapshot it builds ByModel from — keeping model and tool cost at one + // point in time rather than letting the handler read it live against a + // cached ByModel. + ToolCostMicroUSD int64 } // ModelBucket mirrors v1alpha1.ModelCostBucket's token counts without pulling @@ -120,6 +126,11 @@ type Overview struct { ByModel []ModelTokens `json:"byModel"` // ByAgentClass is the per-class rollup, ordered for display. ByAgentClass []ClassTokens `json:"byAgentClass"` + // ToolCostMicroUSD is the summed interactive-toolkit cost (micro-USD) across + // the live snapshot, computed here so it shares a point in time with ByModel. + // Not token-based, so it never appears in ByModel; the budget handler prices + // ByModel and adds this. + ToolCostMicroUSD int64 `json:"toolCostMicroUSD"` // Series24h is a FIXED 24-hour window, oldest first; empty hours are zeroed // buckets rather than gaps. Series24h [buckets]HourBucket `json:"series24h"` @@ -202,6 +213,7 @@ func (e *Engine) compute(ctx context.Context) (Overview, error) { active++ } ov.KPIs.ApprovalsPending += s.PendingApprovals + ov.ToolCostMicroUSD += s.ToolCostMicroUSD // A session WITH per-served-model buckets attributes its tokens to // EACH bucket's own Model, not the session's blended one, so a diff --git a/pkg/web/admind/overview/aggregate_test.go b/pkg/web/admind/overview/aggregate_test.go index 6feecfae..bcbe1731 100644 --- a/pkg/web/admind/overview/aggregate_test.go +++ b/pkg/web/admind/overview/aggregate_test.go @@ -125,9 +125,9 @@ func TestOverviewAggregates(t *testing.T) { snapshot := func() []LiveSession { return []LiveSession{ - {Model: "claude-opus-4-8", Class: "researcher", Active: true, InputTokens: 100, OutputTokens: 50, PendingApprovals: 2}, + {Model: "claude-opus-4-8", Class: "researcher", Active: true, InputTokens: 100, OutputTokens: 50, PendingApprovals: 2, ToolCostMicroUSD: 5_000_000}, {Model: "claude-sonnet-4-6", Class: "writer", Active: false, InputTokens: 200, OutputTokens: 80, PendingApprovals: 1}, - {Model: "claude-opus-4-8", Class: "researcher", Active: true, InputTokens: 30, OutputTokens: 10}, + {Model: "claude-opus-4-8", Class: "researcher", Active: true, InputTokens: 30, OutputTokens: 10, ToolCostMicroUSD: 1_250_000}, } } @@ -185,6 +185,7 @@ func TestOverviewAggregates(t *testing.T) { assert.Equal(t, 3, ov.KPIs.ToolCalls24h, "1+2 tool_use blocks in window") assert.Equal(t, 1, ov.KPIs.Denials24h, "one denied authz_decision in window") assert.Equal(t, 3, ov.KPIs.ApprovalsPending, "2+1 pending approvals summed across live sessions") + assert.Equal(t, int64(6_250_000), ov.ToolCostMicroUSD, "tool cost summed across live sessions from the same snapshot") } // TestOverviewAggregates_ByModelBuckets proves the byModel rollup attributes