diff --git a/packages/api-client/src/spend-analysis.ts b/packages/api-client/src/spend-analysis.ts index a289ca389d..ac49b75239 100644 --- a/packages/api-client/src/spend-analysis.ts +++ b/packages/api-client/src/spend-analysis.ts @@ -20,6 +20,9 @@ export interface SpendAnalysisToolRow { cost_usd: number; share_of_scoped: number; avg_input_tokens: number; + // Optional until the backend honest-attribution rollout reaches every deployment. + cost_attributed_usd?: number; + share_attributed?: number; } export interface SpendAnalysisModelRow { @@ -36,6 +39,15 @@ export interface SpendAnalysisDayRow { cost_usd: number; } +export interface SpendAnalysisInputSizeRow { + bucket: string; + min_input_tokens: number; + generation_count: number; + cost_usd: number; + share_of_scoped: number; + avg_cost_per_generation: number; +} + export interface SpendAnalysisBreakdown { items: TRow[]; truncated: boolean; @@ -48,6 +60,8 @@ export interface SpendAnalysisResponse { by_model: SpendAnalysisBreakdown; // Optional until the backend by_day rollout reaches every deployment. by_day?: SpendAnalysisBreakdown; + // Optional until the backend by_input_size rollout reaches every deployment. + by_input_size?: SpendAnalysisBreakdown; // `top_traces` is still in the backend response shape (always empty) per // posthog/posthog#59796. Renderer code does not consume it; left out of the // TS type so future readers see only what we actually use. diff --git a/packages/core/src/billing/spendAnalysisTypes.ts b/packages/core/src/billing/spendAnalysisTypes.ts index 36ae669812..30ff6ae658 100644 --- a/packages/core/src/billing/spendAnalysisTypes.ts +++ b/packages/core/src/billing/spendAnalysisTypes.ts @@ -20,6 +20,9 @@ export interface SpendAnalysisToolRow { cost_usd: number; share_of_scoped: number; avg_input_tokens: number; + // Optional until the backend honest-attribution rollout reaches every deployment. + cost_attributed_usd?: number; + share_attributed?: number; } export interface SpendAnalysisModelRow { @@ -36,6 +39,15 @@ export interface SpendAnalysisDayRow { cost_usd: number; } +export interface SpendAnalysisInputSizeRow { + bucket: string; + min_input_tokens: number; + generation_count: number; + cost_usd: number; + share_of_scoped: number; + avg_cost_per_generation: number; +} + export interface SpendAnalysisBreakdown { items: TRow[]; truncated: boolean; @@ -48,4 +60,6 @@ export interface SpendAnalysisResponse { by_model: SpendAnalysisBreakdown; // Optional until the backend by_day rollout reaches every deployment. by_day?: SpendAnalysisBreakdown; + // Optional until the backend by_input_size rollout reaches every deployment. + by_input_size?: SpendAnalysisBreakdown; } diff --git a/packages/core/src/billing/spendSuggestions.test.ts b/packages/core/src/billing/spendSuggestions.test.ts new file mode 100644 index 0000000000..5839d6fa93 --- /dev/null +++ b/packages/core/src/billing/spendSuggestions.test.ts @@ -0,0 +1,293 @@ +import { describe, expect, it } from "vitest"; +import type { + SpendAnalysisInputSizeRow, + SpendAnalysisResponse, + SpendAnalysisToolRow, +} from "./spendAnalysisTypes"; +import { deriveSpendSuggestions } from "./spendSuggestions"; + +function makeResponse( + overrides: Partial = {}, +): SpendAnalysisResponse { + return { + summary: { + date_from: "2025-04-23T00:00:00Z", + date_to: "2025-05-23T00:00:00Z", + product: "posthog_code", + total_cost_usd: 100, + event_count: 1000, + scoped_cost_usd: 80, + scoped_event_count: 800, + }, + by_product: { items: [], truncated: false }, + by_tool: { items: [], truncated: false }, + by_model: { items: [], truncated: false }, + ...overrides, + }; +} + +function makeToolRow( + overrides: Partial, +): SpendAnalysisToolRow { + return { + tool: "Bash", + generation_count: 500, + cost_usd: 50, + share_of_scoped: 0.625, + avg_input_tokens: 150_000, + ...overrides, + }; +} + +function makeInputSizeRow( + overrides: Partial, +): SpendAnalysisInputSizeRow { + return { + bucket: "<50k", + min_input_tokens: 0, + generation_count: 100, + cost_usd: 10, + share_of_scoped: 0.1, + avg_cost_per_generation: 0.1, + ...overrides, + }; +} + +describe("deriveSpendSuggestions", () => { + it("returns a single no-spend message when total cost is zero", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + summary: { + date_from: "2025-04-23T00:00:00Z", + date_to: "2025-05-23T00:00:00Z", + product: "posthog_code", + total_cost_usd: 0, + event_count: 0, + scoped_cost_usd: 0, + scoped_event_count: 0, + }, + }), + ); + expect(suggestions).toEqual(["No LLM spend in the selected window."]); + }); + + // The whole point of this PR: once the backend serves honest attribution, + // the client must use `share_attributed` and stop claiming a tool "drives" + // the spend, which is the co-occurrence artifact we're replacing. + it("uses share_attributed and avoids 'drives' wording when attribution is present", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + by_tool: { + items: [ + makeToolRow({ + cost_attributed_usd: 36, + share_attributed: 0.45, + }), + ], + truncated: false, + }, + }), + ); + const toolSuggestion = suggestions.find((s) => s.startsWith("Bash")); + expect(toolSuggestion).toContain("45%"); + expect(toolSuggestion).not.toContain("drives"); + expect(toolSuggestion).toContain("split across the tools it used"); + }); + + // Guards the explicit backwards-compat requirement: an older backend that + // hasn't shipped cost_attributed_usd/share_attributed yet must still surface + // a top-tool suggestion, using the old share_of_scoped-based wording. + it("falls back to share_of_scoped and 'drives' wording when attribution fields are absent", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + by_tool: { + items: [makeToolRow({ share_of_scoped: 0.625 })], + truncated: false, + }, + }), + ); + const toolSuggestion = suggestions.find((s) => s.startsWith("Bash")); + expect(toolSuggestion).toContain("drives 63%"); + }); + + it.each([ + ["at threshold, no suggestion", 0.35, false], + ["above threshold, suggestion fires", 0.36, true], + ])( + "top-tool suggestion (attributed): %s", + (_label, shareAttributed, expectSuggestion) => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + by_tool: { + items: [ + makeToolRow({ + cost_attributed_usd: shareAttributed * 80, + share_attributed: shareAttributed, + }), + ], + truncated: false, + }, + }), + ); + const found = suggestions.some((s) => s.startsWith("Bash")); + expect(found).toBe(expectSuggestion); + }, + ); + + it("does not add a top-tool suggestion when the top row has a null tool", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + by_tool: { + items: [ + makeToolRow({ + tool: null, + share_attributed: 0.9, + cost_attributed_usd: 72, + }), + ], + truncated: false, + }, + }), + ); + expect(suggestions.join(" ")).not.toContain("was involved in"); + }); + + it("surfaces the no-tool suggestion without overstating it as a driver", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + by_tool: { + items: [ + makeToolRow({ tool: "Bash", share_of_scoped: 0.5 }), + makeToolRow({ tool: null, share_of_scoped: 0.2 }), + ], + truncated: false, + }, + }), + ); + const noToolSuggestion = suggestions.find((s) => s.startsWith("20%")); + expect(noToolSuggestion).toContain("no tool call"); + expect(noToolSuggestion).not.toContain("drives"); + }); + + it("recommends trimming context when the >300k bucket exceeds the threshold", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + by_input_size: { + items: [ + makeInputSizeRow({ bucket: "<50k", share_of_scoped: 0.1 }), + makeInputSizeRow({ + bucket: ">300k", + min_input_tokens: 300_000, + share_of_scoped: 0.45, + }), + ], + truncated: false, + }, + }), + ); + const contextSuggestion = suggestions.find((s) => s.includes("300k")); + expect(contextSuggestion).toContain("45%"); + expect(contextSuggestion).toContain("clear between unrelated tasks"); + }); + + // The >300k bucket can stay under the bar while spend actually clusters in + // a smaller bucket (e.g. 200k-300k) -- the top-by-cost bucket should still + // trigger the suggestion using its own numbers. + it("falls back to the top-by-cost bucket when >300k is under threshold", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + by_input_size: { + items: [ + makeInputSizeRow({ bucket: "<50k", share_of_scoped: 0.1 }), + makeInputSizeRow({ + bucket: "200k-300k", + min_input_tokens: 200_000, + share_of_scoped: 0.5, + }), + makeInputSizeRow({ + bucket: ">300k", + min_input_tokens: 300_000, + share_of_scoped: 0.2, + }), + ], + truncated: false, + }, + }), + ); + const contextSuggestion = suggestions.find((s) => + s.includes("200k+ input tokens"), + ); + expect(contextSuggestion).toContain("50%"); + }); + + it("adds no context-size suggestion when every bucket is under threshold", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + by_input_size: { + items: [ + makeInputSizeRow({ bucket: "<50k", share_of_scoped: 0.3 }), + makeInputSizeRow({ + bucket: ">300k", + min_input_tokens: 300_000, + share_of_scoped: 0.3, + }), + ], + truncated: false, + }, + }), + ); + expect( + suggestions.some((s) => s.includes("clear between unrelated tasks")), + ).toBe(false); + }); + + it("adds no context-size suggestion when by_input_size is absent", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + by_tool: { + items: [makeToolRow({ share_of_scoped: 0.5 })], + truncated: false, + }, + }), + ); + expect( + suggestions.some((s) => s.includes("clear between unrelated tasks")), + ).toBe(false); + }); + + it("keeps the codeShare suggestion when this app dominates total spend", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + summary: { + date_from: "2025-04-23T00:00:00Z", + date_to: "2025-05-23T00:00:00Z", + product: "posthog_code", + total_cost_usd: 100, + event_count: 1000, + scoped_cost_usd: 90, + scoped_event_count: 900, + }, + }), + ); + expect(suggestions[0]).toContain("90% of your spend"); + }); + + it("falls back to the generic message when no other suggestion fires", () => { + const suggestions = deriveSpendSuggestions( + makeResponse({ + summary: { + date_from: "2025-04-23T00:00:00Z", + date_to: "2025-05-23T00:00:00Z", + product: "posthog_code", + total_cost_usd: 100, + event_count: 1000, + scoped_cost_usd: 50, + scoped_event_count: 500, + }, + }), + ); + expect(suggestions).toEqual([ + "Your spend is fairly evenly distributed across tools. No single hotspot stands out.", + ]); + }); +}); diff --git a/packages/core/src/billing/spendSuggestions.ts b/packages/core/src/billing/spendSuggestions.ts index 4c18d14e0b..3b249c5543 100644 --- a/packages/core/src/billing/spendSuggestions.ts +++ b/packages/core/src/billing/spendSuggestions.ts @@ -1,5 +1,85 @@ import { formatTokens } from "./spendAnalysisFormat"; -import type { SpendAnalysisResponse } from "./spendAnalysisTypes"; +import type { + SpendAnalysisInputSizeRow, + SpendAnalysisResponse, +} from "./spendAnalysisTypes"; + +// A single turn often calls several tools, and the backend's `share_of_scoped` +// splits full turn cost across every tool that turn touched -- so summing it +// across tools exceeds 100%, and whichever tool is most ubiquitous (Bash) looks +// artificially dominant. `share_attributed` / `cost_attributed_usd` fix this by +// splitting each turn's cost fractionally across its tools, so they reconcile +// to scoped spend. Prefer them once the backend serves them; fall back to the +// old co-occurrence share otherwise. +const TOP_TOOL_SHARE_THRESHOLD = 0.35; +const NO_TOOL_SHARE_THRESHOLD = 0.1; +// The largest-context bucket eating this much of scoped spend is worth calling +// out as the primary lever -- trimming context, not switching tools. +const LARGE_CONTEXT_SHARE_THRESHOLD = 0.4; +const LARGEST_BUCKET_LABEL = ">300k"; + +function hasAttributedCost( + row: SpendAnalysisResponse["by_tool"]["items"][number], +): row is SpendAnalysisResponse["by_tool"]["items"][number] & { + cost_attributed_usd: number; + share_attributed: number; +} { + return ( + typeof row.cost_attributed_usd === "number" && + typeof row.share_attributed === "number" + ); +} + +function topToolSuggestion( + toolItems: SpendAnalysisResponse["by_tool"]["items"], +): string | null { + const top = toolItems[0]; + if (!top?.tool) return null; + + if (hasAttributedCost(top)) { + if (top.share_attributed <= TOP_TOOL_SHARE_THRESHOLD) return null; + return `${top.tool} was involved in ${Math.round(top.share_attributed * 100)}% of your PostHog Code spend when each turn's cost is split across the tools it used, averaging ${formatTokens(top.avg_input_tokens)} input tokens per call.`; + } + + if (top.share_of_scoped <= TOP_TOOL_SHARE_THRESHOLD) return null; + return `${top.tool} drives ${Math.round(top.share_of_scoped * 100)}% of your spend in this app, averaging ${formatTokens(top.avg_input_tokens)} input tokens per call.`; +} + +function noToolSuggestion( + toolItems: SpendAnalysisResponse["by_tool"]["items"], +): string | null { + const noToolRow = toolItems.find((r) => r.tool === null); + if (!noToolRow || noToolRow.share_of_scoped <= NO_TOOL_SHARE_THRESHOLD) { + return null; + } + return `${Math.round(noToolRow.share_of_scoped * 100)}% of spend is generations with no tool call: text-only replies. Tighter prompts or stopping the agent earlier can cut this.`; +} + +/** The bucket to call out: the `>300k` bucket if it clears the threshold, + * otherwise whichever bucket has the highest cost share (a shop can cluster in + * `200k-300k` without much landing in `>300k`). */ +function largestContextSuggestion( + rows: SpendAnalysisInputSizeRow[], +): string | null { + if (rows.length === 0) return null; + + const largestBucket = rows.find((r) => r.bucket === LARGEST_BUCKET_LABEL); + const topByCost = rows.reduce((max, r) => + r.share_of_scoped > max.share_of_scoped ? r : max, + ); + + const candidate = + largestBucket && + largestBucket.share_of_scoped > LARGE_CONTEXT_SHARE_THRESHOLD + ? largestBucket + : topByCost.share_of_scoped > LARGE_CONTEXT_SHARE_THRESHOLD + ? topByCost + : null; + + if (!candidate) return null; + + return `${Math.round(candidate.share_of_scoped * 100)}% of your spend comes from calls with ${formatTokens(candidate.min_input_tokens)}+ input tokens. Trimming context is your biggest lever here: clear between unrelated tasks and avoid re-reading large files or command outputs.`; +} export function deriveSpendSuggestions(data: SpendAnalysisResponse): string[] { const suggestions: string[] = []; @@ -18,20 +98,19 @@ export function deriveSpendSuggestions(data: SpendAnalysisResponse): string[] { ); } - const codeTotal = summary.scoped_cost_usd; - if (codeTotal > 0 && toolItems.length > 0) { - const top = toolItems[0]; - if (top.share_of_scoped > 0.35 && top.tool) { - suggestions.push( - `${top.tool} drives ${Math.round(top.share_of_scoped * 100)}% of your spend in this app, averaging ${formatTokens(top.avg_input_tokens)} input tokens per call.`, - ); - } - const noToolRow = toolItems.find((r) => r.tool === null); - if (noToolRow && noToolRow.share_of_scoped > 0.1) { - suggestions.push( - `${Math.round(noToolRow.share_of_scoped * 100)}% is spent on generations that take no tool action: pure text replies. Consider tighter prompts or stopping the agent earlier.`, - ); - } + if (data.by_input_size) { + const inputSizeSuggestion = largestContextSuggestion( + data.by_input_size.items, + ); + if (inputSizeSuggestion) suggestions.push(inputSizeSuggestion); + } + + if (summary.scoped_cost_usd > 0 && toolItems.length > 0) { + const topSuggestion = topToolSuggestion(toolItems); + if (topSuggestion) suggestions.push(topSuggestion); + + const noToolSuggestionText = noToolSuggestion(toolItems); + if (noToolSuggestionText) suggestions.push(noToolSuggestionText); } if (suggestions.length === 0) { diff --git a/packages/ui/src/features/usage/components/SpendAnalysisSection.tsx b/packages/ui/src/features/usage/components/SpendAnalysisSection.tsx index 6f2998c60b..a36533c69f 100644 --- a/packages/ui/src/features/usage/components/SpendAnalysisSection.tsx +++ b/packages/ui/src/features/usage/components/SpendAnalysisSection.tsx @@ -8,6 +8,7 @@ import { useMemo, useState } from "react"; import { useSpendAnalysis } from "../useSpendAnalysis"; import { ModelBreakdownCards } from "./ModelBreakdownCards"; import { + InputSizeBreakdownCard, ProductBreakdownCard, ToolBreakdownCard, } from "./SpendBreakdownTables"; @@ -95,6 +96,9 @@ export function SpendAnalysisSection() { + {data.by_input_size && ( + + )} ) : null} diff --git a/packages/ui/src/features/usage/components/SpendBreakdownTables.tsx b/packages/ui/src/features/usage/components/SpendBreakdownTables.tsx index 1199e033a7..70ba3ffdb2 100644 --- a/packages/ui/src/features/usage/components/SpendBreakdownTables.tsx +++ b/packages/ui/src/features/usage/components/SpendBreakdownTables.tsx @@ -1,9 +1,10 @@ -import { Stack, Wrench } from "@phosphor-icons/react"; +import { Ruler, Stack, Wrench } from "@phosphor-icons/react"; import { formatTokens, formatUsd, } from "@posthog/core/billing/spendAnalysisFormat"; import type { + SpendAnalysisInputSizeRow, SpendAnalysisProductRow, SpendAnalysisToolRow, } from "@posthog/core/billing/spendAnalysisTypes"; @@ -44,14 +45,26 @@ function BreakdownTable({ export function ToolBreakdownCard({ rows }: { rows: SpendAnalysisToolRow[] }) { if (rows.length === 0) return null; + // The backend rolls out attributed cost per response, not per row: if the + // top row has it, every row does. + const hasAttributedCost = rows[0]?.cost_attributed_usd !== undefined; + return ( } title="By tool" > {rows.slice(0, 10).map((r) => ( @@ -59,6 +72,13 @@ export function ToolBreakdownCard({ rows }: { rows: SpendAnalysisToolRow[] }) { {r.generation_count.toLocaleString()} {formatTokens(r.avg_input_tokens)} {formatUsd(r.cost_usd)} + {hasAttributedCost && ( + + {r.cost_attributed_usd !== undefined + ? formatUsd(r.cost_attributed_usd) + : "—"} + + )} ))} @@ -94,3 +114,32 @@ export function ProductBreakdownCard({ ); } + +export function InputSizeBreakdownCard({ + rows, +}: { + rows: SpendAnalysisInputSizeRow[]; +}) { + if (rows.length === 0) return null; + return ( + } + title="By context size" + > + + {rows.map((r) => ( + + {r.bucket} + {r.generation_count.toLocaleString()} + {formatUsd(r.cost_usd)} + {Math.round(r.share_of_scoped * 100)}% + {formatUsd(r.avg_cost_per_generation)} + + ))} + + + ); +}