Skip to content

fix(mastra): record time_to_first_token for streaming model spans #2214

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Cedric / ViaDézo1er (viadezo1er) merged 2 commits into from
Jul 13, 2026
Merged

fix(mastra): record time_to_first_token for streaming model spans #2214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
144524d
fix(mastra): record time_to_first_token for streaming model spans
viadezo1er Jul 8, 2026
8dc95b5
chore: log metric instead of just the key for mastra e2e test
viadezo1er Jul 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .changeset/mastra-token-usage-ttft.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
"braintrust": patch
---

fix(mastra): Record `time_to_first_token` for streaming Mastra model spans

The Mastra observability exporter now derives `time_to_first_token` (in
seconds) from a streaming model span's `completionStartTime` attribute and its
start time, matching the metric Braintrust surfaces for other streaming LLM
calls.

Ports the TTFT portion of mastra-ai/mastra#11029.
54 changes: 22 additions & 32 deletions e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.log-payloads.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,13 @@
"metadata": {
"scenario": "mastra-instrumentation"
},
"metric_keys": [],
"name": "mastra-instrumentation-root",
"type": "task"
},
{
"metadata": {
"operation": "generate"
},
"metric_keys": [],
"name": "mastra-agent-generate-operation",
"type": null
},
Expand All @@ -22,7 +20,6 @@
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [],
"name": "agent run: 'weather-agent'",
"output": {
"files": [],
Expand Down Expand Up @@ -60,11 +57,12 @@
"model": "mock-model-id",
"provider": "mock-provider"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
],
"metrics": {
"completion_tokens": 20,
"prompt_tokens": 10,
"time_to_first_token": 0,
"tokens": 30
},
"name": "llm: 'mock-model-id'",
"output": {
"files": [],
Expand All @@ -82,11 +80,11 @@
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
],
"metrics": {
"completion_tokens": 20,
"prompt_tokens": 10,
"tokens": 30
},
"name": "step: 0",
"output": {
"text": "The forecast is sunny.",
Expand All @@ -100,7 +98,6 @@
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [],
"name": "chunk: 'text'",
"output": {
"text": "The forecast is sunny."
Expand All @@ -111,7 +108,6 @@
"metadata": {
"operation": "stream"
},
"metric_keys": [],
"name": "mastra-agent-stream-operation",
"type": null
},
Expand All @@ -122,7 +118,6 @@
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [],
"name": "agent run: 'weather-agent'",
"output": {
"files": [],
Expand Down Expand Up @@ -160,11 +155,12 @@
"model": "mock-model-id",
"provider": "mock-provider"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
],
"metrics": {
"completion_tokens": 20,
"prompt_tokens": 10,
"time_to_first_token": 0,
"tokens": 30
},
"name": "llm: 'mock-model-id'",
"output": {
"files": [],
Expand All @@ -182,11 +178,11 @@
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
],
"metrics": {
"completion_tokens": 20,
"prompt_tokens": 10,
"tokens": 30
},
"name": "step: 0",
"output": {
"text": "The forecast is sunny.",
Expand All @@ -200,7 +196,6 @@
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [],
"name": "chunk: 'text'",
"output": {
"text": "The forecast is sunny."
Expand All @@ -211,15 +206,13 @@
"metadata": {
"operation": "tool"
},
"metric_keys": [],
"name": "mastra-tool-operation",
"type": null
},
{
"metadata": {
"operation": "workflow"
},
"metric_keys": [],
"name": "mastra-workflow-operation",
"type": null
},
Expand All @@ -232,7 +225,6 @@
"entity_name": "travel-flow",
"entity_type": "workflow_run"
},
"metric_keys": [],
"name": "workflow run: 'travel-flow'",
"output": {
"forecast": "Sunny in Berlin"
Expand All @@ -248,7 +240,6 @@
"entity_name": "travel-flow",
"entity_type": "workflow_step"
},
"metric_keys": [],
"name": "workflow step: 'lookup-step'",
"output": {
"forecast": "Sunny in Berlin"
Expand All @@ -259,7 +250,6 @@
"metadata": {
"scenario": "mastra-instrumentation"
},
"metric_keys": [],
"name": "mastra-instrumentation-root",
"type": "task"
}
Expand Down
42 changes: 22 additions & 20 deletions e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
]
"metrics": {
"completion_tokens": 20,
"prompt_tokens": 10,
"tokens": 30
}
}
],
"input": {
Expand Down Expand Up @@ -85,11 +85,12 @@
"model": "mock-model-id",
"provider": "mock-provider"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
]
"metrics": {
"completion_tokens": 20,
"prompt_tokens": 10,
"time_to_first_token": 0,
"tokens": 30
}
}
],
"input": "What is the weather in Paris?",
Expand Down Expand Up @@ -138,11 +139,11 @@
"entity_name": "Weather Agent",
"entity_type": "agent"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
]
"metrics": {
"completion_tokens": 20,
"prompt_tokens": 10,
"tokens": 30
}
}
],
"input": {
Expand Down Expand Up @@ -181,11 +182,12 @@
"model": "mock-model-id",
"provider": "mock-provider"
},
"metric_keys": [
"completion_tokens",
"prompt_tokens",
"tokens"
]
"metrics": {
"completion_tokens": 20,
"prompt_tokens": 10,
"time_to_first_token": 0,
"tokens": 30
}
}
],
"input": "Stream the Paris forecast.",
Expand Down
42 changes: 22 additions & 20 deletions e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.span-tree.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,12 @@ span_tree:
│ │ "model": "mock-model-id",
│ │ "provider": "mock-provider"
│ │ }
│ │ metric_keys: [
│ │ "completion_tokens",
│ │ "prompt_tokens",
│ │ "tokens"
│ │ ]
│ │ metrics: {
│ │ "completion_tokens": 20,
│ │ "prompt_tokens": 10,
│ │ "time_to_first_token": 0,
│ │ "tokens": 30
│ │ }
│ │ └── step: 0 [llm]
│ │ output: {
│ │ "text": "The forecast is sunny.",
Expand All @@ -70,11 +71,11 @@ span_tree:
│ │ "entity_name": "Weather Agent",
│ │ "entity_type": "agent"
│ │ }
│ │ metric_keys: [
│ │ "completion_tokens",
│ │ "prompt_tokens",
│ │ "tokens"
│ │ ]
│ │ metrics: {
│ │ "completion_tokens": 20,
│ │ "prompt_tokens": 10,
│ │ "tokens": 30
│ │ }
│ │ └── chunk: 'text' [llm]
│ │ output: {
│ │ "text": "The forecast is sunny."
Expand Down Expand Up @@ -132,11 +133,12 @@ span_tree:
│ │ "model": "mock-model-id",
│ │ "provider": "mock-provider"
│ │ }
│ │ metric_keys: [
│ │ "completion_tokens",
│ │ "prompt_tokens",
│ │ "tokens"
│ │ ]
│ │ metrics: {
│ │ "completion_tokens": 20,
│ │ "prompt_tokens": 10,
│ │ "time_to_first_token": 0,
│ │ "tokens": 30
│ │ }
│ │ └── step: 0 [llm]
│ │ output: {
│ │ "text": "The forecast is sunny.",
Expand All @@ -147,11 +149,11 @@ span_tree:
│ │ "entity_name": "Weather Agent",
│ │ "entity_type": "agent"
│ │ }
│ │ metric_keys: [
│ │ "completion_tokens",
│ │ "prompt_tokens",
│ │ "tokens"
│ │ ]
│ │ metrics: {
│ │ "completion_tokens": 20,
│ │ "prompt_tokens": 10,
│ │ "tokens": 30
│ │ }
│ │ └── chunk: 'text' [llm]
│ │ output: {
│ │ "text": "The forecast is sunny."
Expand Down
23 changes: 20 additions & 3 deletions e2e/scenarios/mastra-instrumentation/scenario.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,21 @@ function scrubMastraProviderTimestamps(value: Json): Json {
return value;
}

function summarizeMetrics(
metrics: CapturedLogEvent["metrics"],
): Record<string, Json> | undefined {
if (!metrics) return undefined;

const entries = Object.entries(metrics)
.filter(
([key, value]) => key !== "start" && key !== "end" && value !== undefined,
)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, value]) => [key, value as Json] as const);

return entries.length > 0 ? Object.fromEntries(entries) : undefined;
}

function summarizeMastraPayload(event: CapturedLogEvent): Record<string, Json> {
const metadata = event.row.metadata as Record<string, unknown> | undefined;
const pickedMetadata = Object.fromEntries(
Expand All @@ -104,13 +119,15 @@ function summarizeMastraPayload(event: CapturedLogEvent): Record<string, Json> {
const result: Record<string, Json> = {
metadata:
Object.keys(pickedMetadata).length > 0 ? (pickedMetadata as Json) : null,
metric_keys: Object.keys(event.metrics ?? {})
.filter((key) => key !== "start" && key !== "end")
.sort(),
name: event.span.name ?? null,
type: event.span.type ?? null,
};

const metrics = summarizeMetrics(event.metrics);
if (metrics) {
result.metrics = metrics;
}

if (event.input !== undefined && event.input !== null) {
result.input = scrubMastraProviderTimestamps(event.input as Json);
}
Expand Down
Loading
Loading