From 67c501144226caab0a6a1ad84c7c16ede6a80614 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Wed, 7 Oct 2026 12:51:00 +0100 Subject: [PATCH] feat(framework): show tool call cwd on Braintrust tool spans --- .../scripts/upload-braintrust.test.ts | 49 +++++++++++++++++++ apps/framework/scripts/upload-braintrust.ts | 8 ++- packages/core/src/index.ts | 2 + packages/core/src/parsers/adapt.test.ts | 8 +++ packages/core/src/parsers/adapt.ts | 1 + 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/apps/framework/scripts/upload-braintrust.test.ts b/apps/framework/scripts/upload-braintrust.test.ts index b8d7269e..dcee13b3 100644 --- a/apps/framework/scripts/upload-braintrust.test.ts +++ b/apps/framework/scripts/upload-braintrust.test.ts @@ -7,6 +7,7 @@ import { unwrapShell, type SpanSink, tokenMetrics, + transcriptSchema, utcStamp, } from './upload-braintrust.js'; @@ -333,6 +334,54 @@ describe('logTranscript', () => { }); }); +describe('tool span cwd', () => { + const toolMetadata = (transcript: unknown) => { + const metadata: unknown[] = []; + const sink = (): SpanSink => ({ + startSpan: sink, + log(event) { + if (event.metadata && 'tool_name' in event.metadata) { + metadata.push(event.metadata); + } + }, + end() {}, + }); + logTranscript(sink(), { + prompt: 'go', + agentReport: '', + checks: [], + judgeCalls: [], + passed: true, + modelId: 'm', + startTime: 100, + endTime: 120, + agentEndTime: 110, + scoringEndTime: 120, + toolLabels: [], + transcript: transcriptSchema.parse(transcript), + }); + return metadata; + }; + + it('puts a call cwd in tool span metadata and omits it otherwise', () => { + expect( + toolMetadata([ + { type: 'tool_call', name: 'Bash', ts: 101_000, cwd: '/work/a' }, + { type: 'tool_call', name: 'Bash', ts: 102_000 }, + ]) + ).toEqual([{ tool_name: 'Bash', cwd: '/work/a' }, { tool_name: 'Bash' }]); + }); + + it('parses transcript parts with and without cwd', () => { + const parsed = transcriptSchema.parse([ + { type: 'tool_call', name: 'Bash', input: {}, cwd: '/work/a' }, + { type: 'tool_call', name: 'Bash', input: {} }, + ]); + expect(parsed[0]).toMatchObject({ cwd: '/work/a' }); + expect(parsed[1]).not.toHaveProperty('cwd'); + }); +}); + describe('judge spans', () => { it('nests judge calls under the score span', () => { const spans: Record[] = []; diff --git a/apps/framework/scripts/upload-braintrust.ts b/apps/framework/scripts/upload-braintrust.ts index 55788433..be1e2d17 100644 --- a/apps/framework/scripts/upload-braintrust.ts +++ b/apps/framework/scripts/upload-braintrust.ts @@ -61,11 +61,12 @@ const transcriptPartSchema = z.discriminatedUnion('type', [ ts: z.number().optional(), resultTs: z.number().optional(), id: z.string().optional(), + cwd: z.string().optional(), requestId: z.string().optional(), usage: requestUsageSchema.optional(), }), ]); -const transcriptSchema = z.array(transcriptPartSchema).catch([]); +export const transcriptSchema = z.array(transcriptPartSchema).catch([]); const judgeCallsSchema = z.array(judgeCallSchema).catch([]); type TranscriptPart = z.infer; @@ -595,7 +596,10 @@ export function logTranscript( input: part.input, // The span name adds a label, so keep the raw name filterable. // https://github.com/braintrustdata/braintrust-spec/blob/b068e39112e081e45b6070e035877f1e2e83f9b7/skills/instrumentation-spec/references/features/skill-load-metadata.md - metadata: { tool_name: part.name }, + metadata: { + tool_name: part.name, + ...(part.cwd ? { cwd: part.cwd } : {}), + }, ...(part.output !== undefined ? { output: part.output } : {}), ...(part.error ? { error: part.error } : {}), }); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index d8e4a5eb..d715b4da 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -184,6 +184,8 @@ export type TranscriptPart = resultTs?: number; /** Correlates the call with its result, as `TranscriptEvent.tool.id`. */ id?: string; + /** Absolute directory the call ran in, when the agent records one per call. */ + cwd?: string; requestId?: string; usage?: RequestUsage; }; diff --git a/packages/core/src/parsers/adapt.test.ts b/packages/core/src/parsers/adapt.test.ts index 1a339134..bfd96c0e 100644 --- a/packages/core/src/parsers/adapt.test.ts +++ b/packages/core/src/parsers/adapt.test.ts @@ -33,4 +33,12 @@ describe('adaptTranscript', () => { expect(second.cwd).toBeUndefined(); expect(second.resultTs).toBeUndefined(); }); + + it("copies a call's cwd onto its transcript part only when present", () => { + const parts = adaptTranscript(events).transcript.filter( + (part) => part.type === 'tool_call' + ); + expect(parts[0]).toMatchObject({ cwd: '/work/client-a' }); + expect(parts[1]).not.toHaveProperty('cwd'); + }); }); diff --git a/packages/core/src/parsers/adapt.ts b/packages/core/src/parsers/adapt.ts index 605fb358..466b75b4 100644 --- a/packages/core/src/parsers/adapt.ts +++ b/packages/core/src/parsers/adapt.ts @@ -79,6 +79,7 @@ export function adaptTranscript(events: TranscriptEvent[]): AdaptedTranscript { ...timed(event.timestamp), ...(resolved?.ts ? { resultTs: resolved.ts } : {}), ...(event.tool.id ? { id: event.tool.id } : {}), + ...(event.tool.cwd ? { cwd: event.tool.cwd } : {}), ...(event.requestId ? { requestId: event.requestId } : {}), ...(event.usage ? { usage: event.usage } : {}), });