diff --git a/src/event-processor.ts b/src/event-processor.ts index 64b9ec6..ead8e86 100644 --- a/src/event-processor.ts +++ b/src/event-processor.ts @@ -16,6 +16,11 @@ function generateUUID(): string { return crypto.randomUUID() } +interface ExplicitSkillRequestMetadata extends Record { + loaded_skill_names: string[] + loaded_skills: Array<{ name: string }> +} + function skillLoadMetadata(tool: string, args: unknown): Record | undefined { if (tool !== "skill") return undefined const name = @@ -23,10 +28,65 @@ function skillLoadMetadata(tool: string, args: unknown): Record ? args.name : undefined return { + tool_kind: "skill", skill_name: name, } } +function skillNameFromArgs(args: unknown): string | undefined { + return args && typeof args === "object" && "name" in args && typeof args.name === "string" + ? args.name + : undefined +} + +function normalizeExplicitSkillName(name: string): string | undefined { + const normalized = name.trim().replace(/[),.;]+$/, "") + return normalized ? normalized : undefined +} + +function explicitSkillRequestMetadata( + names: readonly string[], +): ExplicitSkillRequestMetadata | undefined { + const seen = new Set() + const loaded_skill_names: string[] = [] + for (const name of names) { + const normalized = normalizeExplicitSkillName(name) + if (!normalized || seen.has(normalized)) continue + seen.add(normalized) + loaded_skill_names.push(normalized) + } + if (loaded_skill_names.length === 0) return undefined + return { + loaded_skill_names, + loaded_skills: loaded_skill_names.map((name) => ({ name })), + } +} + +function explicitSkillNamesFromUserMessage(message: string): string[] { + const names: string[] = [] + for (const match of message.matchAll(/(?:^|\s)\/skills(?::|\s+)([A-Za-z0-9_.:-]+)/g)) { + names.push(match[1] ?? "") + } + return explicitSkillRequestMetadata(names)?.loaded_skill_names ?? [] +} + +function isExplicitSkillsCommand(message: string): boolean { + return /(?:^|\s)\/skills(?=$|\s|:)/.test(message) +} + +function skillLoadTrigger( + explicitSkillsCommand: boolean | undefined, + explicitSkillNames: readonly string[] | undefined, + tool: string, + args: unknown, +): "explicit" | undefined { + if (tool !== "skill") return undefined + const name = skillNameFromArgs(args) + if (name === undefined) return undefined + if (explicitSkillNames?.includes(name)) return "explicit" + return explicitSkillsCommand ? "explicit" : undefined +} + function skillToolSpanName(tool: string, title: string, args: unknown, fallback: string): string { if (tool !== "skill") return fallback const name = @@ -48,6 +108,8 @@ interface SessionState { currentInput?: string currentOutput?: string currentMessageId?: string + currentTurnExplicitSkillsCommand?: boolean + currentTurnExplicitSkillNames?: string[] // Joined system prompt captured from experimental.chat.system.transform systemPrompt?: string // Parent-child session tracking (for subagents) @@ -163,6 +225,8 @@ export class EventProcessor { state.currentTurnSpanId = generateUUID() state.currentOutput = undefined state.currentInput = userMessage + state.currentTurnExplicitSkillsCommand = isExplicitSkillsCommand(userMessage) + state.currentTurnExplicitSkillNames = explicitSkillNamesFromUserMessage(userMessage) const now = this.clock.now() const nowSeconds = msToSeconds(now) @@ -178,6 +242,7 @@ export class EventProcessor { metadata: { turn_number: state.turnNumber, model: model ? model.modelID : undefined, + ...explicitSkillRequestMetadata(state.currentTurnExplicitSkillNames), }, metrics: { start: nowSeconds, @@ -332,6 +397,35 @@ export class EventProcessor { // output value, so extract the text from content[] as a last resort. const rawOutput = capturedOutput !== undefined ? capturedOutput : extractToolOutput(output) const formattedName = this.formatToolName(tool, title) + const trigger = skillLoadTrigger( + state.currentTurnExplicitSkillsCommand, + state.currentTurnExplicitSkillNames, + tool, + toolArgs, + ) + const skillName = skillNameFromArgs(toolArgs) + if ( + trigger === "explicit" && + skillName !== undefined && + !state.currentTurnExplicitSkillNames?.includes(skillName) + ) { + state.currentTurnExplicitSkillNames = [ + ...(state.currentTurnExplicitSkillNames ?? []), + skillName, + ] + const explicitSkillMetadata = explicitSkillRequestMetadata( + state.currentTurnExplicitSkillNames, + ) + if (explicitSkillMetadata !== undefined) { + await this.spanSink.insertSpan({ + id: state.currentTurnSpanId, + span_id: state.currentTurnSpanId, + root_span_id: state.effectiveRootSpanId, + metadata: explicitSkillMetadata, + _is_merge: true, + }) + } + } const toolSpan: SpanData = { id: generateUUID(), span_id: toolSpanId, @@ -345,6 +439,7 @@ export class EventProcessor { title, reasoning: reasoning || undefined, ...skillLoadMetadata(tool, toolArgs), + ...(trigger !== undefined ? { skill_load_trigger: trigger } : {}), }, metrics: { start: startTime, diff --git a/src/tracing.test.ts b/src/tracing.test.ts index 2d38004..662897c 100644 --- a/src/tracing.test.ts +++ b/src/tracing.test.ts @@ -244,40 +244,92 @@ describe("Event to Span Transformation", () => { it("normalizes native skill tool calls as skill load spans", async () => { const sessionId = "ses_skill_tool" const messageId = "msg_1" + const skillSession = session( + sessionId, + sessionCreated(sessionId), + chatMessage("Use the review skill"), + toolCallPart(sessionId, messageId, "call_skill", "skill", { name: "review" }), + toolExecute("call_skill", "skill", "review", { name: "review" }, "Loaded skill review"), + textPart(sessionId, messageId, "Loaded the review skill."), + messageCompleted(sessionId, messageId, { tokens: { input: 20, output: 15 } }), + sessionIdle(sessionId), + ) - await assertEventsProduceTree( - session( - sessionId, - sessionCreated(sessionId), - chatMessage("Use the review skill"), - toolCallPart(sessionId, messageId, "call_skill", "skill", { name: "review" }), - toolExecute("call_skill", "skill", "review", { name: "review" }, "Loaded skill review"), - textPart(sessionId, messageId, "Loaded the review skill."), - messageCompleted(sessionId, messageId, { tokens: { input: 20, output: 15 } }), - sessionIdle(sessionId), - ), - { - span_attributes: { name: "OpenCode: test-project", type: "task" }, - children: [ - { - span_attributes: { name: "Turn 1", type: "task" }, - children: [ - { - span_attributes: { name: "skill: review", type: "tool" }, - metadata: { - tool_name: "skill", - call_id: "call_skill", - skill_name: "review", - }, - }, - { - span_attributes: { name: "anthropic/claude-3-haiku", type: "llm" }, + await assertEventsProduceTree(skillSession, { + span_attributes: { name: "OpenCode: test-project", type: "task" }, + children: [ + { + span_attributes: { name: "Turn 1", type: "task" }, + children: [ + { + span_attributes: { name: "skill: review", type: "tool" }, + metadata: { + tool_name: "skill", + tool_kind: "skill", + call_id: "call_skill", + skill_name: "review", }, - ], - }, - ], - }, + }, + { + span_attributes: { name: "anthropic/claude-3-haiku", type: "llm" }, + }, + ], + }, + ], + }) + + const tree = await eventsToTree(skillSession) + const turn = tree?.children[0] + const skillSpan = turn?.children.find((child) => child.name === "skill: review") + expect(turn?.metadata?.loaded_skill_names).toBeUndefined() + expect(turn?.metadata?.loaded_skills).toBeUndefined() + expect(skillSpan?.metadata?.skill_load_trigger).toBeUndefined() + }) + + it("marks /skills skill tool calls as explicit", async () => { + const sessionId = "ses_explicit_skill_tool" + const messageId = "msg_1" + const skillSession = session( + sessionId, + sessionCreated(sessionId), + chatMessage("/skills review"), + toolCallPart(sessionId, messageId, "call_skill", "skill", { name: "review" }), + toolExecute("call_skill", "skill", "review", { name: "review" }, "Loaded skill review"), + textPart(sessionId, messageId, "Loaded the review skill."), + messageCompleted(sessionId, messageId, { tokens: { input: 20, output: 15 } }), + sessionIdle(sessionId), ) + + const tree = await eventsToTree(skillSession) + const turn = tree?.children[0] + const skillSpan = turn?.children.find((child) => child.name === "skill: review") + + expect(turn?.metadata?.loaded_skill_names).toEqual(["review"]) + expect(turn?.metadata?.loaded_skills).toEqual([{ name: "review" }]) + expect(skillSpan?.metadata?.skill_load_trigger).toBe("explicit") + }) + + it("backfills bare /skills commands from the selected skill tool call", async () => { + const sessionId = "ses_bare_explicit_skill_tool" + const messageId = "msg_1" + const skillSession = session( + sessionId, + sessionCreated(sessionId), + chatMessage("/skills"), + toolCallPart(sessionId, messageId, "call_skill", "skill", { name: "review" }), + toolExecute("call_skill", "skill", "review", { name: "review" }, "Loaded skill review"), + textPart(sessionId, messageId, "Loaded the review skill."), + messageCompleted(sessionId, messageId, { tokens: { input: 20, output: 15 } }), + sessionIdle(sessionId), + ) + + const tree = await eventsToTree(skillSession) + const turn = tree?.children[0] + const skillSpan = turn?.children.find((child) => child.name === "skill: review") + + expect(turn?.metadata?.loaded_skill_names).toEqual(["review"]) + expect(turn?.metadata?.loaded_skills).toEqual([{ name: "review" }]) + expect(skillSpan?.metadata?.skill_load_trigger).toBe("explicit") }) it("session -> turn -> multiple tool calls", async () => { diff --git a/src/tracing.ts b/src/tracing.ts index 48f5bf1..3cfa456 100644 --- a/src/tracing.ts +++ b/src/tracing.ts @@ -22,6 +22,11 @@ function generateUUID(): string { return crypto.randomUUID() } +interface ExplicitSkillRequestMetadata extends Record { + loaded_skill_names: string[] + loaded_skills: Array<{ name: string }> +} + function skillLoadMetadata(tool: string, args: unknown): Record | undefined { if (tool !== "skill") return undefined const name = @@ -29,10 +34,65 @@ function skillLoadMetadata(tool: string, args: unknown): Record ? args.name : undefined return { + tool_kind: "skill", skill_name: name, } } +function skillNameFromArgs(args: unknown): string | undefined { + return args && typeof args === "object" && "name" in args && typeof args.name === "string" + ? args.name + : undefined +} + +function normalizeExplicitSkillName(name: string): string | undefined { + const normalized = name.trim().replace(/[),.;]+$/, "") + return normalized ? normalized : undefined +} + +function explicitSkillRequestMetadata( + names: readonly string[], +): ExplicitSkillRequestMetadata | undefined { + const seen = new Set() + const loaded_skill_names: string[] = [] + for (const name of names) { + const normalized = normalizeExplicitSkillName(name) + if (!normalized || seen.has(normalized)) continue + seen.add(normalized) + loaded_skill_names.push(normalized) + } + if (loaded_skill_names.length === 0) return undefined + return { + loaded_skill_names, + loaded_skills: loaded_skill_names.map((name) => ({ name })), + } +} + +function explicitSkillNamesFromUserMessage(message: string): string[] { + const names: string[] = [] + for (const match of message.matchAll(/(?:^|\s)\/skills(?::|\s+)([A-Za-z0-9_.:-]+)/g)) { + names.push(match[1] ?? "") + } + return explicitSkillRequestMetadata(names)?.loaded_skill_names ?? [] +} + +function isExplicitSkillsCommand(message: string): boolean { + return /(?:^|\s)\/skills(?=$|\s|:)/.test(message) +} + +function skillLoadTrigger( + explicitSkillsCommand: boolean | undefined, + explicitSkillNames: readonly string[] | undefined, + tool: string, + args: unknown, +): "explicit" | undefined { + if (tool !== "skill") return undefined + const name = skillNameFromArgs(args) + if (name === undefined) return undefined + if (explicitSkillNames?.includes(name)) return "explicit" + return explicitSkillsCommand ? "explicit" : undefined +} + function skillToolSpanName(tool: string, title: string, args: unknown, fallback: string): string { if (tool !== "skill") return fallback const name = @@ -54,6 +114,8 @@ interface SessionState { currentInput?: string currentOutput?: string currentMessageId?: string + currentTurnExplicitSkillsCommand?: boolean + currentTurnExplicitSkillNames?: string[] // Joined system prompt captured from experimental.chat.system.transform systemPrompt?: string // Parent-child session tracking (for subagents) @@ -877,6 +939,8 @@ export function createTracingHooks( .join("\n") || "" state.currentInput = userMessage + state.currentTurnExplicitSkillsCommand = isExplicitSkillsCommand(userMessage) + state.currentTurnExplicitSkillNames = explicitSkillNamesFromUserMessage(userMessage) const now = wallClock.now() const nowSeconds = msToSeconds(now) state.currentTurnStartTime = now @@ -901,6 +965,7 @@ export function createTracingHooks( typeof messageInput.model === "object" && messageInput.model ? `${(messageInput.model as { modelID?: string }).modelID}` : String(messageInput.model || ""), + ...explicitSkillRequestMetadata(state.currentTurnExplicitSkillNames), }, metrics: { start: nowSeconds, @@ -1039,6 +1104,35 @@ export function createTracingHooks( capturedOutput !== undefined ? capturedOutput : extractToolOutput(result.output ?? result) const toolOutput = typeof rawOutput === "string" ? rawOutput.substring(0, 10000) : rawOutput const formattedName = formatToolName(tool, result.title) + const trigger = skillLoadTrigger( + state.currentTurnExplicitSkillsCommand, + state.currentTurnExplicitSkillNames, + tool, + toolArgs, + ) + const skillName = skillNameFromArgs(toolArgs) + if ( + trigger === "explicit" && + skillName !== undefined && + !state.currentTurnExplicitSkillNames?.includes(skillName) + ) { + state.currentTurnExplicitSkillNames = [ + ...(state.currentTurnExplicitSkillNames ?? []), + skillName, + ] + const explicitSkillMetadata = explicitSkillRequestMetadata( + state.currentTurnExplicitSkillNames, + ) + if (explicitSkillMetadata !== undefined) { + enqueue({ + id: state.currentTurnSpanId, + span_id: state.currentTurnSpanId, + root_span_id: state.effectiveRootSpanId, + metadata: explicitSkillMetadata, + _is_merge: true, + }) + } + } const toolSpan: SpanData = { id: generateUUID(), span_id: toolSpanId, @@ -1052,6 +1146,7 @@ export function createTracingHooks( title: result.title, reasoning: reasoning || undefined, ...skillLoadMetadata(tool, toolArgs), + ...(trigger !== undefined ? { skill_load_trigger: trigger } : {}), }, metrics: { start: startTime,