-
Notifications
You must be signed in to change notification settings - Fork 131
feat: [AI-7392] humanize tool-call titles at the source #980
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
b12a473
9e1853e
14f5cbd
0d359d8
31f297d
693bb8c
2b89566
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| /** | ||
| * Produces a readable, dbt-aware title for a tool call — e.g. "Reading customers | ||
| * model" instead of a bare file path — so any client (chat webview, TUI, ...) can | ||
| * render a descriptive label straight from the tool part's `state.title`. | ||
| * | ||
| * This is the source of truth for tool-call labels: it runs inside the tool | ||
| * execute() wrapper (see `tool/tool.ts`) and rewrites the title every tool | ||
| * returns. Only file-acting tools (whose native title is a bare path) are | ||
| * rewritten; every other tool keeps the rich title it already emits. | ||
| * | ||
| * dbt naming ("model"/"seed"/...) is applied only when the path sits under the | ||
| * matching directory, so it degrades to the plain filename off-dbt. | ||
| */ | ||
|
|
||
| /** File-acting tools whose native title is a bare path → gerund verb. */ | ||
| const FILE_TOOL_VERBS: Record<string, string> = { | ||
| read: "Reading", | ||
| write: "Writing", | ||
| edit: "Editing", | ||
| multiedit: "Editing", | ||
| glob: "Searching", | ||
| grep: "Searching", | ||
| list: "Listing", | ||
| } | ||
|
|
||
| /** dbt directory → singular noun used in the label. */ | ||
| const DBT_DIR_KIND: Record<string, string> = { | ||
| models: "model", | ||
| seeds: "seed", | ||
| macros: "macro", | ||
| snapshots: "snapshot", | ||
| tests: "test", | ||
| analyses: "analysis", | ||
| analysis: "analysis", | ||
| } | ||
|
|
||
| function asString(value: unknown): string | undefined { | ||
| return typeof value === "string" && value.trim() ? value : undefined | ||
| } | ||
|
|
||
| /** dbt file extensions that mark a target as a single dbt node (model/seed/...). */ | ||
| const DBT_FILE_EXT = /\.(sql|ya?ml|csv|py|md)$/i | ||
|
|
||
| /** | ||
| * Turn a file path into a friendly target: | ||
| * - a dbt *file* under a known dbt dir → "<name> <kind>" with the extension stripped | ||
| * - otherwise → the basename (e.g. "dbt_project.yml", "index.ts", or a directory name) | ||
| * | ||
| * The dbt-kind rewrite is gated on a recognized dbt file extension so directory | ||
| * targets (e.g. `list models/staging`) aren't mislabeled as a single model, and | ||
| * the nearest dbt ancestor is matched by scanning right-to-left so a coincidental | ||
| * outer directory name (e.g. a repo called `models/`) doesn't win over the real one. | ||
| */ | ||
| function friendlyTarget(rawPath: string): string { | ||
| const segments = rawPath.replace(/\\/g, "/").replace(/^\.\//, "").split("/").filter(Boolean) | ||
| const base = segments[segments.length - 1] ?? rawPath | ||
| if (DBT_FILE_EXT.test(base)) { | ||
| for (let i = segments.length - 2; i >= 0; i--) { | ||
| const kind = DBT_DIR_KIND[segments[i].toLowerCase()] | ||
| if (kind) { | ||
| return `${base.replace(DBT_FILE_EXT, "")} ${kind}` | ||
| } | ||
| } | ||
| } | ||
| return base | ||
| } | ||
|
|
||
| /** Extract the display target for a given file tool from its input args. */ | ||
| function fileTarget(tool: string, input: Record<string, unknown>): string | undefined { | ||
| if (tool === "glob" || tool === "grep") { | ||
| return asString(input["pattern"]) | ||
| } | ||
| if (tool === "list") { | ||
| const path = asString(input["path"]) | ||
| return path ? friendlyTarget(path) : undefined | ||
| } | ||
| // read / write / edit / multiedit | ||
| const filePath = asString(input["filePath"]) ?? asString(input["path"]) | ||
| return filePath ? friendlyTarget(filePath) : undefined | ||
| } | ||
|
|
||
| /** | ||
| * @param tool the tool id (e.g. "read", "sql_analyze") | ||
| * @param input the tool's input args | ||
| * @param rawTitle the title the tool itself returned (a bare path for file tools, | ||
| * already human-readable for everything else) | ||
| * @returns a humanized label for file tools, otherwise the tool's own title. | ||
| */ | ||
| export function describeToolCall(tool: string, input: unknown, rawTitle?: string): string | undefined { | ||
| const fallback = asString(rawTitle) | ||
| const verb = FILE_TOOL_VERBS[tool] | ||
| if (verb && input && typeof input === "object") { | ||
| const target = fileTarget(tool, input as Record<string, unknown>) | ||
| if (target) return `${verb} ${target}` | ||
| } | ||
| // Non-file / rich-title tools: keep the title the tool already emitted. | ||
| return fallback | ||
| } | ||
|
Comment on lines
+89
to
+98
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| /** | ||
| * Authoritative classification of a tool call's origin, stamped onto the tool | ||
| * part's `state.metadata.source` so clients (chat webview, ...) render the right | ||
| * badge without re-deriving it from tool-name prefixes. | ||
| * | ||
| * - "builtin" — native opencode tools (read/glob/bash/...) | ||
| * - "altimate" — Altimate-provided tools (sql_*, schema_*, finops_*, ...) AND | ||
| * tools from the Datamates MCP server (Altimate-owned, just | ||
| * delivered over MCP) | ||
| * - "mcp" — third-party MCP tools | ||
| * | ||
| * Registry tools and MCP tools are resolved in separate loops (see | ||
| * `session/prompt.ts` resolveTools), so each has its own classifier. The native | ||
| * `skill` tool is a further special case: it loads skills of varying origin, so | ||
| * its badge is classified per-call from the loaded skill's origin (see | ||
| * `skillToolSource`) rather than from the tool id. | ||
| */ | ||
| export type ToolSource = "builtin" | "altimate" | "mcp" | ||
|
|
||
| /** | ||
| * Native opencode tool ids. This set is small and stable; every other tool in | ||
| * the registry is Altimate-provided, so new Altimate tools classify correctly | ||
| * with no per-tool maintenance here. | ||
| */ | ||
| const NATIVE_TOOL_IDS = new Set<string>([ | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [SUGGESTION]: The native
Reply with |
||
| "invalid", | ||
| "question", | ||
| "bash", | ||
| "batch", | ||
| "read", | ||
| "glob", | ||
| "grep", | ||
| "list", | ||
| "edit", | ||
| "write", | ||
| "multiedit", | ||
| "task", | ||
| "webfetch", | ||
| "todowrite", | ||
| "todoread", | ||
| "websearch", | ||
| "codesearch", | ||
| "skill", | ||
| "apply_patch", | ||
| "lsp", | ||
| "plan_exit", | ||
| "plan_enter", | ||
| "StructuredOutput", | ||
| ]) | ||
|
|
||
| /** MCP client names that are Altimate-owned (Datamates as an MCP server). */ | ||
| const ALTIMATE_MCP_PREFIXES = ["datamate"] | ||
|
|
||
| /** Classify a registry tool (never an MCP tool) as builtin vs Altimate. */ | ||
| export function registryToolSource(id: string): ToolSource { | ||
| return NATIVE_TOOL_IDS.has(id) ? "builtin" : "altimate" | ||
| } | ||
|
|
||
| /** | ||
| * Classify an MCP tool by its `<client>_<tool>` key: Altimate (Datamates) vs third-party. | ||
| * Matches the parsed `<client>` segment (before the first `_`), not the whole key, so a | ||
| * third-party client that merely starts with "datamate" (e.g. `datamatex_foo`) isn't | ||
| * mislabeled Altimate. Accepts the exact name, its plural (`datamates`), and a | ||
| * `datamate-…` prefixed client. | ||
| */ | ||
| export function mcpToolSource(key: string): ToolSource { | ||
| const underscore = key.indexOf("_") | ||
| const client = (underscore === -1 ? key : key.slice(0, underscore)).toLowerCase() | ||
| return ALTIMATE_MCP_PREFIXES.some((p) => client === p || client === `${p}s` || client.startsWith(`${p}-`)) | ||
| ? "altimate" | ||
| : "mcp" | ||
| } | ||
|
|
||
| /** | ||
| * Classify a skill-load (the native `skill` tool) by the loaded skill's origin, | ||
| * which the skill tool reports on its metadata (see `tool/skill.ts`). Altimate | ||
| * ships its skills bundled with the CLI ("builtin" origin) — those wear the | ||
| * Altimate mark. User-authored global/project skills stay neutral so the badge | ||
| * doesn't over-claim third-party or personal skills. Origin arrives as `unknown` | ||
| * (client metadata), so anything other than "builtin" is treated as neutral. | ||
| */ | ||
| export function skillToolSource(origin: unknown): ToolSource { | ||
| return origin === "builtin" ? "altimate" : "builtin" | ||
| } | ||
|
|
||
| /** | ||
| * Best-effort readable title for an MCP tool call, from its `<client>_<tool>` | ||
| * key — e.g. "datamates_jira_get_issue" → "Jira Get Issue". Strips the leading | ||
| * client segment and Title-Cases the rest. (Richer per-call titles are the MCP | ||
| * server's job; this is the fallback so MCP rows aren't a bare snake_case id.) | ||
| */ | ||
| export function humanizeMcpTitle(key: string): string { | ||
| const withoutClient = key.includes("_") ? key.slice(key.indexOf("_") + 1) : key | ||
| const words = (withoutClient || key).split(/[_-]+/).filter(Boolean) | ||
| if (words.length === 0) return key | ||
| return words.map((w) => w.charAt(0).toUpperCase() + w.slice(1)).join(" ") | ||
| } | ||
| Original file line number | Diff line number | Diff line change | ||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -17,10 +17,22 @@ import os from "os" | |||||||||||||
|
|
||||||||||||||
| const MAX_DISPLAY_SKILLS = 50 | ||||||||||||||
|
|
||||||||||||||
| // altimate_change start — classifySkillSource helper for skill telemetry | ||||||||||||||
| function classifySkillSource(location: string): "builtin" | "global" | "project" { | ||||||||||||||
| if (location.includes("node_modules") || location.includes(".altimate/builtin")) return "builtin" | ||||||||||||||
| if (location.startsWith(os.homedir())) return "global" | ||||||||||||||
| // altimate_change start — classifySkillSource helper for skill telemetry + source badge | ||||||||||||||
| export function classifySkillSource(location: string): "builtin" | "global" | "project" { | ||||||||||||||
| // Normalize separators so `.altimate/builtin` / homedir prefix match on Windows too. | ||||||||||||||
| const normalized = location.replace(/\\/g, "/") | ||||||||||||||
| // Embedded skills load with a `builtin:<name>/SKILL.md` location and Altimate's | ||||||||||||||
| // bundled skills land under `~/.altimate/builtin/` — both are Altimate-shipped. | ||||||||||||||
| // The node_modules match is scoped to Altimate-owned packages (`@altimateai/*` and | ||||||||||||||
| // the `altimate-code` package) so a third-party skill installed under some other | ||||||||||||||
| // `node_modules/<pkg>` isn't tagged as Altimate. | ||||||||||||||
| if ( | ||||||||||||||
| normalized.startsWith("builtin:") || | ||||||||||||||
| /\/node_modules\/(@altimateai\/|altimate-code\/)/.test(normalized) || | ||||||||||||||
| normalized.includes(".altimate/builtin") | ||||||||||||||
| ) | ||||||||||||||
| return "builtin" | ||||||||||||||
| if (normalized.startsWith(os.homedir().replace(/\\/g, "/"))) return "global" | ||||||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P3: The new home-directory classification uses Prompt for AI agents
Suggested change
|
||||||||||||||
| return "project" | ||||||||||||||
| } | ||||||||||||||
|
coderabbitai[bot] marked this conversation as resolved.
|
||||||||||||||
| // altimate_change end | ||||||||||||||
|
|
@@ -147,6 +159,10 @@ export const SkillTool = Tool.define("skill", async (ctx) => { | |||||||||||||
| const followups = SkillFollowups.format(skill.name) | ||||||||||||||
| // altimate_change end | ||||||||||||||
|
|
||||||||||||||
| // altimate_change start — classify origin once, reused for telemetry and the source badge | ||||||||||||||
| const skillOrigin = classifySkillSource(skill.location) | ||||||||||||||
| // altimate_change end | ||||||||||||||
|
|
||||||||||||||
| // altimate_change start — telemetry instrumentation for skill loading with trigger classification | ||||||||||||||
| try { | ||||||||||||||
| Telemetry.track({ | ||||||||||||||
|
|
@@ -155,7 +171,7 @@ export const SkillTool = Tool.define("skill", async (ctx) => { | |||||||||||||
| session_id: ctx.sessionID, | ||||||||||||||
| message_id: ctx.messageID, | ||||||||||||||
| skill_name: skill.name, | ||||||||||||||
| skill_source: classifySkillSource(skill.location), | ||||||||||||||
| skill_source: skillOrigin, | ||||||||||||||
| duration_ms: Date.now() - startTime, | ||||||||||||||
| trigger: Telemetry.classifySkillTrigger(ctx.extra), | ||||||||||||||
| has_followups: followups.length > 0, | ||||||||||||||
|
|
@@ -188,6 +204,8 @@ export const SkillTool = Tool.define("skill", async (ctx) => { | |||||||||||||
| metadata: { | ||||||||||||||
| name: skill.name, | ||||||||||||||
| dir, | ||||||||||||||
| // altimate_change — origin drives the source badge (see altimate/tool-source.ts skillToolSource) | ||||||||||||||
| skillOrigin, | ||||||||||||||
| }, | ||||||||||||||
| } | ||||||||||||||
| // altimate_change end | ||||||||||||||
|
|
||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| import { describe, test, expect } from "bun:test" | ||
| import { describeToolCall } from "../../src/altimate/tool-label" | ||
|
|
||
| describe("describeToolCall", () => { | ||
| test("humanizes reads of a dbt model into 'Reading <name> model'", () => { | ||
| expect(describeToolCall("read", { filePath: "models/customers.sql" }, "models/customers.sql")).toBe( | ||
| "Reading customers model", | ||
| ) | ||
| expect( | ||
| describeToolCall("read", { filePath: "models/staging/stg_customers.sql" }, "models/staging/stg_customers.sql"), | ||
| ).toBe("Reading stg_customers model") | ||
| }) | ||
|
|
||
| test("maps other dbt directories to their noun", () => { | ||
| expect(describeToolCall("edit", { filePath: "macros/cents_to_dollars.sql" }, "macros/cents_to_dollars.sql")).toBe( | ||
| "Editing cents_to_dollars macro", | ||
| ) | ||
| expect(describeToolCall("write", { filePath: "analyses/rollup.sql" }, "analyses/rollup.sql")).toBe( | ||
| "Writing rollup analysis", | ||
| ) | ||
| expect(describeToolCall("read", { filePath: "seeds/raw_customers.csv" }, "seeds/raw_customers.csv")).toBe( | ||
| "Reading raw_customers seed", | ||
| ) | ||
| }) | ||
|
|
||
| test("falls back to the filename for non-dbt paths (never a false 'model')", () => { | ||
| expect(describeToolCall("read", { filePath: "dbt_project.yml" }, "dbt_project.yml")).toBe("Reading dbt_project.yml") | ||
| expect(describeToolCall("read", { filePath: "src/index.ts" }, "src/index.ts")).toBe("Reading index.ts") | ||
| }) | ||
|
|
||
| test("labels glob / grep / list by their target", () => { | ||
| expect(describeToolCall("glob", { pattern: "**/*.sql" }, "12 matches")).toBe("Searching **/*.sql") | ||
| expect(describeToolCall("grep", { pattern: "customer_id" }, "3 matches")).toBe("Searching customer_id") | ||
| expect(describeToolCall("list", { path: "models" }, "models/")).toBe("Listing models") | ||
| }) | ||
|
|
||
| test("list on a nested dbt subdirectory keeps the dir name (not a false '<x> model')", () => { | ||
| // A directory listing under models/ is a folder of many models, not one model. | ||
| expect(describeToolCall("list", { path: "models/staging" }, "models/staging/")).toBe("Listing staging") | ||
| }) | ||
|
|
||
| test("matches the nearest dbt ancestor, not a coincidental outer directory", () => { | ||
| // `models` appears higher in the path but the file lives under `macros`. | ||
| expect( | ||
| describeToolCall("read", { filePath: "models/project/macros/util.sql" }, "models/project/macros/util.sql"), | ||
| ).toBe("Reading util macro") | ||
| }) | ||
|
|
||
| test("humanizes python and markdown dbt files", () => { | ||
| expect(describeToolCall("read", { filePath: "models/py_model.py" }, "models/py_model.py")).toBe( | ||
| "Reading py_model model", | ||
| ) | ||
| expect(describeToolCall("read", { filePath: "models/customers.md" }, "models/customers.md")).toBe( | ||
| "Reading customers model", | ||
| ) | ||
| }) | ||
|
|
||
| test("keeps the tool's own title for non-file / rich-title tools", () => { | ||
| expect( | ||
| describeToolCall("sql_analyze", { filePath: "models/customers.sql" }, "Analyze: 2 issues [high]"), | ||
| ).toBe("Analyze: 2 issues [high]") | ||
| expect(describeToolCall("bash", { command: "dbt build" }, "Run full dbt build")).toBe("Run full dbt build") | ||
| // apply_patch carries a diff, not a path, so it keeps its own per-file title. | ||
| expect(describeToolCall("apply_patch", { patch: "*** Update File: models/x.sql" }, "# Patched x.sql")).toBe( | ||
| "# Patched x.sql", | ||
| ) | ||
| }) | ||
|
|
||
| test("falls back to the raw title when a file tool has no usable path", () => { | ||
| expect(describeToolCall("read", {}, "some title")).toBe("some title") | ||
| expect(describeToolCall("read", undefined, "some title")).toBe("some title") | ||
| }) | ||
| }) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P2: When
listtargets the worktree root (no path argument or empty relative path),fileTarget()returnsundefinedandasString(rawTitle)also returnsundefined(sincepath.relative(worktree, worktree)is""). The??fallback intool.tsthen yields the original empty-string title, producing a blank UI label. Consider producing a fallback like"Listing ."when a file tool has a verb but neither a usable target nor a non-empty raw title.Prompt for AI agents