From 97add84ace782dfb962dafb90a41d1a7fb2f965c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 27 Feb 2026 07:25:58 +0000 Subject: [PATCH 1/2] Add annotate tool for highlighting changed elements Adds a new `annotate` tool that the model calls after completing all code edits to highlight what was changed. Each annotation includes a label and a one-sentence description. The system prompt instructs the model to call it exactly once before returning its final response. Backend: tool schema, runtime handler, and input summarizer. Frontend: amber-themed display in AgentActivity with BsTag icon. https://claude.ai/code/session_01S3R2LH589n6nrSAJkfeDg8 --- backend/agent/tools/definitions.py | 37 +++++++++++++++++++ backend/agent/tools/runtime.py | 29 +++++++++++++++ backend/agent/tools/summaries.py | 17 +++++++++ backend/prompts/system_prompt.py | 1 + .../src/components/agent/AgentActivity.tsx | 36 ++++++++++++++++++ 5 files changed, 120 insertions(+) diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.py index d29f4c79a..de9680e03 100644 --- a/backend/agent/tools/definitions.py +++ b/backend/agent/tools/definitions.py @@ -101,6 +101,32 @@ def _retrieve_option_schema() -> Dict[str, Any]: } +def _annotate_schema() -> Dict[str, Any]: + return { + "type": "object", + "properties": { + "annotations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string", + "description": "Short label identifying the changed element (e.g. 'Header', 'Login button', 'Hero image').", + }, + "description": { + "type": "string", + "description": "One-sentence description of what was changed.", + }, + }, + "required": ["label", "description"], + }, + } + }, + "required": ["annotations"], + } + + def canonical_tool_definitions( image_generation_enabled: bool = True, ) -> List[CanonicalToolDefinition]: @@ -152,6 +178,17 @@ def canonical_tool_definitions( ), parameters=_retrieve_option_schema(), ), + CanonicalToolDefinition( + name="annotate", + description=( + "Annotate the changed elements to highlight what was modified. " + "Call this tool exactly once, AFTER all code edits are complete " + "and right before returning your final response to the user. " + "Each annotation should identify a changed element and include " + "a short one-sentence description of what changed." + ), + parameters=_annotate_schema(), + ), ] ) return tools diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.py index b16805075..87434e4b0 100644 --- a/backend/agent/tools/runtime.py +++ b/backend/agent/tools/runtime.py @@ -49,6 +49,8 @@ async def execute(self, tool_call: ToolCall) -> ToolExecutionResult: return await self._remove_background(tool_call.arguments) if tool_call.name == "retrieve_option": return self._retrieve_option(tool_call.arguments) + if tool_call.name == "annotate": + return self._annotate(tool_call.arguments) return ToolExecutionResult( ok=False, result={"error": f"Unknown tool: {tool_call.name}"}, @@ -357,6 +359,33 @@ def coerce_int(value: Any) -> Optional[int]: result = {"option_number": resolved_index + 1, "code": code} return ToolExecutionResult(ok=True, result=result, summary=summary) + def _annotate(self, args: Dict[str, Any]) -> ToolExecutionResult: + annotations = args.get("annotations") + if not isinstance(annotations, list) or not annotations: + return ToolExecutionResult( + ok=False, + result={"error": "annotate requires a non-empty annotations list"}, + summary={"error": "Missing annotations"}, + ) + + cleaned: List[Dict[str, str]] = [] + for annotation in annotations: + label = ensure_str(annotation.get("label")) + description = ensure_str(annotation.get("description")) + if label and description: + cleaned.append({"label": label, "description": description}) + + if not cleaned: + return ToolExecutionResult( + ok=False, + result={"error": "No valid annotations provided"}, + summary={"error": "No valid annotations"}, + ) + + summary = {"annotations": cleaned} + result = {"annotations": cleaned} + return ToolExecutionResult(ok=True, result=result, summary=summary) + # Backwards-compatible alias for older imports. AgentToolbox = AgentToolRuntime diff --git a/backend/agent/tools/summaries.py b/backend/agent/tools/summaries.py index c87ba88ed..69aa266d3 100644 --- a/backend/agent/tools/summaries.py +++ b/backend/agent/tools/summaries.py @@ -69,4 +69,21 @@ def summarize_tool_input(tool_call: ToolCall, file_state: AgentFileState) -> Dic "index": args.get("index"), } + if tool_call.name == "annotate": + annotations = args.get("annotations") or [] + if isinstance(annotations, list): + return { + "count": len(annotations), + "annotations": [ + { + "label": ensure_str(a.get("label")), + "description": summarize_text( + ensure_str(a.get("description")), 160 + ), + } + for a in annotations + if isinstance(a, dict) + ], + } + return args diff --git a/backend/prompts/system_prompt.py b/backend/prompts/system_prompt.py index 0fa1d9b94..9da2ab16a 100644 --- a/backend/prompts/system_prompt.py +++ b/backend/prompts/system_prompt.py @@ -18,6 +18,7 @@ - When available, use generate_images to create image URLs from prompts (you may pass multiple prompts). The image generation AI is not capable of generating images with a transparent background. - Use remove_background to remove backgrounds from provided image URLs when needed (you may pass multiple image URLs). - Use retrieve_option to fetch the full HTML for a specific option (1-based option_number) when a user references another option. +- After ALL code edits are complete and right before returning your final response, call annotate exactly once to highlight the elements you changed. Each annotation should have a short label identifying the element and a one-sentence description of what was changed. This helps the user quickly see what was modified. # Stack-specific instructions diff --git a/frontend/src/components/agent/AgentActivity.tsx b/frontend/src/components/agent/AgentActivity.tsx index d01af32d2..828bc175a 100644 --- a/frontend/src/components/agent/AgentActivity.tsx +++ b/frontend/src/components/agent/AgentActivity.tsx @@ -16,6 +16,7 @@ import { BsImage, BsScissors, BsFiles, + BsTag, } from "react-icons/bs"; import ReactMarkdown from "react-markdown"; import { Light as SyntaxHighlighterBase } from "react-syntax-highlighter"; @@ -107,6 +108,9 @@ function getEventIcon(type: AgentEventType, toolName?: string) { if (toolName === "retrieve_option") { return ; } + if (toolName === "annotate") { + return ; + } return ; } @@ -149,6 +153,11 @@ function getEventTitle(event: AgentEvent): string { ? "Retrieving option" : "Retrieved option"; } + if (event.toolName === "annotate") { + return event.status === "running" + ? "Annotating changes" + : "Annotated changes"; + } return event.status === "running" ? "Running tool" : "Tool completed"; } return "Activity"; @@ -345,6 +354,33 @@ function renderToolDetails(event: AgentEvent, variantCode?: string) { )} + {event.toolName === "annotate" && !hasError && ( +
+ {(() => { + const annotations = + (output?.annotations as Array<{ label: string; description: string }>) || + (input?.annotations as Array<{ label: string; description: string }>) || + []; + return annotations.map((annotation, index) => ( +
+ +
+ + {annotation.label} + + + {annotation.description} + +
+
+ )); + })()} +
+ )} + {!event.toolName && !hasError && ( <> {event.input && ( From b0f6f61705f4875aa44eb354665d98350c1658eb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 27 Feb 2026 08:37:51 +0000 Subject: [PATCH 2/2] Use CSS selectors in annotate tool and highlight elements in preview Replace the label field with a CSS selector so annotations can target actual DOM elements. The PreviewComponent now injects a pulsing amber outline into the iframe for each matched selector after the iframe loads. PreviewPane extracts annotations from completed annotate tool events and passes them down. System prompt and tool description updated to instruct the model to provide CSS selectors. https://claude.ai/code/session_01S3R2LH589n6nrSAJkfeDg8 --- backend/agent/tools/definitions.py | 13 ++-- backend/agent/tools/runtime.py | 6 +- backend/agent/tools/summaries.py | 2 +- backend/prompts/system_prompt.py | 2 +- .../src/components/agent/AgentActivity.tsx | 12 ++-- .../components/preview/PreviewComponent.tsx | 68 +++++++++++++++++++ .../src/components/preview/PreviewPane.tsx | 24 ++++++- 7 files changed, 109 insertions(+), 18 deletions(-) diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.py index de9680e03..d48da14b1 100644 --- a/backend/agent/tools/definitions.py +++ b/backend/agent/tools/definitions.py @@ -110,16 +110,16 @@ def _annotate_schema() -> Dict[str, Any]: "items": { "type": "object", "properties": { - "label": { + "selector": { "type": "string", - "description": "Short label identifying the changed element (e.g. 'Header', 'Login button', 'Hero image').", + "description": "CSS selector that uniquely identifies the changed element in the HTML (e.g. '#hero-section', '.navbar', 'button.login-btn', 'header > nav').", }, "description": { "type": "string", "description": "One-sentence description of what was changed.", }, }, - "required": ["label", "description"], + "required": ["selector", "description"], }, } }, @@ -181,11 +181,12 @@ def canonical_tool_definitions( CanonicalToolDefinition( name="annotate", description=( - "Annotate the changed elements to highlight what was modified. " + "Annotate the changed elements to highlight them in the preview. " "Call this tool exactly once, AFTER all code edits are complete " "and right before returning your final response to the user. " - "Each annotation should identify a changed element and include " - "a short one-sentence description of what changed." + "Each annotation requires a CSS selector that uniquely targets " + "the changed element in the HTML, plus a short one-sentence " + "description of what changed." ), parameters=_annotate_schema(), ), diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.py index 87434e4b0..b017002ba 100644 --- a/backend/agent/tools/runtime.py +++ b/backend/agent/tools/runtime.py @@ -370,10 +370,10 @@ def _annotate(self, args: Dict[str, Any]) -> ToolExecutionResult: cleaned: List[Dict[str, str]] = [] for annotation in annotations: - label = ensure_str(annotation.get("label")) + selector = ensure_str(annotation.get("selector")) description = ensure_str(annotation.get("description")) - if label and description: - cleaned.append({"label": label, "description": description}) + if selector and description: + cleaned.append({"selector": selector, "description": description}) if not cleaned: return ToolExecutionResult( diff --git a/backend/agent/tools/summaries.py b/backend/agent/tools/summaries.py index 69aa266d3..2c677876c 100644 --- a/backend/agent/tools/summaries.py +++ b/backend/agent/tools/summaries.py @@ -76,7 +76,7 @@ def summarize_tool_input(tool_call: ToolCall, file_state: AgentFileState) -> Dic "count": len(annotations), "annotations": [ { - "label": ensure_str(a.get("label")), + "selector": ensure_str(a.get("selector")), "description": summarize_text( ensure_str(a.get("description")), 160 ), diff --git a/backend/prompts/system_prompt.py b/backend/prompts/system_prompt.py index 9da2ab16a..7e776ceb8 100644 --- a/backend/prompts/system_prompt.py +++ b/backend/prompts/system_prompt.py @@ -18,7 +18,7 @@ - When available, use generate_images to create image URLs from prompts (you may pass multiple prompts). The image generation AI is not capable of generating images with a transparent background. - Use remove_background to remove backgrounds from provided image URLs when needed (you may pass multiple image URLs). - Use retrieve_option to fetch the full HTML for a specific option (1-based option_number) when a user references another option. -- After ALL code edits are complete and right before returning your final response, call annotate exactly once to highlight the elements you changed. Each annotation should have a short label identifying the element and a one-sentence description of what was changed. This helps the user quickly see what was modified. +- After ALL code edits are complete and right before returning your final response, call annotate exactly once to highlight the elements you changed in the preview. Each annotation needs a CSS selector that uniquely targets the changed element in the HTML (e.g. '#hero-section', '.navbar', 'button.login-btn') and a one-sentence description of what was changed. # Stack-specific instructions diff --git a/frontend/src/components/agent/AgentActivity.tsx b/frontend/src/components/agent/AgentActivity.tsx index 828bc175a..f169be87e 100644 --- a/frontend/src/components/agent/AgentActivity.tsx +++ b/frontend/src/components/agent/AgentActivity.tsx @@ -358,19 +358,19 @@ function renderToolDetails(event: AgentEvent, variantCode?: string) {
{(() => { const annotations = - (output?.annotations as Array<{ label: string; description: string }>) || - (input?.annotations as Array<{ label: string; description: string }>) || + (output?.annotations as Array<{ selector: string; description: string }>) || + (input?.annotations as Array<{ selector: string; description: string }>) || []; return annotations.map((annotation, index) => (
- - {annotation.label} - + + {annotation.selector} + {annotation.description} diff --git a/frontend/src/components/preview/PreviewComponent.tsx b/frontend/src/components/preview/PreviewComponent.tsx index 8be4560c7..de5c5e527 100644 --- a/frontend/src/components/preview/PreviewComponent.tsx +++ b/frontend/src/components/preview/PreviewComponent.tsx @@ -4,11 +4,17 @@ import useThrottle from "../../hooks/useThrottle"; import { useAppStore } from "../../store/app-store"; import { addHighlight, removeHighlight } from "../select-and-edit/utils"; +export interface Annotation { + selector: string; + description: string; +} + interface Props { code: string; device: "mobile" | "desktop"; onScaleChange?: (scale: number) => void; viewMode?: "fit" | "actual"; + annotations?: Annotation[]; } const MOBILE_VIEWPORT_WIDTH = 375; @@ -19,6 +25,7 @@ function PreviewComponent({ device, onScaleChange, viewMode, + annotations, }: Props) { const iframeRef = useRef(null); const wrapperRef = useRef(null); @@ -158,6 +165,67 @@ function PreviewComponent({ } }, [throttledCode]); + // Apply annotation highlights inside the iframe when annotations change + useEffect(() => { + const iframe = iframeRef.current; + if (!iframe || !annotations || annotations.length === 0) return; + + const applyAnnotations = () => { + const doc = iframe.contentWindow?.document; + if (!doc) return; + + // Remove any previously injected annotation styles + doc.getElementById("__annotation-styles")?.remove(); + + // Inject a style tag with a pulsing highlight animation + const style = doc.createElement("style"); + style.id = "__annotation-styles"; + style.textContent = ` + @keyframes __annotation-pulse { + 0%, 100% { outline-color: rgba(245, 158, 11, 0.8); } + 50% { outline-color: rgba(245, 158, 11, 0.3); } + } + .__annotation-highlight { + outline: 2px solid rgba(245, 158, 11, 0.8); + outline-offset: 2px; + animation: __annotation-pulse 2s ease-in-out 3; + } + `; + doc.head.appendChild(style); + + // Query each selector and apply the highlight class + for (const annotation of annotations) { + try { + const el = doc.querySelector(annotation.selector); + if (el) { + el.classList.add("__annotation-highlight"); + el.setAttribute("title", annotation.description); + } + } catch { + // Invalid selector — skip silently + } + } + }; + + // The iframe may not have loaded yet after srcdoc changes, so listen for load + iframe.addEventListener("load", applyAnnotations); + // Also try immediately in case the iframe is already loaded + applyAnnotations(); + + return () => { + iframe.removeEventListener("load", applyAnnotations); + // Clean up highlights from iframe DOM + const doc = iframe.contentWindow?.document; + if (doc) { + doc.getElementById("__annotation-styles")?.remove(); + doc.querySelectorAll(".__annotation-highlight").forEach((el) => { + el.classList.remove("__annotation-highlight"); + el.removeAttribute("title"); + }); + } + }; + }, [annotations]); + return (
{ + if (!currentCommit) return []; + const variant = currentCommit.variants[currentCommit.selectedVariantIndex]; + const events = variant?.agentEvents || []; + for (const event of events) { + if ( + event.type === "tool" && + event.toolName === "annotate" && + event.status === "complete" + ) { + const output = event.output as { annotations?: Array<{ selector: string; description: string }> } | undefined; + if (output?.annotations && Array.isArray(output.annotations)) { + return output.annotations; + } + } + } + return []; + }, [currentCommit]); + return (
@@ -224,6 +245,7 @@ function PreviewPane({ settings, onOpenVersions }: Props) { code={previewCode} device="mobile" viewMode="actual" + annotations={annotations} />