From 089c89a77eb18a148f4f8e34174abbf59d49188e Mon Sep 17 00:00:00 2001 From: Jack Herrington Date: Wed, 14 Oct 2026 12:59:59 -0700 Subject: [PATCH 01/20] feat(generative-ui): implement Open Generative UI draft 0.1 POC and status report --- STATUS.md | 119 ++ examples/generative-ui/index.html | 12 + examples/generative-ui/package.json | 26 + examples/generative-ui/src/App.tsx | 450 +++++++ examples/generative-ui/src/index.css | 13 + examples/generative-ui/src/main.tsx | 14 + examples/generative-ui/tsconfig.json | 16 + examples/generative-ui/vite.config.ts | 254 ++++ packages/ai-generative-ui/README.md | 34 + packages/ai-generative-ui/package.json | 76 ++ packages/ai-generative-ui/src/compute.ts | 101 ++ .../ai-generative-ui/src/drivers/index.ts | 173 +++ .../ai-generative-ui/src/fixtures/index.ts | 621 ++++++++++ packages/ai-generative-ui/src/index.ts | 12 + packages/ai-generative-ui/src/react.ts | 1 + packages/ai-generative-ui/src/react/index.tsx | 1038 +++++++++++++++++ packages/ai-generative-ui/src/schema.ts | 120 ++ packages/ai-generative-ui/src/types.ts | 374 ++++++ packages/ai-generative-ui/src/validator.ts | 368 ++++++ .../ai-generative-ui/tests/compute.test.ts | 47 + .../ai-generative-ui/tests/drivers.test.ts | 55 + .../tests/react-renderer.test.tsx | 126 ++ .../tests/schema-validator.test.ts | 169 +++ packages/ai-generative-ui/tsconfig.json | 11 + packages/ai-generative-ui/vite.config.ts | 40 + pnpm-lock.yaml | 94 +- 26 files changed, 4363 insertions(+), 1 deletion(-) create mode 100644 STATUS.md create mode 100644 examples/generative-ui/index.html create mode 100644 examples/generative-ui/package.json create mode 100644 examples/generative-ui/src/App.tsx create mode 100644 examples/generative-ui/src/index.css create mode 100644 examples/generative-ui/src/main.tsx create mode 100644 examples/generative-ui/tsconfig.json create mode 100644 examples/generative-ui/vite.config.ts create mode 100644 packages/ai-generative-ui/README.md create mode 100644 packages/ai-generative-ui/package.json create mode 100644 packages/ai-generative-ui/src/compute.ts create mode 100644 packages/ai-generative-ui/src/drivers/index.ts create mode 100644 packages/ai-generative-ui/src/fixtures/index.ts create mode 100644 packages/ai-generative-ui/src/index.ts create mode 100644 packages/ai-generative-ui/src/react.ts create mode 100644 packages/ai-generative-ui/src/react/index.tsx create mode 100644 packages/ai-generative-ui/src/schema.ts create mode 100644 packages/ai-generative-ui/src/types.ts create mode 100644 packages/ai-generative-ui/src/validator.ts create mode 100644 packages/ai-generative-ui/tests/compute.test.ts create mode 100644 packages/ai-generative-ui/tests/drivers.test.ts create mode 100644 packages/ai-generative-ui/tests/react-renderer.test.tsx create mode 100644 packages/ai-generative-ui/tests/schema-validator.test.ts create mode 100644 packages/ai-generative-ui/tsconfig.json create mode 100644 packages/ai-generative-ui/vite.config.ts diff --git a/STATUS.md b/STATUS.md new file mode 100644 index 0000000000..0a09a71bf2 --- /dev/null +++ b/STATUS.md @@ -0,0 +1,119 @@ +# Open Generative UI — Implementation Status & Architecture + +**Date**: 2026-10-14 +**Specification**: Open Generative UI (Draft 0.1) +**Workspace**: `packages/ai-generative-ui`, `examples/generative-ui`, and `goose` desktop POC + +--- + +## 1. Specification Overview (Draft 0.1) + +The Open Generative UI specification defines a declarative, portable, model-composed user interface standard for autonomous AI agents. Rather than dumping raw prose, markdown tables, or executing arbitrary/untrusted HTML/JS, the agent composes structured node graphs from a shared component catalog that the host application natively renders. + +### Core Architecture (4 Layers) +``` +┌─────────────────────────────────────────────────────────┐ +│ Driver (§7) │ +│ Produces node graphs: fixture / programmatic / live LLM │ +├─────────────────────────────────────────────────────────┤ +│ Node Graph (§5) │ +│ Versioned, validated, transport-agnostic data │ +├─────────────────────────────────────────────────────────┤ +│ Transport Binding (§4.1) │ +│ AG-UI state events · MCP/ACP tool result · file/HTTP │ +├─────────────────────────────────────────────────────────┤ +│ Host Renderer (§9) │ +│ Maps nodes to host's own components / design system │ +└─────────────────────────────────────────────────────────┘ +``` + +### The 8 Canonical Jobs to be Done +- **U1 (Compare)**: Side-by-side option evaluations with matching attributes (`compare-plans-basic`). +- **U2 (Understand)**: Explorable mechanical/process systems with interactive hotspots (`bicycle-drivetrain`). +- **U3 (Plan & Track)**: Step sequences with timers and completion checklists (`cooking-plan`). +- **U4 (Calculate)**: Parameter exploration bound to **deterministic host compute functions** (`bill-split`, `savingsGrowth`), never LLM arithmetic. +- **U5 (Collect Input)**: Structured forms that return typed intent data back to the agent (`recipe-parameters`). +- **U6 (Decide / Approve)**: Explicit approval boundaries with approve/reject actions (`approval-tool-call`). +- **U7 (Monitor)**: Glanceable metric cards, trends, and progress bars (`spend-monitor`). +- **U8 (Locate)**: Geographic or route itinerary waypoints with notes (`trip-stops`). +- **§6.4 Vector Scenes**: Sandboxed 2D SVG scenes (`chord-diagram-emaj7`) with declared interactivity, semantic paint roles (`accent`, `neutral`, `muted`, `surface`, `danger`), and forbidden script/eval vectors. + +### Safety & Resilience +- **Zero Raw HTML/JS**: Graphs contain only declarative component primitives. Handlers are attached solely by the host renderer. +- **Bounded Repair (§9.3)**: Host validators degrade unknown containers to `Stack`, unknown leaves to `Text`, swap inverted slider/stepper bounds, synthesize missing `fallbackText` (§9.2), drop unsafe SVG path commands, and absorb vector elements placed in `children` into `props.shapes`. +- **Bidirectional Intent Model (§8)**: User actions emit typed payloads (`local`, `submit`, `action`) back to the host/agent without arbitrary networking. + +--- + +## 2. Current Implementation Status + +### A. Core Package: `@tanstack/ai-generative-ui` +Located at `packages/ai-generative-ui`. + +- **Type Definitions (`src/types.ts`)**: + - `GraphDocument`, `UINode`, `GraphProvenance`, `ComputeBinding`, `IntentPayload`, `HostCapability`. + - 22 Catalog component schemas across Layout (`Stack`, `Columns`, `Section`, `Tabs`, `Timeline`), Content (`Text`, `Image`, `Table`, `List`, `Metric`, `Chart`, `Map`, `Diagram`, `Callout`), Input (`Form`, `TextInput`, `NumberInput`, `Select`, `Slider`, `Stepper`, `Toggle`, `Checkbox`, `Checklist`, `Button`, `Choice`), and Vector (`Scene`). +- **Schema & Validation (`src/schema.ts`, `src/validator.ts`)**: + - JSON Schema compliant definitions for `render_ui` tool calling. + - Full bounded repair validator enforcing §9.3 rules and SVG path grammar validation (`/^[ \t\r\n]*([MmLlHhVvCcSsQqTtAaZz][ \t\r\n,0-9.-]*)+$/`). +- **Deterministic Compute Engine (`src/compute.ts`)**: + - Built-in compute functions: `billSplit`, `savingsGrowth`, `sum`. + - Type-safe `ComputeRegistry` allowing custom host arithmetic registration. +- **Drivers (`src/drivers/index.ts`)**: + - `FixtureDriver`: Mock driver delivering the 9 canonical spec fixtures. + - `ProgrammaticDriver`: Fluent builder API for constructing graphs in application code with zero LLM dependency. + - `LiveDriver`: Automated tool schema generator and output processor for model-driven generation. +- **React Host Renderer (`src/react/index.tsx`)**: + - Dynamic host renderer with recursive `NodeRenderer`. + - SVG Vector scene rendering with interactive shape selection and keyboard accessibility. + - Reactive compute bindings automatically recalculating outputs when inputs change. + - Accessible Text Fallback component (`GenerativeUITextFallback`). +- **Test Suite**: + - 16/16 unit tests passing across 4 test suites: + - `tests/schema-validator.test.ts` + - `tests/compute.test.ts` + - `tests/drivers.test.ts` + - `tests/react-renderer.test.tsx` + +--- + +### B. Web Playground: `examples/generative-ui` +Located at `examples/generative-ui`. + +- **Live Multi-Provider LLM Integration**: + - Middleware in `vite.config.ts` supporting OpenAI (`gpt-4o`), Anthropic (`claude-sonnet-4-5-20250929`), Gemini (`gemini-3.8-flash`), xAI (`grok-4.20-0309-non-reasoning`), and OpenRouter via native fetch and `.env.local` keys. + - Automatic tool calling invocation using the `render_ui` specification schema. +- **Playground Features (`src/App.tsx`)**: + - Scenario Selector covering all 9 canonical fixtures (U1–U8 + §6.4.3 chord diagram). + - Live AI prompt bar with sample chips (*Mortgage calculator*, *Streaming tiers*, *Recipe checklist*, *Dmaj9 chord voicing*). + - Bounded Repair banner detailing applied schema repairs. + - Visual vs. Fallback Text mode toggle (§9.2). + - Interactive JSON inspector toggle. + - Real-time Outbound Intent Stream logging and visual confirmation toast. + - Styled with Tailwind CSS v4 using `@source` workspace resolution. + +--- + +### C. Desktop Integration: Goose (`block/goose`) +Located at `/Users/jherr/projects/goose` on branch `feat/generative-ui-poc`. + +- **Linked Package**: `@tanstack/ai-generative-ui` linked via workspace dependency. +- **Timeline Renderer (`ui/desktop/src/components/GenerativeUI/GenerativeUIMessage.tsx`)**: + - Native host renderer mounting inside Goose's conversation message stream. + - Detects `render_ui` tool calls in `ui/desktop/src/components/ToolCallWithResponse.tsx`. + - Handles outbound intents (`action` / `submit`) and routes them back into the active session via Goose's `append()` message channel. +- **Unit & Integration Tests**: + - `GenerativeUIMessage.test.tsx` verifying native rendering across all canonical fixtures with green typechecks and test suites. + +--- + +## 3. Quality Gates Verification + +| Check | Tool / Target | Status | +|-------|---------------|--------| +| Unit Tests | `pnpm --filter @tanstack/ai-generative-ui test:lib` | ✅ 16/16 passing | +| Type Check | `pnpm test:types` | ✅ 101/101 projects passing | +| Monorepo Dependencies | `pnpm test:sherif` | ✅ Clean (0 errors) | +| Dead Code / Unused Deps | `pnpm test:knip` | ✅ Clean (0 issues) | +| Package Build & Types | `vite build` + `tsc` + `publint --strict` | ✅ Clean ESM & DTS | +| Example Build | `pnpm --filter generative-ui-example build` | ✅ Clean bundle | diff --git a/examples/generative-ui/index.html b/examples/generative-ui/index.html new file mode 100644 index 0000000000..af3c9f712a --- /dev/null +++ b/examples/generative-ui/index.html @@ -0,0 +1,12 @@ + + + + + + Open Generative UI — Reference Playground + + +
+ + + diff --git a/examples/generative-ui/package.json b/examples/generative-ui/package.json new file mode 100644 index 0000000000..36865e9f19 --- /dev/null +++ b/examples/generative-ui/package.json @@ -0,0 +1,26 @@ +{ + "name": "generative-ui-example", + "private": true, + "type": "module", + "version": "0.1.0", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "test:types": "tsc --noEmit" + }, + "dependencies": { + "@tanstack/ai-generative-ui": "workspace:*", + "react": "^19.2.3", + "react-dom": "^19.2.3" + }, + "devDependencies": { + "@tailwindcss/vite": "^4.1.18", + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", + "@vitejs/plugin-react": "^5.2.0", + "tailwindcss": "^4.1.18", + "typescript": "5.9.3", + "vite": "^8.2.1" + } +} diff --git a/examples/generative-ui/src/App.tsx b/examples/generative-ui/src/App.tsx new file mode 100644 index 0000000000..eeb87dfd2b --- /dev/null +++ b/examples/generative-ui/src/App.tsx @@ -0,0 +1,450 @@ +import React, { useState, useEffect, useCallback } from 'react' +import { + GenerativeUIRenderer, + GenerativeUITextFallback, + CANONICAL_FIXTURES, + validateGraph, + type GraphDocument, + type IntentPayload, +} from '@tanstack/ai-generative-ui' + +interface IntentLogEntry extends IntentPayload { + timestamp: string +} + +const SAMPLE_PROMPTS = [ + 'Compare 3 subscription tiers for a video streaming service with pricing table, feature matrix, and select buttons', + 'Mortgage calculation tool with loan amount, interest rate, and years inputs with monthly payment calculation', + 'Recipe card for chocolate lava cake with ingredient checklist, prep steps, and a serving size stepper', + 'Guitar chord voicing diagram for Dmaj9 chord with fretboard scene and play button', + 'Spend monitor dashboard with monthly budget progress, metric cards, and category breakdown chart', +] + +export function App() { + const [selectedScenario, setSelectedScenario] = useState('compare-plans-basic') + const [viewMode, setViewMode] = useState<'visual' | 'fallback'>('visual') + const [intentLogs, setIntentLogs] = useState([]) + const [showJson, setShowJson] = useState(false) + + // Live generation state + const [providers, setProviders] = useState([]) + const [selectedProvider, setSelectedProvider] = useState('openai') + const [promptInput, setPromptInput] = useState('') + const [isGenerating, setIsGenerating] = useState(false) + const [generationError, setGenerationError] = useState(null) + const [appliedRepairs, setAppliedRepairs] = useState([]) + const [lastIntent, setLastIntent] = useState(null) + + // Current graph + const [graph, setGraph] = useState(() => CANONICAL_FIXTURES['compare-plans-basic']) + + // Fetch available providers on mount + useEffect(() => { + fetch('/api/providers') + .then((res) => res.json()) + .then((data) => { + if (data.providers && data.providers.length > 0) { + setProviders(data.providers) + setSelectedProvider(data.providers[0]) + } + }) + .catch((err) => { + console.warn('Could not load providers:', err) + }) + }, []) + + const handleSelectScenario = (key: string) => { + setSelectedScenario(key) + const nextFixture = CANONICAL_FIXTURES[key] + if (nextFixture) { + setGraph(JSON.parse(JSON.stringify(nextFixture))) + setAppliedRepairs([]) + setGenerationError(null) + setLastIntent(null) + } + } + + const handleIntent = useCallback((intent: IntentPayload) => { + const entry: IntentLogEntry = { + ...intent, + timestamp: new Date().toLocaleTimeString(), + } + setIntentLogs((prev) => [entry, ...prev.slice(0, 19)]) + setLastIntent(entry) + }, []) + + const handleLiveGenerate = async (e?: React.FormEvent) => { + if (e) e.preventDefault() + if (!promptInput.trim() || isGenerating) return + + setIsGenerating(true) + setGenerationError(null) + setAppliedRepairs([]) + + try { + const res = await fetch('/api/generate', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + prompt: promptInput, + provider: selectedProvider, + }), + }) + + const data = await res.json() + if (!res.ok) { + throw new Error(data.error || 'Failed to generate UI') + } + + if (data.graph) { + setGraph(data.graph) + setSelectedScenario('custom') + if (data.repairs && data.repairs.length > 0) { + setAppliedRepairs(data.repairs) + } + } + } catch (err: any) { + setGenerationError(err.message || 'An error occurred during generation') + } finally { + setIsGenerating(false) + } + } + + const handleTestRepair = () => { + // Malformed graph testing §9.3 bounded repair rules + const malformed = { + specVersion: '0.1', + title: 'Malformed Graph Repair Test', + provenance: { driver: 'live' }, + root: { + id: 'bad-root', + type: 'AlienContainerNode', // Unknown container -> degrades to Stack + children: [ + { + id: 'bad-leaf', + type: 'CyberneticImplantNode', // Unknown leaf -> degrades to Text + }, + { + id: 'broken-slider', + type: 'Slider', + props: { label: 'Inverted Slider', min: 100, max: 10 }, // Inverted bounds -> swapped + }, + { + id: 'broken-scene', + type: 'Scene', + props: { + viewBox: '0 0 200 200', + shapes: [ + { kind: 'circle', cx: 50, cy: 50, r: 20, fill: 'accent' }, + { kind: 'path', d: '' }, // Unsafe path -> dropped + ], + }, + }, + ], + }, + } + + const res = validateGraph(malformed, { repair: true }) + if (res.valid && res.graph) { + setGraph(res.graph) + setSelectedScenario('custom') + setAppliedRepairs(res.repairs) + } + } + + return ( +
+ {/* Top Header */} +
+
+
+ UI +
+
+

+ Open Generative UI Draft 0.1 +

+

Deterministic Computation • Bounded Repair • Structured Intents

+
+
+ + {/* Fixture & Mode Controls */} +
+ + + + {/* View Mode Toggle */} +
+ + +
+ + +
+
+ + {/* Live Generation Panel */} +
+
+
+
+ setPromptInput(e.target.value)} + disabled={isGenerating} + className="w-full bg-slate-950 border border-slate-700 rounded-xl px-4 py-2.5 text-sm text-white placeholder-slate-500 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent disabled:opacity-50" + /> +
+ +
+ + + +
+
+ + {/* Sample Prompt Chips */} +
+ Try: + {SAMPLE_PROMPTS.map((prompt, i) => ( + + ))} +
+ + {/* Status Banners */} + {generationError && ( +
+ ⚠️ {generationError} + +
+ )} + + {appliedRepairs.length > 0 && ( +
+
+ 🛡️ + Bounded Repair Applied ({appliedRepairs.length} fix{appliedRepairs.length > 1 ? 'es' : ''}) +
+
    + {appliedRepairs.map((r, i) => ( +
  • {r}
    • + ))} +
+
+ )} +
+
+ + {/* Main Content Area */} +
+ {/* Render Canvas */} +
+
+
+ + Surface: + TanStack React Host + • + ID: {graph.graphId} (rev {graph.revision}) + + +
+ + {showJson && ( +
+ 
{JSON.stringify(graph, null, 2)}
+
+ )} + + {lastIntent && ( +
+
+ + + Intent Fired ({lastIntent.intentClass}) + + • + {lastIntent.intent} + from node + {lastIntent.nodeId} + {lastIntent.payload !== undefined && ( + + payload: {JSON.stringify(lastIntent.payload)} + + )} +
+ +
+ )} + + {viewMode === 'visual' ? ( + + ) : ( + + )} +
+
+ + {/* Intent Log Inspector (§8) */} + +
+
+ ) +} diff --git a/examples/generative-ui/src/index.css b/examples/generative-ui/src/index.css new file mode 100644 index 0000000000..b3123d6fda --- /dev/null +++ b/examples/generative-ui/src/index.css @@ -0,0 +1,13 @@ +@import "tailwindcss"; +@source "../src"; +@source "../../packages/ai-generative-ui/src"; + +@layer base { + body { + background-color: #020617; + color: #f8fafc; + font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + margin: 0; + padding: 0; + } +} diff --git a/examples/generative-ui/src/main.tsx b/examples/generative-ui/src/main.tsx new file mode 100644 index 0000000000..41b90e21f8 --- /dev/null +++ b/examples/generative-ui/src/main.tsx @@ -0,0 +1,14 @@ +import React from 'react' +import { createRoot } from 'react-dom/client' +import './index.css' +import { App } from './App' + +const container = document.getElementById('root') +if (container) { + const root = createRoot(container) + root.render( + + + , + ) +} diff --git a/examples/generative-ui/tsconfig.json b/examples/generative-ui/tsconfig.json new file mode 100644 index 0000000000..75411a39bc --- /dev/null +++ b/examples/generative-ui/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "useDefineForClassFields": true, + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "strict": true + }, + "include": ["src"] +} diff --git a/examples/generative-ui/vite.config.ts b/examples/generative-ui/vite.config.ts new file mode 100644 index 0000000000..bd37870710 --- /dev/null +++ b/examples/generative-ui/vite.config.ts @@ -0,0 +1,254 @@ +import { existsSync } from 'node:fs' +import { resolve } from 'node:path' +import { defineConfig, type Plugin } from 'vite' +import react from '@vitejs/plugin-react' +import tailwindcss from '@tailwindcss/vite' +import { LiveDriver, validateGraph } from '@tanstack/ai-generative-ui' + +// Load .env.local if present +for (const envFile of ['.env.local', '.env']) { + const p = resolve(import.meta.dirname, envFile) + if (existsSync(p)) { + process.loadEnvFile(p) + } +} + +const SYSTEM_PROMPT = `You are an Open Generative UI agent complying with Open Generative UI Specification (Draft 0.1). +When given a user request, you MUST call the "render_ui" tool to produce a declarative node graph. + +Ground Rules: +1. Always call the "render_ui" tool. Never return plain prose when UI is requested. +2. The UI graph must be composed only from catalog components: + - Layout: Stack, Columns, Section, Tabs, Timeline + - Content: Text, Image, Table, List, Metric, Chart, Map, Diagram, Callout + - Input: Form, TextInput, NumberInput, Select, Slider, Stepper, Toggle, Checkbox, Checklist, Button, Choice + - Vector Scene: Scene (props must include "viewBox", e.g. "0 0 300 360", and "shapes": Array<{ kind: "rect"|"circle"|"line"|"text"|"path", ... }>. ALWAYS put vector shapes inside props.shapes, not in children. Use semantic paint roles: "accent", "neutral", "muted", "surface", "danger", "highlight", or "none") +3. Safety: No arbitrary HTML, scripts, CSS strings, or execution handlers. +4. Deterministic computation: If values are computed (e.g. loan payments, bill splits, taxes), DO NOT author arithmetic in text. Use named host compute functions (e.g. "billSplit", "savingsGrowth", "sum") with input bindings. +5. Provide a meaningful, accessible "fallbackText" summarizing the interface for non-visual surfaces. +6. Set specVersion to "0.1", revision to 1, and provenance.driver to "live".` + +function generativeUiApiPlugin(): Plugin { + return { + name: 'generative-ui-api', + configureServer(server) { + server.middlewares.use(async (req, res, next) => { + if (req.method === 'GET' && req.url === '/api/providers') { + const providers = [] + if (process.env.OPENAI_API_KEY) providers.push('openai') + if (process.env.ANTHROPIC_API_KEY) providers.push('anthropic') + if (process.env.GEMINI_API_KEY) providers.push('gemini') + if (process.env.XAI_API_KEY) providers.push('xai') + if (process.env.OPENROUTER_API_KEY) providers.push('openrouter') + res.setHeader('Content-Type', 'application/json') + res.end(JSON.stringify({ providers })) + return + } + + if (req.method === 'POST' && req.url === '/api/generate') { + let bodyStr = '' + req.on('data', (chunk) => { + bodyStr += chunk + }) + req.on('end', async () => { + try { + const { prompt, provider = 'openai' } = JSON.parse(bodyStr || '{}') + if (!prompt) { + res.statusCode = 400 + res.end(JSON.stringify({ error: 'Missing prompt' })) + return + } + + const toolDef = LiveDriver.getRenderToolDefinition() + let rawGraph: unknown = null + + if (provider === 'openai') { + const apiKey = process.env.OPENAI_API_KEY + if (!apiKey) throw new Error('OPENAI_API_KEY not configured in .env.local') + + const response = await fetch('https://api.openai.com/v1/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: 'gpt-4o', + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: prompt }, + ], + tools: [{ type: 'function', function: toolDef }], + tool_choice: { type: 'function', function: { name: 'render_ui' } }, + }), + }) + + if (!response.ok) { + const errText = await response.text() + throw new Error(`OpenAI error (${response.status}): ${errText}`) + } + + const json = await response.json() + const toolCall = json.choices?.[0]?.message?.tool_calls?.[0] + const argsStr = toolCall?.function?.arguments + rawGraph = typeof argsStr === 'string' ? JSON.parse(argsStr) : argsStr + } else if (provider === 'gemini') { + const apiKey = process.env.GEMINI_API_KEY + if (!apiKey) throw new Error('GEMINI_API_KEY not configured in .env.local') + + const response = await fetch('https://generativelanguage.googleapis.com/v1beta/openai/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: 'gemini-3.8-flash', + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: prompt }, + ], + tools: [{ type: 'function', function: toolDef }], + tool_choice: { type: 'function', function: { name: 'render_ui' } }, + }), + }) + + if (!response.ok) { + const errText = await response.text() + throw new Error(`Gemini error (${response.status}): ${errText}`) + } + + const json = await response.json() + const toolCall = json.choices?.[0]?.message?.tool_calls?.[0] + const argsStr = toolCall?.function?.arguments + rawGraph = typeof argsStr === 'string' ? JSON.parse(argsStr) : argsStr + } else if (provider === 'xai') { + const apiKey = process.env.XAI_API_KEY + if (!apiKey) throw new Error('XAI_API_KEY not configured in .env.local') + + const response = await fetch('https://api.x.ai/v1/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: 'grok-4.20-0309-non-reasoning', + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: prompt }, + ], + tools: [{ type: 'function', function: toolDef }], + tool_choice: { type: 'function', function: { name: 'render_ui' } }, + }), + }) + + if (!response.ok) { + const errText = await response.text() + throw new Error(`xAI error (${response.status}): ${errText}`) + } + + const json = await response.json() + const toolCall = json.choices?.[0]?.message?.tool_calls?.[0] + const argsStr = toolCall?.function?.arguments + rawGraph = typeof argsStr === 'string' ? JSON.parse(argsStr) : argsStr + } else if (provider === 'anthropic') { + const apiKey = process.env.ANTHROPIC_API_KEY + if (!apiKey) throw new Error('ANTHROPIC_API_KEY not configured in .env.local') + + const response = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'x-api-key': apiKey, + 'anthropic-version': '2023-06-01', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: 'claude-sonnet-4-5-20250929', + max_tokens: 4096, + system: SYSTEM_PROMPT, + messages: [{ role: 'user', content: prompt }], + tools: [ + { + name: toolDef.name, + description: toolDef.description, + input_schema: toolDef.parameters, + }, + ], + tool_choice: { type: 'tool', name: 'render_ui' }, + }), + }) + + if (!response.ok) { + const errText = await response.text() + throw new Error(`Anthropic error (${response.status}): ${errText}`) + } + + const json = await response.json() + const toolUse = json.content?.find((c: any) => c.type === 'tool_use') + rawGraph = toolUse?.input + } else if (provider === 'openrouter') { + const apiKey = process.env.OPENROUTER_API_KEY + if (!apiKey) throw new Error('OPENROUTER_API_KEY not configured in .env.local') + + const response = await fetch('https://openrouter.ai/api/v1/chat/completions', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: 'openai/gpt-4o', + messages: [ + { role: 'system', content: SYSTEM_PROMPT }, + { role: 'user', content: prompt }, + ], + tools: [{ type: 'function', function: toolDef }], + tool_choice: { type: 'function', function: { name: 'render_ui' } }, + }), + }) + + if (!response.ok) { + const errText = await response.text() + throw new Error(`OpenRouter error (${response.status}): ${errText}`) + } + + const json = await response.json() + const toolCall = json.choices?.[0]?.message?.tool_calls?.[0] + const argsStr = toolCall?.function?.arguments + rawGraph = typeof argsStr === 'string' ? JSON.parse(argsStr) : argsStr + } else { + throw new Error(`Unknown or unsupported provider: ${provider}`) + } + + if (!rawGraph) { + throw new Error('Model did not return a valid render_ui tool call') + } + + // Run through Bounded Repair & Schema Validation (§9.3, §7.3) + const validation = validateGraph(rawGraph, { repair: true }) + const graph = validation.valid && validation.graph ? validation.graph : LiveDriver.processModelOutput(rawGraph) + + res.setHeader('Content-Type', 'application/json') + res.end(JSON.stringify({ graph, repairs: validation.repairs })) + } catch (err: any) { + res.statusCode = 500 + res.setHeader('Content-Type', 'application/json') + res.end(JSON.stringify({ error: err.message })) + } + }) + return + } + + next() + }) + }, + } +} + +export default defineConfig({ + plugins: [react(), tailwindcss(), generativeUiApiPlugin()], + server: { + port: 3000, + }, +}) diff --git a/packages/ai-generative-ui/README.md b/packages/ai-generative-ui/README.md new file mode 100644 index 0000000000..a18e155a78 --- /dev/null +++ b/packages/ai-generative-ui/README.md @@ -0,0 +1,34 @@ +
+ + + + TanStack AI + +
+ +
+ +# @tanstack/ai-generative-ui + +Reference implementation of the **Open Generative UI Specification (Draft 0.1)** for TanStack AI. + +Generative UI is the missing layer in agent interfaces: the agent composes an interface appropriate to the task, from a shared vocabulary of components the host knows how to render. + +## Core Capabilities + +1. **Portable & Safe**: Clean-room JSON node graph model without arbitrary HTML/JS execution from language models. +2. **Deterministic Computation**: Numbers and aggregates presented by the UI are computed by registered host code (`billSplit`, `savingsGrowth`, etc.), not hallucinatory arithmetic by language models. +3. **Structured Intents**: Bidirectional interaction flows back to the host/agent as structured intents (`local`, `submit`, `action`). +4. **Resilient Degradation**: Unknown components degrade to their children or accessible fallback text; invalid props are clamped to valid ranges. +5. **Declarative Vector Scenes**: `Scene` shape graphs for data-driven diagrams (chord diagrams, topology, workflows) with declared semantic paint roles and interaction. +6. **Fixture & Driver Architecture**: Testable without a model via 9 canonical fixtures covering all 8 use-case jobs plus vector scenes. diff --git a/packages/ai-generative-ui/package.json b/packages/ai-generative-ui/package.json new file mode 100644 index 0000000000..1dfca98d59 --- /dev/null +++ b/packages/ai-generative-ui/package.json @@ -0,0 +1,76 @@ +{ + "name": "@tanstack/ai-generative-ui", + "version": "0.1.0", + "description": "Open Generative UI specification (Draft 0.1) schemas, validator, drivers, fixtures, and React renderer for TanStack AI.", + "author": "Tanner Linsley", + "license": "MIT", + "homepage": "https://tanstack.com/ai", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-generative-ui" + }, + "bugs": { + "url": "https://github.com/TanStack/ai/issues" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + }, + "./react": { + "types": "./dist/esm/react.d.ts", + "import": "./dist/esm/react.js" + } + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "clean": "premove ./build ./dist", + "test:oxlint": "oxlint src --type-aware", + "test:lib": "vitest run", + "test:lib:dev": "pnpm test:lib --watch", + "test:types": "tsc", + "test:build": "publint --strict", + "build": "vite build" + }, + "keywords": [ + "ai", + "tanstack", + "generative-ui", + "declarative-ui", + "react", + "agent-ui" + ], + "peerDependencies": { + "react": "^19.2.3", + "react-dom": "^19.2.3" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-dom": { + "optional": true + } + }, + "devDependencies": { + "@types/react": "^19.2.7", + "@types/react-dom": "^19.2.3", + "premove": "^4.0.0", + "publint": "^0.3.15", + "react": "^19.2.3", + "react-dom": "^19.2.3", + "vite": "^8.2.1", + "vitest": "^4.1.10" + } +} diff --git a/packages/ai-generative-ui/src/compute.ts b/packages/ai-generative-ui/src/compute.ts new file mode 100644 index 0000000000..fd38d6da33 --- /dev/null +++ b/packages/ai-generative-ui/src/compute.ts @@ -0,0 +1,101 @@ +/** + * Open Generative UI Specification (Draft 0.1) + * Host-side deterministic compute registry (§5.3). + */ + +export type ComputeFunction = ( + inputs: Record, + state: Record, +) => Record | any + +export class ComputeRegistry { + private readonly functions = new Map() + + constructor() { + this.registerDefaults() + } + + register(name: string, fn: ComputeFunction): this { + this.functions.set(name, fn) + return this + } + + has(name: string): boolean { + return this.functions.has(name) + } + + get(name: string): ComputeFunction | undefined { + return this.functions.get(name) + } + + list(): Array { + return Array.from(this.functions.keys()) + } + + execute(name: string, inputs: Record, state: Record): any { + const fn = this.functions.get(name) + if (!fn) { + console.warn(`[GenerativeUI] Compute function "${name}" not found in host registry`) + return undefined + } + return fn(inputs, state) + } + + private registerDefaults() { + // U4: Bill Splitter + this.register('billSplit', (inputs) => { + const subtotal = Number(inputs.subtotal ?? 0) + const tipPercent = Number(inputs.tipPercent ?? 18) + const splitCount = Math.max(1, Math.round(Number(inputs.splitCount ?? 1))) + + const tipAmount = Math.round(subtotal * (tipPercent / 100) * 100) / 100 + const total = Math.round((subtotal + tipAmount) * 100) / 100 + const perPerson = Math.round((total / splitCount) * 100) / 100 + + return { + subtotal, + tipPercent, + splitCount, + tipAmount, + total, + perPerson, + } + }) + + // U4: Savings Growth + this.register('savingsGrowth', (inputs) => { + const principal = Number(inputs.principal ?? 1000) + const annualRate = Number(inputs.annualRate ?? 5) / 100 + const years = Number(inputs.years ?? 5) + const monthlyContribution = Number(inputs.monthlyContribution ?? 100) + + const monthlyRate = annualRate / 12 + const months = years * 12 + + let futureValue = principal + let totalContributions = principal + + for (let m = 0; m < months; m++) { + futureValue = (futureValue + monthlyContribution) * (1 + monthlyRate) + totalContributions += monthlyContribution + } + + futureValue = Math.round(futureValue * 100) / 100 + const totalInterest = Math.round((futureValue - totalContributions) * 100) / 100 + + return { + futureValue, + totalContributions, + totalInterest, + } + }) + + // U7: Sum / Aggregates + this.register('sum', (inputs) => { + const items = Array.isArray(inputs.items) ? inputs.items : [] + return items.reduce((acc, curr) => acc + (Number(curr) || 0), 0) + }) + } +} + +export const defaultComputeRegistry = new ComputeRegistry() diff --git a/packages/ai-generative-ui/src/drivers/index.ts b/packages/ai-generative-ui/src/drivers/index.ts new file mode 100644 index 0000000000..c58fa33155 --- /dev/null +++ b/packages/ai-generative-ui/src/drivers/index.ts @@ -0,0 +1,173 @@ +/** + * Open Generative UI Specification (Draft 0.1) + * UIDriver implementations: FixtureDriver, ProgrammaticDriver, LiveDriver (§7). + */ + +import type { UIDriver, UIRequest, GraphDocument, UINode } from '../types' +import { CANONICAL_FIXTURES } from '../fixtures' +import { validateGraph } from '../validator' +import { GRAPH_DOCUMENT_SCHEMA } from '../schema' + +/** + * 7.1 Fixture driver (mock) — the test backbone + * Returns canned, schema-valid graphs without network or LLM dependency. + */ +export class FixtureDriver implements UIDriver { + private readonly fixtures: Record + + constructor(customFixtures: Record = {}) { + this.fixtures = { ...CANONICAL_FIXTURES, ...customFixtures } + } + + async render(request: UIRequest): Promise { + const key = request.scenario || 'compare-plans-basic' + const fixture = this.fixtures[key] + if (!fixture) { + throw new Error( + `[FixtureDriver] Unknown scenario: "${key}". Available scenarios: ${Object.keys(this.fixtures).join(', ')}`, + ) + } + return JSON.parse(JSON.stringify(fixture)) + } + + listScenarios(): Array { + return Object.keys(this.fixtures) + } +} + +/** + * 7.2 Programmatic driver + * Construct graphs directly with type-safety and no model in the loop. + */ +export class ProgrammaticDriver implements UIDriver { + constructor(private readonly builder?: (req: UIRequest) => GraphDocument) {} + + async render(request: UIRequest): Promise { + if (this.builder) { + const graph = this.builder(request) + const res = validateGraph(graph) + if (!res.valid || !res.graph) { + throw new Error(`[ProgrammaticDriver] Built invalid graph: ${res.errors.join(', ')}`) + } + return res.graph + } + + // Default programmatic graph generator + const title = request.job || 'Programmatic Interface' + return { + specVersion: '0.1', + graphId: `g_prog_${Date.now()}`, + revision: 1, + title, + fallbackText: `${title}: Data presented programmatically.`, + provenance: { + driver: 'programmatic', + }, + root: { + id: 'root-stack', + type: 'Stack', + props: { gap: 'md' }, + children: [ + { + id: 'job-heading', + type: 'Text', + props: { content: title, variant: 'heading' }, + }, + ], + }, + } + } + + static createBuilder(title: string, fallbackText: string) { + let nextId = 1 + const rootNodes: Array = [] + + return { + addText(content: string, variant: 'body' | 'heading' | 'caption' = 'body') { + rootNodes.push({ id: `n_${nextId++}`, type: 'Text', props: { content, variant } }) + return this + }, + addMetric(label: string, value: string | number, delta?: string, tone?: 'positive' | 'negative' | 'neutral') { + rootNodes.push({ id: `n_${nextId++}`, type: 'Metric', props: { label, value, delta, tone } }) + return this + }, + addButton(label: string, intent: string, payload?: unknown) { + rootNodes.push({ + id: `n_${nextId++}`, + type: 'Button', + props: { label }, + on: { click: { intent, payload } }, + }) + return this + }, + build(): GraphDocument { + return { + specVersion: '0.1', + graphId: `g_prog_${Date.now()}`, + revision: 1, + title, + fallbackText, + provenance: { driver: 'programmatic' }, + root: { + id: 'root', + type: 'Stack', + props: { gap: 'md' }, + children: rootNodes, + }, + } + }, + } + } +} + +/** + * 7.3 Live driver + * Tool schema definition and output validator for model-driven UI generation. + */ +export class LiveDriver { + /** + * Tool definition for model calling (e.g. OpenAI / Anthropic / Gemini tool schema) + */ + static getRenderToolDefinition() { + return { + name: 'render_ui', + description: + 'Compose an interactive generative user interface to serve the user request. Returns a validated Open Generative UI graph document.', + parameters: GRAPH_DOCUMENT_SCHEMA, + } + } + + /** + * Validates and repairs the raw output emitted by the LLM tool call. + */ + static processModelOutput(rawGraph: unknown): GraphDocument { + const result = validateGraph(rawGraph, { repair: true }) + if (result.valid && result.graph) { + return result.graph + } + + // If unrepairable, produce safe text fallback node + const fallbackText = + typeof (rawGraph as any)?.fallbackText === 'string' + ? (rawGraph as any).fallbackText + : 'The interface could not be rendered.' + + return { + specVersion: '0.1', + graphId: `g_fallback_${Date.now()}`, + revision: 1, + title: 'Interface Notice', + fallbackText, + provenance: { driver: 'live' }, + root: { + id: 'root-fallback', + type: 'Callout', + props: { + tone: 'warning', + title: 'Interface Fallback', + text: fallbackText, + }, + }, + } + } +} diff --git a/packages/ai-generative-ui/src/fixtures/index.ts b/packages/ai-generative-ui/src/fixtures/index.ts new file mode 100644 index 0000000000..36d89758a6 --- /dev/null +++ b/packages/ai-generative-ui/src/fixtures/index.ts @@ -0,0 +1,621 @@ +/** + * Open Generative UI Specification (Draft 0.1) + * Canonical 9 Fixtures covering U1-U8 and Vector Scene (§6.4.3). + */ + +import type { GraphDocument } from '../types' + +/** + * U1: Compare + * Options side by side, same attributes, differences visible. + */ +export const comparePlansBasicFixture: GraphDocument = { + specVersion: '0.1', + graphId: 'g_compare_plans_01', + revision: 1, + title: 'Compare Hosting Plans', + fallbackText: 'Starter: $10/mo, 2 CPUs, 4GB RAM. Pro: $25/mo, 4 CPUs, 8GB RAM, Priority Support. Enterprise: $80/mo, 16 CPUs, 32GB RAM, 24/7 SLA.', + provenance: { + driver: 'fixture', + sourceId: 'compare-plans-basic', + }, + root: { + id: 'root-stack', + type: 'Stack', + props: { gap: 'md' }, + children: [ + { + id: 'heading-text', + type: 'Text', + props: { + content: 'Choose the best plan for your deployment scale:', + variant: 'heading', + }, + }, + { + id: 'plan-columns', + type: 'Columns', + props: { count: 3, gap: 'md' }, + children: [ + { + id: 'plan-starter', + type: 'Section', + props: { title: 'Starter' }, + children: [ + { id: 'starter-price', type: 'Metric', props: { label: 'Monthly', value: '$10', unit: '/mo' } }, + { id: 'starter-cpu', type: 'Text', props: { content: '• 2 vCPUs\n• 4 GB RAM\n• Community Support' } }, + { id: 'starter-btn', type: 'Button', props: { label: 'Select Starter', tone: 'secondary' }, on: { click: { intent: 'selectPlan', payload: { plan: 'starter' } } } }, + ], + }, + { + id: 'plan-pro', + type: 'Section', + props: { title: 'Pro (Recommended)' }, + children: [ + { id: 'pro-price', type: 'Metric', props: { label: 'Monthly', value: '$25', unit: '/mo', tone: 'positive' } }, + { id: 'pro-cpu', type: 'Text', props: { content: '• 4 vCPUs\n• 8 GB RAM\n• Priority Support' } }, + { id: 'pro-btn', type: 'Button', props: { label: 'Select Pro', tone: 'primary' }, on: { click: { intent: 'selectPlan', payload: { plan: 'pro' } } } }, + ], + }, + { + id: 'plan-enterprise', + type: 'Section', + props: { title: 'Enterprise' }, + children: [ + { id: 'ent-price', type: 'Metric', props: { label: 'Monthly', value: '$80', unit: '/mo' } }, + { id: 'ent-cpu', type: 'Text', props: { content: '• 16 vCPUs\n• 32 GB RAM\n• 24/7 Phone SLA' } }, + { id: 'ent-btn', type: 'Button', props: { label: 'Select Enterprise', tone: 'secondary' }, on: { click: { intent: 'selectPlan', payload: { plan: 'enterprise' } } } }, + ], + }, + ], + }, + ], + }, +} + +/** + * U2: Understand + * Explorable system/process with hotspots. + */ +export const bicycleDrivetrainFixture: GraphDocument = { + specVersion: '0.1', + graphId: 'g_bicycle_drivetrain_02', + revision: 1, + title: 'Bicycle Drivetrain Mechanics', + fallbackText: 'Pedal & Crankset transfers rider effort into rotation. Chain transfers torque to the rear cassette. Rear Derailleur changes gear ratios. Rear Wheel delivers traction.', + provenance: { + driver: 'fixture', + sourceId: 'bicycle-drivetrain', + }, + root: { + id: 'root-stack', + type: 'Stack', + props: { gap: 'md' }, + children: [ + { + id: 'overview-callout', + type: 'Callout', + props: { + tone: 'info', + title: 'Mechanical System Overview', + text: 'Select any drivetrain component below to inspect its role and maintenance requirements.', + }, + }, + { + id: 'drivetrain-diagram', + type: 'Diagram', + props: { + title: 'Drivetrain Anatomy', + hotspots: [ + { id: 'crank', label: 'Crankset & Chainring', explanation: 'Converts pedal leg force into rotational chain tension. Inspect teeth for shark-fin wear.' }, + { id: 'chain', label: 'Bicycle Chain', explanation: 'Transfers rotational power over the wheelbase. Lubricate weekly; replace at 0.75% stretch.' }, + { id: 'derailleur', label: 'Rear Derailleur', explanation: 'Maintains chain tension and guides the chain across rear sprockets.' }, + { id: 'cassette', label: 'Rear Cassette', explanation: 'Stack of sprockets offering mechanical gear ratios for climbing and flat sprints.' }, + ], + }, + bind: { selected: 'selectedPart' }, + }, + ], + }, +} + +/** + * U3: Plan & Track + * Sequence with steps that can be followed and checked off. + */ +export const cookingPlanFixture: GraphDocument = { + specVersion: '0.1', + graphId: 'g_cooking_plan_03', + revision: 1, + title: 'Pasta Carbonara Preparation Plan', + fallbackText: 'Step 1: Boil salted pasta water. Step 2: Crisp pancetta/guanciale in skillet. Step 3: Whisk egg yolks, pecorino, black pepper. Step 4: Toss hot pasta off heat with egg mixture.', + provenance: { + driver: 'fixture', + sourceId: 'cooking-plan', + }, + root: { + id: 'root-stack', + type: 'Stack', + props: { gap: 'md' }, + children: [ + { + id: 'timeline-steps', + type: 'Timeline', + props: { currentStep: 2 }, + children: [ + { id: 's1', type: 'Text', props: { content: '1. Prep Station (5 min)', variant: 'heading' } }, + { id: 's2', type: 'Text', props: { content: '2. Crisp Pancetta (8 min)', variant: 'heading' } }, + { id: 's3', type: 'Text', props: { content: '3. Cook Rigatoni (10 min)', variant: 'heading' } }, + { id: 's4', type: 'Text', props: { content: '4. Emulsify & Serve (3 min)', variant: 'heading' } }, + ], + }, + { + id: 'checklist-section', + type: 'Section', + props: { title: 'Preparation Checklist' }, + children: [ + { + id: 'tasks', + type: 'Checklist', + props: { + items: [ + { id: 'water', label: 'Bring 4L water to a boil with 2 tbsp kosher salt', defaultDone: true }, + { id: 'dice', label: 'Dice 150g guanciale or pancetta into 1/2-inch batons', defaultDone: true }, + { id: 'eggs', label: 'Whisk 3 large egg yolks + 1 whole egg with 60g Pecorino Romano', defaultDone: false }, + { id: 'pepper', label: 'Coarsely crush 2 tsp whole black peppercorns', defaultDone: false }, + { id: 'water-reserve', label: 'Reserve 1 cup starchy pasta cooking water before draining', defaultDone: false }, + ], + }, + bind: { state: 'checklistProgress' }, + }, + ], + }, + ], + }, +} + +/** + * U4: Calculate / explore parameters + * Change an input, see results update — computed by code, NOT by LLM! + */ +export const billSplitFixture: GraphDocument = { + specVersion: '0.1', + graphId: 'g_bill_split_04', + revision: 1, + title: 'Dinner Bill Splitter', + fallbackText: 'Calculate tip and bill split per person with subtotal, tip percentage slider, and guest count stepper.', + provenance: { + driver: 'fixture', + sourceId: 'bill-split', + }, + root: { + id: 'root-stack', + type: 'Stack', + props: { gap: 'md' }, + children: [ + { + id: 'inputs-section', + type: 'Section', + props: { title: 'Bill Details' }, + children: [ + { + id: 'subtotal-input', + type: 'NumberInput', + props: { label: 'Subtotal ($)', min: 0, step: 1 }, + bind: { value: 'subtotal' }, + }, + { + id: 'tip-slider', + type: 'Slider', + props: { label: 'Tip Percentage (%)', min: 10, max: 30, step: 1 }, + bind: { value: 'tipPercent' }, + }, + { + id: 'guests-stepper', + type: 'Stepper', + props: { label: 'Number of Guests', min: 1, max: 20, step: 1 }, + bind: { value: 'splitCount' }, + }, + ], + }, + { + id: 'computation-node', + type: 'Stack', + compute: { + name: 'billSplit', + inputs: { + subtotal: 'subtotal', + tipPercent: 'tipPercent', + splitCount: 'splitCount', + }, + output: 'computedBill', + }, + children: [ + { + id: 'results-section', + type: 'Section', + props: { title: 'Calculated Totals (Computed by Host Code)' }, + children: [ + { + id: 'metrics-columns', + type: 'Columns', + props: { count: 3, gap: 'md' }, + children: [ + { + id: 'tip-metric', + type: 'Metric', + props: { label: 'Tip Amount', value: '$18.00' }, + bind: { value: 'computedBill.tipAmount', unit: 'USD' }, + }, + { + id: 'total-metric', + type: 'Metric', + props: { label: 'Total with Tip', value: '$118.00' }, + bind: { value: 'computedBill.total', unit: 'USD' }, + }, + { + id: 'per-person-metric', + type: 'Metric', + props: { label: 'Per Person', value: '$29.50', tone: 'positive' }, + bind: { value: 'computedBill.perPerson', unit: 'USD' }, + }, + ], + }, + ], + }, + ], + }, + ], + }, +} + +/** + * U5: Collect structured input + * Form whose result goes back to the agent as data, not prose. + */ +export const recipeParametersFixture: GraphDocument = { + specVersion: '0.1', + graphId: 'g_recipe_params_05', + revision: 1, + title: 'Recipe Customizer', + fallbackText: 'Configure meal type, dietary restrictions, spice level, and target cooking time.', + provenance: { + driver: 'fixture', + sourceId: 'recipe-parameters', + }, + root: { + id: 'form-root', + type: 'Form', + props: { + submitLabel: 'Generate Tailored Recipe', + submitIntent: 'createCustomRecipe', + }, + children: [ + { + id: 'cuisine-select', + type: 'Select', + props: { + label: 'Cuisine Style', + options: [ + { value: 'italian', label: 'Italian' }, + { value: 'mexican', label: 'Mexican' }, + { value: 'japanese', label: 'Japanese' }, + { value: 'mediterranean', label: 'Mediterranean' }, + ], + }, + bind: { value: 'cuisine' }, + }, + { + id: 'spice-slider', + type: 'Slider', + props: { label: 'Spice Level (1 = Mild, 5 = Fiery)', min: 1, max: 5, step: 1 }, + bind: { value: 'spiceLevel' }, + }, + { + id: 'vegetarian-toggle', + type: 'Toggle', + props: { label: 'Vegetarian Only' }, + bind: { value: 'isVegetarian' }, + }, + { + id: 'notes-input', + type: 'TextInput', + props: { label: 'Pantry Ingredients to Prioritize', placeholder: 'e.g. cherry tomatoes, zucchini, basil' }, + bind: { value: 'ingredients' }, + }, + ], + }, +} + +/** + * U6: Decide / approve + * Proposal presented with consequences and explicit approve/reject act. + */ +export const approvalToolCallFixture: GraphDocument = { + specVersion: '0.1', + graphId: 'g_approval_tool_06', + revision: 1, + title: 'Approval Required: Production Database Migration', + fallbackText: 'Agent requests permission to apply migration "20261014_add_audit_index". Estimated downtime: 0s. Estimated cost: $0. Requires table lock for ~150ms.', + provenance: { + driver: 'fixture', + sourceId: 'approval-tool-call', + }, + root: { + id: 'root-stack', + type: 'Stack', + props: { gap: 'md' }, + children: [ + { + id: 'warning-callout', + type: 'Callout', + props: { + tone: 'warning', + title: 'Action Requires Human Authorization', + text: 'This operation will run against the primary cluster `db-prod-east`. Read traffic will remain active, but writes will briefly queue.', + }, + }, + { + id: 'details-section', + type: 'Section', + props: { title: 'Proposed Tool Call Details' }, + children: [ + { + id: 'diff-text', + type: 'Text', + props: { + content: '```sql\nCREATE INDEX CONCURRENTLY IF NOT EXISTS idx_audit_log_tenant_created\nON audit_logs (tenant_id, created_at DESC);\n```', + variant: 'code', + }, + }, + ], + }, + { + id: 'choice-actions', + type: 'Choice', + props: { + label: 'Decision', + options: [ + { id: 'approve', label: 'Approve & Execute Now', description: 'Proceed immediately with the migration.' }, + { id: 'reject', label: 'Reject / Cancel', description: 'Abort the tool call and notify the agent.' }, + ], + }, + on: { + select: { + intent: 'decideToolCall', + payload: { toolCallId: 'tc_migrate_982' }, + }, + }, + }, + ], + }, +} + +/** + * U7: Monitor + * Live-ish values and trends, glanceable metrics and charts. + */ +export const spendMonitorFixture: GraphDocument = { + specVersion: '0.1', + graphId: 'g_spend_monitor_07', + revision: 1, + title: 'Agent Pod Spend & Health', + fallbackText: 'Daily Spend: $42.50 (+12%). Monthly Forecast: $1,280. Active Agents: 6. 7-day spend trend and running worker table.', + provenance: { + driver: 'fixture', + sourceId: 'spend-monitor', + }, + root: { + id: 'root-stack', + type: 'Stack', + props: { gap: 'md' }, + children: [ + { + id: 'summary-columns', + type: 'Columns', + props: { count: 3, gap: 'md' }, + children: [ + { id: 'metric-daily', type: 'Metric', props: { label: 'Today Spend', value: '$42.50', delta: '+12%', tone: 'positive' } }, + { id: 'metric-forecast', type: 'Metric', props: { label: 'Month Forecast', value: '$1,280', delta: '-4%', tone: 'neutral' } }, + { id: 'metric-agents', type: 'Metric', props: { label: 'Active Agents', value: '6 / 8', tone: 'neutral' } }, + ], + }, + { + id: 'trend-chart', + type: 'Chart', + props: { + kind: 'bar', + title: 'Daily Spend (Past 7 Days)', + unit: 'USD', + data: [ + { label: 'Thu', value: 28 }, + { label: 'Fri', value: 34 }, + { label: 'Sat', value: 18 }, + { label: 'Sun', value: 12 }, + { label: 'Mon', value: 45 }, + { label: 'Tue', value: 39 }, + { label: 'Wed', value: 42.5 }, + ], + }, + }, + { + id: 'workers-table', + type: 'Table', + props: { + columns: [ + { key: 'agent', label: 'Agent Name' }, + { key: 'status', label: 'Status' }, + { key: 'requests', label: 'Tokens Processed' }, + { key: 'cost', label: 'Cost Today' }, + ], + rows: [ + { agent: 'research-scout', status: 'Active', requests: '142,800', cost: '$14.28' }, + { agent: 'code-reviewer', status: 'Active', requests: '89,400', cost: '$18.12' }, + { agent: 'doc-sync', status: 'Idle', requests: '21,000', cost: '$4.20' }, + { agent: 'test-runner', status: 'Active', requests: '32,100', cost: '$5.90' }, + ], + }, + }, + ], + }, +} + +/** + * U8: Locate + * Things in space, with notes and routes. + */ +export const tripStopsFixture: GraphDocument = { + specVersion: '0.1', + graphId: 'g_trip_stops_08', + revision: 1, + title: 'Pacific Coast Highway Itinerary', + fallbackText: 'Stop 1: Monterey Bay Aquarium. Stop 2: Bixby Creek Bridge. Stop 3: Pfeiffer Beach. Stop 4: McWay Falls.', + provenance: { + driver: 'fixture', + sourceId: 'trip-stops', + }, + root: { + id: 'root-stack', + type: 'Stack', + props: { gap: 'md' }, + children: [ + { + id: 'map-view', + type: 'Map', + props: { + center: { lat: 36.3, lng: -121.8 }, + markers: [ + { id: 'm1', label: 'Monterey Bay Aquarium', lat: 36.618, lng: -121.901, note: 'World-class kelp forest exhibit' }, + { id: 'm2', label: 'Bixby Creek Bridge', lat: 36.371, lng: -121.901, note: 'Iconic coastal span viewpoint' }, + { id: 'm3', label: 'Pfeiffer Beach', lat: 36.238, lng: -121.815, note: 'Purple sand & Keyhole Arch' }, + { id: 'm4', label: 'McWay Falls', lat: 36.157, lng: -121.672, note: '80-foot waterfall dropping onto pristine cove' }, + ], + }, + }, + { + id: 'stops-list', + type: 'List', + props: { + ordered: true, + items: [ + { text: 'Monterey Bay Aquarium', subtitle: 'Arrive 09:30 AM — reserve parking ahead' }, + { text: 'Bixby Creek Bridge', subtitle: 'Pull out on northbound shoulder for best photo angle' }, + { text: 'Pfeiffer Beach', subtitle: 'Narrow 2-mile road — cash fee at entry gate' }, + { text: 'McWay Falls Overlook', subtitle: 'Short flat 0.5-mile trail from highway parking' }, + ], + }, + }, + ], + }, +} + +/** + * §6.4.3: Guitar Chord Diagram (Emaj7) + * The canonical Scene workload: geometry derived from tuning & voicing. + */ +export const chordDiagramEmaj7Fixture: GraphDocument = { + specVersion: '0.1', + graphId: 'g_chord_emaj7_09', + revision: 1, + title: 'Guitar Voicing: Open Emaj7', + fallbackText: 'Open Emaj7 chord (0-2-1-1-0-0): Low E (Root), B (5th, 2nd fret), D# (Maj 7th, 1st fret), G# (Maj 3rd, 1st fret), B (5th, open), High E (Root, open).', + provenance: { + driver: 'fixture', + sourceId: 'chord-diagram-emaj7', + }, + root: { + id: 'root-stack', + type: 'Stack', + props: { gap: 'md' }, + children: [ + { + id: 'chord-scene', + type: 'Scene', + props: { + viewBox: '0 0 300 320', + width: 300, + height: 320, + shapes: [ + // Nut (thick top bar) + { kind: 'rect', x: 40, y: 35, width: 220, height: 8, fill: 'neutral' }, + // Fret lines (frets 1 to 4) + { kind: 'line', x1: 40, y1: 100, x2: 260, y2: 100, stroke: 'muted', strokeWidth: 2 }, + { kind: 'line', x1: 40, y1: 160, x2: 260, y2: 160, stroke: 'muted', strokeWidth: 2 }, + { kind: 'line', x1: 40, y1: 220, x2: 260, y2: 220, stroke: 'muted', strokeWidth: 2 }, + { kind: 'line', x1: 40, y1: 280, x2: 260, y2: 280, stroke: 'muted', strokeWidth: 2 }, + // 6 Strings (vertical lines from low E to high e) + { kind: 'line', x1: 50, y1: 35, x2: 50, y2: 280, stroke: 'neutral', strokeWidth: 3 }, // 6th string (E) + { kind: 'line', x1: 86, y1: 35, x2: 86, y2: 280, stroke: 'neutral', strokeWidth: 2.5 }, // 5th string (A) + { kind: 'line', x1: 122, y1: 35, x2: 122, y2: 280, stroke: 'neutral', strokeWidth: 2 }, // 4th string (D) + { kind: 'line', x1: 158, y1: 35, x2: 158, y2: 280, stroke: 'neutral', strokeWidth: 1.5 }, // 3rd string (G) + { kind: 'line', x1: 194, y1: 35, x2: 194, y2: 280, stroke: 'neutral', strokeWidth: 1.2 }, // 2nd string (B) + { kind: 'line', x1: 230, y1: 35, x2: 230, y2: 280, stroke: 'neutral', strokeWidth: 1 }, // 1st string (e) + // Open string markers (circle at top) + { kind: 'circle', id: 'open-6', cx: 50, cy: 20, r: 6, stroke: 'neutral', fill: 'none', label: 'String 6: Open E (Root)' }, + { kind: 'circle', id: 'open-2', cx: 194, cy: 20, r: 6, stroke: 'neutral', fill: 'none', label: 'String 2: Open B (5th)' }, + { kind: 'circle', id: 'open-1', cx: 230, cy: 20, r: 6, stroke: 'neutral', fill: 'none', label: 'String 1: Open E (Root)' }, + // Fingered note dots with declared interactivity (§6.4.2) + { + kind: 'circle', + id: 'note-b-5', + cx: 86, + cy: 130, // Fret 2 + r: 12, + fill: 'accent', + label: 'B — Perfect 5th (5th string, 2nd fret)', + bind: { selected: 'selectedNote' }, + on: { select: 'noteSelected' }, + }, + { + kind: 'circle', + id: 'note-ds-4', + cx: 122, + cy: 70, // Fret 1 + r: 12, + fill: 'accent', + label: 'D♯ — Major 7th (4th string, 1st fret)', + bind: { selected: 'selectedNote' }, + on: { select: 'noteSelected' }, + }, + { + kind: 'circle', + id: 'note-gs-3', + cx: 158, + cy: 70, // Fret 1 + r: 12, + fill: 'accent', + label: 'G♯ — Major 3rd (3rd string, 1st fret)', + bind: { selected: 'selectedNote' }, + on: { select: 'noteSelected' }, + }, + // Text labels for note dots + { kind: 'text', x: 86, y: 134, text: '5', fill: 'surface', sizeRole: 'sm' }, + { kind: 'text', x: 122, y: 74, text: '7', fill: 'surface', sizeRole: 'sm' }, + { kind: 'text', x: 158, y: 74, text: '3', fill: 'surface', sizeRole: 'sm' }, + ], + }, + }, + { + id: 'selected-note-callout', + type: 'Callout', + props: { + tone: 'info', + title: 'Chord Voicing Formula: 1 - 5 - 7 - 3 - 5 - 1', + text: 'Emaj7 creates a warm, dreamy jazz and neo-soul harmony. Select note markers in the diagram to inspect their harmonic interval.', + }, + }, + ], + }, +} + +export const CANONICAL_FIXTURES: Record = { + 'compare-plans-basic': comparePlansBasicFixture, + 'bicycle-drivetrain': bicycleDrivetrainFixture, + 'cooking-plan': cookingPlanFixture, + 'bill-split': billSplitFixture, + 'recipe-parameters': recipeParametersFixture, + 'approval-tool-call': approvalToolCallFixture, + 'spend-monitor': spendMonitorFixture, + 'trip-stops': tripStopsFixture, + 'chord-diagram-emaj7': chordDiagramEmaj7Fixture, +} diff --git a/packages/ai-generative-ui/src/index.ts b/packages/ai-generative-ui/src/index.ts new file mode 100644 index 0000000000..ff240acc13 --- /dev/null +++ b/packages/ai-generative-ui/src/index.ts @@ -0,0 +1,12 @@ +/** + * Open Generative UI Specification (Draft 0.1) + * Reference implementation for TanStack AI. + */ + +export type * from './types' +export * from './schema' +export * from './validator' +export * from './compute' +export * from './fixtures/index' +export * from './drivers/index' +export * from './react/index' diff --git a/packages/ai-generative-ui/src/react.ts b/packages/ai-generative-ui/src/react.ts new file mode 100644 index 0000000000..46f63b7a13 --- /dev/null +++ b/packages/ai-generative-ui/src/react.ts @@ -0,0 +1 @@ +export * from './react/index' diff --git a/packages/ai-generative-ui/src/react/index.tsx b/packages/ai-generative-ui/src/react/index.tsx new file mode 100644 index 0000000000..9092dc9615 --- /dev/null +++ b/packages/ai-generative-ui/src/react/index.tsx @@ -0,0 +1,1038 @@ +/** + * Open Generative UI Specification (Draft 0.1) + * Host React Renderer with catalog components, state binding, and vector scenes. + */ + +import React, { useState, useEffect, useCallback } from 'react' +import type { + GraphDocument, + UINode, + IntentPayload, + SceneShape, + SemanticPaintRole, +} from '../types' +import type { ComputeRegistry} from '../compute'; +import { defaultComputeRegistry } from '../compute' + +export interface GenerativeUIRendererProps { + graph: GraphDocument + onIntent?: (intent: IntentPayload) => void + computeRegistry?: ComputeRegistry + className?: string + initialState?: Record +} + +// Map semantic paint roles to CSS values +const PAINT_ROLES: Record = { + accent: '#2563eb', // blue-600 + muted: '#94a3b8', // slate-400 + danger: '#ef4444', // red-500 + neutral: '#334155', // slate-700 + highlight: '#f59e0b', // amber-500 + surface: '#ffffff', // white + none: 'none', +} + +function resolveStatePath(obj: unknown, path: string): unknown { + if (!obj || !path || typeof obj !== 'object') return undefined + const parts = path.split('.') + let curr: unknown = obj + for (const p of parts) { + if (curr === null || curr === undefined || typeof curr !== 'object') return undefined + curr = (curr as Record)[p] + } + return curr +} + +function setStatePath(obj: Record, path: string, value: unknown): Record { + const parts = path.split('.') + const copy: Record = { ...obj } + let curr: Record = copy + for (let i = 0; i < parts.length - 1; i++) { + const p = parts[i] + if (!p) continue + curr[p] = curr[p] && typeof curr[p] === 'object' ? { ...(curr[p] as Record) } : {} + curr = curr[p] as Record + } + const lastKey = parts[parts.length - 1] + if (lastKey) { + curr[lastKey] = value + } + return copy +} + +export function GenerativeUIRenderer({ + graph, + onIntent, + computeRegistry = defaultComputeRegistry, + className = '', + initialState = {}, +}: GenerativeUIRendererProps) { + // Initialize state with defaults from bindings + const [state, setState] = useState>(() => { + const s: Record = { ...initialState } + // Pre-populate sensible defaults for demo fixtures + if (s.subtotal === undefined) s.subtotal = 100 + if (s.tipPercent === undefined) s.tipPercent = 18 + if (s.splitCount === undefined) s.splitCount = 4 + if (s.spiceLevel === undefined) s.spiceLevel = 3 + if (s.cuisine === undefined) s.cuisine = 'italian' + return s + }) + + // Recompute any bound compute nodes whenever state changes + useEffect(() => { + function evaluateComputes(node: UINode) { + if (node.compute) { + if (typeof node.compute === 'object' && node.compute.name) { + const { name, inputs, output } = node.compute + const resolvedInputs: Record = {} + for (const [inKey, stateKey] of Object.entries(inputs)) { + resolvedInputs[inKey] = resolveStatePath(state, stateKey) + } + const result = computeRegistry.execute(name, resolvedInputs, state) + if (result !== undefined) { + setState((prev) => { + const currentVal = resolveStatePath(prev, output) + if (JSON.stringify(currentVal) !== JSON.stringify(result)) { + return setStatePath(prev, output, result) + } + return prev + }) + } + } + } + if (node.children) { + node.children.forEach(evaluateComputes) + } + } + + evaluateComputes(graph.root) + }, [graph.root, state, computeRegistry]) + + const handleStateChange = useCallback((path: string, value: unknown) => { + setState((prev) => setStatePath(prev, path, value)) + }, []) + + const fireIntent = useCallback( + (nodeId: string, intent: string, intentClass: 'local' | 'submit' | 'action', payload?: unknown) => { + onIntent?.({ + graphId: graph.graphId, + nodeId, + intent, + intentClass, + payload, + state, + }) + }, + [graph.graphId, onIntent, state], + ) + + return ( +
+
+
+

{graph.title}

+ Spec v{graph.specVersion} • Rev {graph.revision} +
+
+ +
+ +
+
+ ) +} + +interface NodeRendererProps { + node: UINode + state: Record + onStateChange: (path: string, value: unknown) => void + onFireIntent: ( + nodeId: string, + intent: string, + intentClass: 'local' | 'submit' | 'action', + payload?: unknown, + ) => void + computeRegistry?: ComputeRegistry +} + +function NodeRenderer({ node, state, onStateChange, onFireIntent, computeRegistry }: NodeRendererProps) { + const { id, type, props = {}, children, bind = {}, on = {} } = node + + // Resolve bound props from state + const resolvedProps = { ...props } + for (const [propKey, statePath] of Object.entries(bind)) { + const val = resolveStatePath(state, statePath) + if (val !== undefined) { + resolvedProps[propKey] = val + } + } + + // 6.1 Layout Components + if (type === 'Stack') { + const isHorizontal = resolvedProps.direction === 'horizontal' + const gapClass = resolvedProps.gap === 'sm' ? 'gap-2' : resolvedProps.gap === 'lg' ? 'gap-6' : 'gap-4' + return ( +
+ {children?.map((child) => ( + + ))} +
+ ) + } + + if (type === 'Columns') { + const count = resolvedProps.count ?? 2 + const gridCols = count === 4 ? 'grid-cols-4' : count === 3 ? 'grid-cols-3' : 'grid-cols-2' + return ( +
+ {children?.map((child) => ( + + ))} +
+ ) + } + + if (type === 'Section') { + return ( +
+

{resolvedProps.title}

+ {resolvedProps.subtitle &&

{resolvedProps.subtitle}

} +
+ {children?.map((child) => ( + + ))} +
+
+ ) + } + + if (type === 'Tabs') { + const tabs: Array<{ id: string; label: string }> = resolvedProps.tabs || [] + const tabKey = bind.tab || `_tab_${id}` + const activeTab = (resolveStatePath(state, tabKey) as string) || resolvedProps.defaultTab || tabs[0]?.id + + return ( +
+
+ {tabs.map((t) => ( + + ))} +
+
+ {children + ?.filter((child) => !child.props?.tabId || child.props.tabId === activeTab) + .map((child) => ( + + ))} +
+
+ ) + } + + if (type === 'Timeline') { + const currentStep = resolvedProps.currentStep ?? 1 + return ( +
+ {children?.map((child, idx) => ( +
+ + +
+ ))} +
+ ) + } + + // 6.2 Content Components + if (type === 'Text') { + const variant = resolvedProps.variant || 'body' + if (variant === 'heading') { + return

{resolvedProps.content}

+ } + if (variant === 'caption') { + return

{resolvedProps.content}

+ } + if (variant === 'code') { + return ( + 
+          {resolvedProps.content}
+
+ ) + } + return

{resolvedProps.content}

+ } + + if (type === 'Callout') { + const tone = resolvedProps.tone || 'info' + const colors = + tone === 'warning' + ? 'bg-amber-50 border-amber-200 text-amber-900' + : tone === 'danger' + ? 'bg-red-50 border-red-200 text-red-900' + : 'bg-blue-50 border-blue-200 text-blue-900' + + return ( +
+ {resolvedProps.title &&
{resolvedProps.title}
} +
{resolvedProps.text}
+
+ ) + } + + if (type === 'Metric') { + const toneClass = + resolvedProps.tone === 'positive' + ? 'text-emerald-600' + : resolvedProps.tone === 'negative' + ? 'text-rose-600' + : 'text-slate-900' + + return ( +
+
{resolvedProps.label}
+
+ {resolvedProps.value} + {resolvedProps.unit && {resolvedProps.unit}} +
+ {resolvedProps.delta && ( +
{resolvedProps.delta}
+ )} +
+ ) + } + + if (type === 'Table') { + const columns: Array<{ key: string; label: string }> = (resolvedProps.columns as Array<{ key: string; label: string }>) || [] + const rows: Array> = (resolvedProps.rows as Array>) || [] + + return ( +
+ + + + {columns.map((col) => ( + + ))} + + + + {rows.map((row, i) => ( + + {columns.map((col) => ( + + ))} + + ))} + +
+ {col.label} +
+ {String(row[col.key] ?? '')} +
+
+ ) + } + + if (type === 'List') { + const items: Array = (resolvedProps.items as Array) || [] + const isOrdered = Boolean(resolvedProps.ordered) + + return ( +
+ {items.map((item, idx) => { + const text = typeof item === 'string' ? item : (item as any)?.text + const subtitle = typeof item === 'object' && item !== null ? (item as any)?.subtitle : undefined + return ( +
+ {isOrdered ? `${idx + 1}.` : '•'} +
+
{text}
+ {subtitle &&
{subtitle}
} +
+
+ ) + })} +
+ ) + } + + if (type === 'Chart') { + const data: Array<{ label: string; value: number }> = resolvedProps.data || [] + const maxValue = Math.max(...data.map((d) => d.value), 1) + + return ( +
+ {resolvedProps.title &&
{resolvedProps.title}
} +
+ {data.map((d, i) => { + const heightPercent = Math.round((d.value / maxValue) * 100) + return ( +
+ {d.value} +
+ {d.label} +
+ ) + })} +
+
+ ) + } + + if (type === 'Map') { + const markers: Array<{ id: string; label: string; note?: string }> = resolvedProps.markers || [] + return ( +
+
📍 Geographic Points of Interest
+
+ {markers.map((m) => ( +
+
📌 {m.label}
+ {m.note &&
{m.note}
} +
+ ))} +
+
+ ) + } + + if (type === 'Diagram') { + const hotspots: Array<{ id: string; label: string; explanation: string }> = resolvedProps.hotspots || [] + const selectedPart = bind.selected ? resolveStatePath(state, bind.selected) : undefined + + return ( +
+ {resolvedProps.title &&

{resolvedProps.title}

} +
+ {hotspots.map((spot) => ( + + ))} +
+ {selectedPart && ( +
+ {hotspots.find((h) => h.id === selectedPart)?.label}:{' '} + {hotspots.find((h) => h.id === selectedPart)?.explanation} +
+ )} +
+ ) + } + + // 6.3 Input and Action Components + if (type === 'Form') { + return ( +
{ + e.preventDefault() + const submitIntent = resolvedProps.submitIntent || 'submit' + onFireIntent(id, submitIntent, 'submit', state) + }} + > + {children?.map((child) => ( + + ))} + + + ) + } + + if (type === 'TextInput') { + const val = bind.value ? resolveStatePath(state, bind.value) ?? '' : resolvedProps.value ?? '' + return ( +
+ + { + if (bind.value) onStateChange(bind.value, e.target.value) + }} + /> +
+ ) + } + + if (type === 'NumberInput') { + const val = bind.value ? resolveStatePath(state, bind.value) ?? 0 : resolvedProps.value ?? 0 + return ( +
+ + { + const num = parseFloat(e.target.value) || 0 + if (bind.value) onStateChange(bind.value, num) + }} + /> +
+ ) + } + + if (type === 'Select') { + const val = bind.value ? resolveStatePath(state, bind.value) ?? '' : resolvedProps.value ?? '' + const options: Array<{ value: string; label: string }> = resolvedProps.options || [] + + return ( +
+ + +
+ ) + } + + if (type === 'Slider') { + const min = resolvedProps.min ?? 0 + const max = resolvedProps.max ?? 100 + const step = resolvedProps.step ?? 1 + const val = bind.value ? resolveStatePath(state, bind.value) ?? min : resolvedProps.value ?? min + + return ( +
+
+ + {val} +
+ { + const num = parseFloat(e.target.value) || min + if (bind.value) onStateChange(bind.value, num) + }} + /> +
+ ) + } + + if (type === 'Stepper') { + const min = resolvedProps.min ?? 1 + const max = resolvedProps.max ?? 100 + const step = resolvedProps.step ?? 1 + const val = bind.value ? resolveStatePath(state, bind.value) ?? min : resolvedProps.value ?? min + + return ( +
+ +
+ + {val} + +
+
+ ) + } + + if (type === 'Toggle' || type === 'Checkbox') { + const isChecked = bind.value ? Boolean(resolveStatePath(state, bind.value)) : false + + return ( + + ) + } + + if (type === 'Checklist') { + const items: Array<{ id: string; label: string; defaultDone?: boolean }> = (resolvedProps.items as Array<{ id: string; label: string; defaultDone?: boolean }>) || [] + const rawState = bind.state ? resolveStatePath(state, bind.state) : undefined + const checklistState = (rawState && typeof rawState === 'object' ? rawState : {}) as Record + + return ( +
+ {items.map((item) => { + const isDone = checklistState[item.id] ?? item.defaultDone ?? false + return ( + + ) + })} +
+ ) + } + + if (type === 'Button') { + const tone = resolvedProps.tone || 'primary' + const isSelected = bind.selected + ? resolveStatePath(state, bind.selected) === (resolvedProps.value ?? id) + : false + const btnClass = + tone === 'danger' + ? 'bg-rose-600 text-white hover:bg-rose-700 active:bg-rose-800' + : tone === 'secondary' + ? 'bg-slate-100 text-slate-800 border border-slate-300 hover:bg-slate-200 active:bg-slate-300' + : 'bg-blue-600 text-white hover:bg-blue-700 active:bg-blue-800' + + return ( + + ) + } + + if (type === 'Choice') { + const options: Array<{ id: string; label: string; description?: string }> = resolvedProps.options || [] + + return ( +
+ {resolvedProps.label &&
{resolvedProps.label}
} +
+ {options.map((opt) => ( + + ))} +
+
+ ) + } + + // 6.4 Vector Scenes (Scene) + if (type === 'Scene') { + const shapes: Array = resolvedProps.shapes || [] + const viewBox = resolvedProps.viewBox || '0 0 400 300' + const width = resolvedProps.width ?? '100%' + const height = resolvedProps.height ?? 'auto' + + return ( +
+ + {shapes.map((shape, idx) => ( + + ))} + +
+ ) + } + + // Degradation for unknown nodes (§9.3): render children if container + if (children && children.length > 0) { + return ( +
+ {children.map((child) => ( + + ))} +
+ ) + } + + // Degradation for unknown leaf nodes + return
[Unsupported node: {type}]
+} + +function SceneShapeRenderer({ + shape, + state, + onStateChange, + onFireIntent, +}: { + shape: SceneShape + state: Record + onStateChange: (path: string, value: unknown) => void + onFireIntent: ( + nodeId: string, + intent: string, + intentClass: 'local' | 'submit' | 'action', + payload?: unknown, + ) => void +}) { + const { kind, id, label, fill, stroke, strokeWidth, opacity, bind, on } = shape + + const rawFill = fill || (kind !== 'line' ? (shape as any).paint : undefined) + const rawStroke = stroke || (kind === 'line' ? (shape as any).paint || 'neutral' : undefined) + const fillColor = rawFill ? ((PAINT_ROLES as Record)[rawFill] ?? rawFill) : 'none' + const strokeColor = rawStroke ? ((PAINT_ROLES as Record)[rawStroke] ?? rawStroke) : 'none' + + const isInteractive = Boolean(bind?.selected || on?.select) + const isSelected = bind?.selected && id ? resolveStatePath(state, bind.selected) === id : false + + const handleInteraction = () => { + if (!isInteractive) return + if (bind?.selected && id) { + onStateChange(bind.selected, id) + } + if (on?.select && id) { + onFireIntent(id, on.select, 'local', { shapeId: id, label }) + } + } + + const commonProps = { + id, + opacity, + style: { + cursor: isInteractive ? 'pointer' : 'default', + transition: 'all 0.15s ease', + transformOrigin: 'center', + }, + ...(isInteractive + ? { + role: 'button', + tabIndex: 0, + 'aria-label': label || id, + onClick: handleInteraction, + onKeyDown: (e: React.KeyboardEvent) => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault() + handleInteraction() + } + }, + } + : {}), + } + + // Highlight selected shape + const appliedFill = isSelected ? PAINT_ROLES.highlight : fillColor + const appliedStroke = isSelected ? '#b45309' : strokeColor + const appliedStrokeWidth = isSelected ? (strokeWidth ?? 1) + 2 : strokeWidth + + if (kind === 'rect') { + return ( + + ) + } + + if (kind === 'circle') { + return ( + + ) + } + + if (kind === 'line') { + return ( + + ) + } + + if (kind === 'polygon') { + return ( + + ) + } + + if (kind === 'path') { + return ( + + ) + } + + if (kind === 'text') { + const fontSize = + shape.sizeRole === 'xs' + ? 10 + : shape.sizeRole === 'sm' + ? 12 + : shape.sizeRole === 'lg' + ? 16 + : 14 + return ( + + {shape.text || (shape as any).content} + + ) + } + + if (kind === 'group') { + return ( + + {shape.children?.map((c, i) => ( + + ))} + + ) + } + + return null +} + +/** + * Text Fallback Component (§9.2) + * For text-only surfaces (CLI, notifications) or accessible degradation. + */ +export function GenerativeUITextFallback({ graph }: { graph: GraphDocument }) { + return ( +
+
{graph.title}
+
{graph.fallbackText}
+
+ ) +} diff --git a/packages/ai-generative-ui/src/schema.ts b/packages/ai-generative-ui/src/schema.ts new file mode 100644 index 0000000000..ffe8da557a --- /dev/null +++ b/packages/ai-generative-ui/src/schema.ts @@ -0,0 +1,120 @@ +/** + * Open Generative UI Specification (Draft 0.1) + * JSON Schema definitions for GraphDocument and catalog types. + */ + +export const GRAPH_DOCUMENT_SCHEMA = { + $schema: 'http://json-schema.org/draft-07/schema#', + title: 'GraphDocument', + type: 'object', + required: ['specVersion', 'graphId', 'revision', 'title', 'fallbackText', 'provenance', 'root'], + properties: { + specVersion: { type: 'string', enum: ['0.1'] }, + graphId: { type: 'string' }, + revision: { type: 'integer', minimum: 1 }, + title: { type: 'string' }, + fallbackText: { type: 'string', minLength: 1 }, + provenance: { + type: 'object', + required: ['driver'], + properties: { + driver: { type: 'string', enum: ['fixture', 'programmatic', 'live'] }, + agentId: { type: 'string' }, + runId: { type: 'string' }, + sourceId: { type: 'string' }, + }, + }, + root: { $ref: '#/definitions/UINode' }, + }, + definitions: { + UINode: { + type: 'object', + required: ['id', 'type'], + properties: { + id: { type: 'string' }, + type: { type: 'string' }, + props: { type: 'object' }, + children: { + type: 'array', + items: { $ref: '#/definitions/UINode' }, + }, + bind: { + type: 'object', + additionalProperties: { type: 'string' }, + }, + compute: { + oneOf: [ + { type: 'string' }, + { + type: 'object', + required: ['name', 'inputs', 'output'], + properties: { + name: { type: 'string' }, + inputs: { type: 'object', additionalProperties: { type: 'string' } }, + output: { type: 'string' }, + }, + }, + ], + }, + on: { + type: 'object', + additionalProperties: { + oneOf: [ + { type: 'string' }, + { + type: 'object', + required: ['intent'], + properties: { + intent: { type: 'string' }, + payload: {}, + }, + }, + ], + }, + }, + }, + }, + }, +} as const + +export const KNOWN_CATALOG_TYPES = new Set([ + // 6.1 Layout + 'Stack', + 'Columns', + 'Section', + 'Tabs', + 'Timeline', + // 6.2 Content + 'Text', + 'Image', + 'Table', + 'List', + 'Metric', + 'Chart', + 'Map', + 'Diagram', + 'Callout', + // 6.3 Input and Action + 'Form', + 'TextInput', + 'NumberInput', + 'Select', + 'Slider', + 'Stepper', + 'Toggle', + 'Checkbox', + 'Checklist', + 'Button', + 'Choice', + // 6.4 Vector Scenes + 'Scene', +]) + +export const CONTAINER_TYPES = new Set([ + 'Stack', + 'Columns', + 'Section', + 'Tabs', + 'Timeline', + 'Form', +]) diff --git a/packages/ai-generative-ui/src/types.ts b/packages/ai-generative-ui/src/types.ts new file mode 100644 index 0000000000..35dddf065a --- /dev/null +++ b/packages/ai-generative-ui/src/types.ts @@ -0,0 +1,374 @@ +/** + * Open Generative UI Specification (Draft 0.1) + * Clean-room TypeScript type definitions. + */ + +export type SpecVersion = '0.1' + +export type SemanticPaintRole = + | 'accent' + | 'muted' + | 'danger' + | 'neutral' + | 'highlight' + | 'surface' + | 'none' + +export type SemanticTone = 'info' | 'warning' | 'danger' | 'neutral' | 'primary' | 'secondary' | 'accent' + +export interface GraphProvenance { + driver: 'fixture' | 'programmatic' | 'live' + agentId?: string + runId?: string + sourceId?: string +} + +export interface GraphDocument { + specVersion: SpecVersion + graphId: string + revision: number + title: string + fallbackText: string + provenance: GraphProvenance + root: UINode +} + +export interface ComputeBinding { + name: string + inputs: Record + output: string +} + +export interface UINode { + id: string + type: string + props?: Record + children?: Array + bind?: Record + compute?: ComputeBinding | string + on?: Record +} + +/* -------------------------------------------------------------------------- */ +/* Catalog Node Props Sketches (§6) */ +/* -------------------------------------------------------------------------- */ + +// 6.1 Layout (Containers) +export interface StackProps { + direction?: 'vertical' | 'horizontal' + gap?: 'none' | 'sm' | 'md' | 'lg' + align?: 'start' | 'center' | 'end' | 'stretch' +} + +export interface ColumnsProps { + count?: 2 | 3 | 4 + gap?: 'sm' | 'md' | 'lg' +} + +export interface SectionProps { + title: string + subtitle?: string + collapsible?: boolean + defaultCollapsed?: boolean +} + +export interface TabsProps { + tabs: Array<{ id: string; label: string }> + defaultTab?: string +} + +export interface TimelineProps { + currentStep?: number +} + +// 6.2 Content +export interface TextProps { + content: string + variant?: 'body' | 'heading' | 'caption' | 'code' +} + +export interface ImageProps { + src: string + alt: string + caption?: string + width?: number | string + height?: number | string +} + +export interface TableColumn { + key: string + label: string + align?: 'left' | 'center' | 'right' +} + +export interface TableProps { + columns: Array + rows: Array> +} + +export interface ListProps { + items: Array + ordered?: boolean +} + +export interface MetricProps { + label: string + value: string | number + delta?: string | number + unit?: string + tone?: 'neutral' | 'positive' | 'negative' +} + +export interface ChartDatum { + label: string + value: number + series?: string +} + +export interface ChartProps { + kind: 'bar' | 'line' | 'pie' + data: Array + title?: string + unit?: string +} + +export interface MapMarker { + id: string + label: string + lat?: number + lng?: number + note?: string +} + +export interface MapProps { + markers: Array + route?: Array<{ lat: number; lng: number }> + center?: { lat: number; lng: number } + zoom?: number +} + +export interface DiagramHotspot { + id: string + label: string + explanation: string + x?: number + y?: number +} + +export interface DiagramProps { + title?: string + hotspots: Array + imageSrc?: string +} + +export interface CalloutProps { + tone: 'info' | 'warning' | 'danger' + text: string + title?: string +} + +// 6.3 Input and Action +export interface FormProps { + submitLabel?: string + submitIntent?: string +} + +export interface TextInputProps { + label: string + placeholder?: string + validation?: { + pattern?: string + minLength?: number + maxLength?: number + } +} + +export interface NumberInputProps { + label: string + min?: number + max?: number + step?: number +} + +export interface SelectOption { + value: string + label: string +} + +export interface SelectProps { + label: string + options: Array +} + +export interface SliderProps { + label: string + min: number + max: number + step?: number +} + +export interface StepperProps { + label: string + min?: number + max?: number + step?: number +} + +export interface ToggleProps { + label: string +} + +export interface CheckboxProps { + label: string +} + +export interface ChecklistItem { + id: string + label: string + defaultDone?: boolean +} + +export interface ChecklistProps { + items: Array +} + +export interface ButtonProps { + label: string + tone?: 'primary' | 'secondary' | 'danger' | 'accent' + disabled?: boolean +} + +export interface ChoiceOption { + id: string + label: string + description?: string +} + +export interface ChoiceProps { + label?: string + options: Array +} + +// 6.4 Vector Scenes (Scene) +export type SceneShapeKind = 'group' | 'rect' | 'circle' | 'line' | 'polygon' | 'path' | 'text' + +export interface BaseSceneShape { + kind: SceneShapeKind + id?: string + label?: string + fill?: SemanticPaintRole + stroke?: SemanticPaintRole + strokeWidth?: number + opacity?: number + bind?: Record + on?: Record +} + +export interface GroupShape extends BaseSceneShape { + kind: 'group' + children: Array +} + +export interface RectShape extends BaseSceneShape { + kind: 'rect' + x: number + y: number + width: number + height: number + rx?: number + ry?: number +} + +export interface CircleShape extends BaseSceneShape { + kind: 'circle' + cx: number + cy: number + r: number +} + +export interface LineShape extends BaseSceneShape { + kind: 'line' + x1: number + y1: number + x2: number + y2: number +} + +export interface PolygonShape extends BaseSceneShape { + kind: 'polygon' + points: string +} + +export interface PathShape extends BaseSceneShape { + kind: 'path' + d: string +} + +export interface TextShape extends BaseSceneShape { + kind: 'text' + x: number + y: number + text: string + sizeRole?: 'xs' | 'sm' | 'base' | 'lg' +} + +export type SceneShape = + | GroupShape + | RectShape + | CircleShape + | LineShape + | PolygonShape + | PathShape + | TextShape + +export interface SceneProps { + viewBox: string + shapes: Array + width?: number | string + height?: number | string +} + +/* -------------------------------------------------------------------------- */ +/* Intents (§8) */ +/* -------------------------------------------------------------------------- */ + +export type IntentClass = 'local' | 'submit' | 'action' + +export interface IntentPayload { + graphId: string + nodeId: string + intent: string + intentClass: IntentClass + payload?: T + state: Record +} + +/* -------------------------------------------------------------------------- */ +/* Drivers & Requests (§7) */ +/* -------------------------------------------------------------------------- */ + +export interface UIRequest { + scenario?: string + job: string + data?: unknown + context?: Record +} + +export interface UIDriver { + render: (request: UIRequest) => Promise +} + +/* -------------------------------------------------------------------------- */ +/* Host Renderer Contract (§9) */ +/* -------------------------------------------------------------------------- */ + +export interface HostCapability { + catalogVersion: string + implementedNodes: Array + computeFunctions: Array + supportedBindings: Array + sceneLimits?: { + maxShapes?: number + maxPathLength?: number + } +} diff --git a/packages/ai-generative-ui/src/validator.ts b/packages/ai-generative-ui/src/validator.ts new file mode 100644 index 0000000000..3725055c7e --- /dev/null +++ b/packages/ai-generative-ui/src/validator.ts @@ -0,0 +1,368 @@ +/** + * Open Generative UI Specification (Draft 0.1) + * Graph Document validator with §9.3 bounded repair rules. + */ + +import type { GraphDocument, UINode, SceneShape, GraphProvenance, ComputeBinding } from './types' +import { KNOWN_CATALOG_TYPES } from './schema' + +export interface ValidationResult { + valid: boolean + graph?: GraphDocument + errors: Array + repairs: Array +} + +export interface ValidationOptions { + repair?: boolean +} + +const SVG_PATH_REGEX = /^[ \t\r\n]*([MmLlHhVvCcSsQqTtAaZz][ \t\r\n,0-9.-]*)+$/ + +export function isValidSvgPath(d: string): boolean { + if (!d || typeof d !== 'string') return false + if (d.includes('<') || d.includes('>') || d.toLowerCase().includes('script')) return false + return SVG_PATH_REGEX.test(d.trim()) +} + +export function validateGraph(input: unknown, options: ValidationOptions = {}): ValidationResult { + const repair = options.repair ?? true + const errors: Array = [] + const repairs: Array = [] + + if (!input || typeof input !== 'object' || Array.isArray(input)) { + return { valid: false, errors: ['GraphDocument must be a non-null object'], repairs } + } + + const raw = input as Record + + // Spec version check + if (raw.specVersion !== '0.1') { + if (repair) { + repairs.push(`Repaired specVersion from ${String(raw.specVersion)} to "0.1"`) + } else { + errors.push(`Invalid specVersion: expected "0.1", received ${String(raw.specVersion)}`) + } + } + + // Graph ID + let graphId: string + if (typeof raw.graphId === 'string' && raw.graphId.length > 0) { + graphId = raw.graphId + } else if (repair) { + graphId = `g_${Math.random().toString(36).slice(2, 10)}` + repairs.push(`Generated missing graphId: ${graphId}`) + } else { + graphId = '' + errors.push('Missing or invalid graphId') + } + + // Revision + let revision = Number(raw.revision) + if (!Number.isInteger(revision) || revision < 1) { + if (repair) { + revision = 1 + repairs.push('Repaired revision to 1') + } else { + errors.push('Missing or invalid revision (must be positive integer)') + } + } + + // Title + let title: string + if (typeof raw.title === 'string' && raw.title.length > 0) { + title = raw.title + } else if (repair) { + title = 'Generative UI View' + repairs.push('Defaulted missing title') + } else { + title = '' + errors.push('Missing or invalid title') + } + + // Provenance + let provenance: GraphProvenance + const rawProvenance = raw.provenance as Record | undefined + if (!rawProvenance || typeof rawProvenance !== 'object') { + if (repair) { + provenance = { driver: 'live' } + repairs.push('Created default provenance with driver="live"') + } else { + provenance = { driver: 'live' } + errors.push('Missing or invalid provenance') + } + } else { + const driver = rawProvenance.driver as string + if (!['fixture', 'programmatic', 'live'].includes(driver)) { + if (repair) { + provenance = { driver: 'live' } + repairs.push('Repaired invalid provenance.driver to "live"') + } else { + provenance = { driver: 'live' } + errors.push(`Invalid provenance.driver: ${String(driver)}`) + } + } else { + provenance = { + driver: driver as 'fixture' | 'programmatic' | 'live', + agentId: typeof rawProvenance.agentId === 'string' ? rawProvenance.agentId : undefined, + runId: typeof rawProvenance.runId === 'string' ? rawProvenance.runId : undefined, + sourceId: typeof rawProvenance.sourceId === 'string' ? rawProvenance.sourceId : undefined, + } + } + } + + // Root validation + if (!raw.root || typeof raw.root !== 'object') { + errors.push('Missing or invalid root node') + return { valid: false, errors, repairs } + } + + // Text fallback collector (for synthesizing fallbackText if missing) + const extractedText: Array = [] + + // Recursive node validator and repairer + function processNode(nodeRawInput: unknown, path: string): UINode | null { + if (!nodeRawInput || typeof nodeRawInput !== 'object') { + errors.push(`Invalid node at ${path}: expected object`) + return null + } + + const nodeRaw = nodeRawInput as Record + let id: string + if (typeof nodeRaw.id === 'string' && nodeRaw.id.length > 0) { + id = nodeRaw.id + } else if (repair) { + id = `n_${Math.random().toString(36).slice(2, 8)}` + repairs.push(`Generated missing node id at ${path}: ${id}`) + } else { + id = '' + errors.push(`Missing id at ${path}`) + } + + let type = String(nodeRaw.type || '') + let props: Record = + nodeRaw.props && typeof nodeRaw.props === 'object' + ? { ...(nodeRaw.props as Record) } + : {} + + // Text extraction for fallback + if (type === 'Text' && typeof props.content === 'string') { + extractedText.push(props.content) + } else if (type === 'Callout' && typeof props.text === 'string') { + extractedText.push(props.text) + } else if (type === 'Button' && typeof props.label === 'string') { + extractedText.push(`[${props.label}]`) + } + + // Degradation for unknown types (§9.3) + if (!KNOWN_CATALOG_TYPES.has(type)) { + if (repair) { + if (Array.isArray(nodeRaw.children) && nodeRaw.children.length > 0) { + repairs.push(`Degraded unknown container type "${type}" at ${path} to "Stack"`) + type = 'Stack' + } else { + repairs.push(`Degraded unknown leaf type "${type}" at ${path} to "Text"`) + type = 'Text' + props = { content: `[Unsupported component: ${type}]`, variant: 'caption' } + } + } else { + errors.push(`Unknown catalog node type "${type}" at ${path}`) + } + } + + // Repair type-specific props + if (type === 'Slider') { + let min = Number(props.min ?? 0) + let max = Number(props.max ?? 100) + if (isNaN(min)) min = 0 + if (isNaN(max)) max = 100 + if (min > max) { + if (repair) { + const temp = min + min = max + max = temp + repairs.push(`Swapped inverted Slider min/max at ${path}`) + } else { + errors.push(`Slider min (${min}) cannot exceed max (${max}) at ${path}`) + } + } + props.min = min + props.max = max + } else if (type === 'Scene') { + if (!props.viewBox || typeof props.viewBox !== 'string') { + if (props.width && props.height) { + props.viewBox = `0 0 ${props.width} ${props.height}` + repairs.push(`Derived Scene viewBox "${props.viewBox}" from width/height at ${path}`) + } else if (repair) { + props.viewBox = '0 0 400 300' + repairs.push(`Defaulted missing Scene viewBox at ${path}`) + } else { + errors.push(`Missing Scene viewBox at ${path}`) + } + } + + // Bounded Repair (§9.3): absorb vector shapes placed into children into props.shapes + if (Array.isArray(nodeRaw.children) && nodeRaw.children.length > 0) { + const potentialShapes = (nodeRaw.children as Array>).filter( + (c) => + c && + typeof c === 'object' && + (c.kind || ['group', 'rect', 'circle', 'line', 'polygon', 'path', 'text'].includes(String(c.type || '').toLowerCase())), + ) + + if (potentialShapes.length > 0) { + const convertedShapes: Array = potentialShapes.map((c) => { + const kind = (c.kind || String(c.type || '').toLowerCase()) as string + const childProps = c.props && typeof c.props === 'object' ? (c.props as Record) : {} + const { type: _t, props: _p, ...rest } = c + return { + kind, + ...childProps, + ...rest, + } + }) + + const existingShapes = Array.isArray(props.shapes) ? props.shapes : [] + props.shapes = [...existingShapes, ...convertedShapes] + repairs.push(`Normalized ${potentialShapes.length} vector shape(s) from Scene children into props.shapes at ${path}`) + nodeRaw.children = (nodeRaw.children as Array).filter((c) => !potentialShapes.includes(c as Record)) + } + } + + if (Array.isArray(props.shapes)) { + props.shapes = validateSceneShapes(props.shapes, path, repairs, errors, repair) + } else { + props.shapes = [] + } + } + + // Process children + let processedChildren: Array | undefined + if (Array.isArray(nodeRaw.children)) { + processedChildren = [] + for (let i = 0; i < nodeRaw.children.length; i++) { + const child = processNode(nodeRaw.children[i], `${path}.children[${i}]`) + if (child) { + processedChildren.push(child) + } + } + } + + const cleanNode: UINode = { + id, + type, + props, + } + + if (processedChildren && processedChildren.length > 0) { + cleanNode.children = processedChildren + } + + if (nodeRaw.bind && typeof nodeRaw.bind === 'object') { + cleanNode.bind = nodeRaw.bind as Record + } + + if (nodeRaw.compute) { + cleanNode.compute = nodeRaw.compute as ComputeBinding | string + } + + if (nodeRaw.on && typeof nodeRaw.on === 'object') { + cleanNode.on = nodeRaw.on as Record + } + + return cleanNode + } + + function validateSceneShapes( + shapes: Array, + path: string, + repairsList: Array, + errorsList: Array, + canRepair: boolean, + ): Array { + const validShapes: Array = [] + + for (let i = 0; i < shapes.length; i++) { + const s = shapes[i] as Record | null | undefined + if (!s || typeof s !== 'object') continue + + let kind = String(s.kind ?? s.type ?? '').toLowerCase() + if (!['group', 'rect', 'circle', 'line', 'polygon', 'path', 'text'].includes(kind)) { + if (canRepair) { + repairsList.push(`Dropped unknown Scene shape kind "${kind}" at ${path}.shapes[${i}]`) + continue + } else { + errorsList.push(`Unknown Scene shape kind "${kind}" at ${path}.shapes[${i}]`) + continue + } + } + + s.kind = kind + if (kind === 'line' && !s.stroke) { + s.stroke = 'neutral' + } + if (kind === 'circle' && !s.fill) { + s.fill = 'accent' + } + + // Path grammar safety check + if (kind === 'path') { + const pathData = typeof s.d === 'string' ? s.d : '' + if (!isValidSvgPath(pathData)) { + if (canRepair) { + repairsList.push(`Dropped invalid or unsafe SVG path shape at ${path}.shapes[${i}]`) + continue + } else { + errorsList.push(`Invalid SVG path data at ${path}.shapes[${i}]`) + continue + } + } + } + + if (kind === 'group' && Array.isArray(s.children)) { + s.children = validateSceneShapes(s.children, `${path}.shapes[${i}]`, repairsList, errorsList, canRepair) + } + + // oxlint-disable-next-line eslint-js/no-restricted-syntax -- shape kind and geometry validated dynamically above + validShapes.push(s as unknown as SceneShape) + } + + return validShapes + } + + const cleanRoot = processNode(raw.root, 'root') + if (!cleanRoot) { + return { valid: false, errors, repairs } + } + + // Fallback text check and repair + let fallbackText: string + if (typeof raw.fallbackText === 'string' && raw.fallbackText.trim().length > 0) { + fallbackText = raw.fallbackText + } else if (repair) { + fallbackText = extractedText.length > 0 ? `${title}: ${extractedText.join(' ')}` : title + repairs.push(`Synthesized missing fallbackText: "${fallbackText}"`) + } else { + fallbackText = '' + errors.push('Missing or empty required fallbackText (§9.2)') + } + + const valid = errors.length === 0 + + const graph: GraphDocument = { + specVersion: '0.1', + graphId, + revision, + title, + fallbackText, + provenance, + root: cleanRoot, + } + + return { + valid, + graph: valid ? graph : undefined, + errors, + repairs, + } +} diff --git a/packages/ai-generative-ui/tests/compute.test.ts b/packages/ai-generative-ui/tests/compute.test.ts new file mode 100644 index 0000000000..cafad45775 --- /dev/null +++ b/packages/ai-generative-ui/tests/compute.test.ts @@ -0,0 +1,47 @@ +import { describe, it, expect } from 'vitest' +import { ComputeRegistry, defaultComputeRegistry } from '../src/index' + +describe('Compute Registry (§5.3)', () => { + it('executes billSplit compute function deterministically without model floating point issues', () => { + const inputs = { + subtotal: 100, + tipPercent: 20, + splitCount: 4, + } + + const result = defaultComputeRegistry.execute('billSplit', inputs, {}) + expect(result).toEqual({ + subtotal: 100, + tipPercent: 20, + splitCount: 4, + tipAmount: 20, + total: 120, + perPerson: 30, + }) + }) + + it('handles billSplit with odd guest counts and decimal tips', () => { + const inputs = { + subtotal: 97.43, + tipPercent: 18, + splitCount: 3, + } + + const result = defaultComputeRegistry.execute('billSplit', inputs, {}) + expect(result.tipAmount).toBe(17.54) + expect(result.total).toBe(114.97) + expect(result.perPerson).toBe(38.32) + }) + + it('allows registering and executing custom compute functions', () => { + const registry = new ComputeRegistry() + registry.register('currencyConvert', (inputs) => { + const amount = Number(inputs.amount ?? 0) + const rate = Number(inputs.rate ?? 1.1) + return Math.round(amount * rate * 100) / 100 + }) + + const result = registry.execute('currencyConvert', { amount: 50, rate: 1.08 }, {}) + expect(result).toBe(54) + }) +}) diff --git a/packages/ai-generative-ui/tests/drivers.test.ts b/packages/ai-generative-ui/tests/drivers.test.ts new file mode 100644 index 0000000000..507e0be32c --- /dev/null +++ b/packages/ai-generative-ui/tests/drivers.test.ts @@ -0,0 +1,55 @@ +import { describe, it, expect } from 'vitest' +import { + FixtureDriver, + ProgrammaticDriver, + LiveDriver, +} from '../src/index' + +describe('UIDriver Implementations (§7)', () => { + it('FixtureDriver returns all canonical fixtures by scenario ID', async () => { + const driver = new FixtureDriver() + const scenarios = driver.listScenarios() + expect(scenarios).toHaveLength(9) + + for (const sc of scenarios) { + const graph = await driver.render({ job: 'test', scenario: sc }) + expect(graph.specVersion).toBe('0.1') + expect(graph.graphId).toBeDefined() + expect(graph.root).toBeDefined() + } + }) + + it('ProgrammaticDriver creates valid graph using builder API', async () => { + const builder = ProgrammaticDriver.createBuilder( + 'Server Resource Dashboard', + 'Memory: 8GB, CPU: 4 cores, Button: Reboot', + ) + .addText('Server Health Overview', 'heading') + .addMetric('CPU Usage', '42%', '+3%', 'positive') + .addButton('Restart Service', 'restartServer', { serverId: 'srv-01' }) + + const graph = builder.build() + expect(graph.specVersion).toBe('0.1') + expect(graph.title).toBe('Server Resource Dashboard') + expect(graph.root.children).toHaveLength(3) + }) + + it('LiveDriver outputs tool schema and handles model fallback', () => { + const toolDef = LiveDriver.getRenderToolDefinition() + expect(toolDef.name).toBe('render_ui') + expect(toolDef.parameters.required).toContain('root') + + const repaired = LiveDriver.processModelOutput({ + title: 'Generated View', + fallbackText: 'Plain text fallback', + root: { + id: 'n1', + type: 'Text', + props: { content: 'Generated text' }, + }, + }) + + expect(repaired.specVersion).toBe('0.1') + expect(repaired.fallbackText).toBe('Plain text fallback') + }) +}) diff --git a/packages/ai-generative-ui/tests/react-renderer.test.tsx b/packages/ai-generative-ui/tests/react-renderer.test.tsx new file mode 100644 index 0000000000..4ea4f2a79e --- /dev/null +++ b/packages/ai-generative-ui/tests/react-renderer.test.tsx @@ -0,0 +1,126 @@ +// @vitest-environment happy-dom +;(globalThis as any).IS_REACT_ACT_ENVIRONMENT = true +import { describe, it, expect, vi } from 'vitest' +import { act } from 'react' +import { createRoot } from 'react-dom/client' +import { + GenerativeUIRenderer, + comparePlansBasicFixture, + billSplitFixture, + chordDiagramEmaj7Fixture, +} from '../src/index' + +describe('GenerativeUIRenderer React Component', () => { + it('renders compare plans fixture with columns and buttons', async () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + + const onIntent = vi.fn() + + await act(async () => { + root.render( + , + ) + }) + + expect(container.textContent).toContain('Compare Hosting Plans') + expect(container.textContent).toContain('Starter') + expect(container.textContent).toContain('Pro (Recommended)') + expect(container.textContent).toContain('Enterprise') + + const proBtn = Array.from(container.querySelectorAll('button')).find((b) => + b.textContent?.includes('Select Pro'), + ) + expect(proBtn).toBeDefined() + + await act(async () => { + proBtn?.click() + }) + + expect(onIntent).toHaveBeenCalledTimes(1) + expect(onIntent).toHaveBeenCalledWith( + expect.objectContaining({ + graphId: 'g_compare_plans_01', + intent: 'selectPlan', + intentClass: 'action', + payload: { plan: 'pro' }, + }), + ) + + root.unmount() + container.remove() + }) + + it('evaluates compute bindings and updates display values', async () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + + await act(async () => { + root.render( + , + ) + }) + + expect(container.textContent).toContain('Dinner Bill Splitter') + expect(container.textContent).toContain('Calculated Totals (Computed by Host Code)') + + // 100 subtotal, 20% tip = $20 tip, $120 total, $30 per person + expect(container.textContent).toContain('Tip Amount') + expect(container.textContent).toContain('Total with Tip') + expect(container.textContent).toContain('Per Person') + + root.unmount() + container.remove() + }) + + it('renders vector scene and handles interactive shape selection (§6.4.2)', async () => { + const container = document.createElement('div') + document.body.appendChild(container) + const root = createRoot(container) + + const onIntent = vi.fn() + + await act(async () => { + root.render( + , + ) + }) + + expect(container.textContent).toContain('Guitar Voicing: Open Emaj7') + + const svg = container.querySelector('svg') + expect(svg).toBeDefined() + + const interactiveNote = container.querySelector('[id="note-b-5"]') as SVGElement + expect(interactiveNote).toBeDefined() + expect(interactiveNote.getAttribute('role')).toBe('button') + expect(interactiveNote.getAttribute('aria-label')).toContain('B — Perfect 5th') + + await act(async () => { + interactiveNote.dispatchEvent(new MouseEvent('click', { bubbles: true })) + }) + + expect(onIntent).toHaveBeenCalledWith( + expect.objectContaining({ + nodeId: 'note-b-5', + intent: 'noteSelected', + intentClass: 'local', + payload: expect.objectContaining({ shapeId: 'note-b-5' }), + }), + ) + + root.unmount() + container.remove() + }) +}) diff --git a/packages/ai-generative-ui/tests/schema-validator.test.ts b/packages/ai-generative-ui/tests/schema-validator.test.ts new file mode 100644 index 0000000000..5f1e9c3ce2 --- /dev/null +++ b/packages/ai-generative-ui/tests/schema-validator.test.ts @@ -0,0 +1,169 @@ +import { describe, it, expect } from 'vitest' +import { + validateGraph, + isValidSvgPath, + CANONICAL_FIXTURES, +} from '../src/index' + +describe('Schema Validator and Bounded Repair (§9.3)', () => { + it('validates all 9 canonical fixtures successfully', () => { + for (const [name, fixture] of Object.entries(CANONICAL_FIXTURES)) { + const result = validateGraph(fixture, { repair: false }) + expect(result.valid, `Fixture "${name}" should be schema-valid: ${result.errors.join(', ')}`).toBe(true) + expect(result.errors).toHaveLength(0) + expect(result.graph?.fallbackText).toBeDefined() + expect(result.graph?.fallbackText.length).toBeGreaterThan(0) + } + }) + + it('degrades unknown container type to Stack', () => { + const unknownContainer = { + specVersion: '0.1', + graphId: 'g_test', + revision: 1, + title: 'Unknown Container Test', + fallbackText: 'Testing degradation', + provenance: { driver: 'fixture' }, + root: { + id: 'n_mystery', + type: 'HoloDeckContainer', + children: [ + { id: 'n_child', type: 'Text', props: { content: 'Child item' } }, + ], + }, + } + + const result = validateGraph(unknownContainer, { repair: true }) + expect(result.valid).toBe(true) + expect(result.graph?.root.type).toBe('Stack') + expect(result.graph?.root.children?.[0]?.id).toBe('n_child') + expect(result.repairs.some((r) => r.includes('Degraded unknown container'))).toBe(true) + }) + + it('degrades unknown leaf node to Text', () => { + const unknownLeaf = { + specVersion: '0.1', + graphId: 'g_test', + revision: 1, + title: 'Unknown Leaf Test', + fallbackText: 'Testing degradation', + provenance: { driver: 'fixture' }, + root: { + id: 'n_mystery_leaf', + type: 'QuantumWidget', + }, + } + + const result = validateGraph(unknownLeaf, { repair: true }) + expect(result.valid).toBe(true) + expect(result.graph?.root.type).toBe('Text') + expect(result.repairs.some((r) => r.includes('Degraded unknown leaf'))).toBe(true) + }) + + it('repairs inverted slider min/max bounds', () => { + const invertedSlider = { + specVersion: '0.1', + graphId: 'g_slider', + revision: 1, + title: 'Slider Test', + fallbackText: 'Inverted slider', + provenance: { driver: 'fixture' }, + root: { + id: 'n_slider', + type: 'Slider', + props: { min: 100, max: 10 }, + }, + } + + const result = validateGraph(invertedSlider, { repair: true }) + expect(result.valid).toBe(true) + expect(result.graph?.root.props?.min).toBe(10) + expect(result.graph?.root.props?.max).toBe(100) + expect(result.repairs.some((r) => r.includes('Swapped inverted Slider'))).toBe(true) + }) + + it('validates SVG path grammar and drops malicious/broken paths (§6.4.1)', () => { + expect(isValidSvgPath('M 10 10 L 20 20 Z')).toBe(true) + expect(isValidSvgPath('M0,0 L10,10 C20,20 40,40 50,50 Z')).toBe(true) + expect(isValidSvgPath('')).toBe(false) + expect(isValidSvgPath('javascript:doBadStuff()')).toBe(false) + + const sceneWithBadPath = { + specVersion: '0.1', + graphId: 'g_scene', + revision: 1, + title: 'Scene Test', + fallbackText: 'Scene with bad path', + provenance: { driver: 'fixture' }, + root: { + id: 'n_scene', + type: 'Scene', + props: { + viewBox: '0 0 200 200', + shapes: [ + { kind: 'circle', cx: 50, cy: 50, r: 10 }, + { kind: 'path', d: '' }, + ], + }, + }, + } + + const result = validateGraph(sceneWithBadPath, { repair: true }) + expect(result.valid).toBe(true) + const shapes = result.graph?.root.props?.shapes + expect(shapes).toHaveLength(1) + expect(shapes[0].kind).toBe('circle') + expect(result.repairs.some((r) => r.includes('Dropped invalid or unsafe SVG path'))).toBe(true) + }) + + it('synthesizes missing fallbackText', () => { + const missingFallback = { + specVersion: '0.1', + graphId: 'g_text', + revision: 1, + title: 'Checkout Summary', + provenance: { driver: 'fixture' }, + root: { + id: 'n_text', + type: 'Text', + props: { content: 'Total is $45.00' }, + }, + } + + const result = validateGraph(missingFallback, { repair: true }) + expect(result.valid).toBe(true) + expect(result.graph?.fallbackText).toContain('Total is $45.00') + expect(result.repairs.some((r) => r.includes('Synthesized missing fallbackText'))).toBe(true) + }) + + it('normalizes vector shapes accidentally placed in Scene children into props.shapes', () => { + const sceneWithChildren = { + specVersion: '0.1', + graphId: 'g_guitar', + revision: 1, + title: 'Guitar Diagram', + fallbackText: 'Guitar diagram', + provenance: { driver: 'live' }, + root: { + id: 'chord-diagram', + type: 'Scene', + props: { width: 300, height: 360 }, + children: [ + { type: 'rect', x: 20, y: 20, width: 200, height: 300 }, + { type: 'line', x1: 20, y1: 50, x2: 220, y2: 50 }, + { type: 'circle', cx: 80, cy: 100, r: 10, fill: 'accent' }, + ], + }, + } + + const result = validateGraph(sceneWithChildren, { repair: true }) + expect(result.valid).toBe(true) + expect(result.graph?.root.type).toBe('Scene') + expect(result.graph?.root.children).toBeUndefined() + expect(result.graph?.root.props?.shapes).toHaveLength(3) + expect(result.graph?.root.props?.shapes[0].kind).toBe('rect') + expect(result.graph?.root.props?.shapes[1].kind).toBe('line') + expect(result.graph?.root.props?.shapes[2].kind).toBe('circle') + expect(result.repairs.some((r) => r.includes('Normalized 3 vector shape(s)'))).toBe(true) + }) +}) diff --git a/packages/ai-generative-ui/tsconfig.json b/packages/ai-generative-ui/tsconfig.json new file mode 100644 index 0000000000..16f72d0975 --- /dev/null +++ b/packages/ai-generative-ui/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "jsx": "react-jsx", + "lib": ["ES2022", "DOM"], + "rootDir": "../.." + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-generative-ui/vite.config.ts b/packages/ai-generative-ui/vite.config.ts new file mode 100644 index 0000000000..9ae81061b9 --- /dev/null +++ b/packages/ai-generative-ui/vite.config.ts @@ -0,0 +1,40 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' with { type: 'json' } + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + globals: true, + environment: 'happy-dom', + include: ['tests/**/*.test.ts', 'tests/**/*.test.tsx'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.test.tsx', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.{ts,tsx}'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: [ + './src/index.ts', + './src/react.ts', + ], + srcDir: './src', + cjs: false, + }), +) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bb0a8207db..3894e5ec43 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -174,6 +174,40 @@ importers: specifier: ^8.2.1 version: 8.2.1(@types/node@24.10.3)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0) + examples/generative-ui: + dependencies: + '@tanstack/ai-generative-ui': + specifier: workspace:* + version: link:../../packages/ai-generative-ui + react: + specifier: ^19.2.3 + version: 19.2.3 + react-dom: + specifier: ^19.2.3 + version: 19.2.3(react@19.2.3) + devDependencies: + '@tailwindcss/vite': + specifier: ^4.1.18 + version: 4.1.18(vite@8.3.0(@types/node@24.13.6)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0)) + '@types/react': + specifier: ^19.2.7 + version: 19.3.0 + '@types/react-dom': + specifier: ^19.2.3 + version: 19.2.3(@types/react@19.3.0) + '@vitejs/plugin-react': + specifier: ^5.2.0 + version: 5.2.0(vite@8.3.0(@types/node@24.13.6)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0)) + tailwindcss: + specifier: ^4.1.18 + version: 4.1.18 + typescript: + specifier: 5.9.3 + version: 5.9.3 + vite: + specifier: ^8.2.1 + version: 8.3.0(@types/node@24.13.6)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0) + examples/react/basic-chat: dependencies: '@tailwindcss/vite': @@ -2524,6 +2558,33 @@ importers: specifier: ^4.2.0 version: 4.3.6 + packages/ai-generative-ui: + devDependencies: + '@types/react': + specifier: ^19.2.7 + version: 19.3.0 + '@types/react-dom': + specifier: ^19.2.3 + version: 19.2.3(@types/react@19.3.0) + premove: + specifier: ^4.0.0 + version: 4.0.0 + publint: + specifier: ^0.3.15 + version: 0.3.16 + react: + specifier: ^19.2.3 + version: 19.2.3 + react-dom: + specifier: ^19.2.3 + version: 19.2.3(react@19.2.3) + vite: + specifier: ^8.2.1 + version: 8.3.0(@types/node@24.13.6)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0) + vitest: + specifier: ^4.1.10 + version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@24.13.6)(@vitest/coverage-v8@4.1.10(vitest@4.1.10))(happy-dom@20.11.2)(jsdom@27.4.0(@noble/hashes@2.3.0))(vite@8.3.0(@types/node@24.13.6)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0)) + packages/ai-grok: dependencies: '@tanstack/ai-utils': @@ -3456,7 +3517,7 @@ importers: devDependencies: '@google/genai': specifier: ^2.10.0 - version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)) + version: 2.10.0(@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.6.5)) '@tanstack/ai': specifier: workspace:* version: link:../ai @@ -37960,6 +38021,37 @@ snapshots: - msw optional: true + vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@24.13.6)(@vitest/coverage-v8@4.1.10(vitest@4.1.10))(happy-dom@20.11.2)(jsdom@27.4.0(@noble/hashes@2.3.0))(vite@8.3.0(@types/node@24.13.6)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0)): + dependencies: + '@vitest/expect': 4.1.11 + '@vitest/mocker': 4.1.11(vite@8.3.0(@types/node@24.13.6)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0)) + '@vitest/pretty-format': 4.1.11 + '@vitest/runner': 4.1.11 + '@vitest/snapshot': 4.1.11 + '@vitest/spy': 4.1.11 + '@vitest/utils': 4.1.11 + es-module-lexer: 2.3.2 + expect-type: 1.4.0 + magic-string: 0.30.21 + obug: 2.2.1 + pathe: 2.0.3 + picomatch: 4.0.7 + std-env: 4.2.0 + tinybench: 2.9.0 + tinyexec: 1.3.1 + tinyglobby: 0.2.17 + tinyrainbow: 3.1.1 + vite: 8.3.0(@types/node@24.13.6)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0) + why-is-node-running: 2.3.0 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + '@types/node': 24.13.6 + '@vitest/coverage-v8': 4.1.10(vitest@4.1.10) + happy-dom: 20.11.2 + jsdom: 27.4.0(@noble/hashes@2.3.0) + transitivePeerDependencies: + - msw + vitest@4.1.11(@opentelemetry/api@1.9.1)(@types/node@24.13.6)(@vitest/coverage-v8@4.1.10)(happy-dom@20.11.2)(jsdom@27.4.0(@noble/hashes@2.3.0))(vite@8.3.0(@types/node@24.13.6)(esbuild@0.28.1)(jiti@2.7.0)(less@4.6.6)(sass@1.101.0)(terser@5.44.1)(tsx@4.23.12)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.11 From 58847c14ed4dfe8273af8a42ad455040cc58fccb Mon Sep 17 00:00:00 2001 From: Jack Herrington Date: Wed, 14 Oct 2026 16:08:35 -0700 Subject: [PATCH 02/20] fix(generative-ui): repair rendering + interactivity, elevate design, harden live generation Browser-driven pass over the Open Generative UI POC. Everything below was verified in a real browser against the running playground and live provider calls. Critical bugs fixed - Host renderer was completely unstyled: the example's Tailwind `@source` pointed at a non-existent `examples/packages/...` path, so none of the renderer's utility classes were generated. Corrected to resolve the package src. - Chart bars were invisible (percentage height inside an auto-height flex column). Rebuilt as a fixed-plot gradient bar chart. - Bind-less inputs were inert. A live model commonly omits `bind.value` / `bind.state` and item ids, addressing inputs by node id. Inputs now resolve an implicit `.value` (and Checklist an implicit `.state` with index-based item keys), seeded from props on mount, so stepper/slider/ checklist are interactive regardless. Section/Button/Text/Metric now tolerate common prop-name synonyms (label/title/heading, label/text, content/text). - Live calculators never computed: the render_ui schema never told the model which compute functions exist. Added a compute catalog to the system prompt, expanded the registry (loanPayment/mortgage, percentOf) with input-key aliasing, and flatten object results to the state root so nested and scalar bindings both resolve. - Fixed pre-existing prefer-const / non-null-assertion lint errors. Design elevation - Reworked the host renderer into a cohesive light design system (real Chart, projected SVG Map with route path + numbered pins, dashboard Metric cards, stepped Timeline, badged "Recommended" Section) and the playground shell (Bricolage Grotesque / Hanken Grotesk / JetBrains Mono, engineering-grid + radial-glow background, staggered surface entrance). Regression artifact - Checked in the real-world graph that surfaced the bind-less bug as the `recipe-card` canonical fixture + a "Live-shape: Recipe Card" playground scenario, locked down by a unit test. 21/21 unit tests, package + example typecheck, oxlint (0 errors), and builds all green. Updated STATUS.md to reflect verified reality. Co-Authored-By: Claude Opus 4.8 (1M context) --- STATUS.md | 57 +- examples/generative-ui/src/App.tsx | 23 +- examples/generative-ui/src/index.css | 64 +- examples/generative-ui/vite.config.ts | 9 +- packages/ai-generative-ui/src/compute.ts | 211 ++++-- .../ai-generative-ui/src/fixtures/index.ts | 88 +++ packages/ai-generative-ui/src/react/index.tsx | 614 +++++++++++++----- packages/ai-generative-ui/src/validator.ts | 2 +- .../ai-generative-ui/tests/compute.test.ts | 53 +- .../ai-generative-ui/tests/drivers.test.ts | 3 +- .../tests/react-renderer.test.tsx | 76 ++- 11 files changed, 954 insertions(+), 246 deletions(-) diff --git a/STATUS.md b/STATUS.md index 0a09a71bf2..aea225ea8c 100644 --- a/STATUS.md +++ b/STATUS.md @@ -57,23 +57,29 @@ Located at `packages/ai-generative-ui`. - JSON Schema compliant definitions for `render_ui` tool calling. - Full bounded repair validator enforcing §9.3 rules and SVG path grammar validation (`/^[ \t\r\n]*([MmLlHhVvCcSsQqTtAaZz][ \t\r\n,0-9.-]*)+$/`). - **Deterministic Compute Engine (`src/compute.ts`)**: - - Built-in compute functions: `billSplit`, `savingsGrowth`, `sum`. + - Built-in compute functions: `billSplit`, `loanPayment` (+ `mortgage`/`calculateMortgage` aliases), `savingsGrowth`, `percentOf`, `sum`. + - Input-key aliasing (e.g. `loanAmount`/`principal`/`amount` all resolve) so a model's phrasing still hits the right parameter. + - `describe()` / `describeComputeForPrompt()` emit a machine-readable catalog of functions, their inputs, and output shapes — injected into the live system prompt so models bind to functions that actually exist (§7.3) instead of inventing names. - Type-safe `ComputeRegistry` allowing custom host arithmetic registration. - **Drivers (`src/drivers/index.ts`)**: - - `FixtureDriver`: Mock driver delivering the 9 canonical spec fixtures. + - `FixtureDriver`: Mock driver delivering the 9 canonical spec fixtures plus a stored live-shape regression fixture (`recipe-card`). - `ProgrammaticDriver`: Fluent builder API for constructing graphs in application code with zero LLM dependency. - `LiveDriver`: Automated tool schema generator and output processor for model-driven generation. - **React Host Renderer (`src/react/index.tsx`)**: - - Dynamic host renderer with recursive `NodeRenderer`. + - Dynamic host renderer with recursive `NodeRenderer`, styled as a cohesive, production-grade light design system (the reference host surface). + - Real data visualisation: a fixed-plot gradient bar `Chart`, a geographically-projected SVG `Map` (lat/lng → viewport with route path + numbered pins), dashboard `Metric` cards with signed delta pills, and a stepped `Timeline` with done/current/upcoming states. + - `Section` renders a highlighted, badged variant when its title/`highlighted` flag marks it "Recommended". - SVG Vector scene rendering with interactive shape selection and keyboard accessibility. - - Reactive compute bindings automatically recalculating outputs when inputs change. + - Reactive compute bindings automatically recalculating outputs when inputs change — results are written at the declared `output` path **and** flattened to the state root, so both nested (`loan.monthlyPayment`) and scalar (`monthlyPayment`) model bindings resolve. + - **Flexible input binding**: an input's value path is an explicit `bind.value` or the implicit `.value` convention a live model naturally emits; state is seeded from each input's declared `props.value` on mount so computes have real inputs on first render. + - Prop-name tolerance for common model variance (`label`/`text`/`title` on buttons, `content`/`text` on text and callouts) and a clean em-dash empty state on unpopulated metrics. - Accessible Text Fallback component (`GenerativeUITextFallback`). - **Test Suite**: - - 16/16 unit tests passing across 4 test suites: + - 21/21 unit tests passing across 4 test suites: - `tests/schema-validator.test.ts` - - `tests/compute.test.ts` + - `tests/compute.test.ts` (incl. `loanPayment` amortization, aliases, 0% edge case, `percentOf`, catalog integrity) - `tests/drivers.test.ts` - - `tests/react-renderer.test.tsx` + - `tests/react-renderer.test.tsx` (incl. a bind-less stepper + id-less checklist regression) --- @@ -84,7 +90,7 @@ Located at `examples/generative-ui`. - Middleware in `vite.config.ts` supporting OpenAI (`gpt-4o`), Anthropic (`claude-sonnet-4-5-20250929`), Gemini (`gemini-3.8-flash`), xAI (`grok-4.20-0309-non-reasoning`), and OpenRouter via native fetch and `.env.local` keys. - Automatic tool calling invocation using the `render_ui` specification schema. - **Playground Features (`src/App.tsx`)**: - - Scenario Selector covering all 9 canonical fixtures (U1–U8 + §6.4.3 chord diagram). + - Scenario Selector covering all 9 canonical fixtures (U1–U8 + §6.4.3 chord diagram) plus the stored "Live-shape: Recipe Card" regression scenario. - Live AI prompt bar with sample chips (*Mortgage calculator*, *Streaming tiers*, *Recipe checklist*, *Dmaj9 chord voicing*). - Bounded Repair banner detailing applied schema repairs. - Visual vs. Fallback Text mode toggle (§9.2). @@ -111,9 +117,36 @@ Located at `/Users/jherr/projects/goose` on branch `feat/generative-ui-poc`. | Check | Tool / Target | Status | |-------|---------------|--------| -| Unit Tests | `pnpm --filter @tanstack/ai-generative-ui test:lib` | ✅ 16/16 passing | -| Type Check | `pnpm test:types` | ✅ 101/101 projects passing | -| Monorepo Dependencies | `pnpm test:sherif` | ✅ Clean (0 errors) | -| Dead Code / Unused Deps | `pnpm test:knip` | ✅ Clean (0 issues) | +| Unit Tests | `pnpm --filter @tanstack/ai-generative-ui test:lib` | ✅ 21/21 passing | +| Type Check (package) | `nx test:types ai-generative-ui` | ✅ Passing | +| Type Check (example) | `nx test:types generative-ui-example` | ✅ Passing | +| Lint | `pnpm test:oxlint` | ✅ 0 errors (pre-existing `no-explicit-any` warnings only) | | Package Build & Types | `vite build` + `tsc` + `publint --strict` | ✅ Clean ESM & DTS | | Example Build | `pnpm --filter generative-ui-example build` | ✅ Clean bundle | + +--- + +## 4. This Session: Fixes, Hardening & Browser Verification + +Every change below was driven and verified in a real browser (Chrome automation) against the running playground and live provider calls. + +### Bugs found & fixed +- **Host renderer was completely unstyled (critical).** `examples/generative-ui/src/index.css` pointed `@source` at `../../packages/...`, which resolves to a non-existent `examples/packages/...`. Tailwind generated none of the renderer's utility classes, so the host card rendered with no background, padding, or component styling. Corrected to `../../../packages/ai-generative-ui/src`. +- **`Chart` bars were invisible.** Bars used a percentage height inside an auto-height flex column (circular height dependency → collapse). Rebuilt as a fixed-height plot with pixel-computed gradient bars, a baseline, and hover states. +- **Live calculators never computed.** The `render_ui` tool schema never told the model which compute functions exist, so it invented names (`calculateMortgage`) that silently no-op'd. Added a compute catalog to the system prompt, expanded the registry (loan/mortgage, percentage), and added input-key aliasing. Also supported the implicit `.value` binding pattern and on-mount state seeding so a prefilled input reaches its bound compute. +- **Bind-less inputs were inert.** A live model frequently omits `bind.value`/`bind.state` and item `id`s (addressing inputs by node id). A `Stepper`/`Slider`/`Checklist` without an explicit bind did nothing. Inputs now resolve an implicit `.value` (and `Checklist` an implicit `.state` with index-based item keys), so they are interactive and persist regardless. Covered by a regression test. +- **Prop-name drift.** `Section` titles via `label`, button labels via `text`, etc. were dropped on the floor. Components now tolerate the common synonyms (`title`/`label`/`heading`, `label`/`text`/`title`, `content`/`text`). +- **`prefer-const` lint error** in `validator.ts` and non-null-assertion lint errors in `compute.ts` (both now clean). + +### Design elevation +Reworked the host renderer into a cohesive "technical atelier" design system (refined neutrals, cobalt accent, hairline borders, layered shadows, tabular numerals) and the playground shell into a designed developer surface (Bricolage Grotesque / Hanken Grotesk / JetBrains Mono, engineering-grid + radial-glow background, staggered surface entrance). New real visualisations for `Chart`, `Map`, `Metric`, and `Timeline`; a badged "Recommended" `Section` variant. + +### Browser-verified behaviours +- All 9 canonical fixtures render correctly (U1–U8 + §6.4.3 vector scene). +- U4 bill-split and live-generated tip/mortgage calculators compute **reactively** via host code — confirmed a slider drag updates tip `$18 → $27` and total `$118 → $127` live. +- Structured intents (§8) fire and log on hotspot/plan/map interactions. +- Bounded repair (§9.3) applies all 7 expected fixes to the malformed test graph (degradations, swapped slider bounds, dropped unsafe SVG path, synthesized `fallbackText`). +- Live generation returns valid, rendering graphs across all five providers (OpenAI, Anthropic, Gemini, xAI, OpenRouter). + +### Stored regression artifact +The real-world graph that first surfaced the bind-less interactivity bug is now checked in verbatim as the `recipe-card` canonical fixture and a **"Live-shape: Recipe Card"** scenario in the playground dropdown, so the tricky pattern (inputs by node id, no `bind`, no item ids, section titles via `label`) can be reproduced deterministically without the LLM and is locked down by a unit test. diff --git a/examples/generative-ui/src/App.tsx b/examples/generative-ui/src/App.tsx index eeb87dfd2b..9179d6fcb7 100644 --- a/examples/generative-ui/src/App.tsx +++ b/examples/generative-ui/src/App.tsx @@ -161,8 +161,8 @@ export function App() { UI
-

- Open Generative UI Draft 0.1 +

+ Open Generative UI Draft 0.1

Deterministic Computation • Bounded Repair • Structured Intents

@@ -188,6 +188,7 @@ export function App() { + {/* View Mode Toggle */} @@ -328,7 +329,7 @@ export function App() { {/* Main Content Area */}
{/* Render Canvas */} -
+
@@ -379,15 +380,13 @@ export function App() {
)} - {viewMode === 'visual' ? ( - - ) : ( - - )} +
+ {viewMode === 'visual' ? ( + + ) : ( + + )} +
diff --git a/examples/generative-ui/src/index.css b/examples/generative-ui/src/index.css index b3123d6fda..8d61d4611f 100644 --- a/examples/generative-ui/src/index.css +++ b/examples/generative-ui/src/index.css @@ -1,13 +1,71 @@ @import "tailwindcss"; +@import url('https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:opsz,wght@12..96,400..800&family=Hanken+Grotesk:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500;600&display=swap'); @source "../src"; -@source "../../packages/ai-generative-ui/src"; +@source "../../../packages/ai-generative-ui/src"; + +@theme { + --font-sans: 'Hanken Grotesk', ui-sans-serif, system-ui, -apple-system, sans-serif; + --font-mono: 'JetBrains Mono', ui-monospace, 'SF Mono', Menlo, monospace; + --font-display: 'Bricolage Grotesque', 'Hanken Grotesk', sans-serif; +} @layer base { + html { + -webkit-font-smoothing: antialiased; + text-rendering: optimizeLegibility; + } body { - background-color: #020617; + background-color: #05080f; + background-image: + radial-gradient(55rem 38rem at 82% -12%, rgba(37, 99, 235, 0.14), transparent 60%), + radial-gradient(48rem 38rem at -8% 6%, rgba(99, 102, 241, 0.10), transparent 55%); + background-attachment: fixed; color: #f8fafc; - font-family: system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + font-family: var(--font-sans); margin: 0; padding: 0; } } + +/* Distinctive display type for product-level headings */ +.font-display { + font-family: var(--font-display); +} + +/* The host-rendered card title adopts the display face without coupling the package to a font */ +.open-gen-ui-container > header h2 { + font-family: var(--font-display); + letter-spacing: -0.025em; +} + +/* Subtle engineering-grid texture behind the render canvas */ +.canvas-grid { + background-image: + linear-gradient(rgba(148, 163, 184, 0.045) 1px, transparent 1px), + linear-gradient(90deg, rgba(148, 163, 184, 0.045) 1px, transparent 1px); + background-size: 32px 32px; + background-position: center top; +} + +.no-scrollbar::-webkit-scrollbar { + display: none; +} +.no-scrollbar { + -ms-overflow-style: none; + scrollbar-width: none; +} + +/* One orchestrated entrance for the rendered surface */ +@keyframes surface-rise { + from { + opacity: 0; + transform: translateY(10px) scale(0.995); + } + to { + opacity: 1; + transform: translateY(0) scale(1); + } +} +.animate-surface-rise { + animation: surface-rise 0.45s cubic-bezier(0.22, 1, 0.36, 1) both; +} diff --git a/examples/generative-ui/vite.config.ts b/examples/generative-ui/vite.config.ts index bd37870710..2e5c74004f 100644 --- a/examples/generative-ui/vite.config.ts +++ b/examples/generative-ui/vite.config.ts @@ -3,7 +3,7 @@ import { resolve } from 'node:path' import { defineConfig, type Plugin } from 'vite' import react from '@vitejs/plugin-react' import tailwindcss from '@tailwindcss/vite' -import { LiveDriver, validateGraph } from '@tanstack/ai-generative-ui' +import { LiveDriver, validateGraph, describeComputeForPrompt } from '@tanstack/ai-generative-ui' // Load .env.local if present for (const envFile of ['.env.local', '.env']) { @@ -24,7 +24,12 @@ Ground Rules: - Input: Form, TextInput, NumberInput, Select, Slider, Stepper, Toggle, Checkbox, Checklist, Button, Choice - Vector Scene: Scene (props must include "viewBox", e.g. "0 0 300 360", and "shapes": Array<{ kind: "rect"|"circle"|"line"|"text"|"path", ... }>. ALWAYS put vector shapes inside props.shapes, not in children. Use semantic paint roles: "accent", "neutral", "muted", "surface", "danger", "highlight", or "none") 3. Safety: No arbitrary HTML, scripts, CSS strings, or execution handlers. -4. Deterministic computation: If values are computed (e.g. loan payments, bill splits, taxes), DO NOT author arithmetic in text. Use named host compute functions (e.g. "billSplit", "savingsGrowth", "sum") with input bindings. +4. Deterministic computation: If values are computed (e.g. loan payments, bill splits, taxes), DO NOT author arithmetic in text and DO NOT require a button click. Attach a REACTIVE "compute" to a CONTAINER node (Stack/Section/Form) so results update live as inputs change. Shape: "compute": { "name", "inputs": { fnInput: "stateKey" }, "output": "resultKey" }. Only use functions from this list — inventing a name means nothing computes: +${describeComputeForPrompt()} + Wiring rules: + - Give every input (NumberInput/Slider/etc.) an explicit "bind": { "value": "someKey" } and set its starting number via props.value. Reference that SAME "someKey" in the compute inputs. + - A compute output that is an object is written at output + flattened to the root. A Metric shows a computed number via "bind": { "value": "resultKey.fieldName" } (e.g. output "loan" -> Metric bind.value "loan.monthlyPayment"). Do not put the number in props.value. + - Button/label text goes in the "label" prop (not "text"). 5. Provide a meaningful, accessible "fallbackText" summarizing the interface for non-visual surfaces. 6. Set specVersion to "0.1", revision to 1, and provenance.driver to "live".` diff --git a/packages/ai-generative-ui/src/compute.ts b/packages/ai-generative-ui/src/compute.ts index fd38d6da33..994fb42259 100644 --- a/packages/ai-generative-ui/src/compute.ts +++ b/packages/ai-generative-ui/src/compute.ts @@ -1,6 +1,10 @@ /** * Open Generative UI Specification (Draft 0.1) * Host-side deterministic compute registry (§5.3). + * + * The host — never the model — owns arithmetic. The model binds inputs to a + * named function; the host evaluates it deterministically. This keeps numeric + * results correct and auditable regardless of which model composed the graph. */ export type ComputeFunction = ( @@ -8,15 +12,38 @@ export type ComputeFunction = ( state: Record, ) => Record | any +export interface ComputeDescriptor { + /** Human/model-readable summary of what the function returns. */ + description: string + /** Accepted input keys (first listed is canonical; the rest are aliases). */ + inputs: Array + /** Shape of the output: a scalar number, or an object with these keys. */ + output: string +} + +const round2 = (n: number) => Math.round(n * 100) / 100 + +/** Resolve the first defined value among a set of alias keys. */ +function pick(inputs: Record, keys: Array, fallback = 0): number { + for (const k of keys) { + if (inputs[k] !== undefined && inputs[k] !== null && inputs[k] !== '') { + return Number(inputs[k]) + } + } + return fallback +} + export class ComputeRegistry { private readonly functions = new Map() + private readonly descriptors = new Map() constructor() { this.registerDefaults() } - register(name: string, fn: ComputeFunction): this { + register(name: string, fn: ComputeFunction, descriptor?: ComputeDescriptor): this { this.functions.set(name, fn) + if (descriptor) this.descriptors.set(name, descriptor) return this } @@ -32,6 +59,11 @@ export class ComputeRegistry { return Array.from(this.functions.keys()) } + /** Catalog of registered functions, for prompting a model (§7.3). */ + describe(): Array<{ name: string } & ComputeDescriptor> { + return Array.from(this.descriptors.entries()).map(([name, d]) => ({ name, ...d })) + } + execute(name: string, inputs: Record, state: Record): any { const fn = this.functions.get(name) if (!fn) { @@ -42,60 +74,135 @@ export class ComputeRegistry { } private registerDefaults() { - // U4: Bill Splitter - this.register('billSplit', (inputs) => { - const subtotal = Number(inputs.subtotal ?? 0) - const tipPercent = Number(inputs.tipPercent ?? 18) - const splitCount = Math.max(1, Math.round(Number(inputs.splitCount ?? 1))) - - const tipAmount = Math.round(subtotal * (tipPercent / 100) * 100) / 100 - const total = Math.round((subtotal + tipAmount) * 100) / 100 - const perPerson = Math.round((total / splitCount) * 100) / 100 - + // U4: Bill splitter — subtotal + tip, divided by guests. + this.register( + 'billSplit', + (inputs) => { + const subtotal = pick(inputs, ['subtotal', 'amount', 'bill']) + const tipPercent = pick(inputs, ['tipPercent', 'tip', 'gratuity'], 18) + const splitCount = Math.max(1, Math.round(pick(inputs, ['splitCount', 'guests', 'people'], 1))) + const tipAmount = round2(subtotal * (tipPercent / 100)) + const total = round2(subtotal + tipAmount) + return { + subtotal, + tipPercent, + splitCount, + tipAmount, + total, + perPerson: round2(total / splitCount), + } + }, + { + description: 'Split a bill: applies a tip percentage to a subtotal and divides across guests.', + inputs: ['subtotal', 'tipPercent', 'splitCount'], + output: '{ subtotal, tipPercent, splitCount, tipAmount, total, perPerson }', + }, + ) + + // U4: Fixed-rate loan / mortgage — standard amortized monthly payment. + const loanPaymentFn: ComputeFunction = (inputs) => { + const principal = pick(inputs, ['loanAmount', 'principal', 'amount']) + const annualRatePct = pick(inputs, ['annualInterestRate', 'interestRate', 'rate', 'apr']) + const termYears = pick(inputs, ['termYears', 'years', 'term', 'loanTerm'], 30) + const monthlyRate = annualRatePct / 100 / 12 + const months = Math.max(1, Math.round(termYears * 12)) + const monthlyPayment = + monthlyRate === 0 + ? principal / months + : (principal * monthlyRate) / (1 - Math.pow(1 + monthlyRate, -months)) + const totalPaid = monthlyPayment * months return { - subtotal, - tipPercent, - splitCount, - tipAmount, - total, - perPerson, + monthlyPayment: round2(monthlyPayment), + totalPaid: round2(totalPaid), + totalInterest: round2(totalPaid - principal), + months, } - }) - - // U4: Savings Growth - this.register('savingsGrowth', (inputs) => { - const principal = Number(inputs.principal ?? 1000) - const annualRate = Number(inputs.annualRate ?? 5) / 100 - const years = Number(inputs.years ?? 5) - const monthlyContribution = Number(inputs.monthlyContribution ?? 100) - - const monthlyRate = annualRate / 12 - const months = years * 12 - - let futureValue = principal - let totalContributions = principal - - for (let m = 0; m < months; m++) { - futureValue = (futureValue + monthlyContribution) * (1 + monthlyRate) - totalContributions += monthlyContribution - } - - futureValue = Math.round(futureValue * 100) / 100 - const totalInterest = Math.round((futureValue - totalContributions) * 100) / 100 - - return { - futureValue, - totalContributions, - totalInterest, - } - }) - - // U7: Sum / Aggregates - this.register('sum', (inputs) => { - const items = Array.isArray(inputs.items) ? inputs.items : [] - return items.reduce((acc, curr) => acc + (Number(curr) || 0), 0) - }) + } + const loanPaymentDesc: ComputeDescriptor = { + description: 'Monthly payment for a fixed-rate amortized loan or mortgage.', + inputs: ['loanAmount', 'annualInterestRate', 'termYears'], + output: '{ monthlyPayment, totalPaid, totalInterest, months }', + } + this.register('loanPayment', loanPaymentFn, loanPaymentDesc) + + // U4: Savings growth — compound interest with monthly contributions. + this.register( + 'savingsGrowth', + (inputs) => { + const principal = pick(inputs, ['principal', 'initial', 'startingBalance'], 1000) + const annualRate = pick(inputs, ['annualRate', 'rate', 'apy'], 5) / 100 + const years = pick(inputs, ['years', 'term'], 5) + const monthlyContribution = pick(inputs, ['monthlyContribution', 'contribution', 'monthly'], 100) + const monthlyRate = annualRate / 12 + const months = Math.max(0, Math.round(years * 12)) + let futureValue = principal + let totalContributions = principal + for (let m = 0; m < months; m++) { + futureValue = (futureValue + monthlyContribution) * (1 + monthlyRate) + totalContributions += monthlyContribution + } + return { + futureValue: round2(futureValue), + totalContributions: round2(totalContributions), + totalInterest: round2(futureValue - totalContributions), + } + }, + { + description: 'Future value of savings with compound interest and recurring monthly contributions.', + inputs: ['principal', 'annualRate', 'years', 'monthlyContribution'], + output: '{ futureValue, totalContributions, totalInterest }', + }, + ) + + // Generic: percentage of a value (tax, discount, commission, etc.). + this.register( + 'percentOf', + (inputs) => { + const value = pick(inputs, ['value', 'amount', 'base']) + const percent = pick(inputs, ['percent', 'rate', 'pct']) + const amount = round2(value * (percent / 100)) + return { amount, total: round2(value + amount), base: value, percent } + }, + { + description: 'Apply a percentage to a value — tax, discount, tip, or commission.', + inputs: ['value', 'percent'], + output: '{ amount, total, base, percent }', + }, + ) + + // U7: Sum / aggregate a list of numbers. + this.register( + 'sum', + (inputs) => { + const items = Array.isArray(inputs.items) ? inputs.items : [] + const total = items.reduce((acc, curr) => acc + (Number(curr) || 0), 0) + return round2(total) + }, + { + description: 'Sum an array of numbers. Pass the array as the "items" input.', + inputs: ['items'], + output: 'number (the total)', + }, + ) + + // Aliases so a model can use the obvious name and still hit a real function. + this.register('mortgage', loanPaymentFn, loanPaymentDesc) + this.register('calculateMortgage', loanPaymentFn, loanPaymentDesc) } } export const defaultComputeRegistry = new ComputeRegistry() + +/** + * Render the registry's catalog as a prompt fragment, so a live model binds + * inputs to functions that actually exist on the host (§7.3). Without this the + * model invents plausible names (e.g. "calculateMortgage") that never execute. + */ +export function describeComputeForPrompt( + registry: ComputeRegistry = defaultComputeRegistry, +): string { + const lines = registry.describe().map((f) => { + return `- "${f.name}"(${f.inputs.join(', ')}) -> ${f.output}: ${f.description}` + }) + return lines.join('\n') +} diff --git a/packages/ai-generative-ui/src/fixtures/index.ts b/packages/ai-generative-ui/src/fixtures/index.ts index 36d89758a6..59e4e3ac24 100644 --- a/packages/ai-generative-ui/src/fixtures/index.ts +++ b/packages/ai-generative-ui/src/fixtures/index.ts @@ -608,6 +608,93 @@ export const chordDiagramEmaj7Fixture: GraphDocument = { }, } +/** + * Live-generation shape regression fixture. + * + * A real model emitted this: inputs are addressed by node id with NO `bind`, + * checklist items have NO `id`, and sections title themselves via `label` + * rather than `title`. The host renderer must still make the stepper and + * checklist interactive (implicit `.value` / `.state`) and show + * the section headings. Kept verbatim as a guard against regressing that path. + */ +export const recipeCardFixture: GraphDocument = { + specVersion: '0.1', + graphId: 'recipe-card-lava', + revision: 1, + title: 'Chocolate Lava Cake Recipe', + fallbackText: + 'Interactive recipe card for chocolate lava cake. Includes an ingredient checklist, numbered preparation steps, and a stepper to adjust serving size.', + provenance: { driver: 'fixture', sourceId: 'recipe-card' }, + root: { + id: 'lavaCakeRecipe', + type: 'Stack', + props: { gap: 'md' }, + children: [ + { + id: 'ingredientSection', + type: 'Section', + props: { label: 'Ingredients' }, + children: [ + { + id: 'ingredientChecklist', + type: 'Checklist', + props: { + items: [ + { label: '1/2 cup (1 stick) unsalted butter' }, + { label: '1 cup bittersweet chocolate chips or chopped chocolate' }, + { label: "1 cup confectioners' sugar" }, + { label: '2 large eggs' }, + { label: '2 large egg yolks' }, + { label: '1 teaspoon vanilla extract' }, + { label: '1/4 cup all-purpose flour' }, + { label: '1/4 teaspoon salt' }, + { label: 'Optional: Powdered sugar or ice cream for serving' }, + ], + }, + }, + ], + }, + { + id: 'prepStepsSection', + type: 'Section', + props: { label: 'Preparation Steps' }, + children: [ + { + id: 'prepSteps', + type: 'List', + props: { + ordered: true, + items: [ + 'Preheat the oven to 425°F (220°C) and butter four ramekins.', + 'Melt the butter with the chocolate in a double boiler or microwave until smooth.', + 'Stir in the sugar until combined.', + 'Add the eggs and egg yolks; mix well.', + 'Stir in the vanilla extract.', + 'Gently fold in the flour and salt until just combined.', + 'Divide the batter evenly among the prepared ramekins.', + 'Place ramekins on a baking sheet and bake for 12-14 minutes until the edges are firm but the center is still soft.', + 'Let the cakes cool for 1 minute before serving. Optionally, serve with a dusting of powdered sugar or a scoop of ice cream.', + ], + }, + }, + ], + }, + { + id: 'servingSizeSection', + type: 'Section', + props: { label: 'Serving Size' }, + children: [ + { + id: 'servingSizeStepper', + type: 'Stepper', + props: { label: 'Adjust Servings', value: 4, min: 1, max: 12, step: 1 }, + }, + ], + }, + ], + }, +} + export const CANONICAL_FIXTURES: Record = { 'compare-plans-basic': comparePlansBasicFixture, 'bicycle-drivetrain': bicycleDrivetrainFixture, @@ -618,4 +705,5 @@ export const CANONICAL_FIXTURES: Record = { 'spend-monitor': spendMonitorFixture, 'trip-stops': tripStopsFixture, 'chord-diagram-emaj7': chordDiagramEmaj7Fixture, + 'recipe-card': recipeCardFixture, } diff --git a/packages/ai-generative-ui/src/react/index.tsx b/packages/ai-generative-ui/src/react/index.tsx index 9092dc9615..3d08e9d560 100644 --- a/packages/ai-generative-ui/src/react/index.tsx +++ b/packages/ai-generative-ui/src/react/index.tsx @@ -61,6 +61,52 @@ function setStatePath(obj: Record, path: string, value: unknown return copy } +/** + * The state path an input reads and writes: an explicit `bind.value`, or the + * implicit `.value` convention a live model naturally reaches for + * (it references an input by id, e.g. compute input "loanAmountInput.value"). + */ +function valueBindPath(node: Pick): string { + const explicit = node.bind?.value + return typeof explicit === 'string' && explicit ? explicit : `${node.id}.value` +} + +/** + * Walk the graph once and seed state from inputs that declare an initial value + * in props. Without this, a value shown in an input (e.g. a prefilled loan + * amount) would never reach a bound compute on first render. + */ +function collectInitialBoundState( + node: UINode, + acc: Record, +): Record { + let next = acc + const p = node.props ?? {} + const seed = p.value ?? p.defaultValue ?? p.default + if (seed !== undefined) { + const path = valueBindPath(node) + if (resolveStatePath(next, path) === undefined) next = setStatePath(next, path, seed) + } + // Checklist: seed its state object from each item's defaultDone. Mirrors the + // renderer's implicit `${id}.state` path and index-based item keys. + if (node.type === 'Checklist') { + const statePath = (typeof node.bind?.state === 'string' && node.bind.state) || `${node.id}.state` + const items = node.props?.items + if (Array.isArray(items) && resolveStatePath(next, statePath) === undefined) { + const checklistState: Record = {} + items.forEach((item, idx) => { + const key = item && typeof item === 'object' && typeof item.id === 'string' ? item.id : String(idx) + checklistState[key] = Boolean(item?.defaultDone) + }) + if (Object.keys(checklistState).length > 0) next = setStatePath(next, statePath, checklistState) + } + } + node.children?.forEach((child) => { + next = collectInitialBoundState(child, next) + }) + return next +} + export function GenerativeUIRenderer({ graph, onIntent, @@ -77,7 +123,10 @@ export function GenerativeUIRenderer({ if (s.splitCount === undefined) s.splitCount = 4 if (s.spiceLevel === undefined) s.spiceLevel = 3 if (s.cuisine === undefined) s.cuisine = 'italian' - return s + // Seed state from the graph's own declared input defaults so computes have + // real inputs on first render (a live model often sets an input's initial + // value via props AND binds it to a state key). + return collectInitialBoundState(graph.root, s) }) // Recompute any bound compute nodes whenever state changes @@ -93,11 +142,17 @@ export function GenerativeUIRenderer({ const result = computeRegistry.execute(name, resolvedInputs, state) if (result !== undefined) { setState((prev) => { - const currentVal = resolveStatePath(prev, output) - if (JSON.stringify(currentVal) !== JSON.stringify(result)) { - return setStatePath(prev, output, result) + // Write the result at the declared output path (fixtures bind to + // e.g. "computedBill.perPerson"), and also flatten object keys to + // the state root so scalar-style bindings a live model may emit + // (e.g. output "monthlyPayment") resolve too. + let next = setStatePath(prev, output, result) + if (result && typeof result === 'object' && !Array.isArray(result)) { + for (const [k, v] of Object.entries(result)) { + next = setStatePath(next, k, v) + } } - return prev + return JSON.stringify(next) === JSON.stringify(prev) ? prev : next }) } } @@ -130,17 +185,20 @@ export function GenerativeUIRenderer({ return (
-
-
-

{graph.title}

- Spec v{graph.specVersion} • Rev {graph.revision} +
+
+

{graph.title}

+ + + Spec v{graph.specVersion} · Rev {graph.revision} +
-
+
-

{resolvedProps.title}

- {resolvedProps.subtitle &&

{resolvedProps.subtitle}

} -
+
+ {(cleanTitle || recommended) && ( +
+ {cleanTitle && ( +

{cleanTitle}

+ )} + {recommended && ( + + ★ Recommended + + )} +
+ )} + {resolvedProps.subtitle &&

{resolvedProps.subtitle}

} +
{children?.map((child) => ( - {children?.map((child, idx) => ( -
- - -
- ))} +
+ {steps.map((child, idx) => { + const stepNum = idx + 1 + const done = stepNum < currentStep + const current = stepNum === currentStep + const isLast = idx === steps.length - 1 + return ( +
+ {!isLast && ( + + )} + + {done ? '✓' : stepNum} + +
+ +
+
+ ) + })}
) } @@ -309,35 +408,47 @@ function NodeRenderer({ node, state, onStateChange, onFireIntent, computeRegistr // 6.2 Content Components if (type === 'Text') { const variant = resolvedProps.variant || 'body' + const content = resolvedProps.content ?? resolvedProps.text ?? resolvedProps.value ?? '' if (variant === 'heading') { - return

{resolvedProps.content}

+ return

{content}

} if (variant === 'caption') { - return

{resolvedProps.content}

+ return

{content}

} if (variant === 'code') { return ( - 
-          {resolvedProps.content}
+        
+          {content}
         

       )
     }
-    return 
{resolvedProps.content}

+    return 
{content}

   }
 
   if (type === 'Callout') {
     const tone = resolvedProps.tone || 'info'
     const colors =
       tone === 'warning'
-        ? 'bg-amber-50 border-amber-200 text-amber-900'
+        ? 'bg-amber-50/80 border-amber-200/80 text-amber-900'
         : tone === 'danger'
-          ? 'bg-red-50 border-red-200 text-red-900'
-          : 'bg-blue-50 border-blue-200 text-blue-900'
+          ? 'bg-rose-50/80 border-rose-200/80 text-rose-900'
+          : tone === 'success'
+            ? 'bg-emerald-50/80 border-emerald-200/80 text-emerald-900'
+            : 'bg-blue-50/70 border-blue-200/70 text-blue-900'
+    const accent =
+      tone === 'warning' ? 'bg-amber-400' : tone === 'danger' ? 'bg-rose-500' : tone === 'success' ? 'bg-emerald-500' : 'bg-blue-500'
+    const icon = tone === 'warning' ? '!' : tone === 'danger' ? '✕' : tone === 'success' ? '✓' : 'i'
 
     return (
-      

-        {resolvedProps.title && 
{resolvedProps.title}
}
-        
{resolvedProps.text}

+      

+        
+        
+          {icon}
+        
+        

+          {resolvedProps.title && 
{resolvedProps.title}
}
+          
{resolvedProps.text ?? resolvedProps.content ?? resolvedProps.body}

+        

       

     )
   }
@@ -349,16 +460,33 @@ function NodeRenderer({ node, state, onStateChange, onFireIntent, computeRegistr
         : resolvedProps.tone === 'negative'
           ? 'text-rose-600'
           : 'text-slate-900'
-
+    const deltaStr =
+      resolvedProps.delta != null ? String(resolvedProps.delta).trim() : ''
+    const deltaUp = deltaStr.startsWith('+')
+    const deltaDown = deltaStr.startsWith('-')
+    const deltaClass = deltaUp
+      ? 'text-emerald-700 bg-emerald-50 border-emerald-200/70'
+      : deltaDown
+        ? 'text-rose-700 bg-rose-50 border-rose-200/70'
+        : 'text-slate-500 bg-slate-50 border-slate-200/70'
+
+    const metricValue = resolvedProps.value ?? resolvedProps.content ?? resolvedProps.text
+    const hasValue = metricValue !== undefined && metricValue !== null && metricValue !== ''
     return (
-      

-        
{resolvedProps.label}

-        

-          {resolvedProps.value}
-          {resolvedProps.unit && {resolvedProps.unit}}
+      

+        
{resolvedProps.label}

+        

+          {hasValue ? metricValue : '—'}
+          {hasValue && resolvedProps.unit && {resolvedProps.unit}}
         

-        {resolvedProps.delta && (
-          
{resolvedProps.delta}

+        {deltaStr && (
+          

+            
+              {deltaUp && ▲}
+              {deltaDown && ▼}
+              {deltaStr}
+            
+          

         )}
       

     )
@@ -369,22 +497,22 @@ function NodeRenderer({ node, state, onStateChange, onFireIntent, computeRegistr
     const rows: Array> = (resolvedProps.rows as Array>) || []
 
     return (
-      

-        
-          
-            
+      

+        

+          
+            
               {columns.map((col) => (
-                
               ))}
             
-          
+          
             {rows.map((row, i) => (
-              
-                {columns.map((col) => (
-                  
+                {columns.map((col, ci) => (
+                  
                 ))}
@@ -422,41 +550,176 @@ function NodeRenderer({ node, state, onStateChange, onFireIntent, computeRegistr
   if (type === 'Chart') {
     const data: Array<{ label: string; value: number }> = resolvedProps.data || []
     const maxValue = Math.max(...data.map((d) => d.value), 1)
+    const PLOT_HEIGHT = 156
 
     return (
-      

-        {resolvedProps.title && 
{resolvedProps.title}
}
-        

+      

+        {resolvedProps.title && (
+          

+            {resolvedProps.title}
+          

+        )}
+        

           {data.map((d, i) => {
-            const heightPercent = Math.round((d.value / maxValue) * 100)
+            const barHeight = Math.max(4, Math.round((d.value / maxValue) * (PLOT_HEIGHT - 24)))
             return (
-              

-                {d.value}
+              

+                
+                  {d.value}
+                
                 

-                {d.label}
               

             )
           })}
         

-      

+        

+          {data.map((d, i) => (
+            
+              {d.label}
+            
+          ))}
+        

+      

     )
   }
 
   if (type === 'Map') {
-    const markers: Array<{ id: string; label: string; note?: string }> = resolvedProps.markers || []
+    const markers: Array<{ id: string; label: string; note?: string; lat?: number; lng?: number }> =
+      resolvedProps.markers || []
+    const geoMarkers = markers.filter(
+      (m) => typeof m.lat === 'number' && typeof m.lng === 'number',
+    )
+    const selectedMarker = bind.selected ? resolveStatePath(state, bind.selected) : undefined
+
+    const selectMarker = (m: { id: string; label: string }) => {
+      if (bind.selected) onStateChange(bind.selected, m.id)
+      onFireIntent(id, 'selectStop', 'local', { stopId: m.id, label: m.label })
+    }
+
+    // Project lat/lng into an SVG viewport when we have real coordinates.
+    const W = 560
+    const H = 280
+    const PAD = 42
+    let projected: Array<{ m: (typeof markers)[number]; x: number; y: number }> = []
+    if (geoMarkers.length > 0) {
+      const lats = geoMarkers.map((m) => m.lat as number)
+      const lngs = geoMarkers.map((m) => m.lng as number)
+      const minLat = Math.min(...lats)
+      const maxLat = Math.max(...lats)
+      const minLng = Math.min(...lngs)
+      const maxLng = Math.max(...lngs)
+      const latSpan = maxLat - minLat || 0.02
+      const lngSpan = maxLng - minLng || 0.02
+      projected = geoMarkers.map((m) => ({
+        m,
+        x: PAD + ((((m.lng as number) - minLng) / lngSpan) * (W - 2 * PAD)),
+        y: PAD + ((((maxLat - (m.lat as number)) / latSpan) * (H - 2 * PAD))),
+      }))
+    }
+    const routeD = projected
+      .map((p, i) => `${i === 0 ? 'M' : 'L'} ${p.x.toFixed(1)} ${p.y.toFixed(1)}`)
+      .join(' ')
+
     return (
-      

-        
📍 Geographic Points of Interest

-        

-          {markers.map((m) => (
-            

-              
📌 {m.label}

-              {m.note && 
{m.note}
}
+      

+        {projected.length > 0 ? (
+          

+            
+              
+                
+                  
+                
+                
+                  
+                  
+                
+              
+              
+              
+              {routeD && (
+                
+              )}
+              {projected.map((p, i) => {
+                const isSel = selectedMarker === p.m.id
+                return (
+                   selectMarker(p.m)}
+                    onKeyDown={(e) => {
+                      if (e.key === 'Enter' || e.key === ' ') {
+                        e.preventDefault()
+                        selectMarker(p.m)
+                      }
+                    }}
+                  >
+                    
+                    
+                    
+                      {i + 1}
+                    
+                  
+                )
+              })}
+            
+            

+              {resolvedProps.title || 'Route Map'}
             

-          ))}
+          

+        ) : null}
+        

+          {markers.map((m, i) => {
+            const isSel = selectedMarker === m.id
+            return (
+              
+            )
+          })}
         

       

     )
@@ -464,23 +727,24 @@ function NodeRenderer({ node, state, onStateChange, onFireIntent, computeRegistr
 
   if (type === 'Diagram') {
     const hotspots: Array<{ id: string; label: string; explanation: string }> = resolvedProps.hotspots || []
-    const selectedPart = bind.selected ? resolveStatePath(state, bind.selected) : undefined
+    const selectedPath = (typeof bind.selected === 'string' && bind.selected) || `${id}.selected`
+    const selectedPart = resolveStatePath(state, selectedPath)
 
     return (
-      

+      

         {resolvedProps.title && 
{resolvedProps.title}
}
         

           {hotspots.map((spot) => (
             
-          {val}
+          {val}
           
@@ -659,17 +932,16 @@ function NodeRenderer({ node, state, onStateChange, onFireIntent, computeRegistr
   }
 
   if (type === 'Toggle' || type === 'Checkbox') {
-    const isChecked = bind.value ? Boolean(resolveStatePath(state, bind.value)) : false
+    const vp = valueBindPath(node)
+    const isChecked = Boolean(resolveStatePath(state, vp))
 
     return (
-      

+                
                   {col.label}
                 

           

+              

                     {String(row[col.key] ?? '')}