diff --git a/AGENTS.md b/AGENTS.md index fd0373eb..2479cbba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,8 +28,8 @@ This file provides high-signal context, architectural invariants, and authoring | Goal | Element | Recommended Bundle | Enforced Budget | | :--- | :--- | :--- | :---: | -| `.glb` / `.usdz` 3D viewer + 1-tap AR handoff (no scripting) | `` | `three-live-viewer.min.js` | `< 31 KB gz` | -| Procedural ` @@ -191,7 +191,8 @@ talks, and your code model writes the edit through an `edit_scene` tool, so edit quality never depends on the voice model. ```html - @@ -256,7 +257,7 @@ material preset. The preset is the offline fallback, so the scene always renders the same way without a network: ```html - + + "#; + let (_, ok_diags) = validate_scene_source(ok_html, "ok.html"); + assert!( + !ok_diags.iter().any(|d| d.code == "TL401" || d.code == "TL403"), + "Expected zero TL401/TL403 diagnostics when capabilities are granted, got: {:?}", + ok_diags + ); + + let missing_html = r#" + + + + "#; + let (_, diags) = validate_scene_source(missing_html, "missing.html"); + assert!(!diags.iter().any(|d| d.code == "TL401")); + let tl403: Vec<&str> = + diags.iter().filter(|d| d.code == "TL403").map(|d| d.message.as_str()).collect(); + assert!(tl403.iter().any(|m| m.contains("xr-hands"))); + assert!(tl403.iter().any(|m| m.contains("shared-room"))); + assert!(tl403.iter().any(|m| m.contains("persistent-anchors"))); + assert!(tl403.iter().any(|m| m.contains("userData.xr.splatUrl"))); + assert!(tl403.iter().any(|m| m.contains("userData.xr.audio.src"))); + assert!(tl403.iter().any(|m| m.contains("userData.xr.holdToAsk"))); + + // Audit regressions: `texture-pack`, `xr-mode="vr"`, `allow-client-scripts` + `-scripts` + // negation, and multi-element HTML capability isolation. + let lockdown_html = r#" + + + + "#; + let (_, lock_diags) = validate_scene_source(lockdown_html, "lockdown.html"); + assert!( + lock_diags.iter().any(|d| d.code == "TL401"), + "capabilities=\"none\" must override allow-client-scripts and emit TL401" + ); + assert!(lock_diags.iter().any(|d| d.code == "TL403" && d.message.contains("texture-pack"))); + assert!(lock_diags + .iter() + .any(|d| d.code == "TL403" && d.message.contains("xr-mode=\"vr\""))); + + let multi_html = r#" + + + + + + + "#; + let (_, multi_diags) = validate_scene_source(multi_html, "multi.html"); + assert!( + multi_diags + .iter() + .any(|d| d.code == "TL403" && d.message.contains("RestrictedSplat")), + "Restricted sibling element must still receive TL403 even when another element has capabilities=\"all\"" + ); + assert!(!multi_diags.iter().any(|d| d.code == "TL403" && d.message.contains("AllowedBox"))); + } } diff --git a/docs/llm-guide.md b/docs/llm-guide.md index ebe0f5fd..c6728fd5 100644 --- a/docs/llm-guide.md +++ b/docs/llm-guide.md @@ -42,7 +42,7 @@ You are authoring 3D and Spatial Computing scenes using Google's `three-live` an ### 2. Procedural ` - + ``` ## Bundles diff --git a/three-live/examples/index.html b/three-live/examples/index.html index dadfd191..86a05d12 100644 --- a/three-live/examples/index.html +++ b/three-live/examples/index.html @@ -744,7 +744,7 @@

three-live AI-First Native 3D & AR

- 77.9 KB gz + 80.9 KB gz Zero Dependencies WebGL2 PBR Apple USDZ + WebXR @@ -769,7 +769,7 @@

three-live AI-First Native 3D & AR

Connect AI Provider (Bring Your Own Account) let voiceStopHandle = null; // Storage helpers with window.name fallback so saved keys survive refreshes - // on sandboxed hosts like x20web.corp.google.com (Content-Security-Policy: sandbox allow-scripts). + // on opaque-origin sandboxed static hosts (Content-Security-Policy: sandbox allow-scripts). function readWindowNameStore() { try { const raw = window.name || ""; diff --git a/three-live/src/capabilities.ts b/three-live/src/capabilities.ts new file mode 100644 index 00000000..5ed55d71 --- /dev/null +++ b/three-live/src/capabilities.ts @@ -0,0 +1,534 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import {getSharedRegistry} from './plugin'; + +/** Canonical capability names shared across Web, Android, and iOS. */ +export type ThreeLiveCapabilityName = + | 'scripts' + | 'network' + | 'remote_content' + | 'camera' + | 'microphone' + | 'sensors' + | 'storage' + | 'ai' + | 'external_handoff'; + +/** Network scheme and optional origin allowlist configuration. */ +export interface ThreeLiveNetworkPolicy { + readonly enabled: boolean; + readonly schemes: ReadonlySet; + readonly explicitSchemes: boolean; + readonly origins: ReadonlySet | null; +} + +/** Normalized capability policy evaluated at each sensitive access point. */ +export interface ThreeLiveCapabilities { + readonly scripts: boolean; + readonly network: ThreeLiveNetworkPolicy; + readonly remoteContent: boolean; + readonly camera: boolean; + readonly microphone: boolean; + readonly sensors: boolean; + readonly storage: boolean; + readonly ai: boolean; + readonly externalHandoff: boolean; +} + +/** Flexible input accepted by `el.capabilities` and `ThreeLiveElement.defaultCapabilities`. */ +export interface ThreeLiveCapabilitiesInit { + readonly scripts?: boolean; + readonly network?: + | boolean + | { + readonly enabled?: boolean; + readonly schemes?: Iterable; + readonly explicitSchemes?: boolean; + readonly origins?: Iterable | null; + }; + readonly remoteContent?: boolean; + readonly remote_content?: boolean; + readonly camera?: boolean; + readonly microphone?: boolean; + readonly sensors?: boolean; + readonly storage?: boolean; + readonly ai?: boolean; + readonly externalHandoff?: boolean; + readonly external_handoff?: boolean; +} + +/** Non-nullable input accepted by {@link parseCapabilities} and `el.capabilities`. */ +export type ThreeLiveCapabilitiesInput = + | string + | ThreeLiveCapabilities + | ThreeLiveCapabilitiesInit; + +/** Payload delivered to `three-capability-denied` events and `onCapabilityDenied` callbacks. */ +export interface CapabilityDeniedDetail { + readonly capability: ThreeLiveCapabilityName; + readonly feature: string; + readonly message: string; + readonly url?: string; +} + +type BoolCapKey = Exclude; + +const DEFAULT_SCHEMES: ReadonlySet = new Set(['https', 'wss']); +const NON_NET_RE = /^(?:data|blob|asset|content|file|android\.resource):/i; +const LOOPBACK_RE = /^(?:localhost|127\.0\.0\.1|\[::1\])$/i; +const BOOL_KEYS: readonly BoolCapKey[] = [ + 'scripts', + 'remoteContent', + 'camera', + 'microphone', + 'sensors', + 'storage', + 'ai', + 'externalHandoff', +]; +const WIRE_TO_KEY: Readonly> = { + scripts: 'scripts', + remote_content: 'remoteContent', + camera: 'camera', + microphone: 'microphone', + sensors: 'sensors', + storage: 'storage', + ai: 'ai', + external_handoff: 'externalHandoff', +}; + +/** Normalizes an origin string to `scheme://host[:port]`. */ +export function normalizeCapabilityOrigin(raw: string): string | null { + try { + const u = new URL(raw.trim()); + const s = u.protocol.slice(0, -1).toLowerCase(); + if (!s || !u.hostname) return null; + const p = + ((s === 'https' || s === 'wss') && u.port === '443') || + ((s === 'http' || s === 'ws') && u.port === '80') + ? '' + : u.port; + return `${s}://${u.hostname.toLowerCase()}${p ? ':' + p : ''}`; + } catch { + return null; + } +} + +/** Sanitizes a URL for diagnostics by stripping credentials, query strings, and hashes. */ +export function sanitizeDiagnosticUrl(raw: string): string { + const full = raw.trim().slice(0, 8192); + if (NON_NET_RE.test(full)) { + return `${full.slice(0, full.indexOf(':') + 1)}`; + } + const basePart = full.split(/[?#]/)[0]; + const baseHref = + (typeof location !== 'undefined' && location.href) || 'https://localhost/'; + for (const candidate of [full, basePart]) { + try { + const u = new URL(candidate, baseHref); + u.username = u.password = u.search = u.hash = ''; + return u.href.slice(0, 256); + } catch { + // Fall through to credential-stripping fallback. + } + } + return basePart + .replace(/\/\/[^/@\s]*@/, '//') + .replace(/^([a-z][a-z0-9+.-]*:\/\/)[^/\s:@]+:[^/\s@]*/i, '$1') + .slice(0, 128); +} + +function makeNet( + enabled: boolean, + schemes: ReadonlySet = DEFAULT_SCHEMES, + explicitSchemes = false, + origins: ReadonlySet | null = null, +): ThreeLiveNetworkPolicy { + return Object.freeze({enabled, schemes, explicitSchemes, origins}); +} + +/** Default secure policy: all capabilities disabled. */ +export const CAPABILITIES_NONE: ThreeLiveCapabilities = Object.freeze({ + scripts: false, + network: makeNet(false), + remoteContent: false, + camera: false, + microphone: false, + sensors: false, + storage: false, + ai: false, + externalHandoff: false, +}); + +/** Preset for static `.glb` / `.usdz` viewing and 1-tap AR (`viewer`). */ +export const CAPABILITIES_VIEWER: ThreeLiveCapabilities = Object.freeze({ + ...CAPABILITIES_NONE, + network: makeNet(true), + remoteContent: true, + camera: true, + externalHandoff: true, +}); + +/** Preset enabling all capabilities (`all`), intended for trusted demos and playgrounds. */ +export const CAPABILITIES_ALL: ThreeLiveCapabilities = Object.freeze({ + scripts: true, + network: makeNet(true), + remoteContent: true, + camera: true, + microphone: true, + sensors: true, + storage: true, + ai: true, + externalHandoff: true, +}); + +/** Parses an attribute string or init object into a normalized {@link ThreeLiveCapabilities}. */ +export function parseCapabilities( + input: ThreeLiveCapabilitiesInput | null | undefined, + base: ThreeLiveCapabilities = CAPABILITIES_NONE, +): ThreeLiveCapabilities { + if (input == null) return base; + if (typeof input === 'string') { + const trimmed = input.trim(); + if (!trimmed) return CAPABILITIES_NONE; + const flags: Record = { + scripts: base.scripts, + remoteContent: base.remoteContent, + camera: base.camera, + microphone: base.microphone, + sensors: base.sensors, + storage: base.storage, + ai: base.ai, + externalHandoff: base.externalHandoff, + }; + let netOn = base.network.enabled; + let schemes = new Set(base.network.schemes); + let explicit = base.network.explicitSchemes; + let origins = base.network.origins ? new Set(base.network.origins) : null; + let originsSpecified = false; + + for (const raw of trimmed.split(/\s+/)) { + if (!raw) continue; + const neg = raw.length > 1 && raw[0] === '-'; + const tok = neg ? raw.slice(1) : raw; + const val = !neg; + const [head, originPart = ''] = tok.split('@'); + const [rawName, schemePart = ''] = head.toLowerCase().split(':'); + const name = rawName.replace(/-/g, '_'); + if (name === 'none' && val) { + for (const k of BOOL_KEYS) flags[k] = false; + netOn = explicit = false; + schemes = new Set(DEFAULT_SCHEMES); + origins = null; + originsSpecified = false; + } else if (name === 'all' || name === '*') { + for (const k of BOOL_KEYS) flags[k] = val; + netOn = val; + explicit = false; + schemes = new Set(DEFAULT_SCHEMES); + origins = null; + originsSpecified = false; + } else if (name === 'viewer') { + netOn = + flags.remoteContent = + flags.camera = + flags.externalHandoff = + val; + } else if (name === 'network') { + netOn = val; + if (val && schemePart) { + const list = schemePart + .split(/[|,]/) + .map((s) => s.trim().replace(/:$/, '')) + .filter(Boolean); + if (list.length) { + schemes = new Set(list); + explicit = true; + } + } else if (val && !originPart) { + schemes = new Set(DEFAULT_SCHEMES); + explicit = false; + origins = null; + originsSpecified = false; + } + if (val && originPart) { + if (!originsSpecified) { + origins = new Set(); + originsSpecified = true; + } + origins ??= new Set(); + for (const o of originPart.split(/[|,]/)) { + const norm = normalizeCapabilityOrigin(o); + if (norm) origins.add(norm); + } + } + } else if (name in WIRE_TO_KEY) { + flags[WIRE_TO_KEY[name]] = val; + } + } + return { + ...flags, + network: makeNet(netOn, schemes, explicit, origins), + }; + } + + const init = input as ThreeLiveCapabilitiesInit; + let network = base.network; + if (typeof init.network === 'boolean') { + network = makeNet(init.network); + } else if (init.network && typeof init.network === 'object') { + const n = init.network; + const hasS = n.schemes != null; + const schemes = hasS + ? new Set( + Array.from(n.schemes!, (s) => + s.trim().replace(/:$/, '').toLowerCase(), + ).filter(Boolean), + ) + : new Set(base.network.schemes); + let origins = base.network.origins; + if (n.origins === null) { + origins = null; + } else if (n.origins !== undefined) { + const set = new Set(); + for (const o of n.origins) { + const norm = normalizeCapabilityOrigin(o); + if (norm) set.add(norm); + } + origins = set; + } + network = makeNet( + n.enabled ?? true, + schemes, + n.explicitSchemes ?? hasS, + origins, + ); + } + return { + scripts: init.scripts ?? base.scripts, + network, + remoteContent: + init.remoteContent ?? init.remote_content ?? base.remoteContent, + camera: init.camera ?? base.camera, + microphone: init.microphone ?? base.microphone, + sensors: init.sensors ?? base.sensors, + storage: init.storage ?? base.storage, + ai: init.ai ?? base.ai, + externalHandoff: + init.externalHandoff ?? init.external_handoff ?? base.externalHandoff, + }; +} + +/** Returns true when `url` uses a non-network scheme (`data:`, `blob:`, `file:`, etc.). */ +export function isNonNetworkUrl(url: string): boolean { + return NON_NET_RE.test(url.trim()); +} + +/** Checks whether `caps.network` permits connecting to `rawUrl`. */ +export function isNetworkUrlPermitted( + caps: ThreeLiveCapabilities, + rawUrl?: string, +): boolean { + if (!caps.network.enabled) return false; + if (!rawUrl) return true; + const t = rawUrl.trim(); + if (!t) return false; + if (isNonNetworkUrl(t)) return true; + try { + const base = + (typeof document !== 'undefined' && document.baseURI) || + (typeof location !== 'undefined' && location.href) || + 'https://localhost/'; + const u = new URL(t, base); + const s = u.protocol.slice(0, -1).toLowerCase(); + if (isNonNetworkUrl(`${s}:`)) return true; + const loopback = + (s === 'http' || s === 'ws') && LOOPBACK_RE.test(u.hostname); + if ( + !caps.network.schemes.has(s) && + (caps.network.explicitSchemes || !loopback) + ) { + return false; + } + if (caps.network.origins !== null) { + const origin = normalizeCapabilityOrigin(u.href); + if (!origin || !caps.network.origins.has(origin)) return false; + } + return true; + } catch { + return false; + } +} + +/** Evaluates whether `caps` grants `capability` (and optional `url`). */ +export function hasCapabilityInPolicy( + caps: ThreeLiveCapabilities, + capability: ThreeLiveCapabilityName, + url?: string, +): boolean { + if (capability === 'network') return isNetworkUrlPermitted(caps, url); + if (capability === 'remote_content') { + return Boolean( + (url && isNonNetworkUrl(url)) || + (caps.remoteContent && isNetworkUrlPermitted(caps, url)), + ); + } + if (capability === 'ai') return caps.ai && isNetworkUrlPermitted(caps, url); + return caps[WIRE_TO_KEY[capability]]; +} + +interface CapabilityHostLike { + __tlEl?: boolean; + capabilities?: ThreeLiveCapabilitiesInput | null; + allowClientScripts?: boolean; + getAttribute?: (name: string) => string | null; + hasAttribute?: (name: string) => boolean; + dispatchEvent?: (event: Event) => boolean; + fireEvent?: (type: string, detail?: unknown) => void; + onCapabilityDenied?: ((detail: CapabilityDeniedDetail) => void) | null; + __tlCapsOverride?: ThreeLiveCapabilities | null; + __tlCapsInit?: ThreeLiveCapabilitiesInit | null; +} + +const deniedDedup = new WeakMap>(); + +/** Clears deduplicated denial records for `host`. */ +export function resetCapabilityDenials(host?: object | null): void { + if (host) deniedDedup.delete(host); +} + +/** Resolves the effective {@link ThreeLiveCapabilities} for `host` (or global default). */ +export function resolveHostCapabilities(host?: unknown): ThreeLiveCapabilities { + const reg = getSharedRegistry(); + let base = reg.defaultCapabilities ?? CAPABILITIES_NONE; + if (reg.allowClientScripts && !base.scripts) base = {...base, scripts: true}; + if (!host || typeof host !== 'object') return base; + const h = host as CapabilityHostLike; + const allowAttr = + typeof h.hasAttribute === 'function' && + h.hasAttribute('allow-client-scripts') && + h.getAttribute?.('allow-client-scripts') !== 'false'; + const allowScripts = + allowAttr || (!h.__tlEl && h.allowClientScripts === true); + const effectiveBase = + allowScripts && !base.scripts ? {...base, scripts: true} : base; + const capAttr = + typeof h.getAttribute === 'function' + ? h.getAttribute('capabilities') + : null; + if (h.__tlCapsInit) { + return parseCapabilities(h.__tlCapsInit, effectiveBase); + } + if (h.__tlCapsOverride) return h.__tlCapsOverride; + if ( + !h.__tlEl && + h.capabilities !== undefined && + typeof h.capabilities !== 'function' + ) { + return parseCapabilities(h.capabilities, effectiveBase); + } + return capAttr !== null + ? parseCapabilities(capAttr, effectiveBase) + : effectiveBase; +} + +function reportDenial( + host: unknown, + capability: ThreeLiveCapabilityName, + feature: string, + url?: string, + dedup = false, +): void { + const safeUrl = url ? sanitizeDiagnosticUrl(url) : undefined; + if (dedup && host && typeof host === 'object') { + let seen = deniedDedup.get(host); + if (!seen) deniedDedup.set(host, (seen = new Set())); + const key = `${capability}:${feature}:${safeUrl ?? ''}`; + if (seen.has(key)) return; + seen.add(key); + } + const message = + `[Three-Live] Capability '${capability}' denied for ${feature}` + + (safeUrl ? ` (${safeUrl})` : '') + + `. Grant '${capability}' in capabilities to enable.`; + const detail: CapabilityDeniedDetail = safeUrl + ? {capability, feature, message, url: safeUrl} + : {capability, feature, message}; + console?.warn?.(message); + getSharedRegistry().onCapabilityDenied?.(detail); + if (host && typeof host === 'object') { + const h = host as CapabilityHostLike; + h.onCapabilityDenied?.(detail); + if (typeof h.fireEvent === 'function') { + h.fireEvent('three-capability-denied', detail); + } else if ( + typeof h.dispatchEvent === 'function' && + typeof CustomEvent === 'function' + ) { + h.dispatchEvent( + new CustomEvent('three-capability-denied', { + bubbles: true, + composed: true, + detail, + }), + ); + } + } +} + +/** + * Checks `capability` at a sensitive point of access. Logs a diagnostic, + * dispatches `capability-denied`, and returns `false` without throwing. + */ +export function checkHostCapability( + host: unknown, + capability: ThreeLiveCapabilityName, + feature: string, + options?: {readonly url?: string; readonly dedup?: boolean}, +): boolean { + const caps = resolveHostCapabilities(host); + const url = options?.url; + const dedup = options?.dedup ?? false; + if (capability === 'remote_content') { + if (url && isNonNetworkUrl(url)) return true; + if (!caps.remoteContent) { + reportDenial(host, 'remote_content', feature, url, dedup); + return false; + } + if (!isNetworkUrlPermitted(caps, url)) { + reportDenial(host, 'network', feature, url, dedup); + return false; + } + return true; + } + if (capability === 'ai') { + if (!caps.ai) { + reportDenial(host, 'ai', feature, url, dedup); + return false; + } + if (!isNetworkUrlPermitted(caps, url)) { + reportDenial(host, 'network', feature, url, dedup); + return false; + } + return true; + } + if (!hasCapabilityInPolicy(caps, capability, url)) { + reportDenial(host, capability, feature, url, dedup); + return false; + } + return true; +} + +/** Installs {@link checkHostCapability} onto the shared plugin registry. */ +export function ensureCapabilityChecker(): void { + const reg = getSharedRegistry(); + reg.checkCapability = checkHostCapability; + reg.hasCapability = (host, capability, url) => + hasCapabilityInPolicy(resolveHostCapabilities(host), capability, url); +} + +ensureCapabilityChecker(); diff --git a/three-live/src/core.ts b/three-live/src/core.ts index 60bcdf2c..a905c980 100644 --- a/three-live/src/core.ts +++ b/three-live/src/core.ts @@ -17,6 +17,7 @@ * Exports the WebGL2 PBR renderer (three-view) and Three.js authoring engine (three-author). */ +export * from './capabilities'; export * from './three-author'; export * from './three-convert'; export * from './three-view'; diff --git a/three-live/src/index.ts b/three-live/src/index.ts index fc88e32f..dc432284 100644 --- a/three-live/src/index.ts +++ b/three-live/src/index.ts @@ -39,13 +39,31 @@ ThreeLiveElement.registerScriptRunner(authorScriptRunner); registerPlugin(contactShadowPlugin); export { + CAPABILITIES_ALL, + CAPABILITIES_NONE, + CAPABILITIES_VIEWER, + checkHostCapability, defineModelViewer, + hasCapabilityInPolicy, + isNetworkUrlPermitted, + isNonNetworkUrl, + normalizeCapabilityOrigin, + parseCapabilities, + resetCapabilityDenials, + resolveHostCapabilities, + sanitizeDiagnosticUrl, ThreeLiveElement, type ARLaunchOptions, type ARLaunchResult, type ARMode, type ARProvider, type CameraState, + type CapabilityDeniedDetail, + type ThreeLiveCapabilities, + type ThreeLiveCapabilitiesInit, + type ThreeLiveCapabilitiesInput, + type ThreeLiveCapabilityName, + type ThreeLiveNetworkPolicy, } from './three-live-element'; export { evaluateToneMapping, diff --git a/three-live/src/plugin.ts b/three-live/src/plugin.ts index 20aee0c4..f26bd804 100644 --- a/three-live/src/plugin.ts +++ b/three-live/src/plugin.ts @@ -17,6 +17,13 @@ * Provides extensible hooks across lifecycle, glTF parsing, shaders, and rendering. */ +import type { + CapabilityDeniedDetail, + checkHostCapability, + ThreeLiveCapabilities, + ThreeLiveCapabilityName, +} from './capabilities'; + export interface ThreeLivePlugin { readonly name: string; onRegister?(): void; @@ -30,7 +37,8 @@ export interface ThreeLivePlugin { onCameraChange?(viewer: any, element?: any): void; onElementDisconnected?(element: any): void; onParseGLB?(glbData: any): void; - onLoadModel?(viewer: any, gltf: any): void; + onLoadModel?(viewer: any, gltf?: any): void; + onSceneLoaded?(viewer: any): void; onClearScene?(viewer: any): void; onMeshSelect?(viewer: any, hit: any): void; onResolveShader?( @@ -90,6 +98,7 @@ export interface ThreeLivePlugin { // tslint:disable-next-line:no-any onContextRestored?(gl: WebGL2RenderingContext, viewer: any): void; onDestroy?(gl: WebGL2RenderingContext, viewer: any): void; + onDispose?(viewer: any): void; } /** @@ -187,6 +196,18 @@ export interface SharedThreeLiveRegistry { * permitted unless explicitly enabled. */ allowClientScripts?: boolean; + /** Process/page-wide default capability policy (`CAPABILITIES_NONE` by default). */ + defaultCapabilities?: ThreeLiveCapabilities; + /** Process/page-wide listener invoked whenever any capability check is denied. */ + onCapabilityDenied?: ((detail: CapabilityDeniedDetail) => void) | null; + /** Shared capability check installed by `capabilities.ts` so add-on bundles add ~0 B. */ + checkCapability?: typeof checkHostCapability; + /** Silent capability query installed by `capabilities.ts` (no denial event). */ + hasCapability?: ( + host: unknown, + capability: ThreeLiveCapabilityName, + url?: string, + ) => boolean; /** * `makePngDeflater` from `png-deflate.ts`, published by `three-author.ts` so * host-side plugins (AI texture packs) reuse it instead of bundling a copy. @@ -446,3 +467,49 @@ export function registerPlugin(plugin: ThreeLivePlugin): void { export function getRegisteredPlugins(): readonly ThreeLivePlugin[] { return getSharedRegistry().plugins; } + +/** + * Evaluates `capability` at a plugin access point via the shared registry + * (with a fail-closed fallback when loaded without `ThreeLiveElement`). + */ +export function checkPluginCapability( + host: unknown, + capability: ThreeLiveCapabilityName, + feature: string, + options?: {readonly url?: string; readonly dedup?: boolean}, +): boolean { + const reg = getSharedRegistry(); + if (reg.checkCapability) { + return reg.checkCapability(host, capability, feature, options); + } + const h = (host && typeof host === 'object' ? host : null) as { + checkCapability?: ( + c: string, + f: string, + o?: {readonly url?: string; readonly dedup?: boolean}, + ) => boolean; + } | null; + if (typeof h?.checkCapability === 'function') { + return h.checkCapability(capability, feature, options); + } + return false; +} + +/** + * Evaluates whether `capability` is currently granted on `host` without + * logging a warning or dispatching `three-capability-denied`. + */ +export function hasPluginCapability( + host: unknown, + capability: ThreeLiveCapabilityName, + url?: string, +): boolean { + const reg = getSharedRegistry(); + if (reg.hasCapability) return reg.hasCapability(host, capability, url); + const h = (host && typeof host === 'object' ? host : null) as { + hasCapability?: (c: string, u?: string) => boolean; + } | null; + return typeof h?.hasCapability === 'function' + ? h.hasCapability(capability, url) + : false; +} diff --git a/three-live/src/plugins/ai-image.ts b/three-live/src/plugins/ai-image.ts index 3b133ebb..03640941 100644 --- a/three-live/src/plugins/ai-image.ts +++ b/three-live/src/plugins/ai-image.ts @@ -15,6 +15,7 @@ * and verified against `three-live/tests/fixtures/image_cases.json`. */ +import {checkPluginCapability} from '../plugin'; import { type AIChatContentPart, type AIChatMessage, @@ -463,7 +464,12 @@ export function buildVisualTurnMessages( ...(q ? [{type: 'image_url' as const, image_url: {url: q}}] : []), { type: 'text', - text: buildVisualRefinementPrompt(turn, totalTurns, Boolean(q), metricsNote), + text: buildVisualRefinementPrompt( + turn, + totalTurns, + Boolean(q), + metricsNote, + ), }, ]; return [ @@ -533,7 +539,10 @@ export async function renderComparisonSheetDataUrl( const ph = Math.min(512, Math.max(128, refImg.naturalHeight)); const pw = Math.min( 768, - Math.max(128, Math.round((ph * refImg.naturalWidth) / refImg.naturalHeight)), + Math.max( + 128, + Math.round((ph * refImg.naturalWidth) / refImg.naturalHeight), + ), ); const host = document.createElement('div'); const glCanvas = document.createElement('canvas'); @@ -548,8 +557,8 @@ export async function renderComparisonSheetDataUrl( viewer.loadModel(model); const cam = parseScriptCamera(code); if (cam.fov) viewer.setFieldOfView?.(cam.fov); - const tgt: [number, number, number] = - cam.target ?? viewer.bounds?.center ?? [0, 0, 0]; + const tgt: [number, number, number] = cam.target ?? + viewer.bounds?.center ?? [0, 0, 0]; if (cam.target) viewer.setTarget?.(tgt[0], tgt[1], tgt[2]); if (cam.eye) { const dx = cam.eye[0] - tgt[0]; @@ -693,7 +702,12 @@ function downscaleDrawableToDataUrl( const scale = longest > maxEdge ? maxEdge / longest : 1; const targetW = Math.max(1, Math.round(width * scale)); const targetH = Math.max(1, Math.round(height * scale)); - const tryUrl = (obj: {toDataURL?: (type?: string, quality?: number) => string} | null | undefined) => { + const tryUrl = ( + obj: + | {toDataURL?: (type?: string, quality?: number) => string} + | null + | undefined, + ) => { try { const u = obj?.toDataURL?.('image/jpeg', 0.88); return u && isAllowedImageUrl(u) ? u : null; @@ -736,6 +750,7 @@ export async function normalizeImageInput( readonly maxEdge?: number; readonly signal?: AbortSignal; readonly fetch?: typeof fetch; + readonly host?: unknown; } = {}, ): Promise { const checkAbort = () => { @@ -754,6 +769,20 @@ export async function normalizeImageInput( if (!s) { throw new Error('compileFromImage: image input is empty.'); } + if ( + options.host !== undefined && + !/^(?:data|blob):/i.test(s) && + !checkPluginCapability( + options.host, + 'remote_content', + 'compileFromImage', + { + url: s, + }, + ) + ) { + throw new Error("compileFromImage: capability 'remote_content' denied."); + } if (/^data:/i.test(s)) { if (!isAllowedImageUrl(s)) { throw new Error( @@ -895,11 +924,20 @@ export async function compileFromImage( image: AIImageInput, options: CompileFromImageOptions = {}, ): Promise { - if (!el?.allowClientScripts) { + if (!checkPluginCapability(el, 'scripts', 'compileFromImage')) { throw new Error( 'compileFromImage requires client scripts: add the allow-client-scripts attribute.', ); } + if ( + typeof image === 'string' && + !/^(?:data|blob):/i.test(image.trim()) && + !checkPluginCapability(el, 'remote_content', 'compileFromImage', { + url: image.trim(), + }) + ) { + throw new Error("compileFromImage: capability 'remote_content' denied."); + } const provider = resolveProvider(el, options, 'compileFromImage'); const prevSource = currentSource(el); @@ -927,7 +965,10 @@ export async function compileFromImage( ); const visualTurns = Math.max( 0, - Math.min(MAX_VISUAL_TURNS, Math.floor(Number(options.visualTurns ?? 0)) || 0), + Math.min( + MAX_VISUAL_TURNS, + Math.floor(Number(options.visualTurns ?? 0)) || 0, + ), ); let code = ''; @@ -956,7 +997,12 @@ export async function compileFromImage( }; } try { - const res = await dryRunCompile(el, candidate, signal, 'compileFromImage'); + const res = await dryRunCompile( + el, + candidate, + signal, + 'compileFromImage', + ); const warn = [...res.warnings]; const summary = sceneSummary(res.model); if ( @@ -993,9 +1039,14 @@ export async function compileFromImage( maxEdge: options.maxEdge, signal, fetch: options.fetch, + host: el, }); if (signal.aborted) throw signal.reason; - const initialMessages = buildImageMessages(imageUrl, prompt, options.priors); + const initialMessages = buildImageMessages( + imageUrl, + prompt, + options.priors, + ); const messages = [...initialMessages]; el.fireEvent?.('three-live-image-start', {prompt, hasPriors, visualTurns}); diff --git a/three-live/src/plugins/ai-texture.ts b/three-live/src/plugins/ai-texture.ts index 4fe6a85e..d7ec5b82 100644 --- a/three-live/src/plugins/ai-texture.ts +++ b/three-live/src/plugins/ai-texture.ts @@ -20,7 +20,7 @@ * hot-swapping (by recompiling the current scene with the updated pack). */ -import {getSharedRegistry} from '../plugin'; +import {checkPluginCapability, getSharedRegistry} from '../plugin'; /** `format` field of a serialized texture pack. */ export const TEXTURE_PACK_FORMAT = 'three-live-texture-pack'; @@ -153,7 +153,10 @@ export interface GenerateTexturesOptions { export interface GenerateTexturesResult { readonly generated: readonly string[]; readonly cached: readonly string[]; - readonly failed: ReadonlyArray<{readonly key: string; readonly error: string}>; + readonly failed: ReadonlyArray<{ + readonly key: string; + readonly error: string; + }>; readonly pack: TexturePackJSON; } @@ -211,7 +214,9 @@ function bytesToBase64(bytes: Uint8Array): string { return btoa(s); } -async function toBytes(data: Blob | ArrayBuffer | Uint8Array): Promise { +async function toBytes( + data: Blob | ArrayBuffer | Uint8Array, +): Promise { if (data instanceof Uint8Array) return data; if (data instanceof ArrayBuffer) return new Uint8Array(data); if (data && typeof (data as Blob).arrayBuffer === 'function') { @@ -250,7 +255,8 @@ export function createAIImageProvider( if (signal?.aborted) onAbort(); signal?.addEventListener('abort', onAbort, {once: true}); const timer = setTimeout( - () => ctrl.abort(new Error(`ai-image: timed out after ${timeoutMs} ms`)), + () => + ctrl.abort(new Error(`ai-image: timed out after ${timeoutMs} ms`)), timeoutMs, ); try { @@ -288,7 +294,10 @@ export function createAIImageProvider( } const url = typeof item?.url === 'string' ? item.url : ''; if (url.startsWith('data:image/')) { - return base64ToBytes(url.slice(url.indexOf(',') + 1), MAX_SOURCE_IMAGE_BYTES); + return base64ToBytes( + url.slice(url.indexOf(',') + 1), + MAX_SOURCE_IMAGE_BYTES, + ); } if (url) { let sameOrigin = false; @@ -300,8 +309,11 @@ export function createAIImageProvider( 'ai-image: the endpoint returned an image URL; use response_format "b64_json" (or allowResponseUrls for same-origin URLs).', ); } - const img = await doFetch(new URL(url, endpoint).href, {signal: ctrl.signal}); - if (!img.ok) throw new Error(`ai-image: HTTP ${img.status} fetching the image.`); + const img = await doFetch(new URL(url, endpoint).href, { + signal: ctrl.signal, + }); + if (!img.ok) + throw new Error(`ai-image: HTTP ${img.status} fetching the image.`); const bytes = new Uint8Array(await img.arrayBuffer()); if (bytes.length > MAX_SOURCE_IMAGE_BYTES) { throw new Error('ai-image: image exceeds 16 MB.'); @@ -318,10 +330,17 @@ export function createAIImageProvider( } function imageKind(b: Uint8Array): 'png' | 'jpeg' | 'webp' | null { - if (b.length >= 8 && b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47) { + if ( + b.length >= 8 && + b[0] === 0x89 && + b[1] === 0x50 && + b[2] === 0x4e && + b[3] === 0x47 + ) { return 'png'; } - if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return 'jpeg'; + if (b.length >= 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) + return 'jpeg'; if ( b.length >= 12 && b[0] === 0x52 && @@ -361,7 +380,11 @@ export const browserImageCodec: TextureImageCodec = { ctx.drawImage(bmp, 0, 0); bmp.close?.(); const data = ctx.getImageData(0, 0, width, height).data; - return {width, height, rgba: new Uint8Array(data.buffer, data.byteOffset, data.byteLength)}; + return { + width, + height, + rgba: new Uint8Array(data.buffer, data.byteOffset, data.byteLength), + }; }, async encodePng(rgba, size) { const enc = registry().encodeRgbaPngDataUri; @@ -389,10 +412,16 @@ export function resampleArea( const out = new Uint8Array(ow * oh * 4); for (let y = 0; y < oh; y++) { const sy0 = Math.floor((y * sh) / oh); - const sy1 = Math.min(sh, Math.max(sy0 + 1, Math.floor(((y + 1) * sh) / oh))); + const sy1 = Math.min( + sh, + Math.max(sy0 + 1, Math.floor(((y + 1) * sh) / oh)), + ); for (let x = 0; x < ow; x++) { const sx0 = Math.floor((x * sw) / ow); - const sx1 = Math.min(sw, Math.max(sx0 + 1, Math.floor(((x + 1) * sw) / ow))); + const sx1 = Math.min( + sw, + Math.max(sx0 + 1, Math.floor(((x + 1) * sw) / ow)), + ); let r = 0; let g = 0; let b = 0; @@ -436,7 +465,10 @@ export function seamBlend(rgba: Uint8Array, n: number): Uint8Array { const s = (((y + half) % n) * n + ((x + half) % n)) * 4; for (let c = 0; c < 3; c++) { const a = rgba[o + c]; - out[o + c] = Math.max(0, Math.min(255, Math.floor(a + (rgba[s + c] - a) * w + 0.5))); + out[o + c] = Math.max( + 0, + Math.min(255, Math.floor(a + (rgba[s + c] - a) * w + 0.5)), + ); } out[o + 3] = 255; } @@ -484,20 +516,27 @@ export async function parseTexturePack( const out = new Map(); for (const key of keys) { const e = p.entries[key]; - if (!/^[0-9a-f]{16}$/.test(key)) throw new Error(`texture pack: invalid key '${key}'`); + if (!/^[0-9a-f]{16}$/.test(key)) + throw new Error(`texture pack: invalid key '${key}'`); const spec = textureSpec(e?.spec); if (!spec) throw new Error(`texture pack: entry ${key}: invalid spec`); if (spec.key !== key) { - throw new Error(`texture pack: entry ${key}: key does not match its spec (${spec.key})`); + throw new Error( + `texture pack: entry ${key}: key does not match its spec (${spec.key})`, + ); } const url = typeof e?.albedo === 'string' ? e.albedo : ''; const prefix = 'data:image/png;base64,'; if (!url.startsWith(prefix)) { - throw new Error(`texture pack: entry ${key}: albedo must be a data:image/png;base64 URL`); + throw new Error( + `texture pack: entry ${key}: albedo must be a data:image/png;base64 URL`, + ); } const png = base64ToBytes(url.slice(prefix.length), MAX_ALBEDO_PNG_BYTES); if (png.length > MAX_ALBEDO_PNG_BYTES || imageKind(png) !== 'png') { - throw new Error(`texture pack: entry ${key}: albedo is not a PNG within 6 MB`); + throw new Error( + `texture pack: entry ${key}: albedo is not a PNG within 6 MB`, + ); } const img = await codec.decode(png); if (img.width !== spec.size || img.height !== spec.size) { @@ -508,7 +547,9 @@ export async function parseTexturePack( const rgba = new Uint8Array(img.rgba); for (let i = 3; i < rgba.length; i += 4) rgba[i] = 255; const provenance = - e.provenance && typeof e.provenance === 'object' ? e.provenance : undefined; + e.provenance && typeof e.provenance === 'object' + ? e.provenance + : undefined; out.set(key, { key, spec: {prompt: spec.prompt, size: spec.size, seed: spec.seed}, @@ -547,7 +588,9 @@ function gltfJson(model: unknown): any { if (dv.getUint32(0, true) === 0x46546c67) { try { return JSON.parse( - new TextDecoder().decode(new Uint8Array(model, 20, dv.getUint32(12, true))), + new TextDecoder().decode( + new Uint8Array(model, 20, dv.getUint32(12, true)), + ), ); } catch {} } @@ -574,7 +617,9 @@ export function textureRequests(model: unknown): TextureMiss[] { } /** IndexedDB cache (`three-live-textures`), or null where IndexedDB is unavailable. */ -export function indexedDbTextureCache(maxBytes = MAX_CACHE_BYTES): TextureCache | null { +export function indexedDbTextureCache( + maxBytes = MAX_CACHE_BYTES, +): TextureCache | null { try { if (typeof indexedDB === 'undefined' || !indexedDB) return null; } catch { @@ -590,7 +635,10 @@ export function indexedDbTextureCache(maxBytes = MAX_CACHE_BYTES): TextureCache req.onsuccess = () => resolve(req.result); req.onerror = () => reject(req.error); })); - const run = (mode: IDBTransactionMode, fn: (s: IDBObjectStore) => IDBRequest | void) => + const run = ( + mode: IDBTransactionMode, + fn: (s: IDBObjectStore) => IDBRequest | void, + ) => db().then( (d) => new Promise((resolve, reject) => { @@ -606,7 +654,9 @@ export function indexedDbTextureCache(maxBytes = MAX_CACHE_BYTES): TextureCache // tslint:disable-next-line:no-any Stored record. const rec: any = await run('readonly', (s) => s.get(key)); if (!rec || !(rec.rgba instanceof Uint8Array)) return null; - void run('readwrite', (s) => s.put({...rec, t: Date.now()})).catch(() => {}); + void run('readwrite', (s) => s.put({...rec, t: Date.now()})).catch( + () => {}, + ); return rec as TextureCacheRecord; } catch { return null; @@ -652,7 +702,9 @@ export interface AITextureElement { texturePack?: TexturePackJSON | null; textureCache?: TextureCache | null; textureCodec?: TextureImageCodec; - generateTextures?: (o?: GenerateTexturesOptions) => Promise; + generateTextures?: ( + o?: GenerateTexturesOptions, + ) => Promise; exportTexturePack?: () => Promise; __tlTex?: TextureState; } @@ -663,7 +715,10 @@ interface TextureState { failed: Set; inflight: Map>; lastModel: unknown; - lastRun: {kind: 'script'; value: string} | {kind: 'url'; value: string} | null; + lastRun: + | {kind: 'script'; value: string} + | {kind: 'url'; value: string} + | null; cache: TextureCache | null | undefined; loadSeq: number; } @@ -684,7 +739,9 @@ function state(el: AITextureElement): TextureState { }); } -function compileMap(el: AITextureElement): Record | null { +function compileMap( + el: AITextureElement, +): Record | null { const st = el.__tlTex; if (!st || st.pack.size === 0) return null; const out: Record = {}; @@ -710,7 +767,8 @@ function ensureRunnerWrapped(): void { reg.scriptRunner = { __tlTexWrapped: true, // tslint:disable-next-line:no-any Pass-through runner arguments. - execute: (s: string, sb: string, cb: any, o?: unknown) => r.execute(s, sb, cb, withPack(o)), + execute: (s: string, sb: string, cb: any, o?: unknown) => + r.execute(s, sb, cb, withPack(o)), ...(typeof r.executeUrl === 'function' ? { // tslint:disable-next-line:no-any Pass-through runner arguments. @@ -722,7 +780,10 @@ function ensureRunnerWrapped(): void { } /** Runs `fn` (which must call the script runner synchronously) with the element's pack. */ -export function withElementTexturePack(el: AITextureElement, fn: () => T): T { +export function withElementTexturePack( + el: AITextureElement, + fn: () => T, +): T { ensureRunnerWrapped(); const prev = pendingPack; pendingPack = compileMap(el); @@ -749,6 +810,9 @@ function resolveImageProvider( function cacheFor(el: AITextureElement): TextureCache | null { if (el.textureCache !== undefined) return el.textureCache; + if (!checkPluginCapability(el, 'storage', 'aiTextureCache', {dedup: true})) { + return null; + } const st = state(el); if (st.cache === undefined) st.cache = indexedDbTextureCache(); return st.cache; @@ -764,7 +828,8 @@ function recompile(el: AITextureElement): Promise { 'script[type="text/three"], script[type="text/three-live"]', )?.textContent; const run = - state(el).lastRun ?? (inline ? ({kind: 'script', value: inline} as const) : null); + state(el).lastRun ?? + (inline ? ({kind: 'script', value: inline} as const) : null); if (!run) return Promise.resolve(); return run.kind === 'url' && typeof el.loadScriptFromSrc === 'function' ? el.loadScriptFromSrc(run.value) @@ -778,7 +843,10 @@ function addEntry(el: AITextureElement, entry: TexturePackEntry): void { st.failed.delete(entry.key); } -async function fromCache(el: AITextureElement, misses: readonly TextureMiss[]): Promise { +async function fromCache( + el: AITextureElement, + misses: readonly TextureMiss[], +): Promise { const cache = cacheFor(el); if (!cache) return []; const hits: string[] = []; @@ -789,8 +857,17 @@ async function fromCache(el: AITextureElement, misses: readonly TextureMiss[]): rec.spec?.size === m.spec.size && rec.rgba?.length === m.spec.size * m.spec.size * 4 ) { - addEntry(el, {key: m.key, spec: m.spec, rgba: rec.rgba, provenance: rec.provenance}); - el.fireEvent?.('three-live-texture', {key: m.key, meshes: m.meshes, source: 'cache'}); + addEntry(el, { + key: m.key, + spec: m.spec, + rgba: rec.rgba, + provenance: rec.provenance, + }); + el.fireEvent?.('three-live-texture', { + key: m.key, + meshes: m.meshes, + source: 'cache', + }); hits.push(m.key); } } @@ -811,7 +888,8 @@ async function generateOne( const raw = await toBytes( await provider.generateImage({...m.spec, signal}), ); - if (raw.length > MAX_SOURCE_IMAGE_BYTES) throw new Error('generated image exceeds 16 MB'); + if (raw.length > MAX_SOURCE_IMAGE_BYTES) + throw new Error('generated image exceeds 16 MB'); const rgba = prepareAlbedo(await codecFor(el).decode(raw), m.spec.size); const provenance = { provider: provider.name ?? 'custom', @@ -820,7 +898,11 @@ async function generateOne( }; addEntry(el, {key: m.key, spec: m.spec, rgba, provenance}); void cacheFor(el)?.put({key: m.key, spec: m.spec, rgba, provenance}); - el.fireEvent?.('three-live-texture', {key: m.key, meshes: m.meshes, source: 'provider'}); + el.fireEvent?.('three-live-texture', { + key: m.key, + meshes: m.meshes, + source: 'provider', + }); return true; } catch (e: unknown) { st.failed.add(m.key); @@ -838,7 +920,11 @@ async function generateOne( return job; } -async function runPool(items: readonly T[], n: number, fn: (t: T) => Promise) { +async function runPool( + items: readonly T[], + n: number, + fn: (t: T) => Promise, +) { let i = 0; const workers = Array.from({length: Math.min(n, items.length)}, async () => { while (i < items.length) await fn(items[i++]); @@ -856,7 +942,9 @@ export function onElementCompiled(el: AITextureElement, model: unknown): void { } catch { return; } - const pending = misses.filter((m) => !st.failed.has(m.key) && !st.inflight.has(m.key)); + const pending = misses.filter( + (m) => !st.failed.has(m.key) && !st.inflight.has(m.key), + ); if (!pending.length) return; void (async () => { const hits = await fromCache(el, pending); @@ -865,11 +953,25 @@ export function onElementCompiled(el: AITextureElement, model: unknown): void { await recompile(el).catch(() => {}); return; } - if ((el.getAttribute?.('ai-textures') || 'auto').toLowerCase() === 'manual') return; + if ((el.getAttribute?.('ai-textures') || 'auto').toLowerCase() === 'manual') + return; const provider = resolveImageProvider(el); if (!provider) return; - el.fireEvent?.('three-live-texture-start', {keys: pending.map((m) => m.key)}); - const results = await Promise.all(pending.map((m) => generateOne(el, provider, m))); + const ep = el.getAttribute?.('ai-image-endpoint') || undefined; + if ( + !checkPluginCapability(el, 'ai', 'aiTextures', { + url: el.aiImageProvider ? undefined : ep, + dedup: true, + }) + ) { + return; + } + el.fireEvent?.('three-live-texture-start', { + keys: pending.map((m) => m.key), + }); + const results = await Promise.all( + pending.map((m) => generateOne(el, provider, m)), + ); const made = results.filter(Boolean).length; if (made) { if (!hinted) { @@ -904,7 +1006,9 @@ async function setPack(el: AITextureElement, json: unknown): Promise { const parsed = await parseTexturePack(json, codecFor(el)); if (seq !== st.loadSeq) return; st.pack = parsed; - st.packJson = (typeof json === 'string' ? JSON.parse(json) : json) as TexturePackJSON; + st.packJson = ( + typeof json === 'string' ? JSON.parse(json) : json + ) as TexturePackJSON; st.failed.clear(); await recompile(el).catch(() => {}); } catch (e: unknown) { @@ -918,9 +1022,15 @@ async function setPack(el: AITextureElement, json: unknown): Promise { async function loadPackUrl(el: AITextureElement, url: string): Promise { try { + if (!checkPluginCapability(el, 'remote_content', 'texture-pack', {url})) { + throw new Error("texture-pack: capability 'remote_content' denied."); + } const base = typeof document !== 'undefined' ? document.baseURI : undefined; - const res = await fetch(new URL(url, base).href, {credentials: 'same-origin'}); - if (!res.ok) throw new Error(`HTTP ${res.status} loading texture-pack "${url}"`); + const res = await fetch(new URL(url, base).href, { + credentials: 'same-origin', + }); + if (!res.ok) + throw new Error(`HTTP ${res.status} loading texture-pack "${url}"`); await setPack(el, await res.json()); } catch (e: unknown) { el.fireEvent?.('three-live-texture-error', { @@ -959,15 +1069,36 @@ export async function generateTextures( 'generateTextures: no image provider. Set el.aiImageProvider, or the ai-image-endpoint (and ai-image-model) attributes.', ); } + const ep = + options.provider || el.aiImageProvider + ? undefined + : el.getAttribute?.('ai-image-endpoint') || undefined; + if ( + !checkPluginCapability( + el, + 'ai', + 'generateTextures', + ep ? {url: ep} : undefined, + ) + ) { + throw new Error("generateTextures: capability 'ai' denied."); + } el.fireEvent?.('three-live-texture-start', {keys: todo.map((m) => m.key)}); - const n = Math.max(1, Math.min(4, Math.floor(Number(options.concurrency ?? 2)) || 2)); + const n = Math.max( + 1, + Math.min(4, Math.floor(Number(options.concurrency ?? 2)) || 2), + ); const onError = (ev: {key: string; error: string}) => failed.push(ev); await runPool(todo, n, async (m) => { st.failed.delete(m.key); if (options.force) st.pack.delete(m.key); const ok = await generateOne(el, provider, m, options.signal); if (ok) generated.push(m.key); - else onError({key: m.key, error: 'generation failed (see three-live-texture-error)'}); + else + onError({ + key: m.key, + error: 'generation failed (see three-live-texture-error)', + }); }); } if (cached.length || generated.length) await recompile(el).catch(() => {}); @@ -975,7 +1106,9 @@ export async function generateTextures( } /** Serializes the element's current texture pack. */ -export async function exportTexturePack(el: AITextureElement): Promise { +export async function exportTexturePack( + el: AITextureElement, +): Promise { const st = state(el); return (st.packJson ??= await serializeTexturePack(st.pack, codecFor(el))); } @@ -990,16 +1123,23 @@ export function installTextureSupport(el: AITextureElement): void { set: (v: unknown) => void setPack(el, v), configurable: true, }); - el.generateTextures = (o?: GenerateTexturesOptions) => generateTextures(el, o); + el.generateTextures = (o?: GenerateTexturesOptions) => + generateTextures(el, o); el.exportTexturePack = () => exportTexturePack(el); - const inline = el.querySelector?.('script[type="application/three-live-texture-pack"]'); + const inline = el.querySelector?.( + 'script[type="application/three-live-texture-pack"]', + ); if (inline?.textContent?.trim()) { void setPack(el, inline.textContent); } const url = el.getAttribute?.('texture-pack'); if (url) void loadPackUrl(el, url); const node = el as unknown as Node; - if (typeof MutationObserver !== 'undefined' && node && typeof node.nodeType === 'number') { + if ( + typeof MutationObserver !== 'undefined' && + node && + typeof node.nodeType === 'number' + ) { new MutationObserver(() => { const next = el.getAttribute?.('texture-pack'); if (next) void loadPackUrl(el, next); diff --git a/three-live/src/plugins/ai-voice.ts b/three-live/src/plugins/ai-voice.ts index 1bd8f68f..67597d7d 100644 --- a/three-live/src/plugins/ai-voice.ts +++ b/three-live/src/plugins/ai-voice.ts @@ -26,17 +26,22 @@ * at a short-lived ephemeral-token URL your server mints. */ -import type {EditSceneResult, ThreeLiveAIProvider} from './ai'; +import {checkPluginCapability} from '../plugin'; +import { + requireAICap, + type EditSceneResult, + type ThreeLiveAIProvider, +} from './ai'; import { - type AISelectableElement, - type AISelectionMode, - type AISelectProvider, describeSelection, getSelection, resolveSelection, resolveSelectionWithModel, sceneSelectEntries, setSelection, + type AISelectableElement, + type AISelectionMode, + type AISelectProvider, } from './ai-select'; import {hideSelectionChips, showSelectionChips} from './ai-select-chips'; import {parseVoiceCommand, type SelectMatch} from './ai-select-match'; @@ -289,7 +294,8 @@ export const VOICE_TOOLS: readonly AIVoiceTool[] = [ properties: { query: { type: 'string', - description: 'The user\'s words for the objects, e.g. "the lamp next to the sofa".', + description: + 'The user\'s words for the objects, e.g. "the lamp next to the sofa".', }, names: { type: 'array', @@ -391,9 +397,16 @@ export function mergeSession( if (k === '__proto__' || k === 'constructor' || k === 'prototype') continue; const prev = out[k]; out[k] = - v && typeof v === 'object' && !Array.isArray(v) && - prev && typeof prev === 'object' && !Array.isArray(prev) - ? mergeSession(prev as Record, v as Record) + v && + typeof v === 'object' && + !Array.isArray(v) && + prev && + typeof prev === 'object' && + !Array.isArray(prev) + ? mergeSession( + prev as Record, + v as Record, + ) : v; } return out; @@ -434,7 +447,9 @@ export const realtimeAdapter: AIVoiceAdapter = { tool_choice: 'auto', ...(config.model ? {model: config.model} : {}), }; - return [{type: 'session.update', session: mergeSession(base, config.overrides)}]; + return [ + {type: 'session.update', session: mergeSession(base, config.overrides)}, + ]; }, audio(b64) { return {type: 'input_audio_buffer.append', audio: b64}; @@ -529,8 +544,7 @@ export const realtimeAdapter: AIVoiceAdapter = { return [ { type: 'error', - message: - str(obj(m['error'])['message']) || 'Realtime server error', + message: str(obj(m['error'])['message']) || 'Realtime server error', }, ]; default: @@ -626,7 +640,8 @@ export const geminiLiveAdapter: AIVoiceAdapter = { }); } const err = obj(m['error']); - if (str(err['message'])) out.push({type: 'error', message: str(err['message'])}); + if (str(err['message'])) + out.push({type: 'error', message: str(err['message'])}); return out; }, }; @@ -638,7 +653,11 @@ const ADAPTERS: Record = { /** Encodes Int16 samples as little-endian base64. */ export function pcm16ToBase64(samples: Int16Array): string { - const bytes = new Uint8Array(samples.buffer, samples.byteOffset, samples.byteLength); + const bytes = new Uint8Array( + samples.buffer, + samples.byteOffset, + samples.byteLength, + ); let binary = ''; for (let i = 0; i < bytes.length; i += 0x8000) { binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); @@ -682,7 +701,10 @@ export function base64ToFloat(b64: string): Float32Array { type AudioCtor = new (opts?: {sampleRate?: number}) => AudioContext; function audioContextCtor(): AudioCtor | undefined { - const g = globalThis as {AudioContext?: AudioCtor; webkitAudioContext?: AudioCtor}; + const g = globalThis as { + AudioContext?: AudioCtor; + webkitAudioContext?: AudioCtor; + }; return g.AudioContext ?? g.webkitAudioContext; } @@ -694,12 +716,19 @@ export function createBrowserMicrophone(): AIMicrophoneSource { return { async start(rate, onChunk) { const Ctor = audioContextCtor(); - const media = (globalThis as {navigator?: Navigator}).navigator?.mediaDevices; + const media = (globalThis as {navigator?: Navigator}).navigator + ?.mediaDevices; if (!Ctor || !media?.getUserMedia) { - throw new Error('connectVoice: microphone capture is not available here.'); + throw new Error( + 'connectVoice: microphone capture is not available here.', + ); } stream = await media.getUserMedia({ - audio: {echoCancellation: true, noiseSuppression: true, channelCount: 1}, + audio: { + echoCancellation: true, + noiseSuppression: true, + channelCount: 1, + }, }); ctx = new Ctor(); const source = ctx.createMediaStreamSource(stream); @@ -788,11 +817,20 @@ export class AIVoiceSession { private userText = ''; private modelText = ''; private readonly onSelect = (e: Event) => { - const d = (e as CustomEvent<{mesh?: string | null; meshes?: readonly string[]; source?: string}>) - .detail; + const d = ( + e as CustomEvent<{ + mesh?: string | null; + meshes?: readonly string[]; + source?: string; + }> + ).detail; // Voice selections come back in the select_meshes tool result instead. if (d?.source === 'voice') return; - const meshes = Array.isArray(d?.meshes) ? d.meshes : d?.mesh ? [d.mesh] : []; + const meshes = Array.isArray(d?.meshes) + ? d.meshes + : d?.mesh + ? [d.mesh] + : []; this.sendContext(describeSelection(meshes)); }; @@ -807,10 +845,20 @@ export class AIVoiceSession { `connectVoice: unknown provider '${name}' (use ai-realtime or gemini-live).`, ); } + if (!options.transport && !options.url) { + throw new Error('connectVoice: pass a wss:// url or a transport.'); + } + if (!options.transport && options.url && !isAllowedVoiceUrl(options.url)) { + throw new Error( + 'connectVoice: url must use wss:// (or ws:// on localhost).', + ); + } + requireAICap(el, 'ai', 'connectVoice', options.url); + if (options.microphone !== false) { + requireAICap(el, 'microphone', 'connectVoice'); + } this.adapter = adapter; - if (options.transport) this.transport = options.transport; - else if (options.url) this.transport = createVoiceWebSocket(options.url); - else throw new Error('connectVoice: pass a wss:// url or a transport.'); + this.transport = options.transport ?? createVoiceWebSocket(options.url!); this.mic = options.microphone === false ? null @@ -845,7 +893,16 @@ export class AIVoiceSession { if (selected.length) this.sendContext(describeSelection(selected)); try { await this.mic?.start(this.adapter.inputRate, (b64) => { - if (this.state !== 'closed') this.transport.send(this.adapter.audio(b64)); + if (this.state === 'closed') return; + if ( + !checkPluginCapability(this.el, 'microphone', 'connectVoice', { + dedup: true, + }) + ) { + this.close(); + return; + } + this.transport.send(this.adapter.audio(b64)); }); } catch (e: unknown) { this.setState('error', {error: (e as Error)?.message || String(e)}); @@ -858,8 +915,18 @@ export class AIVoiceSession { /** Sends a typed message (e.g. a text fallback) and asks for a reply. */ sendText(text: string): void { - const t = String(text ?? '').trim().slice(0, MAX_VOICE_TEXT); + const t = String(text ?? '') + .trim() + .slice(0, MAX_VOICE_TEXT); if (!t || this.state === 'closed') return; + if ( + !checkPluginCapability(this.el, 'ai', 'connectVoice', { + url: this.options.url, + dedup: true, + }) + ) { + return; + } for (const msg of this.adapter.text(t, true)) this.transport.send(msg); } @@ -885,7 +952,10 @@ export class AIVoiceSession { for (const msg of this.adapter.text(text, false)) this.transport.send(msg); } - private setState(state: AIVoiceState, extra: Partial = {}): void { + private setState( + state: AIVoiceState, + extra: Partial = {}, + ): void { this.state = state; emit(this.el, {state, ...extra}); } @@ -904,9 +974,9 @@ export class AIVoiceSession { break; case 'transcript': if (ev.speaker === 'user') { - this.userText = (ev.isFinal ? ev.text : this.userText + ev.text).slice( - -MAX_VOICE_TEXT, - ); + this.userText = ( + ev.isFinal ? ev.text : this.userText + ev.text + ).slice(-MAX_VOICE_TEXT); emit(this.el, { state: this.state, transcript: this.userText, @@ -914,21 +984,33 @@ export class AIVoiceSession { }); if (ev.isFinal) this.userText = ''; } else { - this.modelText = (ev.isFinal ? ev.text : this.modelText + ev.text).slice( - -MAX_VOICE_TEXT, - ); - emit(this.el, {state: this.state, reply: this.modelText, isFinal: ev.isFinal}); + this.modelText = ( + ev.isFinal ? ev.text : this.modelText + ev.text + ).slice(-MAX_VOICE_TEXT); + emit(this.el, { + state: this.state, + reply: this.modelText, + isFinal: ev.isFinal, + }); if (ev.isFinal) this.modelText = ''; } break; case 'turnComplete': // Gemini Live never marks user transcripts final; close them here. if (this.userText) { - emit(this.el, {state: this.state, transcript: this.userText, isFinal: true}); + emit(this.el, { + state: this.state, + transcript: this.userText, + isFinal: true, + }); this.userText = ''; } if (this.modelText) { - emit(this.el, {state: this.state, reply: this.modelText, isFinal: true}); + emit(this.el, { + state: this.state, + reply: this.modelText, + isFinal: true, + }); this.modelText = ''; } if (this.state === 'speaking' || this.state === 'thinking') { @@ -936,7 +1018,8 @@ export class AIVoiceSession { } break; case 'toolCall': - if (++calls <= MAX_TOOL_CALLS) void this.runTool(ev.id, ev.name, ev.args); + if (++calls <= MAX_TOOL_CALLS) + void this.runTool(ev.id, ev.name, ev.args); break; case 'error': emit(this.el, {state: this.state, error: ev.message}); @@ -953,7 +1036,9 @@ export class AIVoiceSession { let output: Record; try { if (name === 'edit_scene') { - const instruction = str(args['instruction']).trim().slice(0, MAX_VOICE_TEXT); + const instruction = str(args['instruction']) + .trim() + .slice(0, MAX_VOICE_TEXT); if (!instruction) throw new Error('instruction is empty'); if (typeof this.el.editScene !== 'function') { throw new Error('scene editing is not available on this element'); @@ -968,20 +1053,25 @@ export class AIVoiceSession { this.setState('thinking', {result}); } else if (name === 'select_meshes') { const rawMode = str(args['mode']); - const mode: AISelectionMode = (['replace', 'add', 'remove', 'clear'] as const).includes( - rawMode as 'replace', - ) + const mode: AISelectionMode = ( + ['replace', 'add', 'remove', 'clear'] as const + ).includes(rawMode as 'replace') ? (rawMode as AISelectionMode) : 'replace'; const names = Array.isArray(args['names']) - ? (args['names'] as unknown[]).filter((n): n is string => typeof n === 'string') + ? (args['names'] as unknown[]).filter( + (n): n is string => typeof n === 'string', + ) : []; const sel = names.length && mode !== 'clear' ? selectNamesByVoice(this.el, names, mode) : await selectByVoice(this.el, str(args['query']), { mode, - provider: this.options.editProvider ?? this.el.__tlSelectProvider?.() ?? null, + provider: + this.options.editProvider ?? + this.el.__tlSelectProvider?.() ?? + null, }); output = { changed: sel.applied, @@ -993,7 +1083,9 @@ export class AIVoiceSession { }; } else if (name === 'undo_edit') { const undone = (await this.el.undoEdit?.()) ?? false; - output = undone ? {undone: true} : {undone: false, error: 'nothing to undo'}; + output = undone + ? {undone: true} + : {undone: false, error: 'nothing to undo'}; } else { output = {error: `unknown tool ${name}`}; } @@ -1034,7 +1126,14 @@ interface RecognitionLike { interimResults: boolean; continuous: boolean; maxAlternatives: number; - onresult: ((e: {resultIndex: number; results: ArrayLike & {isFinal: boolean}>}) => void) | null; + onresult: + | ((e: { + resultIndex: number; + results: ArrayLike< + ArrayLike<{transcript: string}> & {isFinal: boolean} + >; + }) => void) + | null; onerror: ((e: {error?: string}) => void) | null; onend: (() => void) | null; start(): void; @@ -1053,7 +1152,11 @@ export function createWebSpeechRecognizer(): AISpeechRecognizer | null { listen({lang, signal, onPartial}) { return new Promise((resolve, reject) => { const rec = new Ctor(); - rec.lang = lang || (globalThis as {document?: Document}).document?.documentElement?.lang || 'en-US'; + rec.lang = + lang || + (globalThis as {document?: Document}).document?.documentElement + ?.lang || + 'en-US'; rec.interimResults = true; rec.continuous = false; rec.maxAlternatives = 1; @@ -1084,7 +1187,13 @@ export function createWebSpeechRecognizer(): AISpeechRecognizer | null { }; rec.onerror = (e) => { if (e?.error === 'no-speech' || e?.error === 'aborted') return; - done(() => reject(new Error(`Speech recognition failed: ${e?.error || 'unknown error'}`))); + done(() => + reject( + new Error( + `Speech recognition failed: ${e?.error || 'unknown error'}`, + ), + ), + ); }; rec.onend = () => done(() => resolve((finalText || latest).trim())); if (signal?.aborted) return onAbort(); @@ -1146,7 +1255,8 @@ export function describeVoiceSelection( ? `Removed ${speakList(match.names)}. ${meshes.length} selected.` : `Removed ${speakList(match.names)}. Nothing is selected.`; } - if (mode === 'add') return `Added ${speakList(match.names)}. ${meshes.length} selected.`; + if (mode === 'add') + return `Added ${speakList(match.names)}. ${meshes.length} selected.`; return match.names.length === 1 ? `Selected ${speakableName(match.names[0])}.` : `Selected ${match.names.length}: ${speakList(match.names)}.`; @@ -1173,45 +1283,79 @@ export function describeVoiceSelection( export async function selectByVoice( el: AIVoiceElement, query: string, - options: {mode?: AISelectionMode; provider?: AISelectProvider | null; signal?: AbortSignal} = {}, + options: { + mode?: AISelectionMode; + provider?: AISelectProvider | null; + signal?: AbortSignal; + } = {}, ): Promise { const mode = options.mode ?? 'replace'; const q = String(query ?? '').slice(0, MAX_VOICE_TEXT); if (mode === 'clear') { setSelection(el, [], {mode: 'clear', source: 'voice', query: q || 'clear'}); - return {mode, query: q, match: null, applied: true, tier: 1, meshes: [], say: describeVoiceSelection(mode, null, [], q)}; + return { + mode, + query: q, + match: null, + applied: true, + tier: 1, + meshes: [], + say: describeVoiceSelection(mode, null, [], q), + }; } const within = mode === 'remove' ? getSelection(el) : null; if (within && !within.length) { - return {mode, query: q, match: null, applied: false, tier: 1, meshes: [], say: 'Nothing is selected.'}; + return { + mode, + query: q, + match: null, + applied: false, + tier: 1, + meshes: [], + say: 'Nothing is selected.', + }; } let match = resolveSelection(el, q, {within}); let tier: 1 | 2 = 1; let question = ''; - const provider = options.provider === undefined ? (el.__tlSelectProvider?.() ?? null) : options.provider; if ( - provider && q.trim() && match.confidence === 'none' && - (match.reason === 'relation' || match.reason === 'no_match') + (match.reason === 'relation' || match.reason === 'no_match') && + checkPluginCapability(el, 'ai', 'selectByVoice', {dedup: true}) ) { - try { - const r = await resolveSelectionWithModel(el, q, provider, {within, signal: options.signal}); - tier = 2; - question = r.question; - match = - r.confidence === 'high' - ? {ids: [], names: r.names, confidence: 'high', candidates: [], reason: 'model'} - : { - ids: [], - names: [], - confidence: r.names.length ? 'ambiguous' : 'none', - candidates: r.names.slice(0, 10), - reason: 'model', - }; - } catch (e: unknown) { - if (options.signal?.aborted) throw e; - // Tier 2 failed (network, quota): fall back to the tier-1 answer. + const provider = + options.provider === undefined + ? (el.__tlSelectProvider?.() ?? null) + : options.provider; + if (provider) { + try { + const r = await resolveSelectionWithModel(el, q, provider, { + within, + signal: options.signal, + }); + tier = 2; + question = r.question; + match = + r.confidence === 'high' + ? { + ids: [], + names: r.names, + confidence: 'high', + candidates: [], + reason: 'model', + } + : { + ids: [], + names: [], + confidence: r.names.length ? 'ambiguous' : 'none', + candidates: r.names.slice(0, 10), + reason: 'model', + }; + } catch (e: unknown) { + if (options.signal?.aborted) throw e; + // Tier 2 failed (network, quota): fall back to the tier-1 answer. + } } } let applied = false; @@ -1220,7 +1364,10 @@ export async function selectByVoice( applied = true; } const meshes = getSelection(el); - const say = !applied && question ? question : describeVoiceSelection(mode, match, meshes, q); + const say = + !applied && question + ? question + : describeVoiceSelection(mode, match, meshes, q); return {mode, query: q, match, applied, tier, meshes, say}; } @@ -1231,17 +1378,50 @@ export function selectNamesByVoice( mode: AISelectionMode = 'replace', ): AIVoiceSelection { const known = new Set(sceneSelectEntries(el).map((e) => e.name)); - const valid = names.filter((n) => typeof n === 'string' && known.has(n)).slice(0, 50); - const query = names.filter((n) => typeof n === 'string').join(', ').slice(0, 200); + const valid = names + .filter((n) => typeof n === 'string' && known.has(n)) + .slice(0, 50); + const query = names + .filter((n) => typeof n === 'string') + .join(', ') + .slice(0, 200); if (!valid.length) { - const match: SelectMatch = {ids: [], names: [], confidence: 'none', candidates: [], reason: 'no_match'}; + const match: SelectMatch = { + ids: [], + names: [], + confidence: 'none', + candidates: [], + reason: 'no_match', + }; const meshes = getSelection(el); - return {mode, query, match, applied: false, tier: 1, meshes, say: describeVoiceSelection(mode, match, meshes, query)}; + return { + mode, + query, + match, + applied: false, + tier: 1, + meshes, + say: describeVoiceSelection(mode, match, meshes, query), + }; } setSelection(el, valid, {mode, source: 'voice', query}); - const match: SelectMatch = {ids: [], names: valid, confidence: 'high', candidates: [], reason: 'exact'}; + const match: SelectMatch = { + ids: [], + names: valid, + confidence: 'high', + candidates: [], + reason: 'exact', + }; const meshes = getSelection(el); - return {mode, query, match, applied: true, tier: 1, meshes, say: describeVoiceSelection(mode, match, meshes, query)}; + return { + mode, + query, + match, + applied: true, + tier: 1, + meshes, + say: describeVoiceSelection(mode, match, meshes, query), + }; } /** Short spoken summary of an edit result. */ @@ -1265,7 +1445,13 @@ async function applyPendingEdit( try { result = await el.editScene(instruction); } catch (e: unknown) { - result = {applied: false, code: '', repairs: 0, warnings: [], error: (e as Error)?.message || String(e)}; + result = { + applied: false, + code: '', + repairs: 0, + warnings: [], + error: (e as Error)?.message || String(e), + }; } emit(el, {state: 'idle', transcript: instruction, isFinal: true, result}); if (options.speak !== false) speak(describeEditResult(result), options.lang); @@ -1280,8 +1466,12 @@ export async function startVoiceEdit( options: AIVoiceEditOptions = {}, ): Promise { if (typeof el.editScene !== 'function') { - throw new Error('startVoiceEdit needs three-live-ai.min.js (el.editScene).'); + throw new Error( + 'startVoiceEdit needs three-live-ai.min.js (el.editScene).', + ); } + requireAICap(el, 'microphone', 'startVoiceEdit'); + requireAICap(el, 'ai', 'startVoiceEdit'); const recognizer = options.recognizer ?? createWebSpeechRecognizer(); if (!recognizer) { throw new Error( @@ -1313,21 +1503,27 @@ export async function startVoiceEdit( } let instruction = transcript; let selection: AIVoiceSelection | undefined; - const command = options.talkToSelect === false ? null : parseVoiceCommand(transcript); + const command = + options.talkToSelect === false ? null : parseVoiceCommand(transcript); if (command && command.kind !== 'edit') { emit(el, {state: 'thinking', transcript, isFinal: true}); try { - selection = await selectByVoice(el, command.kind === 'clear' ? '' : command.query, { - mode: command.mode, - signal: options.signal, - }); + selection = await selectByVoice( + el, + command.kind === 'clear' ? '' : command.query, + { + mode: command.mode, + signal: options.signal, + }, + ); } catch (e: unknown) { emit(el, {state: 'error', error: (e as Error)?.message || String(e)}); throw e; } const pending = command.kind === 'mixed' ? command.instruction : undefined; // Edit right away only when the reference was resolved locally (tier 1) and confidently. - const editNow = pending !== undefined && selection.applied && selection.tier === 1; + const editNow = + pending !== undefined && selection.applied && selection.tier === 1; // Chips only without a realtime session; there, the voice model reads the selection back. const chips = options.chips !== false && !el.__tlVoiceSession; if (chips) { @@ -1337,7 +1533,13 @@ export async function startVoiceEdit( sel.applied || sel.mode === 'clear' ? {pending: editNow ? undefined : pending} : sel.match?.candidates.length - ? {question: sel.say, candidates: sel.match.candidates, mode: sel.mode, query: sel.query, pending} + ? { + question: sel.say, + candidates: sel.match.candidates, + mode: sel.mode, + query: sel.query, + pending, + } : {}, (text) => applyPendingEdit(el, text, options), ); @@ -1359,7 +1561,12 @@ export async function startVoiceEdit( } else if (options.chips !== false && el.__tlChips) { hideSelectionChips(el); } - emit(el, {state: 'editing', transcript, isFinal: true, ...(selection ? {selection} : {})}); + emit(el, { + state: 'editing', + transcript, + isFinal: true, + ...(selection ? {selection} : {}), + }); let result: EditSceneResult; try { result = await el.editScene(instruction, {signal: options.signal}); @@ -1372,7 +1579,13 @@ export async function startVoiceEdit( error: (e as Error)?.message || String(e), }; } - emit(el, {state: 'idle', transcript, isFinal: true, result, ...(selection ? {selection} : {})}); + emit(el, { + state: 'idle', + transcript, + isFinal: true, + result, + ...(selection ? {selection} : {}), + }); if (options.speak !== false) { const said = describeEditResult(result); speak(selection ? `${selection.say} ${said}` : said, options.lang); @@ -1383,7 +1596,8 @@ export async function startVoiceEdit( /** Enables or disables tap-to-select (`ai-select-on-tap`). */ export function setSelectOnTap(el: AIVoiceElement, enabled: boolean): void { if (!enabled) { - if (el.__tlTapHandler) el.removeEventListener?.('meshclick', el.__tlTapHandler); + if (el.__tlTapHandler) + el.removeEventListener?.('meshclick', el.__tlTapHandler); if (el.__tlTapDown) el.removeEventListener?.('pointerdown', el.__tlTapDown); el.__tlTapHandler = null; el.__tlTapDown = null; @@ -1407,11 +1621,20 @@ export function setSelectOnTap(el: AIVoiceElement, enabled: boolean): void { const longPress = downAt > 0 && Date.now() - downAt >= LONG_PRESS_MS; const current = getSelection(el as AISelectableElement); if (additive || longPress) { - setSelection(el as AISelectableElement, [name], {mode: 'toggle', source: 'tap'}); + setSelection(el as AISelectableElement, [name], { + mode: 'toggle', + source: 'tap', + }); } else if (current.length === 1 && current[0] === name) { - setSelection(el as AISelectableElement, [], {mode: 'clear', source: 'tap'}); + setSelection(el as AISelectableElement, [], { + mode: 'clear', + source: 'tap', + }); } else { - setSelection(el as AISelectableElement, [name], {mode: 'replace', source: 'tap'}); + setSelection(el as AISelectableElement, [name], { + mode: 'replace', + source: 'tap', + }); } additive = false; downAt = 0; @@ -1444,7 +1667,9 @@ function createMicSvg(): Element { */ export function installVoiceButton(el: AIVoiceElement): void { if (el.__tlVoiceButton || typeof document === 'undefined') return; - const custom = el.querySelector?.('[slot="voice-button"]') as HTMLElement | null; + const custom = el.querySelector?.( + '[slot="voice-button"]', + ) as HTMLElement | null; const btn = custom ?? document.createElement('button'); if (!custom) { btn.setAttribute('type', 'button'); @@ -1473,7 +1698,8 @@ export function installVoiceButton(el: AIVoiceElement): void { let busy: AbortController | null = null; const setActive = (on: boolean) => { btn.setAttribute('aria-pressed', on ? 'true' : 'false'); - if (!custom) btn.style.background = on ? 'rgba(217,48,37,0.9)' : 'rgba(32,33,36,0.85)'; + if (!custom) + btn.style.background = on ? 'rgba(217,48,37,0.9)' : 'rgba(32,33,36,0.85)'; }; btn.addEventListener('click', (e) => { e.stopPropagation(); @@ -1485,15 +1711,26 @@ export function installVoiceButton(el: AIVoiceElement): void { return; } setActive(true); - const provider = (el.getAttribute?.('ai-voice-provider') || 'ai-realtime') as AIVoiceProviderName; - connectVoice(el, {provider, url, model: el.getAttribute?.('ai-voice-model') || undefined}) + const provider = (el.getAttribute?.('ai-voice-provider') || + 'ai-realtime') as AIVoiceProviderName; + connectVoice(el, { + provider, + url, + model: el.getAttribute?.('ai-voice-model') || undefined, + }) .then((s) => { - el.addEventListener?.('three-live-voice', function onVoice(ev: Event) { - if ((ev as CustomEvent).detail?.state === 'closed') { - el.removeEventListener?.('three-live-voice', onVoice); - if (el.__tlVoiceSession !== s) setActive(false); - } - }); + el.addEventListener?.( + 'three-live-voice', + function onVoice(ev: Event) { + if ( + (ev as CustomEvent).detail?.state === + 'closed' + ) { + el.removeEventListener?.('three-live-voice', onVoice); + if (el.__tlVoiceSession !== s) setActive(false); + } + }, + ); }) .catch(() => setActive(false)); return; diff --git a/three-live/src/plugins/ai.ts b/three-live/src/plugins/ai.ts index 8f98874c..a50987ca 100644 --- a/three-live/src/plugins/ai.ts +++ b/three-live/src/plugins/ai.ts @@ -24,7 +24,13 @@ * proxy (or a local model), or pass short-lived credentials via `headers`. */ -import {getSharedRegistry, type ThreeLivePlugin} from '../plugin'; +import type {ThreeLiveCapabilityName} from '../capabilities'; +import { + checkPluginCapability, + getSharedRegistry, + hasPluginCapability, + type ThreeLivePlugin, +} from '../plugin'; import { type AIImageInput, compileFromImage, @@ -309,10 +315,7 @@ export function createAIChatProvider( const baseModel = options.model?.trim() || undefined; const candidateModels: Array = options.fallbackModels ?.length - ? [ - baseModel, - ...options.fallbackModels.filter((m) => m && m !== baseModel), - ] + ? [baseModel, ...options.fallbackModels.filter((m) => m && m !== baseModel)] : [baseModel]; const explicitFirstByteTimeout = Number(options.firstByteTimeoutMs) > 0 @@ -799,19 +802,35 @@ export function selectionSentence(selected: readonly string[]): string { return `The user selected the meshes named ${selected.map((m) => `"${m}"`).join(', ')}; the instruction refers to them.\n`; } +/** Throws when `cap` is not granted on `el`. */ +export function requireAICap( + el: unknown, + cap: ThreeLiveCapabilityName, + feature: string, + url?: string, +): void { + if (!checkPluginCapability(el, cap, feature, url ? {url} : undefined)) { + throw new Error(`${feature}: capability '${cap}' denied.`); + } +} + export function resolveProvider( el: AIEditableElement, options: {readonly provider?: ThreeLiveAIProvider}, caller = 'editScene', ): ThreeLiveAIProvider { const explicit = options.provider ?? el.aiProvider; - if (explicit && typeof explicit.complete === 'function') return explicit; + if (explicit && typeof explicit.complete === 'function') { + requireAICap(el, 'ai', caller); + return explicit; + } const endpoint = el.getAttribute?.('ai-endpoint'); if (!endpoint) { throw new Error( `${caller}: no AI provider. Set el.aiProvider, or the ai-endpoint (and ai-model) attributes.`, ); } + requireAICap(el, 'ai', caller, endpoint); const kind = (el.getAttribute?.('ai-provider') || 'ai-chat').toLowerCase(); if (kind !== 'ai-chat') { throw new Error( @@ -843,7 +862,7 @@ export async function editScene( ): Promise { const text = String(instruction ?? '').trim(); if (!text) throw new Error('editScene: instruction is empty.'); - if (!el?.allowClientScripts) { + if (!checkPluginCapability(el, 'scripts', 'editScene')) { throw new Error( 'editScene requires client scripts: add the allow-client-scripts attribute.', ); @@ -1000,6 +1019,17 @@ export const aiPlugin: ThreeLivePlugin = { .querySelectorAll?.('three-live, xr-live') .forEach((node) => aiPlugin.onElementConnected?.(node)); }, + onAttributeChanged(host: unknown, name: string): void { + if (name !== 'capabilities' || !host) return; + const voiceEl = host as AIVoiceElement; + if ( + voiceEl.__tlVoiceSession && + (!hasPluginCapability(host, 'ai') || + !hasPluginCapability(host, 'microphone')) + ) { + voiceEl.__tlVoiceSession.close(); + } + }, onElementConnected(host: unknown): void { const el = host as AIEditableElement; if (!el || el.__tlAiInstalled) return; @@ -1017,7 +1047,7 @@ export const aiPlugin: ThreeLivePlugin = { startVoiceEdit(voiceEl, options); voiceEl.__tlSelectProvider = () => { try { - return resolveProvider(el, {}); + return resolveProvider(el, {}, 'selectByVoice'); } catch { return null; } diff --git a/three-live/src/plugins/envmap.ts b/three-live/src/plugins/envmap.ts index e72dd318..66ea1a54 100644 --- a/three-live/src/plugins/envmap.ts +++ b/three-live/src/plugins/envmap.ts @@ -21,7 +21,7 @@ */ import {mat4Invert} from '../math'; -import type {ThreeLivePlugin} from '../plugin'; +import {checkPluginCapability, type ThreeLivePlugin} from '../plugin'; const viewSHMap = new WeakMap(); @@ -109,6 +109,21 @@ async function setTex( prop: 'environmentTexture' | 'skyboxTexture', imageOrUrl: string | HTMLImageElement | ImageBitmap | null, ): Promise { + if (typeof imageOrUrl === 'string') { + const trimmed = imageOrUrl.trim(); + if (!trimmed || /^(?:neutral|legacy)$/i.test(trimmed)) { + if (viewer[prop]) viewer.gl.deleteTexture(viewer[prop]); + viewer[prop] = null; + return; + } + if ( + !checkPluginCapability(viewer?.container, 'remote_content', prop, { + url: trimmed, + }) + ) { + return; + } + } const tex = imageOrUrl ? await loadEnvironmentTexture(viewer.gl, imageOrUrl) : null; diff --git a/three-live/src/plugins/postprocessing.ts b/three-live/src/plugins/postprocessing.ts index 69c66d97..de714c5f 100644 --- a/three-live/src/plugins/postprocessing.ts +++ b/three-live/src/plugins/postprocessing.ts @@ -23,12 +23,12 @@ // outline pass around `selected-mesh` (when `outline-selected` is set) or // any node declaring `userData.xr.outline`. -import {createProgram} from '../gl-program'; import { HOST_BLOOM_DEFAULT_RADIUS, HOST_BLOOM_DEFAULT_THRESHOLD, parseNumericAttr, } from '../constants'; +import {createProgram} from '../gl-program'; import {mat3NormalFromMat4, mat4Create, mat4Multiply} from '../math'; import type {ThreeLivePlugin} from '../plugin'; import type {ThreeView} from '../three-view'; @@ -110,7 +110,7 @@ fragColor=vec4(c,1.0); // the page backdrop (premultiplied `rgb > alpha`): using `max(r, g, b)` would // make the browser scale the CSS backdrop by `1 - max` wherever the wide // low-level halo lands, visibly darkening/tinting the background when bloom is -// toggled (the regression fixed in cl/1013876464). White glow still gains +// toggled. White glow still gains // coverage alpha. const FS_KAWASE_UP = `#version 300 es precision highp float; diff --git a/three-live/src/three-live-element.ts b/three-live/src/three-live-element.ts index 365ac782..8fff84c1 100644 --- a/three-live/src/three-live-element.ts +++ b/three-live/src/three-live-element.ts @@ -18,6 +18,27 @@ * responsive layout, mesh inspection, and one-tap spatial/AR launch. */ +import { + CAPABILITIES_ALL, + CAPABILITIES_NONE, + CAPABILITIES_VIEWER, + checkHostCapability, + ensureCapabilityChecker, + hasCapabilityInPolicy, + isNetworkUrlPermitted, + isNonNetworkUrl, + normalizeCapabilityOrigin, + parseCapabilities, + resetCapabilityDenials, + resolveHostCapabilities, + sanitizeDiagnosticUrl, + type CapabilityDeniedDetail, + type ThreeLiveCapabilities, + type ThreeLiveCapabilitiesInit, + type ThreeLiveCapabilitiesInput, + type ThreeLiveCapabilityName, + type ThreeLiveNetworkPolicy, +} from './capabilities'; import { HOST_BLOOM_DEFAULT_RADIUS, HOST_BLOOM_DEFAULT_THRESHOLD, @@ -46,7 +67,28 @@ import { type OrbitLimits, type ToneMappingMode, } from './three-view'; - +ensureCapabilityChecker(); + +export { + CAPABILITIES_ALL, + CAPABILITIES_NONE, + CAPABILITIES_VIEWER, + checkHostCapability, + hasCapabilityInPolicy, + isNetworkUrlPermitted, + isNonNetworkUrl, + normalizeCapabilityOrigin, + parseCapabilities, + resetCapabilityDenials, + resolveHostCapabilities, + sanitizeDiagnosticUrl, + type CapabilityDeniedDetail, + type ThreeLiveCapabilities, + type ThreeLiveCapabilitiesInit, + type ThreeLiveCapabilitiesInput, + type ThreeLiveCapabilityName, + type ThreeLiveNetworkPolicy, +}; export type { DecoderType, MeshDecoder, @@ -60,6 +102,7 @@ export type ARMode = 'quick-look' | 'webxr' | 'scene-viewer'; export interface ARLaunchOptions { model: any; + iosSrc?: string; glbUrl?: string; view?: ThreeView; preferredModes?: ARMode[]; @@ -200,7 +243,7 @@ export interface ThreeLiveElement { /** Custom `` Web Component for declarative 3D/XR rendering. */ export class ThreeLiveElement extends BaseElement { static readonly observedAttributes = - 'src ios-src script-src sandbox allow-client-scripts poster camera-controls camera-orbit camera-target min-camera-orbit max-camera-orbit field-of-view selected-mesh ar ar-modes ar-scale ar-placement exposure shadow-intensity shadow-softness tone-mapping bloom bloom-threshold bloom-radius outline outline-selected outline-color environment-image skybox-image autoplay animation-name variant-name auto-rotate auto-rotate-delay rotation-per-second renderer shadow-maps'.split( + 'src ios-src script-src sandbox capabilities allow-client-scripts poster camera-controls camera-orbit camera-target min-camera-orbit max-camera-orbit field-of-view selected-mesh ar ar-modes ar-scale ar-placement exposure shadow-intensity shadow-softness tone-mapping bloom bloom-threshold bloom-radius outline outline-selected outline-color environment-image skybox-image autoplay animation-name variant-name auto-rotate auto-rotate-delay rotation-per-second renderer shadow-maps'.split( ' ', ); @@ -265,13 +308,46 @@ export class ThreeLiveElement extends BaseElement { * compiled by the sandbox unless explicitly enabled. */ static get allowClientScripts(): boolean { - return Boolean(getSharedRegistry().allowClientScripts); + return Boolean( + getSharedRegistry().allowClientScripts || + getSharedRegistry().defaultCapabilities?.scripts, + ); } static set allowClientScripts(value: boolean) { getSharedRegistry().allowClientScripts = value; } + /** + * Process/page-wide default capability policy (`CAPABILITIES_NONE` by default). + * Individual `` / `` elements override this via `capabilities`. + */ + static get defaultCapabilities(): ThreeLiveCapabilities { + return resolveHostCapabilities(undefined); + } + + static set defaultCapabilities( + value: ThreeLiveCapabilitiesInput | null | undefined, + ) { + getSharedRegistry().defaultCapabilities = parseCapabilities( + value, + CAPABILITIES_NONE, + ); + } + + /** Optional global listener notified whenever a capability check is denied. */ + static get onCapabilityDenied(): + | ((detail: CapabilityDeniedDetail) => void) + | null { + return getSharedRegistry().onCapabilityDenied ?? null; + } + + static set onCapabilityDenied( + value: ((detail: CapabilityDeniedDetail) => void) | null, + ) { + getSharedRegistry().onCapabilityDenied = value; + } + /** * Registers a pluggable procedural script runner (e.g. from `three-live/full` or `three-author`). */ @@ -385,6 +461,14 @@ export class ThreeLiveElement extends BaseElement { private mutationObserver: MutationObserver | null = null; connectedCallback(): void { + for (const prop of ['capabilities', 'allowClientScripts'] as const) { + if (Object.prototype.hasOwnProperty.call(this, prop)) { + const selfRec = this as unknown as Record; + const val = selfRec[prop]; + delete selfRec[prop]; + selfRec[prop] = val; + } + } if (typeof document === 'undefined') return; if (!this.style.display || this.style.display === 'inline') { this.style.display = 'block'; @@ -594,6 +678,18 @@ export class ThreeLiveElement extends BaseElement { if (v) v.shadowMaps = newVal !== null ? this.shadowMaps : undefined; v?.render(); break; + + case 'capabilities': + if (!this._syncingCapsAttr) { + this.__tlCapsOverride = null; + this.__tlCapsInit = null; + } + resetCapabilityDenials(this); + break; + + case 'allow-client-scripts': + resetCapabilityDenials(this); + break; } for (const plugin of ThreeView.getRegisteredPlugins()) { @@ -641,18 +737,88 @@ export class ThreeLiveElement extends BaseElement { this.setAttr('sandbox', value); } + /** Optional per-element callback invoked whenever a capability check is denied. */ + onCapabilityDenied: ((detail: CapabilityDeniedDetail) => void) | null = null; + /** Internal structured capability override set via `el.capabilities = {...}`. */ + __tlCapsOverride: ThreeLiveCapabilities | null = null; + __tlCapsInit: ThreeLiveCapabilitiesInit | null = null; + private _syncingCapsAttr = false; + + /** Effective normalized capability policy for this element. */ + get capabilities(): ThreeLiveCapabilities { + return resolveHostCapabilities(this); + } + + set capabilities(value: ThreeLiveCapabilitiesInput | null | undefined) { + resetCapabilityDenials(this); + this.__tlCapsOverride = null; + if (value == null) { + this.__tlCapsInit = null; + if ( + typeof this.hasAttribute === 'function' && + this.hasAttribute('capabilities') + ) { + this.removeAttribute('capabilities'); + } else { + for (const plugin of ThreeView.getRegisteredPlugins()) { + plugin.onAttributeChanged?.(this, 'capabilities', null, null); + } + } + } else if (typeof value === 'string') { + this.__tlCapsInit = null; + this.setAttr('capabilities', value); + } else { + this.__tlCapsInit = value; + if ( + typeof this.hasAttribute === 'function' && + this.hasAttribute('capabilities') + ) { + this._syncingCapsAttr = true; + try { + this.removeAttribute('capabilities'); + } finally { + this._syncingCapsAttr = false; + } + } else { + for (const plugin of ThreeView.getRegisteredPlugins()) { + plugin.onAttributeChanged?.(this, 'capabilities', null, null); + } + } + } + } + + /** Returns `true` when `capability` (and optional `url`) is granted on this element. */ + hasCapability(capability: ThreeLiveCapabilityName, url?: string): boolean { + const caps = this.capabilities; + if (capability === 'remote_content') { + return hasCapabilityInPolicy(caps, 'remote_content', url); + } + if (capability === 'ai') { + return hasCapabilityInPolicy(caps, 'ai', url); + } + return hasCapabilityInPolicy(caps, capability, url); + } + + /** + * Evaluates `capability` at a sensitive point of access. Logs a diagnostic and + * dispatches `capability-denied` when refused, returning `false` without throwing. + */ + checkCapability( + capability: ThreeLiveCapabilityName, + feature: string, + options?: {readonly url?: string; readonly dedup?: boolean}, + ): boolean { + return checkHostCapability(this, capability, feature, options); + } + /** * Opt-in flag enabling client-modified procedural script strings on this * element (`executeScript(code)` and inline ` diff --git a/xr-live/src/plugins/depth-particles.ts b/xr-live/src/plugins/depth-particles.ts index ba92d807..e055873d 100644 --- a/xr-live/src/plugins/depth-particles.ts +++ b/xr-live/src/plugins/depth-particles.ts @@ -27,15 +27,22 @@ * WebXR depth buffer; collided particles respawn at the top of the volume. */ -import type {ThreeLivePlugin} from '../../../three-live/src/plugin'; +import { + mat4Create, + mat4Invert, + mat4Multiply, +} from '../../../three-live/src/math'; +import { + checkPluginCapability, + type ThreeLivePlugin, +} from '../../../three-live/src/plugin'; import { MAX_PARTICLES_PER_EMITTER, + parseParticles, type ParticleOptions, type ParticlePreset, - parseParticles, } from '../../../three-live/src/xr-trait-schema'; import {extractPosition, getMeshXRTraits} from '../xr-traits'; -import {mat4Create, mat4Invert, mat4Multiply} from '../../../three-live/src/math'; import { cacheXRFrame, clampNum, @@ -60,7 +67,10 @@ export const MAX_EMITTERS = 8; const VERTS_PER_PARTICLE = 8; -const PRESETS: Record = { +const PRESETS: Record< + ParticlePreset, + {color: [number, number, number, number]; size: Vec3} +> = { rain: {color: [0.7, 0.8, 1.0, 0.6], size: [0.002, 0.04, 0.002]}, snow: {color: [1, 1, 1, 0.9], size: [0.01, 0.01, 0.01]}, sparks: {color: [1.0, 0.7, 0.2, 1], size: [0.004, 0.004, 0.004]}, @@ -93,7 +103,10 @@ export interface DepthCamera { * Returns true when world point `p` lies at or behind the real-world surface * seen at its projected pixel (with a 2 cm tolerance). */ -export function isOccludedByDepth(p: readonly number[], cam: DepthCamera): boolean { +export function isOccludedByDepth( + p: readonly number[], + cam: DepthCamera, +): boolean { const m = cam.view; const vx = m[0] * p[0] + m[4] * p[1] + m[8] * p[2] + m[12]; const vy = m[1] * p[0] + m[5] * p[1] + m[9] * p[2] + m[13]; @@ -105,7 +118,12 @@ export function isOccludedByDepth(p: readonly number[], cam: DepthCamera): boole const ndcY = (P[5] * vy + P[9] * vz) / viewDepth; if (Math.abs(ndcX) > 1 || Math.abs(ndcY) > 1) return false; const real = cam.sample((ndcX + 1) * 0.5, (1 - ndcY) * 0.5); - return typeof real === 'number' && Number.isFinite(real) && real > 0.05 && viewDepth >= real - 0.02; + return ( + typeof real === 'number' && + Number.isFinite(real) && + real > 0.05 && + viewDepth >= real - 0.02 + ); } /** CPU particle state for one emitter volume. */ @@ -139,14 +157,21 @@ export class ParticleEmitter { const sp = this.options.speed; const burst = this.options.preset === 'sparks'; this.positions[o] = burst ? cx : cx + (r() - 0.5) * w; - this.positions[o + 1] = burst ? cy : anywhere ? cy + (r() - 0.5) * h : cy + h * 0.5; + this.positions[o + 1] = burst + ? cy + : anywhere + ? cy + (r() - 0.5) * h + : cy + h * 0.5; this.positions[o + 2] = burst ? cz : cz + (r() - 0.5) * d; switch (this.options.preset) { case 'rain': this.velocities.set([0, -4 * sp, 0], o); break; case 'snow': - this.velocities.set([(r() - 0.5) * 0.2, -0.5 * sp, (r() - 0.5) * 0.2], o); + this.velocities.set( + [(r() - 0.5) * 0.2, -0.5 * sp, (r() - 0.5) * 0.2], + o, + ); break; case 'sparks': { const a = r() * Math.PI * 2; @@ -155,7 +180,10 @@ export class ParticleEmitter { break; } case 'confetti': - this.velocities.set([(r() - 0.5) * sp, (0.5 + r()) * sp, (r() - 0.5) * sp], o); + this.velocities.set( + [(r() - 0.5) * sp, (0.5 + r()) * sp, (r() - 0.5) * sp], + o, + ); break; default: { const unreachable: never = this.options.preset; @@ -168,11 +196,17 @@ export class ParticleEmitter { * Advances the simulation. `center` moves the volume with its mesh; when * provided, `floorY` and `depth` collide particles with real geometry. */ - step(dt: number, center: readonly number[], floorY?: number, depth?: DepthCamera): void { + step( + dt: number, + center: readonly number[], + floorY?: number, + depth?: DepthCamera, + ): void { const step = clampNum(dt, 0, 0.1, 1 / 60); this.center = [center[0], center[1], center[2]]; const preset = this.options.preset; - const gravity = preset === 'sparks' || preset === 'confetti' ? -9.8 * 0.4 : 0; + const gravity = + preset === 'sparks' || preset === 'confetti' ? -9.8 * 0.4 : 0; // Every particle uses the same frame-scaled drag coefficient. const drag = preset === 'confetti' ? Math.pow(0.9, step * 60) : 1; const [w, h, d] = this.options.area; @@ -212,35 +246,64 @@ export class ParticleEmitter { /** Writes particle cross-quads (8 verts each) into `out` (model space). */ writeVertices(out: Float32Array): void { - writeParticleVertices(out, this.positions, this.options.count, PRESETS[this.options.preset].size); + writeParticleVertices( + out, + this.positions, + this.options.count, + PRESETS[this.options.preset].size, + ); } } /** Shared cross-quad writer for simulated centers and the static origin template. */ -function writeParticleVertices(out: Float32Array, p: ArrayLike, count: number, size: Vec3): void { - const [sx, sy, sz] = size; - const dz = sz + sx; +function writeParticleVertices( + out: Float32Array, + p: ArrayLike, + count: number, + size: Vec3, +): void { + const [sx, sy, sz] = size; + const dz = sz + sx; - for (let i = 0; i < count; i++) { - const o = i * 3; - let b = i * VERTS_PER_PARTICLE * 3; - const x = p[o]; - const y = p[o + 1]; - const z = p[o + 2]; - // Quad in the XY plane, then quad in the ZY plane (no allocation). - out[b++] = x - sx; out[b++] = y - sy; out[b++] = z; - out[b++] = x + sx; out[b++] = y - sy; out[b++] = z; - out[b++] = x + sx; out[b++] = y + sy; out[b++] = z; - out[b++] = x - sx; out[b++] = y + sy; out[b++] = z; - out[b++] = x; out[b++] = y - sy; out[b++] = z - dz; - out[b++] = x; out[b++] = y - sy; out[b++] = z + dz; - out[b++] = x; out[b++] = y + sy; out[b++] = z + dz; - out[b++] = x; out[b++] = y + sy; out[b++] = z - dz; - } + for (let i = 0; i < count; i++) { + const o = i * 3; + let b = i * VERTS_PER_PARTICLE * 3; + const x = p[o]; + const y = p[o + 1]; + const z = p[o + 2]; + // Quad in the XY plane, then quad in the ZY plane (no allocation). + out[b++] = x - sx; + out[b++] = y - sy; + out[b++] = z; + out[b++] = x + sx; + out[b++] = y - sy; + out[b++] = z; + out[b++] = x + sx; + out[b++] = y + sy; + out[b++] = z; + out[b++] = x - sx; + out[b++] = y + sy; + out[b++] = z; + out[b++] = x; + out[b++] = y - sy; + out[b++] = z - dz; + out[b++] = x; + out[b++] = y - sy; + out[b++] = z + dz; + out[b++] = x; + out[b++] = y + sy; + out[b++] = z + dz; + out[b++] = x; + out[b++] = y + sy; + out[b++] = z - dz; + } } /** Builds the static index/normal buffers for `count` particle cross-quads. */ -export function buildParticleTopology(count: number): {normals: Float32Array; indices: Uint16Array} { +export function buildParticleTopology(count: number): { + normals: Float32Array; + indices: Uint16Array; +} { const normals = new Float32Array(count * VERTS_PER_PARTICLE * 3); const indices = new Uint16Array(count * 12); for (let i = 0; i < count; i++) { @@ -250,7 +313,23 @@ export function buildParticleTopology(count: number): {normals: Float32Array; in normals[(v + k) * 3 + 2] = 1; normals[(v + k + 4) * 3] = 1; } - indices.set([v, v + 1, v + 2, v, v + 2, v + 3, v + 4, v + 5, v + 6, v + 4, v + 6, v + 7], i * 12); + indices.set( + [ + v, + v + 1, + v + 2, + v, + v + 2, + v + 3, + v + 4, + v + 5, + v + 6, + v + 4, + v + 6, + v + 7, + ], + i * 12, + ); } return {normals, indices}; } @@ -277,7 +356,12 @@ export class XRDepthParticlesController { } /** Steps all emitters and syncs their GPU geometry. */ - update(viewer: PluginViewer, dt: number, floorY?: number, depth?: DepthCamera): void { + update( + viewer: PluginViewer, + dt: number, + floorY?: number, + depth?: DepthCamera, + ): void { const meshes = viewer.meshes ?? []; const live = new Set(); for (const mesh of [...meshes]) { @@ -297,22 +381,34 @@ export class XRDepthParticlesController { const emitter = new ParticleEmitter(options, this.entries.size + 1); emitter.reset(center); const gl = viewer.gl; - const instanced = viewer.supportsInstanceOffsets === true && + const instanced = + viewer.supportsInstanceOffsets === true && typeof gl?.drawElementsInstanced === 'function' && typeof gl?.vertexAttribDivisor === 'function'; const count = instanced ? 1 : options.count; const positions = new Float32Array(count * VERTS_PER_PARTICLE * 3); // A zero-centered emitter writes the same cross-quads as the fallback, // including the asymmetric Z extent. Simulation remains on the CPU. - writeParticleVertices(positions, instanced ? [0, 0, 0] : emitter.positions, count, PRESETS[options.preset].size); + writeParticleVertices( + positions, + instanced ? [0, 0, 0] : emitter.positions, + count, + PRESETS[options.preset].size, + ); const topo = buildParticleTopology(count); const pm = createDynamicMesh( viewer, { - positions, normals: topo.normals, indices: topo.indices, + positions, + normals: topo.normals, + indices: topo.indices, translations: instanced ? emitter.positions : undefined, }, - {name: `__xr_particles_${mesh.name ?? 'emitter'}`, color: PRESETS[options.preset].color, pickable: false}, + { + name: `__xr_particles_${mesh.name ?? 'emitter'}`, + color: PRESETS[options.preset].color, + pickable: false, + }, ); entry = {emitter, mesh: pm, key}; this.entries.set(mesh, entry); @@ -332,7 +428,8 @@ export class XRDepthParticlesController { /** Removes every emitter mesh. */ dispose(viewer: PluginViewer | undefined): void { - for (const entry of this.entries.values()) removeDynamicMesh(viewer, entry.mesh); + for (const entry of this.entries.values()) + removeDynamicMesh(viewer, entry.mesh); this.entries.clear(); } } @@ -340,7 +437,9 @@ export class XRDepthParticlesController { const controllers = new WeakMap(); /** Returns (creating on first use) the particle controller for `viewer`. */ -export function getDepthParticlesController(viewer: object): XRDepthParticlesController { +export function getDepthParticlesController( + viewer: object, +): XRDepthParticlesController { let c = controllers.get(viewer); if (!c) { c = new XRDepthParticlesController(); @@ -352,7 +451,14 @@ export function getDepthParticlesController(viewer: object): XRDepthParticlesCon function depthCameraFor(viewer: PluginViewer): DepthCamera | undefined { const host = viewer.container; const dm = host?.depthManager; - if (!dm || host?.xrDepth === 'none' || typeof dm.sampleDepthAndNormal !== 'function') return undefined; + if ( + !dm || + host?.xrDepth === 'none' || + typeof dm.sampleDepthAndNormal !== 'function' || + !checkPluginCapability(host, 'sensors', 'depthParticles', {dedup: true}) + ) { + return undefined; + } // Real-world depth only exists while presenting; use the headset camera. const proj = resolveProjection(viewer); if (!isXRFrameFresh(viewer) || !proj) return undefined; @@ -392,7 +498,12 @@ export const depthParticlesPlugin: ThreeLivePlugin = { onUpdate(viewer: PluginViewer, dt: number): void { if (!isXRFrameFresh(viewer)) stepViewer(viewer, dt); }, - onXRFrame(viewer: unknown, _session: unknown, frame: unknown, refSpace: unknown): void { + onXRFrame( + viewer: unknown, + _session: unknown, + frame: unknown, + refSpace: unknown, + ): void { cacheXRFrame(viewer as PluginViewer, frame, refSpace); stepViewer(viewer as PluginViewer, 1 / 72); }, diff --git a/xr-live/src/plugins/gaze-dwell.ts b/xr-live/src/plugins/gaze-dwell.ts index 8c2e6058..19ba2d7c 100644 --- a/xr-live/src/plugins/gaze-dwell.ts +++ b/xr-live/src/plugins/gaze-dwell.ts @@ -24,6 +24,7 @@ */ import { + checkPluginCapability, getRegisteredPlugins, type ThreeLivePlugin, } from '../../../three-live/src/plugin'; @@ -557,6 +558,9 @@ export class XRGazeDwellController { }); if (opts.action === 'ask') { + if (!checkPluginCapability(element, 'ai', 'holdToAsk', {dedup: true})) { + return; + } const rawTraits = getMeshXRTraits(mesh); const holdOpts = parseHoldToAsk(rawTraits?.holdToAsk) ?? { prompt: 'What is this?', diff --git a/xr-live/src/plugins/gemini-live.ts b/xr-live/src/plugins/gemini-live.ts index ac7749fd..15c14461 100644 --- a/xr-live/src/plugins/gemini-live.ts +++ b/xr-live/src/plugins/gemini-live.ts @@ -31,8 +31,15 @@ * use a short-lived ephemeral token in the `wss://` URL your server mints. */ -import type {ThreeLivePlugin} from '../../../three-live/src/plugin'; -import {type HoldToAskOptions, parseHoldToAsk} from '../../../three-live/src/xr-trait-schema'; +import { + checkPluginCapability, + hasPluginCapability, + type ThreeLivePlugin, +} from '../../../three-live/src/plugin'; +import { + type HoldToAskOptions, + parseHoldToAsk, +} from '../../../three-live/src/xr-trait-schema'; import {getMeshXRTraits} from '../xr-traits'; import {clampNum, emitPluginEvent, isEventClaimed} from './common'; import { @@ -75,7 +82,9 @@ export function isAllowedLiveUrl(raw: string): boolean { if (url.protocol === 'wss:') return true; return ( url.protocol === 'ws:' && - (url.hostname === 'localhost' || url.hostname === '127.0.0.1' || url.hostname === '[::1]') + (url.hostname === 'localhost' || + url.hostname === '127.0.0.1' || + url.hostname === '[::1]') ); } catch { return false; @@ -85,7 +94,9 @@ export function isAllowedLiveUrl(raw: string): boolean { /** Creates a JSON-over-WebSocket transport for a validated `wss:` URL. */ export function createWebSocketTransport(url: string): GeminiLiveTransport { if (!isAllowedLiveUrl(url)) { - throw new Error('gemini-live: only wss:// (or loopback ws://) endpoints are allowed'); + throw new Error( + 'gemini-live: only wss:// (or loopback ws://) endpoints are allowed', + ); } const ws = new WebSocket(url); const queue: string[] = []; @@ -108,7 +119,8 @@ export function createWebSocketTransport(url: string): GeminiLiveTransport { // Decoding a Blob can outlive the controller session that received it. const onMessage = transport.onMessage; try { - const text = typeof e.data === 'string' ? e.data : await (e.data as Blob).text(); + const text = + typeof e.data === 'string' ? e.data : await (e.data as Blob).text(); const parsed: unknown = ws.onmessage && JSON.parse(text); if (parsed && typeof parsed === 'object') { onMessage?.(parsed as Record); @@ -132,7 +144,10 @@ export function bytesToBase64(bytes: Uint8Array): string { /** Accumulates streaming transcription fragments per speaker. */ export class TranscriptionManager { - private readonly partial: Record = {user: '', model: ''}; + private readonly partial: Record = { + user: '', + model: '', + }; /** Appends a fragment and returns the updated (non-final) segment. */ append(speaker: TranscriptSpeaker, fragment: string): TranscriptEvent { @@ -185,7 +200,8 @@ export class PcmPlayer { /** Schedules one PCM chunk after any queued audio. */ enqueue(samples: Float32Array, sampleRate: number): void { if (samples.length === 0) return; - const Ctor = (globalThis as {AudioContext?: typeof AudioContext}).AudioContext; + const Ctor = (globalThis as {AudioContext?: typeof AudioContext}) + .AudioContext; if (!this.ctx && typeof Ctor === 'function') { try { this.ctx = new Ctor(); @@ -257,7 +273,8 @@ export const REPORT_OBJECTS_DECLARATION: Readonly> = { box_2d: { type: 'ARRAY', items: {type: 'NUMBER'}, - description: 'Bounding box [ymin, xmin, ymax, xmax], normalized to 0-1000.', + description: + 'Bounding box [ymin, xmin, ymax, xmax], normalized to 0-1000.', }, }, required: ['label', 'box_2d'], @@ -277,12 +294,18 @@ export function withDetectionTool( setup: Readonly> | undefined, ): Record | undefined { if (!setup || typeof setup !== 'object') return undefined; - const tools = Array.isArray(setup['tools']) ? [...(setup['tools'] as unknown[])] : []; + const tools = Array.isArray(setup['tools']) + ? [...(setup['tools'] as unknown[])] + : []; const declared = tools.some((t) => { const decls = (t as {functionDeclarations?: unknown})?.functionDeclarations; - return Array.isArray(decls) && decls.some((d) => (d as {name?: unknown})?.name === REPORT_OBJECTS_TOOL); + return ( + Array.isArray(decls) && + decls.some((d) => (d as {name?: unknown})?.name === REPORT_OBJECTS_TOOL) + ); }); - if (!declared) tools.push({functionDeclarations: [REPORT_OBJECTS_DECLARATION]}); + if (!declared) + tools.push({functionDeclarations: [REPORT_OBJECTS_DECLARATION]}); return {...setup, tools}; } @@ -311,7 +334,9 @@ export interface GeminiLiveOptions { } interface CaptureHost { - captureViewFrame?: (opts: Record) => Promise<{base64: string; mimeType: string}>; + captureViewFrame?: ( + opts: Record, + ) => Promise<{base64: string; mimeType: string}>; } /** @@ -321,6 +346,7 @@ interface CaptureHost { export class XRGeminiLiveController { readonly transcripts = new TranscriptionManager(); private transport: GeminiLiveTransport | null = null; + private activeUrl: string | null = null; /** Invalidates asynchronous work even when a transport object is reused. */ private sessionId = 0; private frameTimer: ReturnType | null = null; @@ -346,6 +372,10 @@ export class XRGeminiLiveController { return this.transport !== null; } + getActiveUrl(): string | null { + return this.activeUrl; + } + /** Connects (replacing any previous session) and starts frame streaming. */ connect(options: GeminiLiveOptions): void { const element = this.element as object; @@ -359,15 +389,28 @@ export class XRGeminiLiveController { const activeCtrl = controllers.get(element) ?? this; // A close callback or transcript listener may have connected a newer session. if (this.sessionId !== sessionId || activeCtrl !== this) { - if (options.transport !== activeCtrl.transport && options.transport !== prevTransport) { + if ( + options.transport !== activeCtrl.transport && + options.transport !== prevTransport + ) { options.transport?.close(); } return; } + if ( + !checkPluginCapability(this.element, 'ai', 'geminiLive', { + url: options.url, + }) + ) { + options.transport?.close(); + throw new Error("gemini-live: capability 'ai' or 'network' denied."); + } const transport = - options.transport ?? (options.url ? createWebSocketTransport(options.url) : null); + options.transport ?? + (options.url ? createWebSocketTransport(options.url) : null); if (!transport) throw new Error('gemini-live: provide a transport or url'); this.transport = transport; + this.activeUrl = options.url ?? null; const isCurrentSession = this.sessionGuard(); this.speakReplies = options.speakReplies === true; this.playAudio = options.playAudio !== false; @@ -379,14 +422,22 @@ export class XRGeminiLiveController { transport.onClose = () => { if (isCurrentSession()) this.resetSession(false); }; - const setup = this.detectObjects ? withDetectionTool(options.setup) : options.setup; + const setup = this.detectObjects + ? withDetectionTool(options.setup) + : options.setup; try { if (setup) transport.send({setup}); } catch (err) { if (isCurrentSession()) this.resetSession(true); throw err; } - if (options.streamFrames !== false && isCurrentSession()) { + if ( + options.streamFrames !== false && + isCurrentSession() && + checkPluginCapability(this.element, 'camera', 'geminiStreamFrames', { + dedup: true, + }) + ) { const interval = clampNum(options.frameIntervalMs, 250, 10000, 1000); this.frameTimer = setInterval(() => { if (isCurrentSession()) void this.sendFrame(); @@ -402,14 +453,21 @@ export class XRGeminiLiveController { /** Stops streaming and closes the transport. */ disconnect(): void { - if (this.transport || holdState.has(this.element as object)) this.resetSession(true); + if (this.transport || holdState.has(this.element as object)) + this.resetSession(true); } private resetSession(closeTransport: boolean): number { const id = ++this.sessionId; if (this.frameTimer !== null) clearInterval(this.frameTimer); const transport = this.transport; - this.frameTimer = this.pendingFrame = this.lastFrame = this.frameCamera = this.transport = null; + this.frameTimer = + this.pendingFrame = + this.lastFrame = + this.frameCamera = + this.transport = + this.activeUrl = + null; this.turnHasOutputTranscription = false; const finalized = this.transcripts.finalize(); const element = this.element as object; @@ -432,23 +490,44 @@ export class XRGeminiLiveController { const host = this.element as CaptureHost; const transport = this.transport; const current = this.sessionGuard(); - if (!transport || this.pendingFrame || typeof host?.captureViewFrame !== 'function') return Promise.resolve(false); + if ( + !transport || + this.pendingFrame || + typeof host?.captureViewFrame !== 'function' || + !checkPluginCapability(this.element, 'camera', 'geminiStreamFrames', { + dedup: true, + }) + ) { + return Promise.resolve(false); + } let finish!: (sent: boolean) => void; - const pending = new Promise(resolve => { finish = resolve; }); + const pending = new Promise((resolve) => { + finish = resolve; + }); this.pendingFrame = pending; void (async () => { let sent = false; try { - const camera = this.detectObjects ? snapshotDetectionCamera(this.element) : null; - const frame = await host.captureViewFrame!({format: 'image/jpeg', quality: 0.6}); + const camera = this.detectObjects + ? snapshotDetectionCamera(this.element) + : null; + const frame = await host.captureViewFrame!({ + format: 'image/jpeg', + quality: 0.6, + }); if (frame?.base64 && current()) { this.frameCamera = camera; this.lastFrame = {base64: frame.base64, mimeType: frame.mimeType}; - transport.send({realtimeInput: {mediaChunks: [{mimeType: frame.mimeType, data: frame.base64}]}}); + transport.send({ + realtimeInput: { + mediaChunks: [{mimeType: frame.mimeType, data: frame.base64}], + }, + }); sent = true; } - } catch { /* Failed capture cannot admit a dependent visual question. */ } - finally { + } catch { + /* Failed capture cannot admit a dependent visual question. */ + } finally { if (this.pendingFrame === pending) this.pendingFrame = null; finish(sent); } @@ -458,11 +537,27 @@ export class XRGeminiLiveController { /** Sends a chunk of 16-bit little-endian mono PCM microphone audio. */ sendAudioChunk(pcm16: Int16Array, sampleRate = 16000): void { - if (!this.transport || !(pcm16 instanceof Int16Array)) return; + if ( + !this.transport || + !(pcm16 instanceof Int16Array) || + !checkPluginCapability(this.element, 'microphone', 'geminiLiveAudio', { + dedup: true, + }) + ) { + return; + } const rate = Math.round(clampNum(sampleRate, 8000, 48000, 16000)); - const bytes = new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength); + const bytes = new Uint8Array( + pcm16.buffer, + pcm16.byteOffset, + pcm16.byteLength, + ); this.transport.send({ - realtimeInput: {mediaChunks: [{mimeType: `audio/pcm;rate=${rate}`, data: bytesToBase64(bytes)}]}, + realtimeInput: { + mediaChunks: [ + {mimeType: `audio/pcm;rate=${rate}`, data: bytesToBase64(bytes)}, + ], + }, }); } @@ -472,23 +567,42 @@ export class XRGeminiLiveController { const current = this.sessionGuard(); const clipped = String(text ?? '').slice(0, MAX_TEXT_CHARS); if (!transport || !clipped) return; - const canCapture = typeof (this.element as CaptureHost)?.captureViewFrame === 'function'; + if ( + !checkPluginCapability(this.element, 'ai', 'geminiLive', { + url: this.activeUrl ?? undefined, + dedup: true, + }) + ) { + return; + } + const canCapture = + typeof (this.element as CaptureHost)?.captureViewFrame === 'function'; if (options?.captureFrame && !canCapture) return; const pending = - this.pendingFrame ?? (options?.captureFrame && canCapture ? this.sendFrame() : null); + this.pendingFrame ?? + (options?.captureFrame && canCapture ? this.sendFrame() : null); const send = (captured = true) => { if (!captured || !current()) return; const frame = pending ? this.lastFrame : null; // Bind the image and question in one conversation turn. Realtime streams // alone provide no cross-modality ordering guarantee at the service. const parts: Array> = frame - ? [{inlineData: {mimeType: frame.mimeType, data: frame.base64}}, {text: clipped}] + ? [ + {inlineData: {mimeType: frame.mimeType, data: frame.base64}}, + {text: clipped}, + ] : [{text: clipped}]; - transport.send({clientContent: {turns: [{role: 'user', parts}], turnComplete: true}}); + transport.send({ + clientContent: {turns: [{role: 'user', parts}], turnComplete: true}, + }); }; - if (pending) void pending.then(send).catch((error) => { - if (current()) emitPluginEvent(this.element, 'xr-gemini-error', {message: String(error)}); - }); + if (pending) + void pending.then(send).catch((error) => { + if (current()) + emitPluginEvent(this.element, 'xr-gemini-error', { + message: String(error), + }); + }); else send(); } @@ -499,10 +613,15 @@ export class XRGeminiLiveController { const isCurrentSession = this.sessionGuard(); const toolCall = message?.['toolCall']; if (toolCall && typeof toolCall === 'object') { - this.handleToolCall(toolCall as Record, isCurrentSession); + this.handleToolCall( + toolCall as Record, + isCurrentSession, + ); } if (!isCurrentSession()) return; - const content = message?.['serverContent'] as Record | undefined; + const content = message?.['serverContent'] as + | Record + | undefined; if (!content || typeof content !== 'object') return; if (content['interrupted'] === true) { cancelSpeech(); @@ -511,7 +630,8 @@ export class XRGeminiLiveController { if (!isCurrentSession()) return; } const input = (content['inputTranscription'] as {text?: unknown})?.text; - if (typeof input === 'string' && input) this.emitTranscript(this.transcripts.append('user', input)); + if (typeof input === 'string' && input) + this.emitTranscript(this.transcripts.append('user', input)); if (!isCurrentSession()) return; const output = (content['outputTranscription'] as {text?: unknown})?.text; if (typeof output === 'string' && output) { @@ -526,11 +646,17 @@ export class XRGeminiLiveController { const text = (part as {text?: unknown})?.text; // In AUDIO mode the transcript comes from outputTranscription; avoid // double-appending any accompanying text parts. - if (typeof text === 'string' && text && !this.turnHasOutputTranscription) { + if ( + typeof text === 'string' && + text && + !this.turnHasOutputTranscription + ) { this.emitTranscript(this.transcripts.append('model', text)); if (!isCurrentSession()) return; } - const inline = (part as {inlineData?: {mimeType?: unknown; data?: unknown}})?.inlineData; + const inline = ( + part as {inlineData?: {mimeType?: unknown; data?: unknown}} + )?.inlineData; if ( this.playAudio && typeof inline?.data === 'string' && @@ -539,7 +665,10 @@ export class XRGeminiLiveController { inline.data.length <= 8 * 1024 * 1024 ) { try { - this.audio.enqueue(decodePcm16Base64(inline.data), pcmRateFromMime(inline.mimeType)); + this.audio.enqueue( + decodePcm16Base64(inline.data), + pcmRateFromMime(inline.mimeType), + ); } catch { // Ignore malformed audio chunks. } @@ -551,7 +680,8 @@ export class XRGeminiLiveController { for (const ev of this.transcripts.finalize()) { if (!isCurrentSession()) return; this.emitTranscript(ev); - if (isCurrentSession() && ev.speaker === 'model' && this.speakReplies) speak(ev.text); + if (isCurrentSession() && ev.speaker === 'model' && this.speakReplies) + speak(ev.text); } } } @@ -562,7 +692,10 @@ export class XRGeminiLiveController { * through the same validation as the `detections` prop. Other functions * are left to the host. */ - private handleToolCall(toolCall: Record, isCurrentSession: () => boolean): void { + private handleToolCall( + toolCall: Record, + isCurrentSession: () => boolean, + ): void { const calls = toolCall['functionCalls']; const transport = this.transport; if (!this.detectObjects || !transport || !Array.isArray(calls)) return; @@ -572,14 +705,21 @@ export class XRGeminiLiveController { if (c?.name !== REPORT_OBJECTS_TOOL) continue; const objects = (c.args as {objects?: unknown} | null)?.objects; const located = detectObjects( - this.element, detectionsFromList(objects), this.frameCamera ?? undefined, isCurrentSession, + this.element, + detectionsFromList(objects), + this.frameCamera ?? undefined, + isCurrentSession, ); if (!isCurrentSession()) return; - const response: Record = {name: REPORT_OBJECTS_TOOL, response: {ok: true, located: located.length}}; + const response: Record = { + name: REPORT_OBJECTS_TOOL, + response: {ok: true, located: located.length}, + }; if (typeof c.id === 'string') response['id'] = c.id.slice(0, 256); functionResponses.push(response); } - if (functionResponses.length) transport.send({toolResponse: {functionResponses}}); + if (functionResponses.length) + transport.send({toolResponse: {functionResponses}}); } private emitTranscript(ev: TranscriptEvent): void { @@ -588,15 +728,19 @@ export class XRGeminiLiveController { } function speak(text: string): void { - const synth = (globalThis as {speechSynthesis?: SpeechSynthesis}).speechSynthesis; - const Utterance = (globalThis as {SpeechSynthesisUtterance?: typeof SpeechSynthesisUtterance}) - .SpeechSynthesisUtterance; + const synth = (globalThis as {speechSynthesis?: SpeechSynthesis}) + .speechSynthesis; + const Utterance = ( + globalThis as {SpeechSynthesisUtterance?: typeof SpeechSynthesisUtterance} + ).SpeechSynthesisUtterance; if (!synth || typeof Utterance !== 'function') return; synth.speak(new Utterance(text.slice(0, MAX_TEXT_CHARS))); } function cancelSpeech(): void { - (globalThis as {speechSynthesis?: SpeechSynthesis}).speechSynthesis?.cancel?.(); + ( + globalThis as {speechSynthesis?: SpeechSynthesis} + ).speechSynthesis?.cancel?.(); } export {parseHoldToAsk}; @@ -642,12 +786,17 @@ function cancelHold(state: HoldState | undefined): void { const listeners = new WeakMap>(); /** Returns (creating on first use) the Gemini Live controller for `element`. */ -export function getGeminiLiveController(element: object): XRGeminiLiveController { +export function getGeminiLiveController( + element: object, +): XRGeminiLiveController { return controllers.get(element) ?? new XRGeminiLiveController(element); } /** Convenience: connects `element` to Gemini Live and returns its controller. */ -export function connectGeminiLive(element: object, options: GeminiLiveOptions): XRGeminiLiveController { +export function connectGeminiLive( + element: object, + options: GeminiLiveOptions, +): XRGeminiLiveController { const c = getGeminiLiveController(element); c.connect(options); return c; @@ -662,14 +811,23 @@ interface Recognition { function startRecognition(onText: (text: string) => void): Recognition | null { const g = globalThis as Record; const Ctor = (g['SpeechRecognition'] ?? g['webkitSpeechRecognition']) as - | (new () => {lang: string; interimResults: boolean; onresult: ((e: unknown) => void) | null; start(): void; stop(): void; abort?(): void}) + | (new () => { + lang: string; + interimResults: boolean; + onresult: ((e: unknown) => void) | null; + start(): void; + stop(): void; + abort?(): void; + }) | undefined; if (typeof Ctor !== 'function') return null; try { const rec = new Ctor(); rec.interimResults = false; rec.onresult = (e: unknown) => { - const results = (e as {results?: ArrayLike>})?.results; + const results = ( + e as {results?: ArrayLike>} + )?.results; const text = results?.[0]?.[0]?.transcript; if (typeof text === 'string') onText(text); }; @@ -685,11 +843,22 @@ interface SelectDetail { readonly hit?: {mesh?: {name?: string}} | null; } -function finishHold(element: object, meshName: string, options: HoldToAskOptions, transcript: string, current: () => boolean): void { +function finishHold( + element: object, + meshName: string, + options: HoldToAskOptions, + transcript: string, + current: () => boolean, +): void { if (!current()) return; const prompt = buildHoldToAskPrompt(meshName, options, transcript); - emitPluginEvent(element, 'xr-hold-to-ask', {mesh: meshName, transcript, prompt}); - if (current()) controllers.get(element)?.sendText(prompt, {captureFrame: true}); + emitPluginEvent(element, 'xr-hold-to-ask', { + mesh: meshName, + transcript, + prompt, + }); + if (current()) + controllers.get(element)?.sendText(prompt, {captureFrame: true}); } /** `ThreeLivePlugin` wiring press-and-hold object queries to Gemini Live. */ @@ -697,17 +866,33 @@ export const geminiLivePlugin: ThreeLivePlugin = { name: 'xr-live-gemini-live', // tslint:disable-next-line:no-any Host element is ``. onElementConnected(element: any): void { - if (!element || typeof element.addEventListener !== 'function' || listeners.has(element)) return; + if ( + !element || + typeof element.addEventListener !== 'function' || + listeners.has(element) + ) + return; const onStart = (e: Event): void => { if (isEventClaimed(e)) return; const detail = (e as CustomEvent).detail; const mesh = detail?.hit?.mesh; - const options = parseHoldToAsk(mesh ? getMeshXRTraits(mesh)?.holdToAsk : null); + const options = parseHoldToAsk( + mesh ? getMeshXRTraits(mesh)?.holdToAsk : null, + ); if (!mesh || !options) return; + if (!checkPluginCapability(element, 'ai', 'holdToAsk', {dedup: true})) { + return; + } const name = mesh.name ?? ''; const session = getGeminiLiveController(element).sessionGuard(); - const state: HoldState = {timer: null, recognizer: null, active: true, hand: detail?.hand ?? 'screen'}; - const current = () => state.active && holdState.get(element) === state && session(); + const state: HoldState = { + timer: null, + recognizer: null, + active: true, + hand: detail?.hand ?? 'screen', + }; + const current = () => + state.active && holdState.get(element) === state && session(); cancelHold(holdState.get(element)); holdState.set(element, state); state.timer = setTimeout(() => { @@ -722,7 +907,14 @@ export const geminiLivePlugin: ThreeLivePlugin = { cancelHold(state); } }; - state.recognizer = startRecognition(complete); + state.recognizer = checkPluginCapability( + element, + 'microphone', + 'holdToAsk', + {dedup: true}, + ) + ? startRecognition(complete) + : null; if (!current()) cancelHold(state); else if (!state.recognizer) complete(''); }, options.holdMs); @@ -747,8 +939,20 @@ export const geminiLivePlugin: ThreeLivePlugin = { listeners.set(element, bound); }, // tslint:disable-next-line:no-any Host element is ``. + onAttributeChanged(element: any, name: string): void { + if (name !== 'capabilities' || !element) return; + const ctrl = controllers.get(element); + if ( + ctrl?.connected && + !hasPluginCapability(element, 'ai', ctrl.getActiveUrl() ?? undefined) + ) { + ctrl.disconnect(); + } + }, + // tslint:disable-next-line:no-any Host element is ``. onElementDisconnected(element: any): void { - for (const [name, fn] of listeners.get(element) ?? []) element.removeEventListener?.(name, fn); + for (const [name, fn] of listeners.get(element) ?? []) + element.removeEventListener?.(name, fn); listeners.delete(element); cancelHold(holdState.get(element)); holdState.delete(element); diff --git a/xr-live/src/plugins/magnetic-snap.ts b/xr-live/src/plugins/magnetic-snap.ts index a223573b..28ac2bca 100644 --- a/xr-live/src/plugins/magnetic-snap.ts +++ b/xr-live/src/plugins/magnetic-snap.ts @@ -27,6 +27,7 @@ import { mat4Multiply, } from '../../../three-live/src/math'; import { + checkPluginCapability, type ThreeLivePlugin, XR_LIVE_AR_SURFACE_Y, } from '../../../three-live/src/plugin'; @@ -868,7 +869,13 @@ export const magneticSnapPlugin: ThreeLivePlugin = { // tslint:disable-next-line:no-any WebXR session init dictionary. onConfigureXRSessionInit(element: any, sessionInit: any): void { const def = resolveMagneticSnapDefaults(element); - if (def.surface === 'none' || !sessionInit) return; + if ( + def.surface === 'none' || + !sessionInit || + !checkPluginCapability(element, 'sensors', 'magneticSnap', {dedup: true}) + ) { + return; + } const optional = new Set(sessionInit.optionalFeatures ?? []); optional.add('plane-detection'); sessionInit.optionalFeatures = [...optional]; diff --git a/xr-live/src/plugins/persistent-anchors.ts b/xr-live/src/plugins/persistent-anchors.ts index b8fc8100..362a94d8 100644 --- a/xr-live/src/plugins/persistent-anchors.ts +++ b/xr-live/src/plugins/persistent-anchors.ts @@ -23,7 +23,10 @@ * 3. WebXR `XRAnchor` / `restorePersistentAnchor` when supported by the headset */ -import type {ThreeLivePlugin} from '../../../three-live/src/plugin'; +import { + checkPluginCapability, + type ThreeLivePlugin, +} from '../../../three-live/src/plugin'; import { clampNum, parseAnchorPersist, @@ -539,6 +542,14 @@ export class XRPersistentAnchorsController { ): string { const entries = this.collectPersistableMeshes(viewer, element); if (entries.length === 0) return ''; + const capHost = element ?? viewer?.container; + if ( + !checkPluginCapability(capHost, 'storage', 'persistentAnchors', { + dedup: true, + }) + ) { + return ''; + } const poses = this.snapshotPoses(viewer, element); if (poses.size === 0) return ''; const cfg = this.resolveStorageKey(element); @@ -599,6 +610,14 @@ export class XRPersistentAnchorsController { ): {count: number; source: AnchorRestoreSource | null} { const entries = this.collectPersistableMeshes(viewer, element); if (entries.length === 0) return {count: 0, source: null}; + const capHost = element ?? viewer?.container; + if ( + !checkPluginCapability(capHost, 'storage', 'persistentAnchors', { + dedup: true, + }) + ) { + return {count: 0, source: null}; + } // Tier 1: URL hash `#xr-state=...` takes precedence for shared layout links. const hashToken = extractHashAnchorToken(this.getUrl()?.getHash()); @@ -663,6 +682,13 @@ export class XRPersistentAnchorsController { /** Clears persisted state from `localStorage`, URL hash, and WebXR persistent handles. */ // tslint:disable-next-line:no-any Host element + optional XR session. clearPersistedAnchors(element?: any, session?: any): void { + if ( + !checkPluginCapability(element, 'storage', 'persistentAnchors', { + dedup: true, + }) + ) { + return; + } const cfg = this.resolveStorageKey(element); try { this.getStorage()?.removeItem(`${ANCHOR_STORAGE_PREFIX}${cfg.namespace}`); @@ -756,7 +782,13 @@ export class XRPersistentAnchorsController { if ( this.xrRestoredSession !== session && typeof session.restorePersistentAnchor === 'function' && - this.xrUuids.size > 0 + this.xrUuids.size > 0 && + checkPluginCapability( + element ?? viewer?.container, + 'storage', + 'persistentAnchors', + {dedup: true}, + ) ) { this.xrRestoredSession = session; let restoredCount = 0; @@ -823,6 +855,16 @@ export class XRPersistentAnchorsController { session?: any, ): void { if (typeof frame?.createAnchor !== 'function') return; + if ( + !checkPluginCapability( + element ?? viewer?.container, + 'storage', + 'persistentAnchors', + {dedup: true}, + ) + ) { + return; + } // tslint:disable-next-line:no-any Global XRRigidTransform constructor. const RigidTransformCtor = (globalThis as any).XRRigidTransform; if (typeof RigidTransformCtor !== 'function') return; @@ -922,7 +964,15 @@ export const persistentAnchorsPlugin: ThreeLivePlugin = { // tslint:disable-next-line:no-any Host element and WebXR sessionInit. onConfigureXRSessionInit(element: any, sessionInit: any): void { const cfg = resolvePersistentAnchorsConfig(element); - if (!cfg.enabled || !sessionInit) return; + if ( + !cfg.enabled || + !sessionInit || + !checkPluginCapability(element, 'storage', 'persistentAnchors', { + dedup: true, + }) + ) { + return; + } const optional = new Set(sessionInit.optionalFeatures ?? []); optional.add('anchors'); sessionInit.optionalFeatures = Array.from(optional); diff --git a/xr-live/src/plugins/shared-room.ts b/xr-live/src/plugins/shared-room.ts index 05851304..a76e31f5 100644 --- a/xr-live/src/plugins/shared-room.ts +++ b/xr-live/src/plugins/shared-room.ts @@ -30,7 +30,11 @@ import { mat4FromRotationTranslationScale, mat4Multiply, } from '../../../three-live/src/math'; -import type {ThreeLivePlugin} from '../../../three-live/src/plugin'; +import { + checkPluginCapability, + hasPluginCapability, + type ThreeLivePlugin, +} from '../../../three-live/src/plugin'; import { clampNum, safeAnchorId, @@ -481,9 +485,13 @@ function parseBoundedVec3(raw: unknown, maxAbs = 1000): Vec3 | null { */ // tslint:disable-next-line:no-any Host `` element. function isVoiceEnabled(el: any): boolean { - return ( + const optedIn = !!el?.hasAttribute?.('shared-room-voice') && - el.getAttribute?.('shared-room-voice') !== 'false' + el.getAttribute?.('shared-room-voice') !== 'false'; + return ( + optedIn && + checkPluginCapability(el, 'microphone', 'sharedRoomVoice', {dedup: true}) && + checkPluginCapability(el, 'network', 'sharedRoomVoice', {dedup: true}) ); } @@ -984,9 +992,17 @@ export class XRSharedRoomController { customTransport?: SharedRoomTransport, ): boolean { this.lastViewer = viewer ?? this.lastViewer; - this.bindHostListeners(element); + const capHost = element ?? viewer?.container; if (customTransport) { + if ( + !checkPluginCapability(capHost, 'network', 'sharedRoom', {dedup: true}) + ) { + this.disconnectTransport(viewer, element); + this.activeRoomKey = ''; + return false; + } + this.bindHostListeners(element); this.disconnectTransport(viewer, element); this.activeRoomKey = `custom:${roomAttr || 'room'}`; this.attachTransport(customTransport, viewer, element); @@ -1000,6 +1016,19 @@ export class XRSharedRoomController { return false; } + if ( + !checkPluginCapability(capHost, 'network', 'sharedRoom', { + url: target.kind === 'websocket' ? target.url : undefined, + dedup: true, + }) + ) { + this.disconnectTransport(viewer, element); + this.activeRoomKey = ''; + return false; + } + + this.bindHostListeners(element); + const nextKey = target.kind === 'websocket' ? `ws:${target.url}` @@ -1565,6 +1594,15 @@ export class XRSharedRoomController { } if (!this.transport) return; + const capHost = element ?? viewer?.container; + const roomUrl = this.activeRoomKey.startsWith('ws:') + ? this.activeRoomKey.slice(3) + : undefined; + if (!hasPluginCapability(capHost, 'network', roomUrl)) { + this.disconnectTransport(viewer, element, false); + this.activeRoomKey = ''; + return; + } // Sync local `traitSystem` grab lifecycle (`grab` -> 20 Hz `move` -> `release`). // tslint:disable-next-line:no-any Heterogeneous scene meshes. @@ -1731,9 +1769,12 @@ export class XRSharedRoomController { viewer: PluginViewer | undefined, // tslint:disable-next-line:no-any Host `` element. element?: any, + sendLeave = true, ): void { if (this.transport) { - this.sendWire({v: 1, type: 'leave', peerId: this.peerId}); + if (sendLeave) { + this.sendWire({v: 1, type: 'leave', peerId: this.peerId}); + } const t = this.transport; this.transport = null; t.onMessage = undefined; @@ -1852,6 +1893,10 @@ export const sharedRoomPlugin: ThreeLivePlugin = { } else { ctrl.dispose(viewer); } + } else if (name === 'capabilities') { + if (!hasPluginCapability(element, 'network')) { + ctrl.dispose(viewer); + } } else if ((name === 'peer-name' || name === 'peer-color') && viewer) { ctrl.broadcastPoseTick(viewer, element, undefined, true); } diff --git a/xr-live/src/plugins/spatial-audio.ts b/xr-live/src/plugins/spatial-audio.ts index ee9871e7..b6e57463 100644 --- a/xr-live/src/plugins/spatial-audio.ts +++ b/xr-live/src/plugins/spatial-audio.ts @@ -32,16 +32,18 @@ */ import { + checkPluginCapability, + hasPluginCapability, ThreeLivePlugin, ThreeLiveViewer, type MeshHit, } from '../../../three-live/src/plugin'; import { AMBIENT_SOUND_PRESETS, - ONE_SHOT_AUDIO_PRESETS, asRecord, clampNum, computeSpatialAudioMix, + ONE_SHOT_AUDIO_PRESETS, parseAudioEmitter, safeMediaUrl, type AmbientSoundPreset, @@ -62,8 +64,8 @@ function userXR(mesh: unknown): Record | null { export { AMBIENT_SOUND_PRESETS, - ONE_SHOT_AUDIO_PRESETS, computeSpatialAudioMix, + ONE_SHOT_AUDIO_PRESETS, parseAudioEmitter, type AmbientSoundPreset, type AudioDistanceModel, @@ -448,7 +450,7 @@ export class XRSpatialAudioEmitterController { rightGain: 0, }; - this.syncWebAudioNode(idx, state); + this.syncWebAudioNode(idx, state, viewer.container); } } @@ -463,10 +465,25 @@ export class XRSpatialAudioEmitterController { this.syncFromViewer(viewer); const state = this.resolveState(meshIndexOrName); if (!state) return null; + if ( + !state.options.preset && + state.options.src && + !checkPluginCapability( + viewer.container, + 'remote_content', + 'spatialAudio', + { + url: state.options.src, + dedup: true, + }, + ) + ) { + return null; + } state.playing = true; state.triggerCount++; this.update(viewer, 0); - this.restartWebAudioSource(state.meshIndex, state); + this.restartWebAudioSource(state.meshIndex, state, viewer.container); const container = viewer.container; if ( @@ -482,7 +499,7 @@ export class XRSpatialAudioEmitterController { node: state.nodeName, meshIndex: state.meshIndex, preset: state.options.preset ?? null, - src: state.options.src ?? null, + src: state.options.preset ? null : (state.options.src ?? null), gain: state.lastMix.gain, pan: state.lastMix.pan, triggerCount: state.triggerCount, @@ -526,7 +543,9 @@ export class XRSpatialAudioEmitterController { viewer: ThreeLiveViewer, ): Map { const result = new Map(); - const gltf = asRecord(viewer.lastGltf); + const gltf = asRecord( + (viewer as unknown as {gltf?: unknown}).gltf ?? viewer.lastGltf, + ); if (!gltf) return result; const extRoot = asRecord( asRecord(gltf['extensions'])?.['KHR_audio_emitter'], @@ -611,6 +630,7 @@ export class XRSpatialAudioEmitterController { private syncWebAudioNode( idx: number, state: XRSpatialAudioEmitterState, + host?: unknown, ): void { if (!state.playing) { this.stopGraph(idx); @@ -618,8 +638,17 @@ export class XRSpatialAudioEmitterController { } if (!this.ensureAudioContext() || !this.ctx) return; let graph = this.audioGraphs.get(idx); + if ( + graph?.mediaEl && + state.options.src && + !hasPluginCapability(host, 'remote_content', state.options.src) + ) { + state.playing = false; + this.stopGraph(idx); + return; + } if (!graph) { - graph = this.createGraph(state); + graph = this.createGraph(state, host); if (graph) { this.audioGraphs.set(idx, graph); } @@ -645,13 +674,15 @@ export class XRSpatialAudioEmitterController { private restartWebAudioSource( idx: number, state: XRSpatialAudioEmitterState, + host?: unknown, ): void { this.stopGraph(idx); - this.syncWebAudioNode(idx, state); + this.syncWebAudioNode(idx, state, host); } private createGraph( state: XRSpatialAudioEmitterState, + host?: unknown, ): WebAudioEmitterGraph | null { if (!this.ctx || !this.masterGain) return null; try { @@ -696,6 +727,17 @@ export class XRSpatialAudioEmitterController { typeof Audio === 'function' && typeof this.ctx.createMediaElementSource === 'function' ) { + if ( + !checkPluginCapability(host, 'remote_content', 'spatialAudio', { + url: state.options.src, + dedup: true, + }) + ) { + state.playing = false; + gainNode.disconnect?.(); + panNode?.disconnect?.(); + return null; + } const mediaEl = new Audio(state.options.src); mediaEl.crossOrigin = 'anonymous'; mediaEl.loop = state.options.loop; @@ -750,6 +792,10 @@ export function getSpatialAudioController( */ export const spatialAudioPlugin: ThreeLivePlugin = { name: 'xr-live-spatial-audio', + onLoadModel(viewer: ThreeLiveViewer): void { + const ctrl = getSpatialAudioController(viewer); + ctrl.update(viewer, 0); + }, onSceneLoaded(viewer: ThreeLiveViewer): void { const ctrl = getSpatialAudioController(viewer); ctrl.update(viewer, 0); @@ -766,6 +812,16 @@ export const spatialAudioPlugin: ThreeLivePlugin = { ctrl.trigger(viewer, hit.meshIndex); } }, + onClearScene(viewer: ThreeLiveViewer): void { + controllers.get(viewer)?.dispose(); + }, + onDestroy(_gl: WebGL2RenderingContext, viewer: ThreeLiveViewer): void { + const ctrl = controllers.get(viewer); + if (ctrl) { + ctrl.dispose(); + controllers.delete(viewer); + } + }, onDispose(viewer: ThreeLiveViewer): void { const ctrl = controllers.get(viewer); if (ctrl) { diff --git a/xr-live/src/plugins/spatial-measure.ts b/xr-live/src/plugins/spatial-measure.ts index c73d795c..71879e20 100644 --- a/xr-live/src/plugins/spatial-measure.ts +++ b/xr-live/src/plugins/spatial-measure.ts @@ -21,7 +21,10 @@ * tape measure that emits `measurecomplete`. */ -import type {ThreeLivePlugin} from '../../../three-live/src/plugin'; +import { + checkPluginCapability, + type ThreeLivePlugin, +} from '../../../three-live/src/plugin'; import { parseDimensions, type DimensionAxis, @@ -1072,7 +1075,15 @@ export const spatialMeasurePlugin: ThreeLivePlugin = { // tslint:disable-next-line:no-any WebXR session init dictionary. onConfigureXRSessionInit(element: any, sessionInit: any): void { const cfg = resolveSpatialMeasureConfig(element); - if (!cfg.tapeEnabled || !sessionInit) return; + if ( + !cfg.tapeEnabled || + !sessionInit || + !checkPluginCapability(element, 'sensors', 'spatialMeasure', { + dedup: true, + }) + ) { + return; + } const optional = new Set(sessionInit.optionalFeatures ?? []); optional.add('plane-detection'); optional.add('hit-test'); diff --git a/xr-live/src/plugins/splat-loader.ts b/xr-live/src/plugins/splat-loader.ts index b46771de..becfd86d 100644 --- a/xr-live/src/plugins/splat-loader.ts +++ b/xr-live/src/plugins/splat-loader.ts @@ -27,6 +27,7 @@ import {createProgram} from '../../../three-live/src/gl-program'; import {mat3NormalFromMat4, mat4Multiply} from '../../../three-live/src/math'; import { + checkPluginCapability, getSharedRegistry, type SplatGPUFrameOutput, type ThreeLivePlugin, @@ -163,7 +164,8 @@ export async function gunzip( } const stream = new Blob([bytes]).stream().pipeThrough(new DS('gzip')); return readStreamCapped( - stream.getReader(), maxBytes, + stream.getReader(), + maxBytes, 'splat-loader: decompressed .spz exceeds size limit', ); } @@ -595,9 +597,17 @@ export async function loadSplat( url: string, fetchFn: typeof fetch = fetch, allowCrossOrigin = true, + host?: unknown, ): Promise { const resolved = resolveSplatUrl(url, undefined, allowCrossOrigin); if (!resolved) throw new Error('splat-loader: URL rejected'); + if ( + !checkPluginCapability(host, 'remote_content', 'splatUrl', {url: resolved}) + ) { + throw new Error( + `splat-loader: capability 'remote_content' or 'network' denied`, + ); + } const res = await fetchFn(resolved, {credentials: 'same-origin'}); if (!res.ok) throw new Error(`splat-loader: HTTP ${res.status}`); const declared = Number(res.headers?.get?.('content-length') ?? 0); @@ -756,12 +766,21 @@ export function computeSplatViewColors( // Normal-matrix storage is column-major inverse-transpose, which is also // row-major inverse. Initialize the transpose fallback for singular models. const inverse = new Float64Array([ - model[0], model[1], model[2], - model[4], model[5], model[6], - model[8], model[9], model[10], + model[0], + model[1], + model[2], + model[4], + model[5], + model[6], + model[8], + model[9], + model[10], ]); - const [i00, i01, i02, i10, i11, i12, i20, i21, i22] = - mat3NormalFromMat4(inverse, model, 1e-12); + const [i00, i01, i02, i10, i11, i12, i20, i21, i22] = mat3NormalFromMat4( + inverse, + model, + 1e-12, + ); const cx = cameraWorldPos[0]; const cy = cameraWorldPos[1]; const cz = cameraWorldPos[2]; @@ -808,10 +827,7 @@ function sameSplatDepth(a: Float32Array, b: Float32Array): boolean { } /** Exact depth rows share an order, including during the moving-view cooldown. */ -function createSplatSortCache( - gl: WebGL2RenderingContext, - data: SplatData, -) { +function createSplatSortCache(gl: WebGL2RenderingContext, data: SplatData) { const target = gl.ELEMENT_ARRAY_BUFFER; let current: SplatEye | undefined; let previous: SplatEye | undefined; @@ -824,10 +840,11 @@ function createSplatSortCache( eye: string, externalOrder?: Uint32Array | null, ): SortedSplatView | undefined { - const row = matrix[3] === 0 && matrix[7] === 0 && matrix[11] === 0 ? 2 : 3; + const row = + matrix[3] === 0 && matrix[7] === 0 && matrix[11] === 0 ? 2 : 3; if (current?.eye !== eye) { - const next = previous?.eye === eye ? previous : - {eye, key: new Float32Array(4)}; + const next = + previous?.eye === eye ? previous : {eye, key: new Float32Array(4)}; previous = current; current = next; } @@ -847,7 +864,7 @@ function createSplatSortCache( // A first eye may update a shared buffer. A divergent second eye // must split before writing, so it cannot replace the first eye's order. if (!view.order || (previous?.seen && view.order === previous.order)) { - let storage = orders.find(o => o !== previous?.order); + let storage = orders.find((o) => o !== previous?.order); if (!storage) { const buffer = gl.createBuffer(); if (!buffer) return; @@ -881,7 +898,7 @@ function createSplatSortCache( gl.deleteBuffer(view.buffer); } current = previous = undefined; - } + }, }; } @@ -1010,11 +1027,7 @@ export class SplatRenderer { externalGpu?: SplatGPUFrameOutput | null, ): void { const gl = this.gl; - if ( - !this.program || - !this.vao || - this.data.count === 0 - ) { + if (!this.program || !this.vao || this.data.count === 0) { return; } const mvp = this.mvp; @@ -1108,6 +1121,14 @@ export class XRSplatLoader { if (!raw) continue; const url = resolveSplatUrl(raw, undefined, allowRemote); if (!url || !mesh?.transform) continue; + if ( + !checkPluginCapability(viewer.container, 'remote_content', 'splatUrl', { + url, + dedup: true, + }) + ) { + continue; + } live.add(mesh); const existing = this.entries.get(mesh); if (existing?.url === url) continue; @@ -1116,10 +1137,15 @@ export class XRSplatLoader { getSharedRegistry().splatAccelerator?.disposeCloud?.(prevData); } existing?.renderer?.dispose(); - const entry: SplatEntry = {url, data: null, renderer: null, failed: false}; + const entry: SplatEntry = { + url, + data: null, + renderer: null, + failed: false, + }; this.entries.set(mesh, entry); if (!gl) continue; - loadSplat(url, this.fetchFn, allowRemote).then( + loadSplat(url, this.fetchFn, allowRemote, viewer.container).then( (data) => { if (this.entries.get(mesh) === entry) { entry.data = data; diff --git a/xr-live/src/plugins/world-understanding.ts b/xr-live/src/plugins/world-understanding.ts index e3ef8f59..37d149c1 100644 --- a/xr-live/src/plugins/world-understanding.ts +++ b/xr-live/src/plugins/world-understanding.ts @@ -27,17 +27,18 @@ */ import {mat4Create} from '../../../three-live/src/math'; -import type {ThreeLivePlugin} from '../../../three-live/src/plugin'; +import { + checkPluginCapability, + type ThreeLivePlugin, +} from '../../../three-live/src/plugin'; import {extractPosition, getMeshXRTraits, type XRPose6DoF} from '../xr-traits'; import { buildBoxGeometry, + cacheXRFrame, clampNum, createDynamicMesh, emitPluginEvent, horizontalForward, - type PluginViewer, - type Quat, - cacheXRFrame, isXRFrameFresh, removeDynamicMesh, resolveHeadPose, @@ -51,6 +52,8 @@ import { vNorm, vScale, vSub, + type PluginViewer, + type Quat, type Vec3, } from './common'; @@ -85,7 +88,12 @@ export const MAX_DETECTIONS_JSON_CHARS = 16 * 1024; * Malformed entries are skipped; malformed or oversized input yields `[]`. */ export function parseDetections(json: unknown): Detection2D[] { - if (typeof json !== 'string' || !json || json.length > MAX_DETECTIONS_JSON_CHARS) return []; + if ( + typeof json !== 'string' || + !json || + json.length > MAX_DETECTIONS_JSON_CHARS + ) + return []; let raw: unknown; try { raw = JSON.parse(json); @@ -135,7 +143,9 @@ export function snapshotDetectionCamera(element: any): DetectionCamera { const viewer: PluginViewer | undefined = element?._view ?? element?.viewer; return { world: Float32Array.from(resolveHeadPose(viewer).matrix), - projection: Float32Array.from(resolveProjection(viewer) ?? defaultProjection()), + projection: Float32Array.from( + resolveProjection(viewer) ?? defaultProjection(), + ), }; } @@ -163,8 +173,15 @@ export function normalizeBox( if (!Array.isArray(box) || box.length < 4) return null; if (!box.slice(0, 4).every((v) => Number.isFinite(v))) return null; const scale = box.slice(0, 4).some((v) => v > 1.0001) ? 1000 : 1; - const [y0, x0, y1, x1] = box.slice(0, 4).map((v) => clampNum(v / scale, 0, 1, 0)); - return [Math.min(y0, y1), Math.min(x0, x1), Math.max(y0, y1), Math.max(x0, x1)]; + const [y0, x0, y1, x1] = box + .slice(0, 4) + .map((v) => clampNum(v / scale, 0, 1, 0)); + return [ + Math.min(y0, y1), + Math.min(x0, x1), + Math.max(y0, y1), + Math.max(x0, x1), + ]; } /** @@ -181,7 +198,11 @@ export function viewportRay( const ndcY = 1 - v * 2; const p0 = proj[0] || 1; const p5 = proj[5] || 1; - const viewDir: Vec3 = [(ndcX + (proj[8] ?? 0)) / p0, (ndcY + (proj[9] ?? 0)) / p5, -1]; + const viewDir: Vec3 = [ + (ndcX + (proj[8] ?? 0)) / p0, + (ndcY + (proj[9] ?? 0)) / p5, + -1, + ]; const origin = extractPosition(cameraWorld); const far = transformPoint(cameraWorld, viewDir); return {origin, dir: vNorm(vSub(far, origin)), viewDir}; @@ -193,7 +214,12 @@ export function hitTestPlanes( origin: readonly number[], dir: readonly number[], ): {point: Vec3; normal: Vec3; distance: number; plane: TrackedPlane} | null { - let best: {point: Vec3; normal: Vec3; distance: number; plane: TrackedPlane} | null = null; + let best: { + point: Vec3; + normal: Vec3; + distance: number; + plane: TrackedPlane; + } | null = null; for (const plane of planes) { const denom = vDot(plane.normal, dir); if (Math.abs(denom) < 1e-6) continue; @@ -227,7 +253,11 @@ export function unprojectDetectionTo3D( let depth = 0; let source: Detection3D['depthSource'] = 'fallback'; const sampled = sampleDepth?.(u, v); - if (typeof sampled === 'number' && Number.isFinite(sampled) && sampled > 0.05) { + if ( + typeof sampled === 'number' && + Number.isFinite(sampled) && + sampled > 0.05 + ) { depth = sampled; source = 'depth'; } else { @@ -330,7 +360,8 @@ export class XRWorldUnderstandingController { if (!m || m.length !== 16) continue; // The plane's local +Y axis is its normal in WebXR plane detection. next.push({ - orientation: plane.orientation === 'vertical' ? 'vertical' : 'horizontal', + orientation: + plane.orientation === 'vertical' ? 'vertical' : 'horizontal', center: [m[12], m[13], m[14]], normal: vNorm([m[4], m[5], m[6]], [0, 1, 0]), radius: polygonRadius(plane.polygon), @@ -351,10 +382,17 @@ export class XRWorldUnderstandingController { camera?: DetectionCamera, ): Detection3D[] { const world = camera?.world ?? resolveHeadPose(viewer).matrix; - const proj = camera?.projection ?? resolveProjection(viewer) ?? defaultProjection(); + const proj = + camera?.projection ?? resolveProjection(viewer) ?? defaultProjection(); const out: Detection3D[] = []; for (const d of detections.slice(0, MAX_DETECTIONS)) { - const located = unprojectDetectionTo3D(d, world, proj, this.planes, sampleDepth); + const located = unprojectDetectionTo3D( + d, + world, + proj, + this.planes, + sampleDepth, + ); if (located) out.push(located); } return out; @@ -364,19 +402,37 @@ export class XRWorldUnderstandingController { * Casts the gaze ray against tracked planes (then depth) and positions the * reticle mesh. Creates the reticle lazily and hides it on a miss. */ - updateReticle(viewer: PluginViewer, head: XRPose6DoF, sampleDepth?: DepthSampler): void { + updateReticle( + viewer: PluginViewer, + head: XRPose6DoF, + sampleDepth?: DepthSampler, + ): void { const m = head.matrix; - const fwd = vNorm([-m[8], -m[9], -m[10]], horizontalForward(head.quaternion)); - let hit: {point: Vec3; normal: Vec3} | null = hitTestPlanes(this.planes, head.position, fwd); + const fwd = vNorm( + [-m[8], -m[9], -m[10]], + horizontalForward(head.quaternion), + ); + let hit: {point: Vec3; normal: Vec3} | null = hitTestPlanes( + this.planes, + head.position, + fwd, + ); if (!hit) { const d = sampleDepth?.(0.5, 0.5); if (typeof d === 'number' && Number.isFinite(d) && d > 0.05) { - hit = {point: vAdd(head.position, vScale(fwd, d)), normal: vScale(fwd, -1)}; + hit = { + point: vAdd(head.position, vScale(fwd, d)), + normal: vScale(fwd, -1), + }; } } this.reticleHit = hit; // Re-create after a model reload replaced (and freed) `viewer.meshes`. - if (this.reticle && Array.isArray(viewer.meshes) && !viewer.meshes.includes(this.reticle)) { + if ( + this.reticle && + Array.isArray(viewer.meshes) && + !viewer.meshes.includes(this.reticle) + ) { this.reticle = null; } if (!this.reticle) { @@ -388,7 +444,13 @@ export class XRWorldUnderstandingController { } this.reticle.visible = Boolean(hit); if (hit) { - setWorldTRS(viewer, this.reticle, quatFromNormal(hit.normal), hit.point, [0.06, 0.06, 0.004]); + setWorldTRS( + viewer, + this.reticle, + quatFromNormal(hit.normal), + hit.point, + [0.06, 0.06, 0.004], + ); } } @@ -422,7 +484,9 @@ function defaultProjection(): Float32Array { const controllers = new WeakMap(); /** Returns (creating on first use) the controller bound to `key`. */ -export function getWorldUnderstandingController(key: object): XRWorldUnderstandingController { +export function getWorldUnderstandingController( + key: object, +): XRWorldUnderstandingController { let c = controllers.get(key); if (!c) { c = new XRWorldUnderstandingController(); @@ -441,7 +505,10 @@ export interface WorldUnderstandingOptions { /** Reads opt-in flags from the host attribute and any mesh trait. */ // tslint:disable-next-line:no-any Host element and GPU mesh structs. -export function resolveWorldUnderstanding(element: any, meshes: readonly any[] = []): WorldUnderstandingOptions { +export function resolveWorldUnderstanding( + element: any, + meshes: readonly any[] = [], +): WorldUnderstandingOptions { const attr = element?.getAttribute?.('world-understanding'); let planes = attr != null && attr !== 'false'; let meshesOn = false; @@ -453,11 +520,25 @@ export function resolveWorldUnderstanding(element: any, meshes: readonly any[] = meshesOn ||= wu.meshes === true; reticle ||= wu.reticle === true; } - return {enabled: planes || meshesOn || Boolean(reticle), planes, meshes: meshesOn, reticle: Boolean(reticle)}; + return { + enabled: planes || meshesOn || Boolean(reticle), + planes, + meshes: meshesOn, + reticle: Boolean(reticle), + }; } function depthSamplerFor(host: unknown): DepthSampler | undefined { - const dm = (host as {depthManager?: {sampleDepthAndNormal?: (u: number, v: number) => {depthMeters: number} | null}})?.depthManager; + const dm = ( + host as { + depthManager?: { + sampleDepthAndNormal?: ( + u: number, + v: number, + ) => {depthMeters: number} | null; + }; + } + )?.depthManager; if (!dm || typeof dm.sampleDepthAndNormal !== 'function') return undefined; return (u, v) => { try { @@ -505,22 +586,43 @@ export const worldUnderstandingPlugin: ThreeLivePlugin = { onConfigureXRSessionInit(element: any, sessionInit: any): void { const viewer = element?._view ?? element?.viewer; const opts = resolveWorldUnderstanding(element, viewer?.meshes ?? []); - if (!opts.enabled || !sessionInit) return; + if ( + !opts.enabled || + !sessionInit || + !checkPluginCapability(element, 'sensors', 'worldUnderstanding', { + dedup: true, + }) + ) { + return; + } const optional = new Set(sessionInit.optionalFeatures ?? []); if (opts.planes || opts.reticle) optional.add('plane-detection'); if (opts.meshes) optional.add('mesh-detection'); sessionInit.optionalFeatures = [...optional]; }, - onXRFrame(viewer: unknown, _session: unknown, frame: unknown, refSpace: unknown): void { + onXRFrame( + viewer: unknown, + _session: unknown, + frame: unknown, + refSpace: unknown, + ): void { const v = viewer as PluginViewer; const host = v?.container; if (!host) return; const opts = resolveWorldUnderstanding(host, v.meshes ?? []); - if (!opts.enabled) return; + if ( + !opts.enabled || + !checkPluginCapability(host, 'sensors', 'worldUnderstanding', { + dedup: true, + }) + ) { + return; + } const ctrl = getWorldUnderstandingController(host); cacheXRFrame(v, frame, refSpace); ctrl.updateFromXRFrame(frame, refSpace); - if (opts.reticle) ctrl.updateReticle(v, resolveHeadPose(v), depthSamplerFor(host)); + if (opts.reticle) + ctrl.updateReticle(v, resolveHeadPose(v), depthSamplerFor(host)); }, onUpdate(viewer: PluginViewer): void { // Outside an immersive session the reticle has no valid surface to sit on. diff --git a/xr-live/src/viewer.ts b/xr-live/src/viewer.ts index 00ea20fe..7a0c86ae 100644 --- a/xr-live/src/viewer.ts +++ b/xr-live/src/viewer.ts @@ -36,7 +36,28 @@ import { ThreeLiveElement.registerAR({activateAR, createUSDZBlob}); -export {ThreeLiveElement, ThreeView} from '../../three-live/src/viewer'; +export { + CAPABILITIES_ALL, + CAPABILITIES_NONE, + CAPABILITIES_VIEWER, + checkHostCapability, + hasCapabilityInPolicy, + isNetworkUrlPermitted, + isNonNetworkUrl, + normalizeCapabilityOrigin, + parseCapabilities, + resetCapabilityDenials, + resolveHostCapabilities, + sanitizeDiagnosticUrl, + ThreeLiveElement, + ThreeView, + type CapabilityDeniedDetail, + type ThreeLiveCapabilities, + type ThreeLiveCapabilitiesInit, + type ThreeLiveCapabilitiesInput, + type ThreeLiveCapabilityName, + type ThreeLiveNetworkPolicy, +} from '../../three-live/src/viewer'; export * from './xr-input'; export * from './xr-mobile-bridge'; export * from './xr-spatial-slot'; diff --git a/xr-live/src/webgpu/splat-accelerator.ts b/xr-live/src/webgpu/splat-accelerator.ts index 12ea1185..3b4eb58c 100644 --- a/xr-live/src/webgpu/splat-accelerator.ts +++ b/xr-live/src/webgpu/splat-accelerator.ts @@ -13,13 +13,16 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * + */ + +/** * @fileoverview Opt-in WebGPU Gaussian Splatting compute & render accelerator * (`xr-live-webgpu.min.js`) for `` and ``. */ import {mat4Multiply} from '../../../three-live/src/math'; import { + checkPluginCapability, getRegisteredPlugins, getSharedRegistry, type SplatGPUAccelerator, @@ -57,7 +60,7 @@ const MAP_READ = 0x0001; const STAGE_VERTEX = 0x1; const STAGE_COMPUTE = 0x4; -/** Minimal structural WebGPU interfaces (zero external `@webgpu/types` dep). */ +/** Minimal structural WebGPU buffer interface (zero external `@webgpu/types` dep). */ export interface WebGPUBufferLike { getMappedRange(offset?: number, size?: number): ArrayBuffer; mapAsync(mode: number, offset?: number, size?: number): Promise; @@ -65,6 +68,7 @@ export interface WebGPUBufferLike { destroy(): void; } +/** Minimal structural WebGPU compute pass encoder interface. */ export interface WebGPUComputePassEncoderLike { setPipeline(pipeline: unknown): void; setBindGroup(index: number, bindGroup: unknown): void; @@ -72,6 +76,7 @@ export interface WebGPUComputePassEncoderLike { end(): void; } +/** Minimal structural WebGPU render pass encoder interface. */ export interface WebGPURenderPassEncoderLike { setPipeline(pipeline: unknown): void; setBindGroup(index: number, bindGroup: unknown): void; @@ -85,6 +90,7 @@ export interface WebGPURenderPassEncoderLike { end(): void; } +/** Minimal structural WebGPU command encoder interface. */ export interface WebGPUCommandEncoderLike { beginComputePass(descriptor?: unknown): WebGPUComputePassEncoderLike; beginRenderPass(descriptor: unknown): WebGPURenderPassEncoderLike; @@ -98,6 +104,7 @@ export interface WebGPUCommandEncoderLike { finish(): unknown; } +/** Minimal structural WebGPU device interface. */ export interface WebGPUDeviceLike { readonly queue: { writeBuffer( @@ -125,6 +132,7 @@ export interface WebGPUDeviceLike { destroy?(): void; } +/** Minimal structural WebGPU canvas context interface. */ export interface WebGPUCanvasContextLike { configure(config: { device: WebGPUDeviceLike; @@ -265,8 +273,9 @@ export function resetWebGPUDevice(): void { } // tslint:disable-next-line:no-any -const xrOf = (m: any): Record | undefined => - m?.extras?.xr ?? m?.userData?.xr; +function xrOf(m: any): Record | undefined { + return m?.extras?.xr ?? m?.userData?.xr; +} /** Returns true if `viewer.meshes` contains any visible non-splat glTF geometry. */ export function hasVisibleNonSplatMeshes(viewer: SplatViewerLike): boolean { @@ -479,11 +488,11 @@ function sameSig(a: Float32Array, b: Float32Array): boolean { return true; } -const safe = (fn: () => unknown): void => { +function safe(fn: () => unknown): void { try { fn(); } catch {} -}; +} function hideViewerOverlay(st: ViewerState): void { if (st.cv && st.v) { @@ -656,11 +665,14 @@ function ensureOverlay( st.er = true; return null; } - const glCanvas = (viewer.gl as {canvas?: HTMLCanvasElement | null} | undefined) - ?.canvas; + const glCanvas = ( + viewer.gl as {canvas?: HTMLCanvasElement | null} | undefined + )?.canvas; const parent: HTMLElement | null = (glCanvas?.parentElement as HTMLElement | null) ?? - (viewer.container?.appendChild ? (viewer.container as HTMLElement) : null); + (viewer.container?.appendChild + ? (viewer.container as HTMLElement) + : null); if (!parent) { st.er = true; return null; @@ -674,7 +686,9 @@ function ensureOverlay( } const ctx = ( - cv as unknown as {getContext?(id: string): WebGPUCanvasContextLike | null} + cv as unknown as { + getContext?(id: string): WebGPUCanvasContextLike | null; + } ).getContext?.('webgpu') ?? null; if (!ctx) { st.er = true; @@ -929,9 +943,9 @@ export class WebGPUSplatAccelerator implements SplatGPUAccelerator { } } - const owner = (params.mesh && typeof params.mesh === 'object' - ? params.mesh - : viewer) as object; + const owner = ( + params.mesh && typeof params.mesh === 'object' ? params.mesh : viewer + ) as object; const eye = getHybridEye(cRef, viewer, owner, params.eye); if (!eye) return null; let outOrder: Uint32Array | null = null; @@ -1096,13 +1110,23 @@ export class StandaloneWebGPUSplatLoader { if (!raw) continue; const url = resolveSplatUrl(raw, undefined, allowRemote); if (!url || !mesh?.transform) continue; + if ( + !checkPluginCapability( + viewer.container, + 'remote_content', + 'splatUrl', + {url, dedup: true}, + ) + ) { + continue; + } live.add(mesh); const existing = entries.get(mesh); if (existing?.url === url) continue; if (existing?.data) accel.disposeCloud(existing.data); const entry: StandaloneEntry = {url, data: null}; entries.set(mesh, entry); - loadSplat(url, doFetch, allowRemote).then( + loadSplat(url, doFetch, allowRemote, viewer.container).then( (data) => { if (entries.get(mesh) === entry) entry.data = data; }, @@ -1230,7 +1254,9 @@ export const webgpuSplatPlugin: ThreeLivePlugin = { if (getNavigatorGPU()) void acquireWebGPUDevice(); }, onElementDisconnected(element: {viewer?: object | null}): void { - if (element?.viewer) getWebGPUSplatAccelerator().disposeViewer(element.viewer); + if (element?.viewer) { + getWebGPUSplatAccelerator().disposeViewer(element.viewer); + } }, onClearScene(viewer: SplatViewerLike): void { if (viewer) { diff --git a/xr-live/src/xr-extras.ts b/xr-live/src/xr-extras.ts index 0774d469..b6380e78 100644 --- a/xr-live/src/xr-extras.ts +++ b/xr-live/src/xr-extras.ts @@ -21,6 +21,7 @@ */ import {mat4FromRotationTranslationScale} from '../../three-live/src/math'; +import {checkPluginCapability} from '../../three-live/src/plugin'; import type {XRDepthManager} from './xr-depth'; import { extractPosition, @@ -419,9 +420,13 @@ export function registerXRLiveWebMCPTools( scale: extractScale(m.transform), xrTraits: getMeshXRTraits(m) || {}, })); - const headPosition = isWebMCPPoseOmitted(element) - ? undefined - : quantizeHeadPosition(getHeadPose().position); + const headPosition = + isWebMCPPoseOmitted(element) || + !checkPluginCapability(element, 'sensors', 'webmcpPose', { + dedup: true, + }) + ? undefined + : quantizeHeadPosition(getHeadPose().position); return { headPosition, objectCount: meshes.length, @@ -443,6 +448,12 @@ export function registerXRLiveWebMCPTools( required: ['name'], }, execute: async (args: {name: string; distanceMeters?: number}) => { + if (!checkPluginCapability(element, 'sensors', 'webmcpGaze')) { + return { + success: false, + error: "Capability 'sensors' denied for WebMCP gaze placement.", + }; + } const viewer = getViewer(); const mesh = (viewer?.meshes || []).find( (m: any) => m.name === args.name, @@ -475,15 +486,33 @@ export function registerXRLiveWebMCPTools( properties: {includeDepth: {type: 'boolean'}}, }, execute: async (args: {includeDepth?: boolean}) => { + const viewer = getViewer(); + if (!checkPluginCapability(element, 'camera', 'webmcpCapture')) { + throw new Error("Capability 'camera' denied for WebMCP XR capture."); + } markWebMCPCaptureActive(element); const allowDepth = - Boolean(args?.includeDepth) && isWebMCPDepthAllowed(element); - const pose = getHeadPose(); + Boolean(args?.includeDepth) && + isWebMCPDepthAllowed(element) && + checkPluginCapability(element, 'sensors', 'webmcpDepth'); + const canReadPose = checkPluginCapability( + element, + 'sensors', + 'webmcpPose', + {dedup: true}, + ); + const pose = canReadPose + ? getHeadPose() + : { + position: [0, 0, 0] as [number, number, number], + quaternion: [0, 0, 0, 1] as [number, number, number, number], + matrix: new Float32Array(16), + }; const quantizedPose: XRPose6DoF = { ...pose, position: quantizeHeadPosition(pose.position), }; - return captureXRViewFrame(getViewer(), quantizedPose, depthManager, { + return captureXRViewFrame(viewer, quantizedPose, depthManager, { includeDepth: allowDepth, }); }, diff --git a/xr-live/src/xr-input.ts b/xr-live/src/xr-input.ts index 73f15258..34d18faa 100644 --- a/xr-live/src/xr-input.ts +++ b/xr-live/src/xr-input.ts @@ -367,6 +367,7 @@ export class XRInputManager { session: any, frame: any, refSpace: any, + allowHands = true, ): void { if (!viewer || !session || !frame || !refSpace) return; const sources = session.inputSources || []; @@ -389,7 +390,11 @@ export class XRInputManager { // Check 25-joint XRHand pinch state (distance between index-finger-tip and thumb-tip) let handPinchState: boolean | null = null; - if (source.hand && typeof frame.getJointPose === 'function') { + if ( + allowHands && + source.hand && + typeof frame.getJointPose === 'function' + ) { const indexTip = source.hand.get('index-finger-tip'); const thumbTip = source.hand.get('thumb-tip'); if (indexTip && thumbTip) { diff --git a/xr-live/src/xr-live-element.ts b/xr-live/src/xr-live-element.ts index dc0a5982..947957b2 100644 --- a/xr-live/src/xr-live-element.ts +++ b/xr-live/src/xr-live-element.ts @@ -317,10 +317,13 @@ export class XRLiveElement extends ThreeLiveElement { ); // 4. Step analytical heightfield & sphere-to-sphere physics + const useDepth = + this.xrDepth !== 'none' && + this.checkCapability('sensors', 'xrDepth', {dedup: true}); this.physics.step( viewer.meshes || [], dt, - this.xrDepth !== 'none' ? this.depthManager : undefined, + useDepth ? this.depthManager : undefined, activeFloorY, activeTableY, (mesh, _speed) => { @@ -374,11 +377,18 @@ export class XRLiveElement extends ThreeLiveElement { }, customViewer?: any, ): Promise { + const viewer = this._getViewer(customViewer); + if (!this.checkCapability('camera', 'captureViewFrame')) { + throw new Error("Capability 'camera' denied for XR frame capture."); + } + const includeDepth = + Boolean(options?.includeDepth) && + this.checkCapability('sensors', 'captureViewFrameDepth'); return getXRExtras().captureXRViewFrame( - this._getViewer(customViewer), + viewer, this.simulator.getHeadPose(), this.depthManager, - options, + options ? {...options, includeDepth} : undefined, ); } @@ -440,6 +450,7 @@ export class XRLiveElement extends ThreeLiveElement { } const report = await launchBestMobileOrHeadsetAR(source, { + element: this, forceFallback, glbUrl: typeof this.currentModel === 'string' ? this.currentModel : undefined, @@ -516,8 +527,16 @@ export const xrLiveSpatialPlugin: ThreeLivePlugin = { onConfigureXRSessionInit(element: any, sessionInit: any): void { if (!element || !(element instanceof XRLiveElement)) return; - element.depthManager.configureSessionInit(sessionInit, element.xrDepth); - if (element.xrHands !== 'none') { + if ( + element.xrDepth !== 'none' && + element.checkCapability('sensors', 'xrDepth', {dedup: true}) + ) { + element.depthManager.configureSessionInit(sessionInit, element.xrDepth); + } + if ( + element.xrHands !== 'none' && + element.checkCapability('sensors', 'xrHands', {dedup: true}) + ) { const optional = new Set(sessionInit.optionalFeatures || []); optional.add('hand-tracking'); sessionInit.optionalFeatures = Array.from(optional); @@ -537,26 +556,28 @@ export const xrLiveSpatialPlugin: ThreeLivePlugin = { ? Math.min(Math.max((now - prev) / 1000, 1 / 120), 0.05) : 1 / 72; + const allowDepth = + host.xrDepth !== 'none' && + host.checkCapability('sensors', 'xrDepth', {dedup: true}); const viewerPose = frame?.getViewerPose?.(refSpace); const firstView = viewerPose?.views?.[0]; if (firstView) { - host.depthManager.updateFromXRFrame( - viewer, - frame, - firstView, - host.xrDepth !== 'none', - ); + host.depthManager.updateFromXRFrame(viewer, frame, firstView, allowDepth); } viewer.onBeforeRenderXRView = (xrView: any) => { - host.depthManager.updateFromXRFrame( - viewer, - frame, - xrView, - host.xrDepth !== 'none', - ); + host.depthManager.updateFromXRFrame(viewer, frame, xrView, allowDepth); }; - host.inputManager.processXRFrame(viewer, session, frame, refSpace); - if (host.xrHands !== 'none') { + const allowHands = + host.xrHands !== 'none' && + host.checkCapability('sensors', 'xrHands', {dedup: true}); + host.inputManager.processXRFrame( + viewer, + session, + frame, + refSpace, + allowHands, + ); + if (allowHands) { host.gestureClassifier.processXRFrame(host, session, frame, refSpace); } diff --git a/xr-live/src/xr-mobile-bridge.ts b/xr-live/src/xr-mobile-bridge.ts index fde6f19c..47f82262 100644 --- a/xr-live/src/xr-mobile-bridge.ts +++ b/xr-live/src/xr-mobile-bridge.ts @@ -22,6 +22,7 @@ * provided), launches native Google ARCore Scene Viewer via `intent://`. */ +import {checkPluginCapability} from '../../three-live/src/plugin'; import type {SceneOrGLBSource} from '../../three-live/src/three-convert'; import { createSceneViewerIntent, @@ -50,12 +51,15 @@ export const buildAndroidSceneViewerIntentUrl = createSceneViewerIntent; export async function launchBestMobileOrHeadsetAR( source: SceneOrGLBSource | string, options?: { + element?: unknown; glbUrl?: string; quickLook?: QuickLookOptions; sceneViewer?: SceneViewerOptions; forceFallback?: 'ios-quicklook' | 'android-scene-viewer'; }, ): Promise { + const host = + options?.element ?? options?.quickLook?.host ?? options?.sceneViewer?.host; const converter = getXRSceneConverter(); const webxrSupported = options?.forceFallback ? false @@ -65,9 +69,24 @@ export async function launchBestMobileOrHeadsetAR( options?.forceFallback === 'ios-quicklook' || (!options?.forceFallback && !webxrSupported && isQuickLookSupported()) ) { + if (!checkPluginCapability(host, 'external_handoff', 'quick-look')) { + throw new Error( + "Capability 'external_handoff' denied for AR Quick Look.", + ); + } + const qlOpts: QuickLookOptions = {host, ...options?.quickLook}; if (typeof source === 'string') { + if ( + !checkPluginCapability(host, 'remote_content', 'quick-look', { + url: source, + }) + ) { + throw new Error( + "Capability 'remote_content' or 'network' denied for AR Quick Look URL.", + ); + } if (typeof document !== 'undefined') { - launchQuickLook(source, options?.quickLook); + launchQuickLook(source, qlOpts); } return {platform: 'ios-quicklook'}; } @@ -75,7 +94,7 @@ export async function launchBestMobileOrHeadsetAR( ? converter.createUSDZBlob(source) : createUSDZBlob(source); if (typeof document !== 'undefined') { - launchQuickLook(blob, options?.quickLook); + launchQuickLook(blob, qlOpts); } return { platform: 'ios-quicklook', @@ -99,7 +118,19 @@ export async function launchBestMobileOrHeadsetAR( isSceneViewerSupported() && Boolean(hostedGlbUrl)) ) { + if (!checkPluginCapability(host, 'external_handoff', 'scene-viewer')) { + throw new Error("Capability 'external_handoff' denied for Scene Viewer."); + } if (hostedGlbUrl) { + if ( + !checkPluginCapability(host, 'remote_content', 'scene-viewer', { + url: hostedGlbUrl, + }) + ) { + throw new Error( + "Capability 'remote_content' or 'network' denied for Scene Viewer URL.", + ); + } const intentUrl = buildAndroidSceneViewerIntentUrl( hostedGlbUrl, options?.sceneViewer, diff --git a/xr-live/tests/gemini_live_session_test.ts b/xr-live/tests/gemini_live_session_test.ts index db377c5c..85f87b96 100644 --- a/xr-live/tests/gemini_live_session_test.ts +++ b/xr-live/tests/gemini_live_session_test.ts @@ -14,14 +14,16 @@ /** Regression tests for asynchronous work crossing Gemini Live sessions. */ +import {ensureCapabilityChecker} from '../../three-live/src/capabilities'; import { + connectGeminiLive, createWebSocketTransport, geminiLivePlugin, - connectGeminiLive, type GeminiLiveTransport, REPORT_OBJECTS_TOOL, XRGeminiLiveController, } from '../src/plugins/gemini-live'; +ensureCapabilityChecker(); let passed = 0; let failed = 0; @@ -55,22 +57,44 @@ function transport() { const sent: Array> = []; let closed = 0; const socket: GeminiLiveTransport = { - send: (message) => { sent.push(message); }, - close: () => { closed++; socket.onClose?.(); }, + send: (message) => { + sent.push(message); + }, + close: () => { + closed++; + socket.onClose?.(); + }, + }; + return { + socket, + sent, + get closed() { + return closed; + }, }; - return {socket, sent, get closed() { return closed; }}; } function host() { const captures: Array> = []; const events: Array<{type: string; detail: unknown}> = []; // Translation alone is enough to distinguish detection-camera snapshots. - const matrix = new Float32Array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1]); + const matrix = new Float32Array([ + 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, + ]); const element = { - viewer: {container: {simulator: { - active: true, - getHeadPose: () => ({matrix, position: [matrix[12], 0, 0], quaternion: [0, 0, 0, 1]}), - }}}, + capabilities: 'ai network camera microphone sensors', + viewer: { + container: { + simulator: { + active: true, + getHeadPose: () => ({ + matrix, + position: [matrix[12], 0, 0], + quaternion: [0, 0, 0, 1], + }), + }, + }, + }, captureViewFrame() { const capture = deferredFrame(); captures.push(capture); @@ -87,41 +111,68 @@ function host() { const frame = (base64: string): Frame => ({base64, mimeType: 'image/jpeg'}); const detection = { toolCall: { - functionCalls: [{ - id: 'object-1', - name: REPORT_OBJECTS_TOOL, - args: {objects: [{label: 'cup', box_2d: [400, 400, 600, 600]}]}, - }], + functionCalls: [ + { + id: 'object-1', + name: REPORT_OBJECTS_TOOL, + args: {objects: [{label: 'cup', box_2d: [400, 400, 600, 600]}]}, + }, + ], }, }; -async function testPendingCapture(rejectOld: boolean, reuseTransport: boolean): Promise { +async function testPendingCapture( + rejectOld: boolean, + reuseTransport: boolean, +): Promise { const h = host(); const first = transport(); const next = reuseTransport ? first : transport(); const ctrl = new XRGeminiLiveController(h.element); - ctrl.connect({transport: first.socket, streamFrames: false, detectObjects: true}); + ctrl.connect({ + transport: first.socket, + streamFrames: false, + detectObjects: true, + }); const oldFrame = ctrl.sendFrame(); - ctrl.connect({transport: next.socket, streamFrames: false, detectObjects: true}); + ctrl.connect({ + transport: next.socket, + streamFrames: false, + detectObjects: true, + }); h.matrix[12] = 10; const newFrame = ctrl.sendFrame(); const label = `${rejectOld ? 'rejected' : 'resolved'} old capture, ${reuseTransport ? 'reused' : 'new'} transport`; - assert(h.captures.length === 2, `${label}: replacement capture starts immediately`); + assert( + h.captures.length === 2, + `${label}: replacement capture starts immediately`, + ); assert(!(await ctrl.sendFrame()), `${label}: only one capture per session`); if (rejectOld) h.captures[0].reject(new Error('old capture failed')); else h.captures[0].resolve(frame('old')); assert(!(await oldFrame), `${label}: old capture is discarded`); - assert(next.sent.length === 0, `${label}: no stale frame reaches replacement`); + assert( + next.sent.length === 0, + `${label}: no stale frame reaches replacement`, + ); const extraFrame = ctrl.sendFrame(); - assert(h.captures.length === 2, `${label}: old finally cannot release new capture lock`); + assert( + h.captures.length === 2, + `${label}: old finally cannot release new capture lock`, + ); // Settle every request even against the unfixed implementation. for (const capture of h.captures.slice(1)) capture.resolve(frame('new')); const [newSent, extraSent] = await Promise.all([newFrame, extraFrame]); - assert(newSent && !extraSent, `${label}: only the new session's original capture sends`); - assert(next.sent.length === 1 && JSON.stringify(next.sent[0]).includes('"new"'), - `${label}: current frame is sent once`); + assert( + newSent && !extraSent, + `${label}: only the new session's original capture sends`, + ); + assert( + next.sent.length === 1 && JSON.stringify(next.sent[0]).includes('"new"'), + `${label}: current frame is sent once`, + ); ctrl.disconnect(); } @@ -130,9 +181,17 @@ async function testStaleCamera(): Promise { const first = transport(); const next = transport(); const ctrl = new XRGeminiLiveController(h.element); - ctrl.connect({transport: first.socket, streamFrames: false, detectObjects: true}); + ctrl.connect({ + transport: first.socket, + streamFrames: false, + detectObjects: true, + }); const oldFrame = ctrl.sendFrame(); - ctrl.connect({transport: next.socket, streamFrames: false, detectObjects: true}); + ctrl.connect({ + transport: next.socket, + streamFrames: false, + detectObjects: true, + }); h.matrix[12] = 10; const newFrame = ctrl.sendFrame(); h.captures[1]?.resolve(frame('new')); @@ -141,8 +200,12 @@ async function testStaleCamera(): Promise { await oldFrame; h.matrix[12] = 20; next.socket.onMessage?.(detection); - const located = h.events.find((event) => event.type === 'xr-object-detected')?.detail as {x?: number}; - assert(located?.x === 10, 'late old frame cannot replace the new detection camera'); + const located = h.events.find((event) => event.type === 'xr-object-detected') + ?.detail as {x?: number}; + assert( + located?.x === 10, + 'late old frame cannot replace the new detection camera', + ); ctrl.disconnect(); } @@ -153,31 +216,66 @@ function testStaleCallbacks(reuseTransport: boolean): void { const ctrl = new XRGeminiLiveController(h.element); let enqueued = 0; let flushed = 0; - ctrl.audio.enqueue = () => { enqueued++; }; - ctrl.audio.flush = () => { flushed++; }; - ctrl.connect({transport: first.socket, streamFrames: false, detectObjects: true}); + ctrl.audio.enqueue = () => { + enqueued++; + }; + ctrl.audio.flush = () => { + flushed++; + }; + ctrl.connect({ + transport: first.socket, + streamFrames: false, + detectObjects: true, + }); const oldMessage = first.socket.onMessage!; const oldClose = first.socket.onClose!; - ctrl.connect({transport: next.socket, streamFrames: false, detectObjects: true}); + ctrl.connect({ + transport: next.socket, + streamFrames: false, + detectObjects: true, + }); h.events.length = 0; flushed = 0; - oldMessage({...detection, serverContent: { - interrupted: true, - inputTranscription: {text: 'old input'}, - outputTranscription: {text: 'old output'}, - modelTurn: {parts: [{inlineData: {mimeType: 'audio/pcm;rate=24000', data: 'AAA='}}]}, - turnComplete: true, - }}); + oldMessage({ + ...detection, + serverContent: { + interrupted: true, + inputTranscription: {text: 'old input'}, + outputTranscription: {text: 'old output'}, + modelTurn: { + parts: [{inlineData: {mimeType: 'audio/pcm;rate=24000', data: 'AAA='}}], + }, + turnComplete: true, + }, + }); oldClose(); const label = reuseTransport ? 'reused transport' : 'new transport'; assert(h.events.length === 0, `${label}: old callbacks emit no events`); assert(next.sent.length === 0, `${label}: old tool calls send no responses`); - assert(enqueued === 0 && flushed === 0, `${label}: old messages cannot play or interrupt audio`); - assert(ctrl.connected, `${label}: old close cannot disconnect the replacement`); - next.socket.onMessage?.({serverContent: {modelTurn: {parts: [{text: 'new output'}]}, turnComplete: true}}); - assert(h.events.some((event) => event.type === 'xr-transcription' && - (event.detail as {text: string; isFinal: boolean}).text === 'new output' && - (event.detail as {isFinal: boolean}).isFinal), `${label}: current callback still works`); + assert( + enqueued === 0 && flushed === 0, + `${label}: old messages cannot play or interrupt audio`, + ); + assert( + ctrl.connected, + `${label}: old close cannot disconnect the replacement`, + ); + next.socket.onMessage?.({ + serverContent: { + modelTurn: {parts: [{text: 'new output'}]}, + turnComplete: true, + }, + }); + assert( + h.events.some( + (event) => + event.type === 'xr-transcription' && + (event.detail as {text: string; isFinal: boolean}).text === + 'new output' && + (event.detail as {isFinal: boolean}).isFinal, + ), + `${label}: current callback still works`, + ); ctrl.disconnect(); } @@ -190,11 +288,19 @@ async function testClosedSession(remote: boolean): Promise { const pending = ctrl.sendFrame(); if (remote) first.socket.onClose?.(); else ctrl.disconnect(); - oldMessage({serverContent: {modelTurn: {parts: [{text: 'late'}]}, turnComplete: true}}); + oldMessage({ + serverContent: {modelTurn: {parts: [{text: 'late'}]}, turnComplete: true}, + }); h.captures[0].resolve(frame('late')); const label = remote ? 'remote close' : 'disconnect'; - assert(!ctrl.connected && !(await pending), `${label}: pending frame is discarded`); - assert(first.sent.length === 0 && h.events.length === 0, `${label}: late work has no effects`); + assert( + !ctrl.connected && !(await pending), + `${label}: pending frame is discarded`, + ); + assert( + first.sent.length === 0 && h.events.length === 0, + `${label}: late work has no effects`, + ); const next = transport(); ctrl.connect({transport: next.socket, streamFrames: false}); const fresh = ctrl.sendFrame(); @@ -209,14 +315,33 @@ function testTranscriptReset(): void { const next = transport(); const ctrl = new XRGeminiLiveController(h.element); ctrl.connect({transport: first.socket, streamFrames: false}); - first.socket.onMessage?.({serverContent: {outputTranscription: {text: 'unfinished'}}}); + first.socket.onMessage?.({ + serverContent: {outputTranscription: {text: 'unfinished'}}, + }); ctrl.connect({transport: next.socket, streamFrames: false}); - assert(h.events.some((event) => (event.detail as {text?: string; isFinal?: boolean}).text === 'unfinished' && - (event.detail as {isFinal?: boolean}).isFinal), 'reconnect still finalizes the old transcript'); + assert( + h.events.some( + (event) => + (event.detail as {text?: string; isFinal?: boolean}).text === + 'unfinished' && (event.detail as {isFinal?: boolean}).isFinal, + ), + 'reconnect still finalizes the old transcript', + ); h.events.length = 0; - next.socket.onMessage?.({serverContent: {modelTurn: {parts: [{text: 'fresh text'}]}, turnComplete: true}}); - assert(h.events.some((event) => (event.detail as {text?: string; isFinal?: boolean}).text === 'fresh text' && - (event.detail as {isFinal?: boolean}).isFinal), 'old output transcription does not suppress the new text turn'); + next.socket.onMessage?.({ + serverContent: { + modelTurn: {parts: [{text: 'fresh text'}]}, + turnComplete: true, + }, + }); + assert( + h.events.some( + (event) => + (event.detail as {text?: string; isFinal?: boolean}).text === + 'fresh text' && (event.detail as {isFinal?: boolean}).isFinal, + ), + 'old output transcription does not suppress the new text turn', + ); ctrl.disconnect(); } @@ -230,33 +355,51 @@ function testReconnectDuringDisconnect(): void { const dispatch = h.element.dispatchEvent; h.element.dispatchEvent = (event: Event) => { dispatch(event); - const detail = (event as CustomEvent).detail as {text?: string; isFinal?: boolean}; + const detail = (event as CustomEvent).detail as { + text?: string; + isFinal?: boolean; + }; if (!switched && detail?.isFinal && detail.text === 'old input') { switched = true; ctrl.connect({transport: nested.socket, streamFrames: false}); - nested.socket.onMessage?.({serverContent: {outputTranscription: {text: 'new output'}}}); + nested.socket.onMessage?.({ + serverContent: {outputTranscription: {text: 'new output'}}, + }); } return true; }; ctrl.connect({transport: first.socket, streamFrames: false}); - first.socket.onMessage?.({serverContent: { - inputTranscription: {text: 'old input'}, outputTranscription: {text: 'old output'}, - }}); + first.socket.onMessage?.({ + serverContent: { + inputTranscription: {text: 'old input'}, + outputTranscription: {text: 'old output'}, + }, + }); ctrl.connect({transport: outer.socket, streamFrames: false}); ctrl.sendText('current'); - assert(switched && nested.sent.length === 1 && outer.sent.length === 0, - 'reconnect during finalization keeps the newest transport'); - assert(!h.events.some((event) => { - const detail = event.detail as {text?: string; isFinal?: boolean}; - return detail.text === 'old output' && detail.isFinal; - }), 'reconnect during finalization stops remaining old transcript events'); + assert( + switched && nested.sent.length === 1 && outer.sent.length === 0, + 'reconnect during finalization keeps the newest transport', + ); + assert( + !h.events.some((event) => { + const detail = event.detail as {text?: string; isFinal?: boolean}; + return detail.text === 'old output' && detail.isFinal; + }), + 'reconnect during finalization stops remaining old transcript events', + ); ctrl.disconnect(); - assert(first.closed === 1 && outer.closed === 1 && nested.closed === 1, - 'reconnect during finalization closes the superseded and active transports'); - assert(h.events.some((event) => { - const detail = event.detail as {text?: string; isFinal?: boolean}; - return detail.text === 'new output' && detail.isFinal; - }), 'reconnect during finalization preserves the new transcript'); + assert( + first.closed === 1 && outer.closed === 1 && nested.closed === 1, + 'reconnect during finalization closes the superseded and active transports', + ); + assert( + h.events.some((event) => { + const detail = event.detail as {text?: string; isFinal?: boolean}; + return detail.text === 'new output' && detail.isFinal; + }), + 'reconnect during finalization preserves the new transcript', + ); } function testReconnectDuringTransportClose(): void { @@ -266,19 +409,29 @@ function testReconnectDuringTransportClose(): void { const ctrl = new XRGeminiLiveController(h.element); first.socket.close = () => { ctrl.connect({transport: next.socket, streamFrames: false}); - next.socket.onMessage?.({serverContent: {outputTranscription: {text: 'new output'}}}); + next.socket.onMessage?.({ + serverContent: {outputTranscription: {text: 'new output'}}, + }); }; ctrl.connect({transport: first.socket, streamFrames: false}); ctrl.disconnect(); - assert(ctrl.connected && !h.events.some((event) => { - const detail = event.detail as {text?: string; isFinal?: boolean}; - return detail.text === 'new output' && detail.isFinal; - }), 'transport close reconnect does not finalize the new session'); + assert( + ctrl.connected && + !h.events.some((event) => { + const detail = event.detail as {text?: string; isFinal?: boolean}; + return detail.text === 'new output' && detail.isFinal; + }), + 'transport close reconnect does not finalize the new session', + ); ctrl.disconnect(); - assert(next.closed === 1 && h.events.some((event) => { - const detail = event.detail as {text?: string; isFinal?: boolean}; - return detail.text === 'new output' && detail.isFinal; - }), 'new session closes and finalizes normally after transport close reconnect'); + assert( + next.closed === 1 && + h.events.some((event) => { + const detail = event.detail as {text?: string; isFinal?: boolean}; + return detail.text === 'new output' && detail.isFinal; + }), + 'new session closes and finalizes normally after transport close reconnect', + ); // Reconnecting with a pre-built outer transport when close() connects an inner one. const initial = transport(); @@ -291,7 +444,10 @@ function testReconnectDuringTransportClose(): void { ctrl.connect({transport: outer.socket, streamFrames: false}); ctrl.sendText('to inner'); assert( - outer.closed === 1 && inner.closed === 0 && inner.sent.length === 1 && outer.sent.length === 0, + outer.closed === 1 && + inner.closed === 0 && + inner.sent.length === 1 && + outer.sent.length === 0, 'superseded outer transport is closed when transport.close connects an inner session', ); ctrl.disconnect(); @@ -309,7 +465,10 @@ function testReconnectDuringTransportClose(): void { ctrl.connect({transport: replacement.socket, streamFrames: false}); ctrl.sendText('still connected'); assert( - prevClosed === 1 && ctrl.connected && replacement.closed === 0 && replacement.sent.length === 1, + prevClosed === 1 && + ctrl.connected && + replacement.closed === 0 && + replacement.sent.length === 1, 'redundant disconnect during teardown does not abort replacement connect', ); ctrl.disconnect(); @@ -325,7 +484,9 @@ async function testFrameTimers(): Promise { active.add(callbacks.length); return callbacks.length; }) as typeof setInterval; - globalThis.clearInterval = ((id: number) => { active.delete(id); }) as typeof clearInterval; + globalThis.clearInterval = ((id: number) => { + active.delete(id); + }) as typeof clearInterval; const h = host(); const first = transport(); const next = transport(); @@ -333,9 +494,15 @@ async function testFrameTimers(): Promise { try { ctrl.connect({transport: first.socket}); ctrl.connect({transport: next.socket}); - assert(active.size === 1 && active.has(2), 'reconnect replaces the frame timer'); + assert( + active.size === 1 && active.has(2), + 'reconnect replaces the frame timer', + ); callbacks[0](); - assert(h.captures.length === 0, 'a queued old timer cannot capture for the replacement'); + assert( + h.captures.length === 0, + 'a queued old timer cannot capture for the replacement', + ); callbacks[1](); assert(h.captures.length === 1, 'the current frame timer still captures'); h.captures[0].resolve(frame('current timer')); @@ -349,10 +516,15 @@ async function testFrameTimers(): Promise { const closing = transport(); closing.socket.send = () => closing.socket.onClose?.(); ctrl.connect({transport: closing.socket, setup: {model: 'm'}}); - assert(!ctrl.connected && active.size === 0, 'close during setup cannot leave an orphan frame timer'); + assert( + !ctrl.connected && active.size === 0, + 'close during setup cannot leave an orphan frame timer', + ); const throwing = transport(); - throwing.socket.send = () => { throw new Error('setup failed'); }; + throwing.socket.send = () => { + throw new Error('setup failed'); + }; let setupThrew = false; try { ctrl.connect({transport: throwing.socket, setup: {model: 'm'}}); @@ -360,7 +532,10 @@ async function testFrameTimers(): Promise { setupThrew = true; } assert( - setupThrew && !ctrl.connected && throwing.closed === 1 && active.size === 0, + setupThrew && + !ctrl.connected && + throwing.closed === 1 && + active.size === 0, 'throwing setup closes transport and leaves no timer', ); @@ -375,11 +550,17 @@ async function testFrameTimers(): Promise { }; ctrl.connect({transport: reentering.socket, setup: {model: 'm'}}); assert( - ctrl.connected && reentering.closed === 1 && nested.closed === 0 && active.size === 1, + ctrl.connected && + reentering.closed === 1 && + nested.closed === 0 && + active.size === 1, 'reconnect during setup send keeps only the nested session timer and transport', ); ctrl.disconnect(); - assert(nested.closed === 1 && active.size === 0, 'nested session timer and transport close cleanly'); + assert( + nested.closed === 1 && active.size === 0, + 'nested session timer and transport close cleanly', + ); const firstCtrl = new XRGeminiLiveController(h.element); const t1 = transport(); @@ -389,7 +570,11 @@ async function testFrameTimers(): Promise { const secondCtrl = new XRGeminiLiveController(h.element); secondCtrl.connect({transport: t2.socket}); assert( - !firstCtrl.connected && secondCtrl.connected && t1.closed === 1 && t2.closed === 0 && active.size === 1, + !firstCtrl.connected && + secondCtrl.connected && + t1.closed === 1 && + t2.closed === 0 && + active.size === 1, 'connecting a second controller on the same element closes the previous timer and transport', ); const rawT2Close = t2.socket.close.bind(t2.socket); @@ -401,7 +586,11 @@ async function testFrameTimers(): Promise { const supersededCtrl = transport(); thirdCtrl.connect({transport: supersededCtrl.socket}); assert( - thirdCtrl.connected && t2.closed === 1 && supersededCtrl.closed === 1 && t3.closed === 0 && active.size === 1, + thirdCtrl.connected && + t2.closed === 1 && + supersededCtrl.closed === 1 && + t3.closed === 0 && + active.size === 1, 'reentrant connectGeminiLive during cross-controller teardown preserves inner session and closes superseded transport', ); geminiLivePlugin.onElementDisconnected?.(h.element); @@ -421,7 +610,15 @@ function testReentrantReconnect(): void { const originalSpeech = globals['speechSynthesis']; const originalUtterance = globals['SpeechSynthesisUtterance']; try { - for (const trigger of ['input', 'output', 'text', 'interrupted', 'final-user', 'final-model', 'detection']) { + for (const trigger of [ + 'input', + 'output', + 'text', + 'interrupted', + 'final-user', + 'final-model', + 'detection', + ]) { const h = host(); const first = transport(); const next = transport(); @@ -433,64 +630,123 @@ function testReentrantReconnect(): void { let staleSpeech = 0; globals['speechSynthesis'] = { cancel() {}, - speak() { if (replacementStarted) staleSpeech++; }, + speak() { + if (replacementStarted) staleSpeech++; + }, + }; + globals['SpeechSynthesisUtterance'] = class { + constructor(readonly text: string) {} + }; + ctrl.audio.enqueue = () => { + if (replacementStarted) staleAudio++; }; - globals['SpeechSynthesisUtterance'] = class { constructor(readonly text: string) {} }; - ctrl.audio.enqueue = () => { if (replacementStarted) staleAudio++; }; const dispatch = h.element.dispatchEvent; h.element.dispatchEvent = (event: Event) => { dispatch(event); if (replacementStarted) staleEvents++; - const detail = (event as CustomEvent).detail as {speaker?: string; isFinal?: boolean}; - const matches = trigger === 'interrupted' ? event.type === 'xr-gemini-interrupted' : - trigger === 'detection' ? event.type === 'xr-object-detected' : - event.type === 'xr-transcription' && ( - trigger === 'final-user' ? detail.isFinal && detail.speaker === 'user' : - trigger === 'final-model' ? detail.isFinal && detail.speaker === 'model' : - !detail.isFinal && detail.speaker === (trigger === 'input' ? 'user' : 'model')); + const detail = (event as CustomEvent).detail as { + speaker?: string; + isFinal?: boolean; + }; + const matches = + trigger === 'interrupted' + ? event.type === 'xr-gemini-interrupted' + : trigger === 'detection' + ? event.type === 'xr-object-detected' + : event.type === 'xr-transcription' && + (trigger === 'final-user' + ? detail.isFinal && detail.speaker === 'user' + : trigger === 'final-model' + ? detail.isFinal && detail.speaker === 'model' + : !detail.isFinal && + detail.speaker === + (trigger === 'input' ? 'user' : 'model')); if (matches && !switched) { switched = true; - ctrl.connect({transport: next.socket, streamFrames: false, detectObjects: true, speakReplies: true}); + ctrl.connect({ + transport: next.socket, + streamFrames: false, + detectObjects: true, + speakReplies: true, + }); replacementStarted = true; } return true; }; - ctrl.connect({transport: first.socket, streamFrames: false, detectObjects: true, speakReplies: true}); + ctrl.connect({ + transport: first.socket, + streamFrames: false, + detectObjects: true, + speakReplies: true, + }); first.socket.onMessage?.({ - ...(trigger === 'detection' ? {toolCall: {functionCalls: [{ - id: 'old-call', name: REPORT_OBJECTS_TOOL, - args: {objects: [ - {label: 'cup', box_2d: [400, 400, 600, 600]}, - {label: 'plate', box_2d: [200, 200, 300, 300]}, - ]}, - }]}} : {}), + ...(trigger === 'detection' + ? { + toolCall: { + functionCalls: [ + { + id: 'old-call', + name: REPORT_OBJECTS_TOOL, + args: { + objects: [ + {label: 'cup', box_2d: [400, 400, 600, 600]}, + {label: 'plate', box_2d: [200, 200, 300, 300]}, + ], + }, + }, + ], + }, + } + : {}), serverContent: { interrupted: trigger === 'interrupted', - ...(trigger === 'text' ? {} : { - inputTranscription: {text: 'old input'}, - outputTranscription: {text: 'old output'}, - }), - modelTurn: {parts: [ - {text: 'old text', inlineData: {mimeType: 'audio/pcm;rate=24000', data: 'AAA='}}, - {text: 'more old text'}, - ]}, + ...(trigger === 'text' + ? {} + : { + inputTranscription: {text: 'old input'}, + outputTranscription: {text: 'old output'}, + }), + modelTurn: { + parts: [ + { + text: 'old text', + inlineData: {mimeType: 'audio/pcm;rate=24000', data: 'AAA='}, + }, + {text: 'more old text'}, + ], + }, turnComplete: true, }, }); assert(switched && ctrl.connected, `${trigger} listener can reconnect`); - assert(staleEvents === 0 && staleAudio === 0 && staleSpeech === 0, - `${trigger} reconnect stops remaining old events, audio and speech`); - assert(next.sent.length === 0, `${trigger} reconnect does not forward the old tool response`); + assert( + staleEvents === 0 && staleAudio === 0 && staleSpeech === 0, + `${trigger} reconnect stops remaining old events, audio and speech`, + ); + assert( + next.sent.length === 0, + `${trigger} reconnect does not forward the old tool response`, + ); replacementStarted = false; - next.socket.onMessage?.({serverContent: {modelTurn: {parts: [{text: 'new reply'}]}, turnComplete: true}}); - assert(h.events.some((event) => (event.detail as {text?: string}).text === 'new reply'), - `${trigger} reconnect accepts the new reply`); + next.socket.onMessage?.({ + serverContent: { + modelTurn: {parts: [{text: 'new reply'}]}, + turnComplete: true, + }, + }); + assert( + h.events.some( + (event) => (event.detail as {text?: string}).text === 'new reply', + ), + `${trigger} reconnect accepts the new reply`, + ); ctrl.disconnect(); } } finally { if (originalSpeech === undefined) delete globals['speechSynthesis']; else globals['speechSynthesis'] = originalSpeech; - if (originalUtterance === undefined) delete globals['SpeechSynthesisUtterance']; + if (originalUtterance === undefined) + delete globals['SpeechSynthesisUtterance']; else globals['SpeechSynthesisUtterance'] = originalUtterance; } } @@ -503,7 +759,9 @@ async function testDeferredBlobCallback(): Promise { onopen?: () => void; onmessage?: (event: MessageEvent) => Promise; onclose?: () => void; - constructor() { ws = this; } + constructor() { + ws = this; + } send() {} close() {} } @@ -511,20 +769,36 @@ async function testDeferredBlobCallback(): Promise { try { const socket = createWebSocketTransport('wss://example.com/live'); let resolve!: (text: string) => void; - const text = new Promise((ok) => { resolve = ok; }); + const text = new Promise((ok) => { + resolve = ok; + }); let oldMessages = 0; let newMessages = 0; - socket.onMessage = () => { oldMessages++; }; + socket.onMessage = () => { + oldMessages++; + }; const pending = ws.onmessage!({data: {text: () => text}} as MessageEvent); - socket.onMessage = () => { newMessages++; }; + socket.onMessage = () => { + newMessages++; + }; resolve('{}'); await pending; - assert(oldMessages === 1 && newMessages === 0, 'Blob decoding retains the callback that received the frame'); + assert( + oldMessages === 1 && newMessages === 0, + 'Blob decoding retains the callback that received the frame', + ); await ws.onmessage!({data: '{}'} as MessageEvent); - assert(newMessages === 1, 'subsequent WebSocket frames use the current callback'); + assert( + newMessages === 1, + 'subsequent WebSocket frames use the current callback', + ); let resolveAfterClose!: (text: string) => void; - const afterCloseText = new Promise((ok) => { resolveAfterClose = ok; }); - const pendingAfterClose = ws.onmessage!({data: {text: () => afterCloseText}} as MessageEvent); + const afterCloseText = new Promise((ok) => { + resolveAfterClose = ok; + }); + const pendingAfterClose = ws.onmessage!({ + data: {text: () => afterCloseText}, + } as MessageEvent); socket.close(); resolveAfterClose('{}'); await pendingAfterClose; @@ -535,44 +809,79 @@ async function testDeferredBlobCallback(): Promise { } async function testFrameQuestionBinding(): Promise { - const h = host(), first = transport(), next = transport(); + const h = host(), + first = transport(), + next = transport(); const ctrl = new XRGeminiLiveController(h.element); ctrl.connect({transport: first.socket, streamFrames: false}); const pending = ctrl.sendFrame(); ctrl.sendText('Which color?'); - assert(first.sent.length === 0, 'question does not overtake pending canvas capture'); + assert( + first.sent.length === 0, + 'question does not overtake pending canvas capture', + ); h.captures[0].resolve(frame('green')); - await pending; await Promise.resolve(); + await pending; + await Promise.resolve(); const turns = first.sent[1]?.clientContent as any; - assert((first.sent[0]?.realtimeInput as any)?.mediaChunks[0].data === 'green', 'local frame is forwarded before its question'); - assert(turns?.turns[0].parts[0].inlineData.data === 'green' && turns.turns[0].parts[1].text === 'Which color?', - 'image and question share a single conversation turn without a fixed delay'); - const stale = ctrl.sendFrame(); ctrl.sendText('stale question'); + assert( + (first.sent[0]?.realtimeInput as any)?.mediaChunks[0].data === 'green', + 'local frame is forwarded before its question', + ); + assert( + turns?.turns[0].parts[0].inlineData.data === 'green' && + turns.turns[0].parts[1].text === 'Which color?', + 'image and question share a single conversation turn without a fixed delay', + ); + const stale = ctrl.sendFrame(); + ctrl.sendText('stale question'); ctrl.connect({transport: next.socket, streamFrames: false}); - h.captures[1].resolve(frame('old')); await stale; await Promise.resolve(); - assert(next.sent.length === 0, 'capture-dependent question cannot enter a replacement session'); + h.captures[1].resolve(frame('old')); + await stale; + await Promise.resolve(); + assert( + next.sent.length === 0, + 'capture-dependent question cannot enter a replacement session', + ); ctrl.disconnect(); } async function testVisualCaptureFailures(): Promise { - const h = host(), t = transport(); + const h = host(), + t = transport(); const ctrl = new XRGeminiLiveController(h.element); ctrl.connect({transport: t.socket, streamFrames: false}); ctrl.sendText('fresh question', {captureFrame: true}); - assert(h.captures.length === 1 && t.sent.length === 0, 'explicit visual question starts a fresh capture'); + assert( + h.captures.length === 1 && t.sent.length === 0, + 'explicit visual question starts a fresh capture', + ); h.captures[0].reject(new Error('capture unavailable')); - await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); - assert(t.sent.length === 0, 'failed capture never downgrades a visual question to text'); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + assert( + t.sent.length === 0, + 'failed capture never downgrades a visual question to text', + ); delete h.element.captureViewFrame; ctrl.sendText('visual unavailable', {captureFrame: true}); - assert(t.sent.length === 0, 'missing capture capability refuses an explicitly visual question'); + assert( + t.sent.length === 0, + 'missing capture capability refuses an explicitly visual question', + ); ctrl.sendText('ordinary text'); - assert(t.sent.length === 1, 'ordinary text keeps its immediate text-only behavior'); + assert( + t.sent.length === 1, + 'ordinary text keeps its immediate text-only behavior', + ); ctrl.disconnect(); } async function testCaptureReentry(): Promise { - const h = host(), first = transport(), next = transport(); + const h = host(), + first = transport(), + next = transport(); const ctrl = new XRGeminiLiveController(h.element); const capture = h.element.captureViewFrame.bind(h.element); let nested!: Promise; @@ -586,49 +895,77 @@ async function testCaptureReentry(): Promise { }; ctrl.connect({transport: first.socket, streamFrames: false}); const old = ctrl.sendFrame(); - assert(h.captures.length === 2, 'capture host can replace the session and start its own capture'); - assert(!(await ctrl.sendFrame()), 'outer capture cannot overwrite the replacement admission lock'); + assert( + h.captures.length === 2, + 'capture host can replace the session and start its own capture', + ); + assert( + !(await ctrl.sendFrame()), + 'outer capture cannot overwrite the replacement admission lock', + ); h.captures[0].resolve(frame('old')); assert(!(await old), 'capture reentry revokes the old frame'); - assert(!(await ctrl.sendFrame()), 'old completion cannot clear the replacement lock'); + assert( + !(await ctrl.sendFrame()), + 'old completion cannot clear the replacement lock', + ); h.captures[1].resolve(frame('new')); assert(await nested, 'replacement capture completes normally'); - assert(next.sent.length === 1 && JSON.stringify(next.sent[0]).includes('"new"'), 'only replacement pixels reach its transport'); + assert( + next.sent.length === 1 && JSON.stringify(next.sent[0]).includes('"new"'), + 'only replacement pixels reach its transport', + ); ctrl.disconnect(); } function testCloseCleansBeforeReentry(): void { for (const remote of [false, true]) { - const h = host(), first = transport(), next = transport(); + const h = host(), + first = transport(), + next = transport(); const ctrl = new XRGeminiLiveController(h.element); ctrl.connect({transport: first.socket, streamFrames: false}); ctrl.transcripts.append('model', 'old partial'); - let cleaned = 0; ctrl.audio.close = () => { cleaned++; }; + let cleaned = 0; + ctrl.audio.close = () => { + cleaned++; + }; if (remote) first.socket.onClose?.(); else { first.socket.close = () => { assert(cleaned > 0, 'old audio closes before transport-close reentry'); ctrl.connect({transport: next.socket, streamFrames: false}); - next.socket.onMessage?.({serverContent: {modelTurn: {parts: [{text: 'fresh'}]}}}); + next.socket.onMessage?.({ + serverContent: {modelTurn: {parts: [{text: 'fresh'}]}}, + }); }; ctrl.disconnect(); } const remaining = ctrl.transcripts.finalize(); - assert(remote ? remaining.length === 0 : remaining[0]?.text === 'fresh', - 'remote closure and reentrant replacement cannot concatenate old partial transcripts'); + assert( + remote ? remaining.length === 0 : remaining[0]?.text === 'fresh', + 'remote closure and reentrant replacement cannot concatenate old partial transcripts', + ); ctrl.disconnect(); } } async function testHoldSessionLease(): Promise { const globals = globalThis as any; - const timeout = globals.setTimeout, clear = globals.clearTimeout, original = globals.SpeechRecognition; + const timeout = globals.setTimeout, + clear = globals.clearTimeout, + original = globals.SpeechRecognition; let tick!: () => void, recognition: any; - globals.setTimeout = (fn: () => void) => { tick = fn; return 1; }; + globals.setTimeout = (fn: () => void) => { + tick = fn; + return 1; + }; globals.clearTimeout = () => {}; globals.SpeechRecognition = class { onresult: ((e: any) => void) | null = null; - constructor() { recognition = this; } + constructor() { + recognition = this; + } start() {} stop() {} abort() {} @@ -636,26 +973,48 @@ async function testHoldSessionLease(): Promise { try { for (const startSpeech of [false, true]) { const element = new EventTarget() as any; + element.capabilities = 'ai network camera microphone'; element.captureViewFrame = async () => frame('green'); - const first = transport(), next = transport(); + const first = transport(), + next = transport(); geminiLivePlugin.onElementConnected?.(element); - const ctrl = connectGeminiLive(element, {transport: first.socket, streamFrames: false}); - element.dispatchEvent(new CustomEvent('xr-select-start', {detail: {hit: {mesh: { - name: 'old object', userData: {xr: {holdToAsk: {prompt: 'old question'}}}, - }}}})); + const ctrl = connectGeminiLive(element, { + transport: first.socket, + streamFrames: false, + }); + element.dispatchEvent( + new CustomEvent('xr-select-start', { + detail: { + hit: { + mesh: { + name: 'old object', + userData: {xr: {holdToAsk: {prompt: 'old question'}}}, + }, + }, + }, + }), + ); if (startSpeech) tick(); const late = recognition?.onresult; ctrl.connect({transport: next.socket, streamFrames: false}); if (startSpeech) { - assert(recognition.onresult === null, 'reconnect detaches recognition delivery before abort'); + assert( + recognition.onresult === null, + 'reconnect detaches recognition delivery before abort', + ); late?.({results: [[{transcript: 'old speech'}]]}); } else tick(); - await Promise.resolve(); await Promise.resolve(); - assert(next.sent.length === 0, 'old hold timer/recognition cannot send into the new session'); + await Promise.resolve(); + await Promise.resolve(); + assert( + next.sent.length === 0, + 'old hold timer/recognition cannot send into the new session', + ); geminiLivePlugin.onElementDisconnected?.(element); } } finally { - globals.setTimeout = timeout; globals.clearTimeout = clear; + globals.setTimeout = timeout; + globals.clearTimeout = clear; if (original === undefined) delete globals.SpeechRecognition; else globals.SpeechRecognition = original; } @@ -663,7 +1022,8 @@ async function testHoldSessionLease(): Promise { async function main(): Promise { for (const reuseTransport of [false, true]) { - for (const rejectOld of [false, true]) await testPendingCapture(rejectOld, reuseTransport); + for (const rejectOld of [false, true]) + await testPendingCapture(rejectOld, reuseTransport); testStaleCallbacks(reuseTransport); } await testStaleCamera(); diff --git a/xr-live/tests/xr_capabilities_test.ts b/xr-live/tests/xr_capabilities_test.ts new file mode 100644 index 00000000..2c282bea --- /dev/null +++ b/xr-live/tests/xr_capabilities_test.ts @@ -0,0 +1,519 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @fileoverview End-to-end capability security tests for `` and its + * spatial plugins (`splat-loader`, `persistent-anchors`, `shared-room`, + * `spatial-audio`, `gemini-live`, `world-understanding`, `depth-particles`, + * and `xr-mobile-bridge`), including `userData.xr.*` privilege-escalation + * refusal while procedural geometry continues to render cleanly. + */ + +import { + CAPABILITIES_NONE, + CapabilityDeniedDetail, + ensureCapabilityChecker, + resetCapabilityDenials, +} from '../../three-live/src/capabilities'; +import {getSharedRegistry, ThreeLiveViewer} from '../../three-live/src/plugin'; +import {THREE} from '../../three-live/src/three-author'; +import {sceneToCanvas3D} from '../../three-live/src/three-convert'; +import {ThreeLiveElement} from '../../three-live/src/three-live-element'; +import {registerXRTraitSerializer} from '../../three-live/src/xr-trait-schema'; +import {depthParticlesPlugin} from '../src/plugins/depth-particles'; +import {XRGeminiLiveController} from '../src/plugins/gemini-live'; +import {XRPersistentAnchorsController} from '../src/plugins/persistent-anchors'; +import {XRSharedRoomController} from '../src/plugins/shared-room'; +import { + spatialAudioPlugin, + XRSpatialAudioEmitterController, +} from '../src/plugins/spatial-audio'; +import {loadSplat, XRSplatLoader} from '../src/plugins/splat-loader'; +import {worldUnderstandingPlugin} from '../src/plugins/world-understanding'; +import {XRInputManager} from '../src/xr-input'; +import {XRLiveElement} from '../src/xr-live-element'; +import {launchBestMobileOrHeadsetAR} from '../src/xr-mobile-bridge'; +import {XRTraitSystem} from '../src/xr-traits'; + +ensureCapabilityChecker(); +registerXRTraitSerializer(); + +let passed = 0; +let failed = 0; + +function assert(condition: boolean, testName: string): void { + if (condition) { + console.log(` ✅ [PASS] ${testName}`); + passed++; + } else { + console.error(` ❌ [FAIL] ${testName}`); + failed++; + } +} + +function createFakeHost(capabilities = 'scripts'): { + attrs: Map; + host: { + capabilities: string; + denied: CapabilityDeniedDetail[]; + events: Array<{type: string; detail: unknown}>; + captureViewFrame: () => Promise<{base64: string; mimeType: string}>; + getAttribute: (name: string) => string | null; + hasAttribute: (name: string) => boolean; + addEventListener: () => void; + removeEventListener: () => void; + onCapabilityDenied: (d: CapabilityDeniedDetail) => void; + dispatchEvent: (e: Event) => boolean; + }; + viewer: ThreeLiveViewer; +} { + const attrs = new Map([ + ['capabilities', capabilities], + ['persistent-anchors', 'studio'], + ['shared-room-voice', ''], + ['allow-remote-splats', ''], + ]); + const host = { + capabilities, + denied: [] as CapabilityDeniedDetail[], + events: [] as Array<{type: string; detail: unknown}>, + captureViewFrame: async () => ({base64: '/9g=', mimeType: 'image/jpeg'}), + getAttribute: (name: string) => attrs.get(name) ?? null, + hasAttribute: (name: string) => attrs.has(name), + addEventListener: () => {}, + removeEventListener: () => {}, + onCapabilityDenied(d: CapabilityDeniedDetail) { + this.denied.push(d); + }, + dispatchEvent(e: Event) { + this.events.push({ + type: e.type, + detail: (e as CustomEvent).detail, + }); + return true; + }, + }; + const scene = new THREE.Scene(); + const m = new THREE.Mesh( + new THREE.BoxGeometry(1, 1, 1), + new THREE.MeshStandardMaterial({color: 0x38bdf8}), + ); + m.name = 'EscalatorNode'; + m.userData.xr = { + grab: '1-hand', + splatUrl: 'https://cdn.example.com/room.spz', + anchor: {target: 'floor', id: 'escalator-1', persist: true}, + sync: true, + worldUnderstanding: {planes: true, meshes: true, reticle: true}, + particles: {preset: 'sparks', count: 32, collideWithDepth: true}, + audioEmitter: { + src: 'https://cdn.example.com/secret.mp3', + autoplay: true, + playOnSelect: true, + }, + }; + scene.add(m); + const converted = sceneToCanvas3D(scene); + const viewer: ThreeLiveViewer = { + container: host as unknown as HTMLElement, + meshes: [ + { + name: 'EscalatorNode', + nodeIndex: 0, + transform: new Float32Array([ + 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, -1, 1, + ]), + position: [0, 0, -1], + quaternion: [0, 0, 0, 1], + scale: [1, 1, 1], + primitives: [], + extras: ( + converted.gltf.nodes?.[0] as {extras?: Record} + )?.extras, + }, + ], + lastGltf: converted.gltf, + } as unknown as ThreeLiveViewer; + return {attrs, host, viewer}; +} + +async function runXRCapabilitiesTests(): Promise { + console.log('🥽 Running XR-Live Capability Security Tests...\n'); + + ThreeLiveElement.defaultCapabilities = CAPABILITIES_NONE; + ThreeLiveElement.allowClientScripts = false; + getSharedRegistry().onCapabilityDenied = null; + + assert( + XRLiveElement.observedAttributes.includes('capabilities'), + 'XRLiveElement inherits observed capabilities attribute', + ); + + // 1. Author-controlled `userData.xr.*` traits CANNOT escalate privileges when host only grants `scripts` + const {host, viewer} = createFakeHost('scripts'); + assert( + viewer.meshes.length === 1, + 'Procedural mesh still compiles cleanly even when userData.xr declares sensitive traits', + ); + + // 1a. `splatUrl` blocked without `remote_content` + `network` + let splatFetched = false; + const fakeFetch = (async () => { + splatFetched = true; + throw new Error('unexpected splat fetch'); + }) as typeof fetch; + let directRejected = false; + try { + await loadSplat('https://cdn.example.com/room.spz', fakeFetch, true, host); + } catch { + directRejected = true; + } + const splats = new XRSplatLoader(fakeFetch); + splats.sync(viewer as unknown as Parameters[0]); + assert( + directRejected && + !splatFetched && + host.denied.some((d) => d.capability === 'remote_content'), + 'splatUrl and loadSplat refuse remote downloads without remote_content', + ); + + // 1b. `audioEmitter.src` blocked without `remote_content`, while preset+src strips `src` + const audio = new XRSpatialAudioEmitterController(); + audio.update(viewer, 0.016); + assert( + audio.trigger(viewer, 0) === null && + !host.events.some((e) => e.type === 'xraudioplay'), + 'spatialAudio refuses remote audioEmitter.src without remote_content', + ); + + const dualViewer: ThreeLiveViewer = { + container: host as unknown as HTMLElement, + meshes: [ + { + name: 'DualSpeaker', + nodeIndex: 0, + transform: new Float32Array([ + 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1.6, -2, 1, + ]), + position: [0, 1.6, -2], + quaternion: [0, 0, 0, 1], + scale: [1, 1, 1], + extras: { + xr: { + audioEmitter: { + preset: 'hum', + src: 'https://evil.example.com/spy.mp3', + autoplay: false, + playOnSelect: true, + }, + }, + }, + }, + ], + } as unknown as ThreeLiveViewer; + const dualAudio = new XRSpatialAudioEmitterController(); + const mix = dualAudio.trigger(dualViewer, 0); + const playEvt = host.events.find((e) => e.type === 'xraudioplay'); + assert( + mix !== null && (playEvt?.detail as {src: string | null})?.src === null, + 'spatialAudio plays synthesized preset and strips untrusted src when preset is set', + ); + + // 1c. `persistent-anchors` (`localStorage` + URL hash) blocked without `storage` + let storageTouched = false; + const anchors = new XRPersistentAnchorsController( + { + getItem: () => { + storageTouched = true; + return null; + }, + setItem: () => { + storageTouched = true; + }, + removeItem: () => { + storageTouched = true; + }, + }, + { + getPathname: () => '/demo', + getHash: () => '#xr-state=abc', + setHash: () => { + storageTouched = true; + }, + }, + ); + const anchorViewer = viewer as unknown as Parameters< + XRPersistentAnchorsController['save'] + >[0]; + assert( + anchors.save(anchorViewer, host) === '' && + anchors.restore(anchorViewer, host).count === 0 && + !storageTouched && + host.denied.some((d) => d.capability === 'storage'), + 'persistentAnchors refuses localStorage and URL hash access without storage capability', + ); + + // 1d. `shared-room` WebSocket + WebRTC voice signaling blocked without `network` + `microphone` + let wsOpened = false; + const origWS = (globalThis as {WebSocket?: unknown}).WebSocket; + (globalThis as {WebSocket?: unknown}).WebSocket = class FakeWS { + constructor() { + wsOpened = true; + } + send(): void {} + close(): void {} + }; + try { + const room = new XRSharedRoomController('peer-alice'); + const roomViewer = viewer as unknown as Parameters< + XRSharedRoomController['connect'] + >[1]; + const joined = room.connect( + 'wss://relay.example.com/room/alpha?token=SECRET', + roomViewer, + host, + ); + assert( + !joined && + !wsOpened && + host.denied.some( + (d) => d.capability === 'network' && !d.url?.includes('SECRET'), + ), + 'sharedRoom refuses WebSocket connection without network capability', + ); + + host.capabilities = 'scripts network'; + resetCapabilityDenials(host); + const sentWire: string[] = []; + const joinedWithNet = room.connect('room-alpha', roomViewer, host, { + send: (raw: string) => sentWire.push(raw), + close: () => {}, + }); + const rtcSent = room.broadcastRtcSignal('peer-bob', 'offer', { + sdp: 'v=0\r\n', + }); + assert( + joinedWithNet && + !rtcSent && + sentWire.length === 0 && + host.denied.some((d) => d.capability === 'microphone'), + 'sharedRoom.broadcastRtcSignal refuses WebRTC voice signaling without microphone capability', + ); + room.dispose(roomViewer); + } finally { + (globalThis as {WebSocket?: unknown}).WebSocket = origWS; + } + + // 1e. `sensors` gates world-understanding plane/mesh reads & depth-particles hardware depth + host.capabilities = 'scripts'; + resetCapabilityDenials(host); + let sensorsRead = false; + const fakeFrame = { + get detectedPlanes() { + sensorsRead = true; + return new Set(); + }, + getPose: () => ({ + transform: { + matrix: new Float32Array([ + 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1.6, 0, 1, + ]), + }, + }), + getViewerPose: () => ({ + views: [ + { + projectionMatrix: new Float32Array(16), + transform: { + matrix: new Float32Array([ + 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1.6, 0, 1, + ]), + }, + getDepthInformation: () => { + sensorsRead = true; + return null; + }, + }, + ], + }), + }; + worldUnderstandingPlugin.onXRFrame!(viewer, {}, fakeFrame, {}); + depthParticlesPlugin.onXRFrame!(viewer, {}, fakeFrame, {}); + assert( + !sensorsRead && host.denied.some((d) => d.capability === 'sensors'), + 'worldUnderstanding and depthParticles refuse XRFrame plane/depth reads without sensors capability', + ); + + // 1f. `gemini-live` gates session (`ai` + `network`), mic (`microphone`), and frames (`camera`) + const sentMsgs: Array> = []; + const fakeTransport = { + onMessage: undefined as ((m: Record) => void) | undefined, + onClose: undefined as (() => void) | undefined, + send: (m: Record) => { + sentMsgs.push(m); + }, + close: () => {}, + }; + const gemini = new XRGeminiLiveController(host); + let geminiConnectRejected = false; + try { + gemini.connect({transport: fakeTransport, streamFrames: false}); + } catch { + geminiConnectRejected = true; + } + assert( + geminiConnectRejected && + sentMsgs.length === 0 && + host.denied.some((d) => d.capability === 'ai'), + 'XRGeminiLiveController refuses connect without ai capability', + ); + host.capabilities = 'scripts ai network'; + resetCapabilityDenials(host); + gemini.connect({ + transport: fakeTransport, + setup: {model: 'models/gemini-live'}, + streamFrames: false, + }); + assert( + sentMsgs.length === 1, + 'XRGeminiLiveController connects when ai + network granted', + ); + gemini.sendAudioChunk(new Int16Array([1, 2]), 16000); + const frameSentDenied = await gemini.sendFrame(); + assert( + sentMsgs.length === 1 && + !frameSentDenied && + host.denied.some((d) => d.capability === 'microphone') && + host.denied.some((d) => d.capability === 'camera'), + 'XRGeminiLiveController blocks sendAudioChunk without microphone and sendFrame without camera', + ); + host.capabilities = 'scripts ai network microphone camera'; + resetCapabilityDenials(host); + gemini.sendAudioChunk(new Int16Array([1, 2]), 16000); + const frameSentAllowed = await gemini.sendFrame(); + assert( + frameSentAllowed && sentMsgs.length === 3, + 'XRGeminiLiveController sends audio and viewport frames once microphone + camera are granted', + ); + gemini.disconnect(); + + // 1g. `xr-mobile-bridge` gates external AR handoff (`external_handoff`) + host.capabilities = 'scripts'; + resetCapabilityDenials(host); + let handoffDenied = false; + try { + await launchBestMobileOrHeadsetAR('https://cdn.example.com/model.usdz', { + element: host, + forceFallback: 'ios-quicklook', + }); + } catch { + handoffDenied = true; + } + assert( + handoffDenied && + host.denied.some((d) => d.capability === 'external_handoff'), + 'launchBestMobileOrHeadsetAR refuses external Quick Look handoff without external_handoff capability', + ); + + // 1h. Audit Regression Coverage: XRInputManager `allowHands = false` skips `frame.getJointPose`, + // dynamic `ai` revocation blocks `XRGeminiLiveController.sendText`, & `spatialAudioPlugin` hooks + let jointPoseCalled = false; + const inputMgr = new XRInputManager(new XRTraitSystem(), () => {}, undefined); + inputMgr.processXRFrame( + {meshes: []}, + { + inputSources: [ + { + handedness: 'right', + hand: new Map([ + ['index-finger-tip', {}], + ['thumb-tip', {}], + ]), + }, + ], + }, + { + getJointPose: () => { + jointPoseCalled = true; + return null; + }, + }, + {}, + false, + ); + assert( + !jointPoseCalled, + 'XRInputManager.processXRFrame skips XRHand joint pose reads when allowHands is false', + ); + + host.capabilities = 'scripts ai network'; + resetCapabilityDenials(host); + const revMsgs: Array> = []; + let revClosed = false; + gemini.connect({ + transport: { + send: (m) => revMsgs.push(m), + close: () => { + revClosed = true; + }, + }, + streamFrames: false, + }); + gemini.sendText('hello allowed'); + assert( + revMsgs.length === 1, + 'XRGeminiLiveController.sendText sends when ai is granted', + ); + host.capabilities = 'scripts'; + resetCapabilityDenials(host); + gemini.sendText('blocked after revoke'); + assert( + revMsgs.length === 1 && host.denied.some((d) => d.capability === 'ai'), + 'XRGeminiLiveController.sendText is blocked once ai capability is dynamically revoked', + ); + gemini.disconnect(); + assert(revClosed, 'XRGeminiLiveController.disconnect closes transport'); + + const khrViewer = { + container: host, + meshes: [ + { + name: 'Bell', + transform: new Float32Array([ + 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, + ]), + primitives: [], + }, + ], + gltf: { + extensions: { + KHR_audio_emitter: { + audio: [{uri: 'https://cdn.example.com/bell.mp3'}], + sources: [{audio: 0, autoPlay: true}], + emitters: [{type: 'positional', sources: [0]}], + }, + }, + nodes: [{mesh: 0, extensions: {KHR_audio_emitter: {emitter: 0}}}], + }, + } as unknown as ThreeLiveViewer; + spatialAudioPlugin.onLoadModel?.(khrViewer); + spatialAudioPlugin.onDestroy?.({} as WebGL2RenderingContext, khrViewer); + assert( + typeof spatialAudioPlugin.onLoadModel === 'function' && + typeof spatialAudioPlugin.onDestroy === 'function', + 'spatialAudioPlugin exposes onLoadModel/onDestroy hooks and reads viewer.gltf', + ); + + console.log( + `\n🏆 XR Capability Security Test Results: ${passed} Passed, ${failed} Failed.`, + ); + if (failed > 0) process.exit(1); +} + +runXRCapabilitiesTests().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/xr-live/tests/xr_live_test.ts b/xr-live/tests/xr_live_test.ts index 5f9d5e17..723996df 100644 --- a/xr-live/tests/xr_live_test.ts +++ b/xr-live/tests/xr_live_test.ts @@ -804,7 +804,9 @@ async function runXRLiveTests() { } const privacyTools: MockWebMCPTool[] = []; - const privacyAttrs = new Map(); + const privacyAttrs = new Map([ + ['capabilities', 'camera sensors'], + ]); const privacyEvents: string[] = []; const privacyElement = { hasAttribute: (name: string) => privacyAttrs.has(name), @@ -872,8 +874,9 @@ async function runXRLiveTests() { // and avoids cross-process blob: Scene Viewer intents // ========================================================================= const arElement = new XRLiveElement(); - (arElement as any).getAttribute = () => null; - (arElement as any).hasAttribute = () => false; + (arElement as any).getAttribute = (n: string) => + n === 'capabilities' ? 'viewer' : null; + (arElement as any).hasAttribute = (n: string) => n === 'capabilities'; (arElement as any).currentModel = glbBytes; const iosReport = await arElement.enterSpatialAR(undefined, 'ios-quicklook'); assert( @@ -886,6 +889,7 @@ async function runXRLiveTests() { const gAny = globalThis as Record; const prevNavDesc = Object.getOwnPropertyDescriptor(globalThis, 'navigator'); try { + ThreeLiveElement.defaultCapabilities = 'viewer'; Object.defineProperty(globalThis, 'navigator', { value: { userAgent: 'Mozilla/5.0 (Linux; Android 14; Pixel 8) Mobile', @@ -907,6 +911,7 @@ async function runXRLiveTests() { 'Regression X2: Android without WebXR launches Scene Viewer when given an https:// GLB URL', ); } finally { + ThreeLiveElement.defaultCapabilities = null; if (prevNavDesc) { Object.defineProperty(globalThis, 'navigator', prevNavDesc); } else { diff --git a/xr-live/tests/xr_plugins_test.ts b/xr-live/tests/xr_plugins_test.ts index 108cbbba..c5da2792 100644 --- a/xr-live/tests/xr_plugins_test.ts +++ b/xr-live/tests/xr_plugins_test.ts @@ -737,6 +737,7 @@ async function testGeminiLive(): Promise { const sent: Array> = []; const events: Array<{type: string; detail: unknown}> = []; const element = { + capabilities: 'ai network camera microphone sensors', dispatchEvent(e: Event) { events.push({type: e.type, detail: (e as CustomEvent).detail}); return true; @@ -2169,6 +2170,7 @@ async function testPersistentAnchors(): Promise { const restoredEvents: Array<{count: number; source: string}> = []; const makeHost = (urlHashEnabled: boolean) => ({ + capabilities: 'storage', hasAttribute: (n: string) => n === 'persistent-anchors' || (urlHashEnabled && n === 'persist-url-hash'), @@ -2453,14 +2455,13 @@ function testGoldenParity(): void { // tslint:disable-next-line:no-require-imports const path = require('path'); const goldenCandidates = [ + ...(process.env['THREE_LIVE_GOLDEN_DIR'] + ? [process.env['THREE_LIVE_GOLDEN_DIR']] + : []), path.resolve(__dirname, '../../../testdata/golden'), path.resolve(__dirname, '../../testdata/golden'), path.resolve(__dirname, '../testdata/golden'), path.resolve(process.cwd(), 'testdata/golden'), - path.resolve( - process.cwd(), - 'experimental/users/binghamj/three_live/testdata/golden', - ), ]; const goldenDir = goldenCandidates.find((d: string) => @@ -2847,6 +2848,7 @@ function testSharedRoom(): void { const map = new Map>(); const events: Array<{type: string; detail: unknown}> = []; const el: Record = { + capabilities: 'network microphone', getAttribute: (k: string) => (k in attrs ? attrs[k] : null), hasAttribute: (k: string) => k in attrs, setAttribute: (k: string, v: string) => { @@ -3243,6 +3245,7 @@ function testSharedRoom(): void { function auditHost(attrs: Record): any { const events: Array<{type: string; detail: any}> = []; return { + capabilities: 'storage network microphone remote_content sensors', events, hasAttribute: (k: string) => k in attrs, getAttribute: (k: string) => (k in attrs ? attrs[k] : null), diff --git a/xr-live/tests/xr_sim_test.ts b/xr-live/tests/xr_sim_test.ts index 3ea466a8..9b967729 100644 --- a/xr-live/tests/xr_sim_test.ts +++ b/xr-live/tests/xr_sim_test.ts @@ -1338,12 +1338,13 @@ function testMechanicalJoints() { `slider motor translates along +X and clamps at +0.35m (x=${slider.pos[0].toFixed(3)})`, ); assert( - springW.pos[1] < 2.0 && springW.pos[1] > 1.5 && Number.isFinite(springW.pos[1]), + springW.pos[1] < 2.0 && + springW.pos[1] > 1.5 && + Number.isFinite(springW.pos[1]), `spring weight sags under gravity and settles (y=${springW.pos[1].toFixed(3)})`, ); assert( - stiff.pos.every(Number.isFinite) && - Math.abs(stiff.pos[1] - 2.2) < 0.1, + stiff.pos.every(Number.isFinite) && Math.abs(stiff.pos[1] - 2.2) < 0.1, `L-stable stiff spring (k=10000) stays finite without blowup (y=${stiff.pos[1].toFixed(3)})`, ); assert( @@ -1613,14 +1614,13 @@ function testGoldenParity() { // tslint:disable-next-line:no-require-imports const path = require('path'); const dirs = [ + ...(process.env['THREE_LIVE_GOLDEN_DIR'] + ? [process.env['THREE_LIVE_GOLDEN_DIR']] + : []), path.resolve(__dirname, '../../../testdata/golden'), path.resolve(__dirname, '../../testdata/golden'), path.resolve(__dirname, '../testdata/golden'), path.resolve(process.cwd(), 'testdata/golden'), - path.resolve( - process.cwd(), - 'experimental/users/binghamj/three_live/testdata/golden', - ), ]; const dir = dirs.find((d: string) => fs.existsSync(path.join(d, 'sim_scene.json')), diff --git a/xr-live/tests/xr_splat_stream_test.ts b/xr-live/tests/xr_splat_stream_test.ts index 6630b813..523e29da 100644 --- a/xr-live/tests/xr_splat_stream_test.ts +++ b/xr-live/tests/xr_splat_stream_test.ts @@ -17,13 +17,29 @@ import assert from 'node:assert/strict'; import {gzipSync} from 'node:zlib'; -import {gunzip, loadSplat, parseSplat, parseSpzPayload, MAX_SPLAT_FILE_BYTES} from '../src/plugins/splat-loader'; +import { + CAPABILITIES_ALL, + ensureCapabilityChecker, +} from '../../three-live/src/capabilities'; +import {getSharedRegistry} from '../../three-live/src/plugin'; +import { + gunzip, + loadSplat, + MAX_SPLAT_FILE_BYTES, + parseSplat, + parseSpzPayload, +} from '../src/plugins/splat-loader'; + +ensureCapabilityChecker(); +getSharedRegistry().defaultCapabilities = CAPABILITIES_ALL; async function run(): Promise { const bytes = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]); assert.deepEqual(await gunzip(gzipSync(bytes), bytes.length), bytes); - await assert.rejects(gunzip(gzipSync(bytes), bytes.length - 1), - /decompressed \.spz exceeds size limit/); + await assert.rejects( + gunzip(gzipSync(bytes), bytes.length - 1), + /decompressed \.spz exceeds size limit/, + ); await assert.rejects(gunzip(new Uint8Array([31, 139, 8]), 100)); // Minimal .splat with nonzero scales and a valid quaternion. @@ -31,45 +47,81 @@ async function run(): Promise { const view = new DataView(record.buffer); for (const offset of [12, 16, 20]) view.setFloat32(offset, 1, true); record.set([255, 128, 128, 128], 28); - const response = (body: unknown, arrayBuffer?: () => Promise) => ({ - ok: true, status: 200, headers: {get: () => null}, body, arrayBuffer, - }) as unknown as Response; + const response = (body: unknown, arrayBuffer?: () => Promise) => + ({ + ok: true, + status: 200, + headers: {get: () => null}, + body, + arrayBuffer, + }) as unknown as Response; let reads = 0; const chunks = [record.slice(0, 7), record.slice(7)]; - const fetchChunks = async () => response({getReader: () => ({ - read: async () => reads < chunks.length - ? {done: false, value: chunks[reads++]} : {done: true}, - })}); - const streamed = await loadSplat('https://localhost/scene.splat', fetchChunks as typeof fetch); - const fallback = await loadSplat('https://localhost/scene.splat', - (async () => response(null, async () => record.buffer)) as typeof fetch); + const fetchChunks = async () => + response({ + getReader: () => ({ + read: async () => + reads < chunks.length + ? {done: false, value: chunks[reads++]} + : {done: true}, + }), + }); + const streamed = await loadSplat( + 'https://localhost/scene.splat', + fetchChunks as typeof fetch, + ); + const fallback = await loadSplat('https://localhost/scene.splat', (async () => + response(null, async () => record.buffer)) as typeof fetch); assert.equal(streamed.count, 1); assert.deepEqual(streamed, fallback); let cancelled = false; let oversizedReads = 0; const oversized = new Uint8Array(MAX_SPLAT_FILE_BYTES + 1); - const fetchOversized = async () => response({getReader: () => ({ - read: async () => { oversizedReads++; return {done: false, value: oversized}; }, - cancel: async () => { cancelled = true; }, - })}); - await assert.rejects(loadSplat('https://localhost/scene.splat', fetchOversized as typeof fetch), - /file too large/); + const fetchOversized = async () => + response({ + getReader: () => ({ + read: async () => { + oversizedReads++; + return {done: false, value: oversized}; + }, + cancel: async () => { + cancelled = true; + }, + }), + }); + await assert.rejects( + loadSplat('https://localhost/scene.splat', fetchOversized as typeof fetch), + /file too large/, + ); assert.equal(cancelled, true); assert.equal(oversizedReads, 1, 'stop reading immediately after the cap'); - await assert.rejects(loadSplat('https://localhost/scene.splat', - (async () => response(null, async () => oversized.buffer)) as typeof fetch), /file too large/); - const fetchFailed = async () => response({getReader: () => ({ - read: async () => { throw new Error('stream interrupted'); }, - })}); - await assert.rejects(loadSplat('https://localhost/scene.splat', fetchFailed as typeof fetch), - /stream interrupted/); + await assert.rejects( + loadSplat('https://localhost/scene.splat', (async () => + response(null, async () => oversized.buffer)) as typeof fetch), + /file too large/, + ); + const fetchFailed = async () => + response({ + getReader: () => ({ + read: async () => { + throw new Error('stream interrupted'); + }, + }), + }); + await assert.rejects( + loadSplat('https://localhost/scene.splat', fetchFailed as typeof fetch), + /stream interrupted/, + ); // Every packed wxyz quaternion must retain its original Float32 rounding. const records = new Uint8Array(256 * 32); - for (let i = 0; i < records.length; i++) records[i] = (i * 37 + (i >>> 5)) & 255; + for (let i = 0; i < records.length; i++) + records[i] = (i * 37 + (i >>> 5)) & 255; const parsed = parseSplat(records); for (let i = 0; i < parsed.count; i++) { - const q = new Float32Array([1, 2, 3, 0].map(k => (records[i * 32 + 28 + k] - 128) / 128)); + const q = new Float32Array( + [1, 2, 3, 0].map((k) => (records[i * 32 + 28 + k] - 128) / 128), + ); const len = Math.hypot(...q); if (len > 1e-8) for (let k = 0; k < 4; k++) q[k] /= len; else q.set([0, 0, 0, 1]); @@ -87,15 +139,17 @@ async function run(): Promise { const expected = new Float32Array(count * 4); for (let i = 0; i < count; i++) { const largest = i % 4; - const parts = [i * 7 & 1023, i * 19 & 1023, i * 31 & 1023]; - const packed = ((largest << 30) | (parts[2] << 20) | (parts[1] << 10) | parts[0]) >>> 0; + const parts = [(i * 7) & 1023, (i * 19) & 1023, (i * 31) & 1023]; + const packed = + ((largest << 30) | (parts[2] << 20) | (parts[1] << 10) | parts[0]) >>> 0; header.setUint32(rotOffset + i * 4, packed, true); const q = [0, 0, 0, 0]; - let part = 0, sum = 0; + let part = 0, + sum = 0; for (let k = 3; k >= 0; k--) { if (k === largest) continue; const value = parts[part++]; - q[k] = (Math.SQRT1_2 * (value & 511) / 511) * (value & 512 ? -1 : 1); + q[k] = ((Math.SQRT1_2 * (value & 511)) / 511) * (value & 512 ? -1 : 1); sum += q[k] * q[k]; } q[largest] = Math.sqrt(Math.max(0, 1 - sum)); @@ -105,6 +159,11 @@ async function run(): Promise { expected.set(rounded, i * 4); } assert.deepEqual(parseSpzPayload(payload).rotations, expected); - console.log('PASS capped splat streams and allocation-free quaternion decode'); + console.log( + 'PASS capped splat streams and allocation-free quaternion decode', + ); } -run().catch(error => { console.error(error); process.exitCode = 1; }); +run().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/xr-live/tests/xr_splat_webgpu_test.ts b/xr-live/tests/xr_splat_webgpu_test.ts index f821683d..787304ba 100644 --- a/xr-live/tests/xr_splat_webgpu_test.ts +++ b/xr-live/tests/xr_splat_webgpu_test.ts @@ -16,6 +16,7 @@ */ import {strict as assert} from 'node:assert'; +import {ensureCapabilityChecker} from '../../three-live/src/capabilities'; import {mat4Create, mat4Multiply} from '../../three-live/src/math'; import { getSharedRegistry, @@ -47,12 +48,12 @@ import { float32ToFloat16Bits, packSplatBaseBuffer, packSplatSHBuffer, + shWordsPerSplatForDegree, SPLAT_BASE_STRIDE_BYTES, SPLAT_PREP_COMPUTE_WGSL, SPLAT_RENDER_WGSL, SPLAT_SORT_COMPUTE_WGSL, SPLAT_UNIFORM_BYTES, - shWordsPerSplatForDegree, writeSplatUniforms, } from '../src/webgpu/shaders'; import { @@ -67,6 +68,7 @@ import { setWebGPUDeviceForTests, StandaloneWebGPUSplatLoader, } from '../src/webgpu/splat-accelerator'; +ensureCapabilityChecker(); function seededRng(seed = 0x12345678): () => number { let s = seed >>> 0; @@ -125,10 +127,22 @@ function perspectiveVP(camZ = 5): Float32Array { const f = 1.7320508; const nf = 1 / (near - far); const proj = new Float32Array([ - f, 0, 0, 0, - 0, f, 0, 0, - 0, 0, (far + near) * nf, -1, - 0, 0, 2 * far * near * nf, 0, + f, + 0, + 0, + 0, + 0, + f, + 0, + 0, + 0, + 0, + (far + near) * nf, + -1, + 0, + 0, + 2 * far * near * nf, + 0, ]); const view = mat4Create(); view[14] = -camZ; @@ -171,12 +185,9 @@ function evaluateWGSLFp16SHColors( const px = bf[i * 12]; const py = bf[i * 12 + 1]; const pz = bf[i * 12 + 2]; - const wpx = - uf[32] * px + uf[36] * py + uf[40] * pz + uf[44] - uf[60]; - const wpy = - uf[33] * px + uf[37] * py + uf[41] * pz + uf[45] - uf[61]; - const wpz = - uf[34] * px + uf[38] * py + uf[42] * pz + uf[46] - uf[62]; + const wpx = uf[32] * px + uf[36] * py + uf[40] * pz + uf[44] - uf[60]; + const wpy = uf[33] * px + uf[37] * py + uf[41] * pz + uf[45] - uf[61]; + const wpz = uf[34] * px + uf[38] * py + uf[42] * pz + uf[46] - uf[62]; let lx = uf[48] * wpx + uf[49] * wpy + uf[50] * wpz; let ly = uf[52] * wpx + uf[53] * wpy + uf[54] * wpz; let lz = uf[56] * wpx + uf[57] * wpy + uf[58] * wpz; @@ -358,7 +369,10 @@ function createMockWebGPUDevice(opts?: {autoResolveMaps?: boolean}) { }, setBindGroup(_idx: number, bg: unknown) { const typed = bg as { - entries: Array<{binding: number; resource: {buffer: MockBuffer}}>; + entries: Array<{ + binding: number; + resource: {buffer: MockBuffer}; + }>; }; if (typed.entries.length === 8) activePrepBG = typed; }, @@ -370,7 +384,8 @@ function createMockWebGPUDevice(opts?: {autoResolveMaps?: boolean}) { computeSubmissions++; assert.equal(dispatched.length, 13); if (!activePrepBG) return; - const getB = (i: number) => activePrepBG!.entries[i].resource.buffer; + const getB = (i: number) => + activePrepBG!.entries[i].resource.buffer; const uBuf = getB(0).bytes.buffer as ArrayBuffer; const baseBuf = getB(1).bytes.buffer as ArrayBuffer; const shWords = new Uint32Array(getB(2).bytes.buffer); @@ -476,8 +491,12 @@ function createMockWebGPUDevice(opts?: {autoResolveMaps?: boolean}) { } function mockGL(withCanvasParent = true) { - type GLBuf = {order: Uint32Array | null}; - type VAO = {element: GLBuf | null}; + interface GLBuf { + order: Uint32Array | null; + } + interface VAO { + element: GLBuf | null; + } let vao: VAO | null = null; const uploads: number[][] = []; const colorUploads: number[][] = []; @@ -493,7 +512,9 @@ function mockGL(withCanvasParent = true) { return idx >= 0 && idx + 1 < children.length ? children[idx + 1] : null; }, }; - const attrs = new Map(); + const attrs = new Map([ + ['capabilities', 'remote_content network'], + ]); const parent = { style: {position: 'relative'}, children, @@ -702,13 +723,17 @@ function packSplatBytes(data: SplatData): Uint8Array { return out; } -const flushMicrotasks = () => - new Promise((resolve) => setTimeout(resolve, 0)); +function flushMicrotasks(): Promise { + return new Promise((resolve) => setTimeout(resolve, 0)); +} let passed = 0; let failed = 0; -async function test(name: string, fn: () => void | Promise): Promise { +async function test( + name: string, + fn: () => void | Promise, +): Promise { try { await fn(); passed++; @@ -792,11 +817,7 @@ async function main() { ); // Extreme finite range (-2e38 .. +2e38) must not overflow f32 in half-range quantization - const extremePos = new Float32Array([ - 0, 0, -2e38, - 0, 0, 0, - 0, 0, 2e38, - ]); + const extremePos = new Float32Array([0, 0, -2e38, 0, 0, 0, 0, 0, 2e38]); const orthoIdentity = mat4Create(); const extremeOrder = simulateWebGPUSplatSort( {count: 3, positions: extremePos}, @@ -806,11 +827,21 @@ async function main() { // Non-finite positions (NaN -> key 0, +Inf -> key 0, -Inf -> key 65535) const nonFinitePos = new Float32Array([ - 0, 0, -1, - 0, 0, NaN, - 0, 0, -Infinity, - 0, 0, Infinity, - 0, 0, 5, + 0, + 0, + -1, + 0, + 0, + NaN, + 0, + 0, + -Infinity, + 0, + 0, + Infinity, + 0, + 0, + 5, ]); const nfGpu = simulateWebGPUSplatSort( {count: 5, positions: nonFinitePos}, @@ -881,11 +912,7 @@ async function main() { antialiased: false, }); - const gpuColors = evaluateWGSLFp16SHColors( - baseBuf, - packedSH.words, - uBuf, - ); + const gpuColors = evaluateWGSLFp16SHColors(baseBuf, packedSH.words, uBuf); const cpuColors = computeSplatViewColors( data, model, @@ -908,7 +935,11 @@ async function main() { assert.equal(buf.byteLength, 5 * SPLAT_BASE_STRIDE_BYTES); const f32 = new Float32Array(buf); const u32 = new Uint32Array(buf); - const expectedCov = computeSplatCovariance3D(data.scales, data.rotations, 2); + const expectedCov = computeSplatCovariance3D( + data.scales, + data.rotations, + 2, + ); for (let k = 0; k < 6; k++) { assert.ok(Math.abs(f32[2 * 12 + 4 + k] - expectedCov[k]) < 1e-6); } @@ -929,7 +960,9 @@ async function main() { visible: true, primitives: [{vao: {}}], }; - const attrs = new Map(); + const attrs = new Map([ + ['capabilities', 'remote_content network'], + ]); const container = {getAttribute: (k: string) => attrs.get(k) ?? null}; // Pure-splat desktop scene defaults to Fast Path A ('vram-overlay') @@ -1074,12 +1107,13 @@ async function main() { try { const c1 = makeCloud(8, 0, 1); const c2 = makeCloud(10, 0, 2); - const loader = new XRSplatLoader(async (url) => - new Response( - (String(url).includes('one') - ? packSplatBytes(c1) - : packSplatBytes(c2)) as BodyInit, - ), + const loader = new XRSplatLoader( + async (url) => + new Response( + (String(url).includes('one') + ? packSplatBytes(c1) + : packSplatBytes(c2)) as BodyInit, + ), ); const {gl, parent} = mockGL(); const viewer = { @@ -1209,7 +1243,10 @@ async function main() { loader.draw(viewer, vpLeft, 'left', [0, 0, 800, 600], vpLeft); } assert.equal(colorUploads.length, beforeRestore + 1); - assert.deepEqual(colorUploads[colorUploads.length - 1], Array.from(data.colors)); + assert.deepEqual( + colorUploads[colorUploads.length - 1], + Array.from(data.colors), + ); loader.dispose(); } finally { @@ -1791,10 +1828,23 @@ async function main() { }); const {gl, parent, glCanvas} = mockGL(); - (glCanvas as unknown as {width: number; height: number; toDataURL: () => string}).width = 800; - (glCanvas as unknown as {width: number; height: number; toDataURL: () => string}).height = 600; + ( + glCanvas as unknown as { + width: number; + height: number; + toDataURL: () => string; + } + ).width = 800; + ( + glCanvas as unknown as { + width: number; + height: number; + toDataURL: () => string; + } + ).height = 600; const origToDataURL = () => 'data:image/png;base64,RAW_GL'; - (glCanvas as unknown as {toDataURL: () => string}).toDataURL = origToDataURL; + (glCanvas as unknown as {toDataURL: () => string}).toDataURL = + origToDataURL; const data = makeCloud(6, 0, 606); const mesh = {