From 3b969687decaf8f209924d00d72ae56138c4db23 Mon Sep 17 00:00:00 2001 From: Shawn Date: Sun, 15 Nov 2026 10:27:52 -0800 Subject: [PATCH] fix(ios): render outline traits in XRLiveRealityKitSession `userData.xr.outline` shells were only created by `RealityKitHelper.applyXRTraits` -> `configureEntity`, which hosts that drive an `XRLiveRealityKitSession` never call (the iOS demo only calls it when no session can be created). So every outlined node, such as the GripperTarget orb in the Robotic Arm IK preset, rendered with no rim on iOS and visionOS. The session now applies enabled `outline` traits to its bound entities when it binds the scene, matching web, where the post-processing plugin draws outlines whenever the trait is set. It reads the traits through `session.traits(at:)`, like `enablePostProcessing` does for bloom, and does not go through `configureEntity`, so it does not re-add the physics bodies `bindEntities` strips or repeat `installInputTargets` work. Now that sessions carry shells, keep them out of paths that would treat them as scene content: - `recolor` skips shells, so material changes (e.g. cycleMaterialColor) no longer repaint the rim with the model's color. - `bindEntities` skips shells, so re-binding a hierarchy cannot bind a shell as a node's fallback mesh entity. - `installInputTargets` removes the collision that `generateCollisionShapes(recursive: true)` gives shells, so taps hit the model instead of the enlarged rim (and hosts never see a shell as the hit entity). - `XRSimRealityKitBridge.setFlash` and `color(of:)` skip shells, so a hit flash does not swap in a non-culled material that hides the model. Adds `testRealityKitSessionRendersOutlineTraits`, which fails without the `recolor` guard and checks that shells take no collision. --- .../ThreeLiveCompiler/XRLiveRealityKit.swift | 5 + .../XRLiveRealityKitSession.swift | 26 ++++- .../XRSimRealityKitBridge.swift | 3 + .../ThreeCompilerTests.swift | 110 ++++++++++++++++++ 4 files changed, 143 insertions(+), 1 deletion(-) diff --git a/mobile/ios/Sources/ThreeLiveCompiler/XRLiveRealityKit.swift b/mobile/ios/Sources/ThreeLiveCompiler/XRLiveRealityKit.swift index 6005d094..b70d2f44 100644 --- a/mobile/ios/Sources/ThreeLiveCompiler/XRLiveRealityKit.swift +++ b/mobile/ios/Sources/ThreeLiveCompiler/XRLiveRealityKit.swift @@ -298,6 +298,11 @@ visit(entity) } + /// Whether `entity` is an outline shell added by `applyOutline` rather than scene content. + static func isOutlineShell(_ entity: Entity) -> Bool { + entity.name == outlineShellName + } + /// Configures an individual RealityKit `Entity` with native spatial components corresponding /// to its `userData.xr` traits. @MainActor diff --git a/mobile/ios/Sources/ThreeLiveCompiler/XRLiveRealityKitSession.swift b/mobile/ios/Sources/ThreeLiveCompiler/XRLiveRealityKitSession.swift index 9a526193..7202a064 100644 --- a/mobile/ios/Sources/ThreeLiveCompiler/XRLiveRealityKitSession.swift +++ b/mobile/ios/Sources/ThreeLiveCompiler/XRLiveRealityKitSession.swift @@ -147,7 +147,8 @@ var byName: [String: [Entity]] = [:] var meshEntitiesInOrder: [Entity] = [] func visit(_ entity: Entity, depth: Int) { - guard depth < 256 else { return } + // Outline shells from an earlier binding are decoration, never glTF nodes. + guard depth < 256, !RealityKitHelper.isOutlineShell(entity) else { return } // Strip USD-imported rigid-body and anchoring components (`PhysicsBodyAPI` and // `preliminary:anchoring:type` in `.usda` are emitted for standalone AR Quick Look; // in a live session `XRLiveRuntimeSession.stepPhysics` owns rigid-body motion and the @@ -249,6 +250,15 @@ entity, state: state, primitives: primitives, materialIndex: materialIndex, depth: 0) } } + + // Draw `outline` traits, as web post-processing does. Compiled GLBs list parents before + // children and `applyOutline` resets its whole subtree, so a child's own outline wins. + for index in 0.. Bool { @@ -263,6 +273,16 @@ return false } + private static func removeOutlineShellCollision(under entity: Entity) { + for child in entity.children { + if RealityKitHelper.isOutlineShell(child) { + child.components.remove(CollisionComponent.self) + } else { + removeOutlineShellCollision(under: child) + } + } + } + private static func isApproximatelyIdentity(_ m: simd_float4x4) -> Bool { let id = matrix_identity_float4x4 for col in 0..<4 { @@ -296,6 +316,8 @@ where session.isSelectable(index) || session.traits(at: index)?.physics == "dynamic" { if !entity.components.has(CollisionComponent.self) { entity.generateCollisionShapes(recursive: true) + // Taps should land on the model, not on its enlarged outline shell. + Self.removeOutlineShellCollision(under: entity) } #if os(visionOS) entity.components.set(InputTargetComponent(allowedInputTypes: .all)) @@ -666,6 +688,8 @@ private func recolor( _ entity: Entity, state: XRMaterialState, primitives: [Int], materialIndex: Int, depth: Int ) { + // Outline shells keep their own unlit color through material changes. + guard !RealityKitHelper.isOutlineShell(entity) else { return } if var model = entity.components[ModelComponent.self] { let count = model.materials.count for k in 0..=6.0) + guard #available(iOS 18.0, macOS 15.0, visionOS 2.0, *) else { + throw XCTSkip("Outline shells need front-face culling") + } + #else + throw XCTSkip("Outline shells need front-face culling") + #endif + let script = """ + const orb = new THREE.Mesh( + new THREE.SphereGeometry(0.1, 16, 16), + new THREE.MeshStandardMaterial({ color: 0x38bdf8 }) + ); + orb.name = 'Outlined_Orb'; + orb.userData.xr = { outline: { color: '#22c55e', thickness: 0.025 } }; + orb.userData.interactivity = { + onSelect: [ + { action: 'cycleMaterialColor', colors: [[1, 0, 0, 1], [0.9, 0, 0, 1]], duration: 0.1 } + ] + }; + scene.add(orb); + + const box = new THREE.Mesh( + new THREE.BoxGeometry(0.2, 0.2, 0.2), + new THREE.MeshStandardMaterial({ color: 0xa855f7 }) + ); + box.name = 'Plain_Box'; + box.position.set(0.5, 0, 0); + scene.add(box); + """ + let usdz = try await ThreeCompiler.compileUSDZ(script) + let glb = try await ThreeCompiler.compileGLB(script) + let entity = try await RealityKitHelper.loadEntity(fromUSDZ: usdz) + let live = try XCTUnwrap(XRLiveRealityKitSession(glbData: glb, root: entity)) + + func shells(under e: Entity) -> [Entity] { + (RealityKitHelper.isOutlineShell(e) ? [e] : []) + e.children.flatMap { shells(under: $0) } + } + func srgb(_ material: Material?) -> SIMD3? { + let tint: XRPlatformColor + if let m = material as? UnlitMaterial { + tint = m.color.tint + } else if let m = material as? PhysicallyBasedMaterial { + tint = m.baseColor.tint + } else if let m = material as? SimpleMaterial { + tint = m.color.tint + } else { + return nil + } + guard + let space = CGColorSpace(name: CGColorSpace.sRGB), + let c = tint.cgColor.converted(to: space, intent: .defaultIntent, options: nil)? + .components, c.count >= 3 + else { return nil } + return SIMD3(Float(c[0]), Float(c[1]), Float(c[2])) + } + func assertRimColor(_ shell: Entity, _ message: String) throws { + let model = try XCTUnwrap(shell.components[ModelComponent.self]) + XCTAssertTrue(model.materials.first is UnlitMaterial, message) + let rgb = try XCTUnwrap(srgb(model.materials.first), message) + let expected = SIMD3(0x22, 0xc5, 0x5e) / 255 + XCTAssertLessThan(simd_length(rgb - expected), 0.02, "\(message): got \(rgb)") + } + + let orb = try XCTUnwrap(live.entity(forNode: 0)) + let box = try XCTUnwrap(live.entity(forNode: 1)) + XCTAssertEqual(orb.name, "Outlined_Orb") + let orbShells = shells(under: orb) + XCTAssertFalse(orbShells.isEmpty, "Session must add an outline shell to the outlined node") + XCTAssertTrue(shells(under: box).isEmpty, "Nodes without an outline trait get no shell") + for shell in orbShells { + try assertRimColor(shell, "Rim uses the outline color") + } + + // The selectable orb gets collision for taps, but its enlarged shell never takes the hit. + live.installInputTargets() + XCTAssertTrue( + ([orb] + orb.children).contains { + !RealityKitHelper.isOutlineShell($0) && $0.components.has(CollisionComponent.self) + }) + for shell in orbShells { + XCTAssertFalse(shell.components.has(CollisionComponent.self), "Shells must not take hits") + } + + // A select-driven material change recolors the orb but must leave the rim alone. + live.start() + XCTAssertTrue(live.handleTap(on: orb)) + for _ in 0..<30 { + live.step(dt: 1.0 / 60.0) + } + let orbModel = try XCTUnwrap( + ([orb] + orb.children).lazy.compactMap { $0.components[ModelComponent.self] }.first) + let orbColor = try XCTUnwrap(srgb(orbModel.materials.first)) + XCTAssertGreaterThan(orbColor.x, 0.8, "cycleMaterialColor must recolor the orb: \(orbColor)") + for shell in shells(under: orb) { + try assertRimColor(shell, "Material changes must not repaint the rim") + } + + // Re-binding the same hierarchy neither duplicates shells nor binds a shell as a node. + live.stop() + let again = try XCTUnwrap(XRLiveRealityKitSession(glbData: glb, root: entity)) + XCTAssertEqual(shells(under: orb).count, orbShells.count) + XCTAssertTrue(again.entity(forNode: 0) === orb) + XCTAssertTrue(again.entity(forNode: 1) === box) + } #endif }