diff --git a/examples/egocap/backend/egocap_web/viser_scene.py b/examples/egocap/backend/egocap_web/viser_scene.py index 36855be1ad..1e462ab2f6 100644 --- a/examples/egocap/backend/egocap_web/viser_scene.py +++ b/examples/egocap/backend/egocap_web/viser_scene.py @@ -66,6 +66,44 @@ _COLORS = {"left": (60, 140, 255), "right": (255, 140, 60)} +# Joint and bone size in the glove drawing. Larger than a bare point so the skeleton reads at arm's length on a +# projector, small enough that the finger joints stay distinct. +_JOINT_POINT_SIZE = 0.009 +_BONE_LINE_WIDTH = 3.0 +# How much a fingertip is lightened toward white (0 keeps the side colour, 1 is white). +_TIP_LIGHTEN = 0.65 +# A finger's base joint is the side colour darkened to this fraction, so the shade runs knuckle to fingertip. +_BASE_DARKEN = 0.55 + + +def _joint_colors(side: str) -> np.ndarray: + """A (26, 3) uint8 colour per OpenXR hand joint for one side. + + The side keeps its identity (blue left, orange right) while each finger shades from a dark knuckle to a pale + tip, which makes the finger direction and curl readable without a legend. The palm and wrist sit mid-shade.""" + base = np.array(_COLORS[side], dtype=np.float64) + dark, light = base * _BASE_DARKEN, base + (255.0 - base) * _TIP_LIGHTEN + colors = np.tile(dark + (light - dark) * 0.35, (26, 1)) # PALM and WRIST + for chain in _CHAINS: + for rank, joint in enumerate(chain): + colors[joint] = dark + (light - dark) * (rank / (len(chain) - 1)) + return np.clip(colors, 0, 255).astype(np.uint8) + + +_JOINT_COLORS = {side: _joint_colors(side) for side in _COLORS} + +# The floor of the 3D view: a dark slab under a blue-grey grid. Tracking space is ``session_floor`` (y = 0 is the +# floor), so the slab's top face and the grid sit at the origin plane and the person is drawn standing on them. +# The slab is what gives the floor a surface to read against; lines alone float in the empty canvas. +_FLOOR_SIZE_M = 6.0 +_FLOOR_THICKNESS_M = 0.01 +_FLOOR_SURFACE_COLOR = (15, 22, 34) +_FLOOR_CELL_M = 0.25 +_FLOOR_CELL_COLOR = (44, 58, 82) +_FLOOR_SECTION_COLOR = (86, 108, 142) +_ORIGIN_AXES_LENGTH_M = 0.25 + + # The tracking fields this view reads, with the dtype and shape the ic_tracking_v1 layout must declare for each. _HAND_FIELDS = ( ("present", "u1", ()), @@ -203,10 +241,15 @@ class NodePose: @dataclass(frozen=True) class Glove: - """A glove skeleton: the reported valid joint points and the bones between valid joints, in tracking space.""" + """A glove skeleton: the reported valid joint points and the bones between valid joints, in tracking space. + + The colour arrays are presentation only (a shade per joint, and per bone end so a bone blends from one joint's + shade to the next); ``None`` draws the whole glove in one flat colour.""" points: np.ndarray # (N, 3) segments: np.ndarray # (M, 2, 3) + point_colors: np.ndarray | None = None # (N, 3) uint8 + segment_colors: np.ndarray | None = None # (M, 2, 3) uint8 @dataclass(frozen=True) @@ -237,8 +280,11 @@ def _glove(values: dict[str, np.ndarray], side: str) -> Glove | None: valid = values[f"{hand}.joint_valid"].astype(bool) & np.isfinite(positions).all(axis=1) if not valid.any(): return None - segments = np.array([(positions[a], positions[b]) for a, b in HAND_BONES if valid[a] and valid[b]]) - return Glove(positions[valid], segments.reshape(-1, 2, 3)) + bones = [(a, b) for a, b in HAND_BONES if valid[a] and valid[b]] + segments = np.array([(positions[a], positions[b]) for a, b in bones]) + shades = _JOINT_COLORS[side] + segment_colors = np.array([(shades[a], shades[b]) for a, b in bones], dtype=np.uint8) + return Glove(positions[valid], segments.reshape(-1, 2, 3), shades[valid], segment_colors.reshape(-1, 2, 3)) def build_frame(values: dict[str, np.ndarray]) -> SceneFrame: @@ -443,6 +489,7 @@ def _build_nodes(self, server: Any, assets: dict[str, bytes | None]) -> None: self._assets.setdefault(self._profile, assets) scene = server.scene scene.set_up_direction("+y") + self._add_environment(scene) scene.add_frame("/tracking", show_axes=False) head = scene.add_frame("/tracking/head", show_axes=assets["head"] is None, axes_length=0.1, visible=False) self._handles["head"] = head @@ -456,6 +503,32 @@ def _build_nodes(self, server: Any, assets: dict[str, bytes | None]) -> None: ) self._add_models(scene, assets) + def _add_environment(self, scene: Any) -> None: + """The static surroundings: a dark floor slab, a grid on it at y = 0 and the tracking-space origin axes. + + None of it is tracked data, so it is never hidden with the tracked nodes. The floor and axes give the eye a + reference for height and distance, which the bare models on a flat canvas lack.""" + # The slab sits just below y = 0 so its top face is the floor plane and never z-fights the grid on it. + scene.add_box( + "/floor_surface", + color=_FLOOR_SURFACE_COLOR, + dimensions=(_FLOOR_SIZE_M, _FLOOR_THICKNESS_M, _FLOOR_SIZE_M), + position=(0.0, -_FLOOR_THICKNESS_M / 2 - 0.002, 0.0), + ) + scene.add_grid( + "/floor", + width=_FLOOR_SIZE_M, + height=_FLOOR_SIZE_M, + plane="xz", + cell_size=_FLOOR_CELL_M, + cell_thickness=0.6, + cell_color=_FLOOR_CELL_COLOR, + section_size=1.0, + section_thickness=1.2, + section_color=_FLOOR_SECTION_COLOR, + ) + scene.add_frame("/origin", show_axes=True, axes_length=_ORIGIN_AXES_LENGTH_M, axes_radius=0.005) + def _add_models(self, scene: Any, assets: dict[str, bytes | None]) -> None: """Attach one profile's model meshes under the tracked frames (a missing file leaves the frame's axes).""" if assets["head"] is not None: @@ -591,9 +664,23 @@ def _set_geometry(self, side: str, glove: Glove | None) -> None: if glove is None or not len(glove.points): return scene, base, color = self._server.scene, f"/tracking/{side}_glove", _COLORS[side] - handles = [scene.add_point_cloud(f"{base}/joints", glove.points, colors=color, point_size=0.006)] + joint_colors = color if glove.point_colors is None else glove.point_colors + bone_colors = color if glove.segment_colors is None else glove.segment_colors + handles = [ + scene.add_point_cloud( + f"{base}/joints", + glove.points, + colors=joint_colors, + point_size=_JOINT_POINT_SIZE, + point_shape="circle", + ) + ] if len(glove.segments): - handles.append(scene.add_line_segments(f"{base}/bones", glove.segments, colors=color, line_width=2.0)) + handles.append( + scene.add_line_segments( + f"{base}/bones", glove.segments, colors=bone_colors, line_width=_BONE_LINE_WIDTH + ) + ) self._geometry[side] = handles def _place(self, key: str, pose: NodePose | None) -> None: diff --git a/examples/egocap/frontend/README.md b/examples/egocap/frontend/README.md index 9851d5aeb9..6d3d502d86 100644 --- a/examples/egocap/frontend/README.md +++ b/examples/egocap/frontend/README.md @@ -63,6 +63,36 @@ when it isn't. Snapshots arrive about once a second, so a per-frame jitter figur cannot be derived here and is deliberately not shown; measure jitter on the backend or with a direct `/stream{id}` client instead. +## Operator layout + +The page is built for an operator mid-take: the cameras fill it, and everything else is either a glance or one click away. + +- **Status bar (top).** A large recording indicator (`READY`, `REC` with the live take timer, `STOPPING`, `SAVING`, `SET UP`, + `NOT READY`, `RECOVERY`, `NO FEED`), the task in large type, the saved count, the last take's outcome (kept / rejected / + discarded, duration, issue count on hover), and one chip per tracking device and camera. Chips with a problem are listed + first and say why; a healthy chip is just its name. The bar is read-only and derived from the backend's recording snapshot + and capture status (`OperatorBar.tsx`, `deviceHealth.ts`). +- `READY` is claimed only when the backend would admit a take **and** this browser has an operator name and the control token; + until then it says `SET UP` and what is missing. A command in flight shows at once (`STOPPING` while a take is written), + because the backend's own phase only changes when the command finishes. A stale feed shows `NO FEED`, never a retained + `REC`. +- **Camera stage (middle).** It takes every pixel the bar above and the dock below leave. The head camera (the one named + `head*`, otherwise the first) takes the left two-thirds and the others stack on the right; each tile carries its name and + measured / advertised Hz and latency on the picture. A live take puts a red frame around the stage. +- **Control dock (bottom).** Record / Stop-Save / Discard are large; an unset operator or control-token field leads the dock, + and once set shrinks to a quiet line. The dock hides the recording panel's own header, status line and last-result line, + since the bar above shows them. +- **Side panel (right).** Always showing: the 3D tracking view on top and the raw tracking tables and session details beneath + it (Details scrolls inside itself, so neither is pushed off the screen). **Drag the divider** between it and the main column + to choose its width -- pulling left widens the panel and narrows the cameras, status bar and controls, and the other way + round. The divider also works from the keyboard (focus it, then Left / Right arrows, Shift for bigger steps, Home / End for + the limits) and double-click resets it. The width is clamped between 400 px and 1000 px and never leaves the main column + under 560 px; it is remembered in this browser (`localStorage`, silently the default if storage is blocked). The status + bar rearranges itself by the main column's width (chips drop to their own row when it is narrow). On a window under 900 px + there is no room for two columns, so the panel goes below the page and the divider is hidden. +- The 3D view's iframe is never narrower than 640 px: Viser switches to a layout that hides its scene at 576 px and below, and + crashes its own page when resized across that breakpoint. + ## Status presentation `derivePresentation` (`frontend/src/App.tsx`) is the single source every @@ -157,6 +187,14 @@ EGOCAP_VISER_ENABLED=0 uv run uvicorn egocap_web.app:app --app-dir backend --por provision that directory with the application. The view's `diagnostic` always says which device's models it is showing (`models: Pico 4 Ultra`), so a stand-in is never mistaken for the real hardware. - The hands are drawn from the reported tracking-space joints as they arrive; controller models follow the grip pose. + Each glove keeps its side colour (blue left, orange right) and shades every finger from a dark knuckle to a pale + tip, so finger direction and curl read at a glance. Tracking space is `session_floor`, so the view draws a floor slab + and grid at y = 0 (1 m sections, 0.25 m cells) and small axes at the tracking origin. +- The **Normal / Large / Full screen** buttons above the view change its size. Normal is inline in the column, Large + is a centred pop-up over a dimmed page, and Full screen fills the window. Close with the Close / Exit full screen + button, by clicking outside the pop-up, or with Escape (Escape typed inside the iframe never reaches the page, so use + the button after clicking into the view). Changing size never reloads the iframe, so the orbit camera is kept, and + the size is not remembered across page loads. - Viser draws its own control strip down the right edge of its page and 0.2.x cannot hide it, so the page makes the iframe wider and crops that strip (see `.pose-viser-viewport` in `src/styles.css`). The cropped controls can still be reached by keyboard from inside the iframe. Reset view reloads the iframe, which returns the camera to the default diff --git a/examples/egocap/frontend/src/App.operatorLayout.test.tsx b/examples/egocap/frontend/src/App.operatorLayout.test.tsx new file mode 100644 index 0000000000..f7d4a4740f --- /dev/null +++ b/examples/egocap/frontend/src/App.operatorLayout.test.tsx @@ -0,0 +1,262 @@ +/** + * The cameras-first operator page: the status bar, the camera stage, the control dock and the side drawer that + * holds the 3D view and raw data. useCaptureSession is mocked so each test controls the snapshot directly. + */ +import { afterEach, describe, expect, it, vi, type Mock } from "vitest"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import App from "./App"; +import type { PreviewSnapshot } from "./captureProtocol"; +import type { RecordingSnapshot } from "./recordingTypes"; +import { useCaptureSession, type CaptureSessionState } from "./useCaptureSession"; + +vi.mock("./useCaptureSession", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, useCaptureSession: vi.fn() }; +}); + +afterEach(() => { + cleanup(); + window.localStorage.clear(); +}); + +function snapshot(overrides: Partial = {}): PreviewSnapshot { + return { + version: 1, + type: 3, + backend_instance_id: "b1", + revision: 1, + station_label: "Station", + profile_label: "Profile", + connecting: false, + started: { session_id: "s1", running: true, mode: 0, streams: [] }, + status: { session_id: "s1", type: 3, running: true, startup_stage: null, message: null, status: null }, + stopped: null, + error_message: null, + binary_generation: 1, + tracking: null, + cameras: [], + visualization: null, + ...overrides, + }; +} + +function recording(overrides: Partial = {}): RecordingSnapshot { + return { + batchId: "b1", + episodePhase: "IDLE", + currentEpisodeId: null, + currentEpisodeElapsedMs: null, + taskText: "Place the object", + defaultOperator: null, + autoFillOperatorName: false, + savedCount: 3, + lastResult: null, + readyToRecord: true, + revision: 1, + ...overrides, + }; +} + +function mount(overrides: Partial = {}) { + (useCaptureSession as unknown as Mock).mockReturnValue({ + phase: "connected", + snapshot: snapshot(), + receivedAtMs: Date.now(), + receivedAtMonotonic: performance.now(), + diagnostic: null, + trackingSample: null, + recordingSnapshot: recording(), + recordingSnapshotReceivedAtMonotonic: performance.now(), + retry: vi.fn(), + sendRecordingCommand: vi.fn().mockRejectedValue(new Error("not connected")), + operatorToken: "op", + setOperatorToken: vi.fn(), + clearOperatorToken: vi.fn(), + controlToken: "token", + setControlToken: vi.fn(), + clearControlToken: vi.fn(), + ...overrides, + }); + return render(); +} + +describe("operator page layout", () => { + it("puts the status bar, the camera stage, the controls and the side panel on one page", () => { + const { container } = mount(); + expect(screen.getByRole("region", { name: "Operator status" })).not.toBeNull(); + expect(container.querySelector(".op-stage .camera-grid")).not.toBeNull(); + expect(container.querySelector(".op-dock .recording-panel")).not.toBeNull(); + expect(container.querySelector(".op-body > .op-main")).not.toBeNull(); + expect(container.querySelector(".op-body > .op-sidebar")).not.toBeNull(); + expect(screen.getByRole("separator", { name: "Resize side panel" })).not.toBeNull(); + }); + + it("always shows the 3D view above the details, with nothing to toggle", () => { + const { container } = mount({ snapshot: snapshot({ visualization: { status: "ready", url: "http://lab:8085/", diagnostic: null } }) }); + const panels = Array.from(container.querySelectorAll(".op-sidebar .op-panel")).map((p) => p.getAttribute("aria-label")); + expect(panels).toEqual(["3D view", "Details"]); + expect(container.querySelector(".op-sidebar iframe")).not.toBeNull(); + expect(screen.getByText("Tracking data")).not.toBeNull(); + expect(screen.queryByRole("button", { name: "Details" })).toBeNull(); + expect(screen.queryByRole("group", { name: "More views" })).toBeNull(); + }); + + it("shows no sidebar or divider while the feed is unavailable", () => { + const { container } = mount({ phase: "unavailable", snapshot: null, recordingSnapshot: null }); + expect(container.querySelector(".op-sidebar")).toBeNull(); + expect(screen.queryByRole("separator")).toBeNull(); + }); + + it("frames the cameras in red only for a live take", () => { + const live = mount({ recordingSnapshot: recording({ episodePhase: "RECORDING", currentEpisodeElapsedMs: 5000 }) }); + expect(live.container.querySelector(".op-stage")?.className).toContain("op-stage--recording"); + cleanup(); + const idle = mount(); + expect(idle.container.querySelector(".op-stage")?.className).not.toContain("op-stage--recording"); + cleanup(); + const stale = mount({ + phase: "reconnecting", + recordingSnapshot: recording({ episodePhase: "RECORDING", currentEpisodeElapsedMs: 5000 }), + }); + expect(stale.container.querySelector(".op-stage")?.className).not.toContain("op-stage--recording"); + expect(screen.getByText("NO FEED")).not.toBeNull(); + }); + + it("says SET UP, not READY, until an operator and control token are entered", () => { + mount({ operatorToken: null, controlToken: null }); + expect(screen.getByText("SET UP")).not.toBeNull(); + expect(screen.queryByText("READY")).toBeNull(); + }); + + it("shows STOPPING at once when Stop is pressed, before the backend finishes", async () => { + mount({ + recordingSnapshot: recording({ episodePhase: "RECORDING", currentEpisodeId: "e1", currentEpisodeElapsedMs: 8000 }), + sendRecordingCommand: vi.fn().mockReturnValue(new Promise(() => {})), // the backend is still writing the take + }); + expect(screen.getByText("REC")).not.toBeNull(); + fireEvent.click(screen.getByRole("button", { name: /^Stop\/Save/ })); + await waitFor(() => expect(screen.getByText("STOPPING")).not.toBeNull()); + expect(screen.queryByText("REC")).toBeNull(); + }); +}); + + +describe("resizing the side panel", () => { + const widthOf = (container: HTMLElement) => + (container.querySelector(".op-body") as HTMLElement).style.getPropertyValue("--op-sidebar-width"); + const handle = () => screen.getByRole("separator", { name: "Resize side panel" }); + + function windowWidth(width: number) { + Object.defineProperty(window, "innerWidth", { configurable: true, value: width }); + } + + it("starts at the default width and describes itself to assistive tech", () => { + windowWidth(1600); + const { container } = mount(); + expect(widthOf(container)).toBe("460px"); + expect(handle().getAttribute("aria-valuenow")).toBe("460"); + expect(handle().getAttribute("aria-valuemin")).toBe("400"); + expect(handle().getAttribute("aria-valuemax")).toBe("1000"); + expect(handle().getAttribute("aria-orientation")).toBe("vertical"); + }); + + it("starts at the remembered width", () => { + windowWidth(1600); + window.localStorage.setItem("egocap.sidebarWidth", "620"); + const { container } = mount(); + expect(widthOf(container)).toBe("620px"); + }); + + it("widens the panel with ArrowLeft, narrows it with ArrowRight, and takes bigger steps with Shift", () => { + windowWidth(1600); + const { container } = mount(); + fireEvent.keyDown(handle(), { key: "ArrowLeft" }); + expect(widthOf(container)).toBe("484px"); + fireEvent.keyDown(handle(), { key: "ArrowLeft", shiftKey: true }); + expect(widthOf(container)).toBe("580px"); + fireEvent.keyDown(handle(), { key: "ArrowRight" }); + expect(widthOf(container)).toBe("556px"); + }); + + it("jumps to the limits with Home and End, and never past them", () => { + windowWidth(1600); + const { container } = mount(); + fireEvent.keyDown(handle(), { key: "Home" }); + expect(widthOf(container)).toBe("400px"); + fireEvent.keyDown(handle(), { key: "ArrowRight", shiftKey: true }); + expect(widthOf(container)).toBe("400px"); + fireEvent.keyDown(handle(), { key: "End" }); + expect(widthOf(container)).toBe("1000px"); + fireEvent.keyDown(handle(), { key: "ArrowLeft", shiftKey: true }); + expect(widthOf(container)).toBe("1000px"); + }); + + it("ignores other keys", () => { + windowWidth(1600); + const { container } = mount(); + fireEvent.keyDown(handle(), { key: "a" }); + expect(widthOf(container)).toBe("460px"); + }); + + it("resets to the default on double-click", () => { + windowWidth(1600); + const { container } = mount(); + fireEvent.keyDown(handle(), { key: "End" }); + fireEvent.doubleClick(handle()); + expect(widthOf(container)).toBe("460px"); + }); + + it("follows the pointer while dragging: pulling left widens the panel, and the page marks the drag", () => { + windowWidth(1600); + const { container } = mount(); + const body = container.querySelector(".op-body") as HTMLElement; + fireEvent.pointerDown(handle(), { pointerId: 1, clientX: 1140 }); + expect(body.className).toContain("op-body--resizing"); + fireEvent.pointerMove(handle(), { pointerId: 1, clientX: 1000 }); + expect(widthOf(container)).toBe("600px"); // 1600 - 1000: the divider is now 600 px from the right edge + fireEvent.pointerMove(handle(), { pointerId: 1, clientX: 1300 }); + expect(widthOf(container)).toBe("400px"); // pushed right past the minimum: held there + fireEvent.pointerUp(handle(), { pointerId: 1 }); + expect(body.className).not.toContain("op-body--resizing"); + }); + + it("ignores pointer movement when no drag started", () => { + windowWidth(1600); + const { container } = mount(); + fireEvent.pointerMove(handle(), { pointerId: 1, clientX: 900 }); + expect(widthOf(container)).toBe("460px"); + }); + + it("remembers the width once a drag ends, not on every move", () => { + windowWidth(1600); + const setItem = vi.spyOn(Storage.prototype, "setItem"); + mount(); + setItem.mockClear(); + fireEvent.pointerDown(handle(), { pointerId: 1, clientX: 1140 }); + fireEvent.pointerMove(handle(), { pointerId: 1, clientX: 1000 }); + fireEvent.pointerMove(handle(), { pointerId: 1, clientX: 900 }); + expect(setItem).not.toHaveBeenCalled(); + fireEvent.pointerUp(handle(), { pointerId: 1 }); + expect(window.localStorage.getItem("egocap.sidebarWidth")).toBe("700"); + }); + + it("does not overwrite the remembered width when the window refits", () => { + windowWidth(1600); + window.localStorage.setItem("egocap.sidebarWidth", "900"); + mount(); + windowWidth(1000); + fireEvent(window, new Event("resize")); + expect(window.localStorage.getItem("egocap.sidebarWidth")).toBe("900"); + }); + + it("refits to a smaller window so the main column keeps its room", () => { + windowWidth(1600); + const { container } = mount(); + fireEvent.keyDown(handle(), { key: "End" }); + expect(widthOf(container)).toBe("1000px"); + windowWidth(1200); + fireEvent(window, new Event("resize")); + expect(widthOf(container)).toBe("640px"); // 1200 - 560 + windowWidth(1024); + }); +}); diff --git a/examples/egocap/frontend/src/App.tsx b/examples/egocap/frontend/src/App.tsx index 1f8830db3f..0e372fee1b 100644 --- a/examples/egocap/frontend/src/App.tsx +++ b/examples/egocap/frontend/src/App.tsx @@ -1,10 +1,9 @@ -import { useId } from "react"; +import { useEffect, useId, useRef, useState, type CSSProperties } from "react"; import { useCaptureSession, type ConnectionPhase } from "./useCaptureSession"; import type { PreviewSnapshot } from "./captureProtocol"; import { CaptureMode, CaptureStatusType, - DeviceState, ProviderState, deviceStateLabel, providerStateLabel, @@ -18,9 +17,21 @@ import { } from "./captureStatus"; import { CaptureTree } from "./CaptureTree"; import { CameraGrid } from "./CameraGrid"; +import { OperatorBar } from "./OperatorBar"; +import { SidebarResizer } from "./SidebarResizer"; +import { + SIDEBAR_DEFAULT_PX, + SIDEBAR_MIN_PX, + clampSidebarWidth, + loadSidebarWidth, + maxSidebarWidth, + saveSidebarWidth, +} from "./sidebarWidth"; +import { deviceTone, retainedTone, type Tone } from "./deviceHealth"; import { PoseViserPanel } from "./PoseViserPanel"; import { RecordingPanel } from "./RecordingPanel"; import { buildRecordingActions } from "./recordingActions"; +import type { RecordingPhase } from "./recordingUiReducer"; /** Feed health is derived from ConnectionPhase rather than a separate * polling freshness signal, since the connection hook owns delivery @@ -75,8 +86,6 @@ function snapshotToLegacyStatus(snapshot: PreviewSnapshot): CaptureStatusRespons }; } -type Tone = "neutral" | "positive" | "warning" | "critical"; - interface Presentation { headline: string; detail: string | null; @@ -175,24 +184,6 @@ function toneClass(tone: Tone): string { return `tone-${tone}`; } -// Keyed directly off DeviceState's own numeric values, not its label text -- -// cameras and tracking devices both use DeviceState, so one table covers both. -const DEVICE_TONE: Partial> = { - [DeviceState.CONNECTED]: "positive", - [DeviceState.FAILED]: "critical", - [DeviceState.DISCONNECTED]: "critical", - [DeviceState.UNAVAILABLE]: "critical", - [DeviceState.DEGRADED]: "warning", - [DeviceState.INITIALIZING]: "warning", - [DeviceState.UNKNOWN]: "warning", - [DeviceState.DISABLED]: "neutral", -}; - -/** The semantic tone for a tracking device or camera's DeviceState. */ -function deviceTone(status: DeviceStatus["status"] | CameraStatus["status"]): Tone { - return DEVICE_TONE[status] ?? "neutral"; -} - // DeviceState and ProviderState are different enums that happen to share // some numeric values (e.g. ProviderState.AVAILABLE=2 == DeviceState.UNKNOWN=2) // -- kept as a separate table, never looked up through DEVICE_TONE. @@ -280,15 +271,6 @@ function StaleOrErrorBanner({ ); } -/** Downgrades a positive ("current and healthy") tone to neutral when the - * data behind it is retained rather than live -- the same rule the status - * strip applies, now applied consistently to every row/entry that renders - * a tone, not just the headline. Non-positive tones (warning/critical) are - * left alone: a real fault is still worth flagging even in retained data. */ -function retainedTone(tone: Tone, live: boolean): Tone { - return !live && tone === "positive" ? "neutral" : tone; -} - /** One tracking device or camera row in the session/provider details * disclosure. */ function SourceRow({ @@ -459,35 +441,63 @@ export default function App() { const response = snapshot === null ? null : snapshotToLegacyStatus(snapshot); const presentation = derivePresentation(response, phase === "connected"); const recordingActions = buildRecordingActions(sendRecordingCommand); + // A live take frames the cameras in red, so the state reads from across the room; never from a stale feed. + const recording = presentation.live && recordingSnapshot?.episodePhase === "RECORDING"; + const [panelPhase, setPanelPhase] = useState("idle"); + // The side panel is always showing: the 3D view above the raw data. Its width is the operator's to choose by + // dragging the divider; the cameras, status bar and controls take whatever it leaves. The choice is remembered. + const [sidebarWidth, setSidebarWidth] = useState(() => clampSidebarWidth(loadSidebarWidth(), window.innerWidth)); + const [resizing, setResizing] = useState(false); + // The latest clamped width, readable from event handlers without a stale closure (a drag end must save the final value). + const sidebarWidthRef = useRef(sidebarWidth); + const applyWidth = (requested: number, persist: boolean) => { + const next = clampSidebarWidth(requested, window.innerWidth); + sidebarWidthRef.current = next; + setSidebarWidth(next); + // Only a choice the operator made is remembered; a window refit must not overwrite their preference. + if (persist) saveSidebarWidth(next); + }; + useEffect(() => { + // A smaller window can no longer fit the remembered width, and the main column must keep its minimum. + const refit = () => applyWidth(sidebarWidthRef.current, false); + window.addEventListener("resize", refit); + return () => window.removeEventListener("resize", refit); + }, []); + const unavailable = response === null && feedHealth === "unavailable"; return ( -
+
-
- {response === null && feedHealth === "unavailable" ? ( -
-

Status updates unavailable{diagnostic !== null ? `: ${diagnostic}` : "."}

- -
- ) : ( - <> - - - {/* One shared workspace: the existing generic capture display - and the Ego Capture UI recording panel are always-present siblings, - never an alternate page/tab. */} -
-
- +
+
+ {unavailable ? ( +
+

Status updates unavailable{diagnostic !== null ? `: ${diagnostic}` : "."}

+ +
+ ) : ( + <> + + + + {/* The cameras are the page: they take every pixel the status bar above and the controls below leave. */} +
-
-
+
+
+ + )} +
+ {!unavailable && ( + <> + applyWidth(width, !resizing)} + onReset={() => applyWidth(SIDEBAR_DEFAULT_PX, true)} + onDragChange={(dragging) => { + setResizing(dragging); + if (!dragging) saveSidebarWidth(sidebarWidthRef.current); + }} + /> +
-
+ +
+ + +
+ )} -
+
); } diff --git a/examples/egocap/frontend/src/CameraGrid.test.tsx b/examples/egocap/frontend/src/CameraGrid.test.tsx index a8674b4756..56eff9ba1f 100644 --- a/examples/egocap/frontend/src/CameraGrid.test.tsx +++ b/examples/egocap/frontend/src/CameraGrid.test.tsx @@ -65,3 +65,30 @@ describe("CameraGrid rate and latency readout", () => { expect(screen.getByText("Disconnected")).not.toBeNull(); }); }); + +describe("CameraGrid primary camera", () => { + function three(names: string[]): PreviewSnapshot { + const base = makeSnapshot("1", 10); + return { + ...base, + cameras: names.map((name, i) => ({ stream_id: i + 1, name, url: `/stream${i + 1}?a=1`, sequence: "1", age_ms: 10, diagnostic: null })), + }; + } + + it("gives the head camera the primary slot wherever it is listed", () => { + const { container } = render(); + const primary = container.querySelectorAll(".camera-tile--primary"); + expect(primary).toHaveLength(1); + expect(primary[0].textContent).toContain("head_human_camera"); + }); + + it("falls back to the first camera when none is called head", () => { + const { container } = render(); + expect(container.querySelector(".camera-tile--primary")?.textContent).toContain("cam_a"); + }); + + it("tags the grid with how many cameras it holds, capped at four", () => { + const { container } = render(); + expect(container.querySelector(".camera-grid-items")?.className).toContain("camera-grid-items--n4"); + }); +}); diff --git a/examples/egocap/frontend/src/CameraGrid.tsx b/examples/egocap/frontend/src/CameraGrid.tsx index ffc5db6521..b501e6f674 100644 --- a/examples/egocap/frontend/src/CameraGrid.tsx +++ b/examples/egocap/frontend/src/CameraGrid.tsx @@ -19,8 +19,11 @@ function CameraTile({ live, stale, stats, + primary, }: { camera: PreviewSnapshot["cameras"][number]; + /** The one camera the layout gives the most room (the operator's main view). */ + primary: boolean; live: boolean; /** Operator-facing rate/latency line, or null while nothing trustworthy can be shown * (no fresh sample): the tile then shows the name alone rather than a misleading figure. */ @@ -111,7 +114,7 @@ function CameraTile({ const budgetExhausted = currentRequestFailed && retryAttempts >= CAMERA_ERROR_RETRY_LIMIT; return ( -
+
{src !== undefined ? ( {camera.name} ) : ( @@ -179,11 +182,14 @@ export function CameraGrid({ ); } const elapsedSinceReceipt = receivedAtMonotonic === null ? 0 : performance.now() - receivedAtMonotonic; + // The head camera is the operator's main view when there is one by that name; otherwise the first camera is. + const headIndex = cameras.findIndex((cam) => /head/i.test(cam.name)); + const primaryIndex = headIndex === -1 ? 0 : headIndex; return (

Cameras

-
- {cameras.map((cam) => { +
+ {cameras.map((cam, index) => { const effectiveAge = cam.age_ms === null ? null : cam.age_ms + elapsedSinceReceipt; // "Never had a sample" (sequence/age_ms both null) is a distinct, // unalarming initial state -- it must not count as stale, or @@ -209,6 +215,7 @@ export function CameraGrid({ = {}): RecordingSnapshot { + return { + batchId: "b1", + episodePhase: "IDLE", + currentEpisodeId: null, + currentEpisodeElapsedMs: null, + taskText: "Place the object", + defaultOperator: null, + autoFillOperatorName: false, + savedCount: 0, + lastResult: null, + readyToRecord: true, + revision: 1, + ...overrides, + }; +} + +describe("deriveRecIndicator", () => { + it("waits for the first snapshot rather than inventing a state", () => { + expect(deriveRecIndicator(null, true)).toMatchObject({ label: "WAITING", variant: "idle" }); + }); + + it("labels a stale feed as such, whatever the retained snapshot says", () => { + expect(deriveRecIndicator(snap({ episodePhase: "RECORDING" }), false)).toMatchObject({ label: "NO FEED", variant: "stale" }); + }); + + it.each([ + ["RECORDING", "REC", "recording"], + ["STARTING", "STARTING", "busy"], + ["STOPPING", "STOPPING", "busy"], + ["FINALIZING", "SAVING", "busy"], + ["RECOVERY_REQUIRED", "RECOVERY", "warning"], + ] as const)("shows backend phase %s as %s", (episodePhase, label, variant) => { + expect(deriveRecIndicator(snap({ episodePhase }), true)).toMatchObject({ label, variant }); + }); + + it("is READY only when the backend would admit a take and this browser may start one", () => { + expect(deriveRecIndicator(snap(), true)).toMatchObject({ label: "READY", variant: "ready" }); + expect(deriveRecIndicator(snap({ readyToRecord: false }), true)).toMatchObject({ label: "NOT READY" }); + expect(deriveRecIndicator(snap({ readyToRecord: false, taskText: "" }), true).hint).toContain("Set a task"); + }); + + it("asks for whatever setup is missing instead of claiming READY", () => { + const none = deriveRecIndicator(snap(), true, { operatorSet: false, controlTokenSet: false }); + expect(none).toMatchObject({ label: "SET UP", variant: "warning" }); + expect(none.hint).toBe("Enter an operator name and the control token below"); + expect(deriveRecIndicator(snap(), true, { operatorSet: false, controlTokenSet: true }).hint).toBe("Enter an operator name below"); + expect(deriveRecIndicator(snap(), true, { operatorSet: true, controlTokenSet: false }).hint).toBe("Enter the control token below"); + }); + + it("shows a command in flight at once, since the backend's phase only changes when it finishes", () => { + const recording = snap({ episodePhase: "RECORDING" }); + expect(deriveRecIndicator(recording, true, undefined, "stopping")).toMatchObject({ label: "STOPPING", variant: "busy" }); + expect(deriveRecIndicator(snap(), true, undefined, "starting").label).toBe("STARTING"); + expect(deriveRecIndicator(recording, true, undefined, "discarding").label).toBe("DISCARDING"); + expect(deriveRecIndicator(snap(), true, undefined, "restarting").label).toBe("NEW BATCH"); + expect(deriveRecIndicator(recording, true, undefined, "uncertain")).toMatchObject({ label: "UNKNOWN", variant: "warning" }); + // editing a task is not a take-level command, so it leaves the indicator alone + expect(deriveRecIndicator(recording, true, undefined, "editingTask").label).toBe("REC"); + }); + + it("lets a stale feed win over an in-flight command", () => { + expect(deriveRecIndicator(snap(), false, undefined, "stopping").label).toBe("NO FEED"); + }); +}); + +const device = (id: string, status: number, error = "") => ({ + id, + path: `/${id}`, + type: "headset", + description: id, + provider_id: "p", + status, + reason: 0, + error, +}); +const status = (devices: unknown[], cameras: unknown[] = []) => + ({ status: { providers: [], devices, cameras } }) as unknown as CaptureStatusResponse; + +function bar(props: Partial[0]> = {}) { + return render( + , + ); +} + +describe("OperatorBar", () => { + it("shows the state, the task and the saved count", () => { + bar({ recordingSnapshot: snap({ savedCount: 12 }) }); + const region = screen.getByRole("region", { name: "Operator status" }); + expect(within(region).getByText("READY")).not.toBeNull(); + expect(within(region).getByText("Place the object")).not.toBeNull(); + expect(within(region).getByText("12")).not.toBeNull(); + }); + + it("flags an unset task rather than leaving a blank", () => { + bar({ recordingSnapshot: snap({ taskText: "", readyToRecord: false }) }); + expect(screen.getByText("No task set")).not.toBeNull(); + }); + + it("counts up the take in progress and shows nothing while idle", () => { + const { rerender } = bar({ + recordingSnapshot: snap({ episodePhase: "RECORDING", currentEpisodeElapsedMs: 42_000 }), + recordingSnapshotReceivedAtMonotonic: performance.now(), + }); + expect(screen.getByText("REC")).not.toBeNull(); + expect(screen.getByText("0:42")).not.toBeNull(); + rerender( + , + ); + expect(screen.queryByText("0:42")).toBeNull(); + }); + + it("shows no running timer next to NO FEED", () => { + bar({ + live: false, + recordingSnapshot: snap({ episodePhase: "RECORDING", currentEpisodeElapsedMs: 42_000 }), + recordingSnapshotReceivedAtMonotonic: performance.now(), + }); + expect(screen.getByText("NO FEED")).not.toBeNull(); + expect(screen.queryByText("0:42")).toBeNull(); + }); + + it("summarises the last take, with its issues on hover", () => { + bar({ + recordingSnapshot: snap({ + lastResult: { episodeId: "e1", disposition: "REJECTED", issues: ["dropped frames", "short clip"], durationMs: 41_000 }, + }), + }); + const last = screen.getByText(/Rejected/); + expect(last.textContent).toBe("Rejected · 0:41 · 2 issues"); + expect(last.closest(".op-last")?.getAttribute("title")).toBe("dropped frames; short clip"); + }); + + it("never says all devices are fine from a stale feed", () => { + bar({ live: false }); + expect(screen.queryByText("All devices OK")).toBeNull(); + expect(screen.getByText("Last known: no faults")).not.toBeNull(); + }); + + it("says every device is fine, or counts the ones that are not and lists them first", () => { + const { unmount } = bar(); + expect(screen.getByText("All devices OK")).not.toBeNull(); + unmount(); + bar({ + response: status( + [device("openxr/headset", DeviceState.CONNECTED), device("manus/hand/left", DeviceState.DISCONNECTED, "glove asleep")], + [], + ), + }); + expect(screen.getByText("1 need attention")).not.toBeNull(); + const chips = within(screen.getByRole("list", { name: "Device health" })).getAllByRole("listitem"); + expect(chips[0].textContent).toContain("Glove L"); + expect(chips[0].textContent).toContain("glove asleep"); + expect(chips[1].textContent).toBe("Headset"); + }); + + it("says so when no device has been reported yet", () => { + bar({ response: null }); + expect(screen.getByText("none reported yet")).not.toBeNull(); + }); +}); diff --git a/examples/egocap/frontend/src/OperatorBar.tsx b/examples/egocap/frontend/src/OperatorBar.tsx new file mode 100644 index 0000000000..6bffec5aa6 --- /dev/null +++ b/examples/egocap/frontend/src/OperatorBar.tsx @@ -0,0 +1,215 @@ +/** + * The operator's status bar: what the take is doing, what the task is, how many are saved, how the last one went, + * and whether every device and camera is healthy. Read-only -- the recording controls themselves stay in + * RecordingPanel, which this bar never duplicates -- and derived only from the backend's own recording snapshot and + * capture status, so it can never claim a state those facts do not support. + */ + +import { useEffect, useReducer } from "react"; + +import type { CaptureStatusResponse } from "./captureStatus"; +import { chipsNeedingAttention, deriveHealthChips, type HealthChip } from "./deviceHealth"; +import { formatElapsed } from "./RecordingPanel"; +import type { RecordingSnapshot } from "./recordingTypes"; +import type { RecordingPhase } from "./recordingUiReducer"; + +/** How the recording indicator is drawn. */ +export type RecVariant = "recording" | "busy" | "ready" | "idle" | "warning" | "stale"; + +export interface RecIndicator { + /** The big word: REC, READY, SAVING, ... */ + label: string; + variant: RecVariant; + /** A short sentence of why, when the state needs one. */ + hint: string | null; +} + +/** What the operator has entered in this browser. Recording is refused until both are set, whatever the backend's + * own readiness says, so READY must not be claimed without them. */ +export interface OperatorSetup { + operatorSet: boolean; + controlTokenSet: boolean; +} + +/** The recording indicator for one backend snapshot. ``live`` is false when the feed is stale or finished, in which + * case the snapshot is last-known state and is labelled so rather than shown as current. */ +export function deriveRecIndicator( + snapshot: RecordingSnapshot | null, + live: boolean, + setup: OperatorSetup = { operatorSet: true, controlTokenSet: true }, + pendingPhase: RecordingPhase = "idle", +): RecIndicator { + if (snapshot === null) return { label: "WAITING", variant: "idle", hint: "Waiting for the recording service" }; + if (!live) return { label: "NO FEED", variant: "stale", hint: "Showing the last known state" }; + // A command this browser has sent and not yet seen answered. The backend's phase only changes once it finishes + // (a stop can take seconds to write the take), so without this the bar would keep saying REC after Stop. + switch (pendingPhase) { + case "starting": + return { label: "STARTING", variant: "busy", hint: null }; + case "stopping": + return { label: "STOPPING", variant: "busy", hint: "Saving the take" }; + case "discarding": + return { label: "DISCARDING", variant: "busy", hint: null }; + case "restarting": + return { label: "NEW BATCH", variant: "busy", hint: null }; + case "uncertain": + return { label: "UNKNOWN", variant: "warning", hint: "The last command's outcome is unknown -- wait for the status to settle" }; + default: + break; + } + switch (snapshot.episodePhase) { + case "RECORDING": + return { label: "REC", variant: "recording", hint: null }; + case "STARTING": + return { label: "STARTING", variant: "busy", hint: null }; + case "STOPPING": + return { label: "STOPPING", variant: "busy", hint: null }; + case "FINALIZING": + return { label: "SAVING", variant: "busy", hint: "Do not unplug anything yet" }; + case "RECOVERY_REQUIRED": + return { label: "RECOVERY", variant: "warning", hint: "An interrupted take is blocking Record -- start a new batch" }; + case "IDLE": { + if (!snapshot.readyToRecord) { + return { label: "NOT READY", variant: "idle", hint: snapshot.taskText ? null : "Set a task to enable Record" }; + } + // The backend would admit a take, but this browser may not be allowed to start one yet. + if (!setup.operatorSet || !setup.controlTokenSet) { + const missing = [!setup.operatorSet && "an operator name", !setup.controlTokenSet && "the control token"].filter(Boolean); + return { label: "SET UP", variant: "warning", hint: `Enter ${missing.join(" and ")} below` }; + } + return { label: "READY", variant: "ready", hint: null }; + } + default: { + // A phase added to the backend must be handled above; the compiler flags it here, and at runtime stay honest. + const unhandled: never = snapshot.episodePhase; + return { label: "UNKNOWN", variant: "warning", hint: `Unrecognised recording phase: ${String(unhandled)}` }; + } + } +} + +/** The elapsed time of the take in progress, advanced locally between snapshots; null when no take is running. + * The backend's own monotonic figure is the basis, plus the local time since this browser received it, so it stays + * right under clock drift. */ +function liveElapsedMs(snapshot: RecordingSnapshot | null, receivedAtMonotonic: number | null): number | null { + if (snapshot?.currentEpisodeElapsedMs == null || receivedAtMonotonic === null) return null; + return snapshot.currentEpisodeElapsedMs + (performance.now() - receivedAtMonotonic); +} + +interface OperatorBarProps { + response: CaptureStatusResponse | null; + /** False when the feed is stale or finished: everything shown is then last-known, never current. */ + live: boolean; + recordingSnapshot: RecordingSnapshot | null; + recordingSnapshotReceivedAtMonotonic: number | null; + /** The operator name this browser has entered, or null. */ + operatorName: string | null; + /** Whether this browser has entered the recording control token. */ + hasControlToken: boolean; + /** The recording panel's own in-flight phase (see RecordingPanel's onPhaseChange). */ + pendingPhase: RecordingPhase; +} + +/** One device or camera chip: its name, a tone dot, and the reason only when something is wrong. */ +function HealthChipView({ chip }: { chip: HealthChip }) { + const unhealthy = chip.tone === "warning" || chip.tone === "critical"; + return ( +
  • +
    • + ); +} + +/** The read-only status bar above the cameras: recording state, task, counts, last take and device health. */ +export function OperatorBar({ + response, + live, + recordingSnapshot, + recordingSnapshotReceivedAtMonotonic, + operatorName, + hasControlToken, + pendingPhase, +}: OperatorBarProps) { + // Re-render once a second while a take is running so the timer advances; released as soon as it is not. + const [, tick] = useReducer((n: number) => n + 1, 0); + const running = live && recordingSnapshot?.currentEpisodeElapsedMs != null; + useEffect(() => { + if (!running) return undefined; + const id = setInterval(tick, 1000); + return () => clearInterval(id); + }, [running]); + + const rec = deriveRecIndicator(recordingSnapshot, live, { operatorSet: operatorName !== null, controlTokenSet: hasControlToken }, pendingPhase); + // A retained snapshot cannot vouch that the take is still running, so a stale feed shows no clock. + const elapsedMs = live ? liveElapsedMs(recordingSnapshot, recordingSnapshotReceivedAtMonotonic) : null; + const chips = deriveHealthChips(response, live); + const problems = chipsNeedingAttention(chips); + const task = recordingSnapshot?.taskText ?? ""; + const last = recordingSnapshot?.lastResult ?? null; + + return ( +
    +
    +
    + +
    + Task +

    {task === "" ? "No task set" : task}

    +
    + +
    +
    + {recordingSnapshot?.savedCount ?? "—"} + saved +
    + {last !== null && ( +
    0 ? last.issues.join("; ") : undefined} + > + Last take + + {last.disposition === "KEPT" ? "Kept" : last.disposition === "REJECTED" ? "Rejected" : "Discarded"} + {" · "} + {formatElapsed(last.durationMs)} + {last.issues.length > 0 && ` · ${last.issues.length} issue${last.issues.length === 1 ? "" : "s"}`} + +
    + )} +
    + +
    + + {chips.length === 0 + ? "Devices" + : problems.length > 0 + ? `${problems.length} need attention` + : // A retained feed cannot vouch for the devices now, so it never claims they are fine. + live + ? "All devices OK" + : "Last known: no faults"} + + {chips.length === 0 ? ( + none reported yet + ) : ( +
      + {/* Problems first, so a fault is never buried behind a row of healthy chips. */} + {[...problems, ...chips.filter((c) => !problems.includes(c))].map((chip) => ( + + ))} +
    + )} +
    +
    + ); +} diff --git a/examples/egocap/frontend/src/PoseViserPanel.test.tsx b/examples/egocap/frontend/src/PoseViserPanel.test.tsx index 83e6b1252c..2f88f6d67d 100644 --- a/examples/egocap/frontend/src/PoseViserPanel.test.tsx +++ b/examples/egocap/frontend/src/PoseViserPanel.test.tsx @@ -5,7 +5,10 @@ import { cleanup, fireEvent, render, screen } from "@testing-library/react"; import { PoseViserPanel } from "./PoseViserPanel"; import type { VisualizationInfo } from "./captureProtocol"; -afterEach(cleanup); +afterEach(() => { + cleanup(); + document.body.style.overflow = ""; +}); const READY: VisualizationInfo = { status: "ready", url: "http://lab:8085/", diagnostic: null }; @@ -21,11 +24,16 @@ describe("PoseViserPanel", () => { expect(container.innerHTML).toBe(""); }); - it("embeds the advertised URL in a titled iframe with a single Reset view button", () => { + it("embeds the advertised URL in a titled iframe with size choices and a Reset view button", () => { render(panel(READY)); const frame = screen.getByTitle("3D tracking view"); expect(frame.getAttribute("src")).toBe("http://lab:8085/"); - expect(screen.getAllByRole("button").map((b) => b.textContent)).toEqual(["Reset view"]); + expect(screen.getAllByRole("button").map((b) => b.textContent)).toEqual([ + "Normal", + "Large", + "Full screen", + "Reset view", + ]); expect(screen.queryByRole("link")).toBeNull(); }); @@ -76,9 +84,81 @@ describe("PoseViserPanel", () => { expect(screen.queryByRole("button", { name: "Resume shortcuts" })).toBeNull(); }); - it("says the view is unavailable, with the reason, and offers no iframe", () => { + it("says the view is unavailable, with the reason, and offers no iframe or size choices", () => { render(panel({ status: "unavailable", url: null, diagnostic: "the 3D view is bound to loopback" })); expect(screen.getByText(/3D view unavailable: the 3D view is bound to loopback/)).toBeTruthy(); expect(screen.queryByTitle("3D tracking view")).toBeNull(); + expect(screen.queryByRole("group", { name: "3D view size" })).toBeNull(); + }); + + describe("size", () => { + const viewport = () => screen.getByTitle("3D tracking view").parentElement as HTMLElement; + const pressed = () => + screen + .getAllByRole("button") + .filter((b) => b.getAttribute("aria-pressed") === "true") + .map((b) => b.textContent); + + it("starts at the normal size with nothing covering the page", () => { + const { container } = render(panel(READY)); + expect(viewport().className).toContain("pose-viser-viewport--normal"); + expect(pressed()).toEqual(["Normal"]); + expect(container.querySelector(".pose-viser-backdrop")).toBeNull(); + expect(screen.queryByRole("button", { name: /Close|Exit full screen/ })).toBeNull(); + }); + + it("changes size without rebuilding the iframe, so the orbit camera is kept", () => { + render(panel(READY)); + const frame = screen.getByTitle("3D tracking view"); + fireEvent.click(screen.getByRole("button", { name: "Large" })); + expect(viewport().className).toContain("pose-viser-viewport--large"); + expect(screen.getByTitle("3D tracking view")).toBe(frame); + fireEvent.click(screen.getByRole("button", { name: "Full screen" })); + expect(viewport().className).toContain("pose-viser-viewport--full"); + expect(screen.getByTitle("3D tracking view")).toBe(frame); + fireEvent.click(screen.getByRole("button", { name: "Normal" })); + expect(screen.getByTitle("3D tracking view")).toBe(frame); + }); + + it("shows a dimmed backdrop for the pop-up only, and clicking it closes the pop-up", () => { + const { container } = render(panel(READY)); + fireEvent.click(screen.getByRole("button", { name: "Large" })); + const backdrop = container.querySelector(".pose-viser-backdrop") as HTMLElement; + expect(backdrop).not.toBeNull(); + fireEvent.click(backdrop); + expect(viewport().className).toContain("pose-viser-viewport--normal"); + expect(container.querySelector(".pose-viser-backdrop")).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Full screen" })); + expect(container.querySelector(".pose-viser-backdrop")).toBeNull(); // full screen needs no backdrop + }); + + it("labels the close button for the size it closes", () => { + render(panel(READY)); + fireEvent.click(screen.getByRole("button", { name: "Large" })); + fireEvent.click(screen.getByRole("button", { name: "Close" })); + expect(pressed()).toEqual(["Normal"]); + fireEvent.click(screen.getByRole("button", { name: "Full screen" })); + fireEvent.click(screen.getByRole("button", { name: "Exit full screen" })); + expect(pressed()).toEqual(["Normal"]); + }); + + it("closes either overlay with Escape, and locks page scroll only while covering", () => { + render(panel(READY)); + for (const name of ["Large", "Full screen"]) { + fireEvent.click(screen.getByRole("button", { name })); + expect(document.body.style.overflow).toBe("hidden"); + fireEvent.keyDown(window, { key: "Escape" }); + expect(viewport().className).toContain("pose-viser-viewport--normal"); + expect(document.body.style.overflow).toBe(""); + } + }); + + it("never reopens in a pop-up or full screen after a reload", () => { + const { unmount } = render(panel(READY)); + fireEvent.click(screen.getByRole("button", { name: "Large" })); + unmount(); + render(panel(READY)); + expect(pressed()).toEqual(["Normal"]); + }); }); }); diff --git a/examples/egocap/frontend/src/PoseViserPanel.tsx b/examples/egocap/frontend/src/PoseViserPanel.tsx index 8dd3c51560..2564d0b6b6 100644 --- a/examples/egocap/frontend/src/PoseViserPanel.tsx +++ b/examples/egocap/frontend/src/PoseViserPanel.tsx @@ -8,14 +8,27 @@ interface PoseViserPanelProps { backendInstanceId: string | null; } +/** How big the 3D view is drawn: inline in its column, a centred pop-up over the page, or the whole window. */ +export type ViserSize = "normal" | "large" | "full"; + +const SIZES: ReadonlyArray<{ size: ViserSize; label: string }> = [ + { size: "normal", label: "Normal" }, + { size: "large", label: "Large" }, + { size: "full", label: "Full screen" }, +]; + /** The backend's Viser 3D tracking view, embedded by iframe. Purely presentational: it never parses poses and has * no connection of its own to report -- an iframe load event does not prove Viser's live WebSocket is connected. - * The iframe keeps one identity across snapshots, tracking loss and recording changes, so its orbit camera is not - * reset; Reset view reloads it, and the backend frames every newly connected browser on the default camera. */ + * The iframe keeps one identity across snapshots, tracking loss, recording changes and size changes, so its orbit + * camera is not reset; Reset view reloads it, and the backend frames every newly connected browser on the default + * camera. Sizing is CSS on the wrapper only, and the pop-up backdrop is a conditional sibling before the wrapper + * (an empty slot when not shown), so the iframe's position in the tree never changes. The size is not remembered + * across page loads: reopening the page straight into a pop-up or full screen would hide the recording controls. */ export function PoseViserPanel({ visualization, backendInstanceId }: PoseViserPanelProps) { const [retries, setRetries] = useState(0); const frameRef = useRef(null); const [frameFocused, setFrameFocused] = useState(false); + const [size, setSize] = useState("normal"); // Keys typed inside the iframe never reach this page, so the recording shortcuts stop working while it has focus. // The parent window sees only a blur whose active element is the iframe; it is the one signal available here. @@ -30,23 +43,67 @@ export function PoseViserPanel({ visualization, backendInstanceId }: PoseViserPa }; }, []); + // The pop-up and full screen cover the page, so the page behind must not scroll, and Escape closes them. Escape + // typed inside the iframe never reaches this page, which is why a close button is always shown while covering. + const covering = size !== "normal"; + useEffect(() => { + if (!covering) return undefined; + const previousOverflow = document.body.style.overflow; + document.body.style.overflow = "hidden"; + const onKey = (event: KeyboardEvent) => { + if (event.key === "Escape") setSize("normal"); + }; + window.addEventListener("keydown", onKey); + return () => { + document.body.style.overflow = previousOverflow; + window.removeEventListener("keydown", onKey); + }; + }, [covering]); + if (visualization === null || visualization.status === "disabled") return null; + const ready = visualization.status === "ready" && visualization.url !== null; + return (
    -

    3D tracking view

    - {visualization.status === "ready" && visualization.url !== null ? ( +
    +

    3D tracking view

    + {ready && ( +
    + {SIZES.map(({ size: option, label }) => ( + + ))} +
    + )} +
    + {ready ? ( <> + {size === "large" && ( +