From 8d0cf90c335007db4f4cd6b98a4025e94f66535c Mon Sep 17 00:00:00 2001 From: Tom X Nguyen Date: Wed, 9 Sep 2026 12:01:11 +0700 Subject: [PATCH 1/2] feat(native): add direct image pixel updates Stage owned BGRA overrides on existing img nodes using merged GPUI updates. Preserve image identity and bound upload memory and failed-version retries. Add lifecycle, atlas, pixel, React ordering and animated example regressions. --- .changeset/dynamic-img-pixels.md | 8 + README.md | 56 ++++ bun.lock | 6 + examples/dynamic-image.test.tsx | 30 ++ examples/dynamic-image.tsx | 44 +++ packages/native/Cargo.lock | 1 + packages/native/Cargo.toml | 1 + packages/native/index.d.ts | 8 + packages/native/src/custom_elements/img.rs | 16 +- packages/native/src/custom_elements/mod.rs | 2 + packages/native/src/dynamic_image.rs | 242 +++++++++++++++ packages/native/src/dynamic_image_tests.rs | 275 ++++++++++++++++++ packages/native/src/lib.rs | 1 + packages/native/src/renderer.rs | 65 +++++ packages/native/src/retained_tree.rs | 11 +- packages/native/src/test_renderer.rs | 43 +++ packages/react/package.json | 2 + .../src/__tests__/dynamic-image.test.tsx | 165 +++++++++++ packages/react/src/testing.ts | 12 + packages/react/src/types/host.ts | 10 + zed | 2 +- 21 files changed, 992 insertions(+), 8 deletions(-) create mode 100644 .changeset/dynamic-img-pixels.md create mode 100644 examples/dynamic-image.test.tsx create mode 100644 examples/dynamic-image.tsx create mode 100644 packages/native/src/dynamic_image.rs create mode 100644 packages/native/src/dynamic_image_tests.rs create mode 100644 packages/react/src/__tests__/dynamic-image.test.tsx diff --git a/.changeset/dynamic-img-pixels.md b/.changeset/dynamic-img-pixels.md new file mode 100644 index 00000000..fcbd355c --- /dev/null +++ b/.changeset/dynamic-img-pixels.md @@ -0,0 +1,8 @@ +--- +'@gpuix/native': minor +'@gpuix/react': minor +--- + +Add desktop `updateImage(elementId, width, height, bgra)` and `clearImage(elementId)` for existing `` nodes. Pixels are copied from a tightly packed BGRA Uint8Array, rendered through ordinary GPUI img, and released on clear, src change, resize, or destruction. Same-size updates preserve native image identity. Sampling remains linear. Uploads are limited to 4096 pixels per axis (64 MiB) before copying; GPU failures fall back to src and are not retried until a new version is submitted. Upstream atlas search also found [zed-industries/zed#54659](https://github.com/zed-industries/zed/issues/54659) (allocation failures) and [#57516](https://github.com/zed-industries/zed/pull/57516) (tile lifetime); these remain upstream work, not fork changes in this PR. + +Bump GPUI to merged [remorses/zed#4](https://github.com/remorses/zed/pull/4) (`81c99f816b4a5f69d3c014774068034c24d1d7af`) for `RenderImage::from_bgra` and `Window::update_image`. Upstream searches for `from_bgra` and `update_image` found no matching issues/PRs; upstream nearest-neighbor [zed-industries/zed#57393](https://github.com/zed-industries/zed/pull/57393) remains open. Ordinary GPUI Img has no sampling builder at this revision, so that feature is deferred rather than reimplementing its painting in GPUIX. diff --git a/README.md b/README.md index edebe157..1a1b1bf9 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ gpuix completions install | **timeline** | `bun --hot timeline.tsx` | A video-editor timeline: clip dragging, edge trimming with snapping, playhead scrubbing, marquee selection, zoom under the pointer, and a two-axis pan with a frozen ruler and track column | | **mail** | `bun --hot mail.tsx` | A Superhuman-style mail client: three panes, thread list, and a Framer newsletter | | **native-text** | `bun --hot native-text.tsx` | The three native text components with a tab switcher | +| **dynamic-image** | `bun --hot dynamic-image.tsx` | Animated BGRA pixels in one ordinary ``, without React commits per frame | | **counter** | `bun --hot counter.tsx` | The smallest possible app: state, events, hover | | **diff** | `bun --hot diff.tsx` | A diff viewer composed from `
` and `` in JS, for comparison | | **web** | `bun run web` from the repository root | The ChatGPT example rendered in a browser canvas with WebGPU | @@ -2107,6 +2108,61 @@ child. Put the radius on the image. /> ``` +### Dynamic pixels (desktop) + +Use the renderer from `useGpuixRequired()` (or `useGpuix().renderer`) to update +an existing `` without a React render or a JSON/base64 pixel payload: + +```tsx +const renderer = useGpuixRequired() +const image = useRef<{ id: number }>(null) + +useLayoutEffect(() => { + const bgra = new Uint8Array([0, 0, 255, 255]) // one opaque red pixel + renderer.updateImage!(image.current!.id, 1, 1, bgra) +}, [renderer]) + +return +``` + +- `updateImage(elementId, width, height, bgra: Uint8Array): void` copies the + supplied view before returning. Reuse or modify it afterwards; native retains + no JS pointer. `Buffer` and offset `Uint8Array` views also work. +- Pixels are **8-bit BGRA, straight (unpremultiplied) alpha**, matching GPUI's + decoded raster images. Rows run top to bottom, pixels left to right, with + exactly `width * 4` bytes per row and no padding. Alpha `0` is transparent; + alpha `255` is opaque. There is no stride, partial-update, or format option. +- Width and height must be positive integers no larger than **4096** each + (at most **64 MiB** per image). This ingress limit is checked before copying + pixels; it is not a guarantee that GPU memory is available. The view length + must equal **exactly `width * height * 4`**. Invalid dimensions, overflow, + lengths, or IDs throw without replacing the current pixels. An ID must be a + non-negative safe integer identifying a live `` in this renderer. +- Call after React commits, for example from `useLayoutEffect`, `useEffect`, or + an event handler. Earlier React mutations are committed before effects run. + Updates schedule the next native frame; multiple calls before that frame + keep only the latest pixels. The test renderer requires `flush()` to paint. +- Same-size updates keep native image identity and use GPUI's atlas update. + A size change creates a new identity and retires the old atlas entry. GPUI + still performs its normal repaint; no cached-frame presentation is exposed. +- Pixels override `src`. An actual subsequent `src` change (including removing + it) clears the override. Rerendering with the same `src` preserves the pixels. + `clearImage(elementId): void` returns to the current `src`, or the usual + placeholder if absent. Clearing a live img twice is safe; a destroyed ID throws. +- Clear, src changes, resize, and unmount release obsolete atlas entries on the + next native frame. The window owns uploaded images; nothing is shared across + renderers. Unmount needs no manual cleanup beyond stopping your producer. +- This is ordinary `` rendering: `objectFit`, intrinsic sizing, styles, + corner clipping, accessibility, and events still apply. Sampling is currently + **linear only**; nearest-neighbor awaits a GPUI Img builder API. +- A GPU upload failure is logged once for that submitted version and falls + back to `src` (or an empty image). Other images continue uploading. Unrelated + frames do not retry it; a new `updateImage` call allows another attempt. +- These two methods are desktop-only, and absent on the browser renderer. + +See [`examples/dynamic-image.tsx`](./examples/dynamic-image.tsx) for a small +animated color field that reuses one buffer and never commits React per frame. + ### `` `` uses GPUI's **monochrome icon renderer**. Raw `source` works on desktop diff --git a/bun.lock b/bun.lock index cc5a109f..857d02e9 100644 --- a/bun.lock +++ b/bun.lock @@ -83,8 +83,10 @@ "devDependencies": { "@testing-library/react": "^16.3.2", "@types/node": "^25.3.3", + "@types/pngjs": "6.0.5", "@types/react": "^19.2.0", "@types/react-reconciler": "^0.28.0", + "pngjs": "7.0.0", "react-refresh": "^0.18.0", "typescript": "^5.3.0", "vitest": "^4.0.18", @@ -459,6 +461,8 @@ "@types/node": ["@types/node@25.3.3", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ=="], + "@types/pngjs": ["@types/pngjs@6.0.5", "", { "dependencies": { "@types/node": "*" } }, "sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ=="], + "@types/react": ["@types/react@19.2.18", "", { "dependencies": { "csstype": "^3.2.2" } }, "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w=="], "@types/react-reconciler": ["@types/react-reconciler@0.28.9", "", { "peerDependencies": { "@types/react": "*" } }, "sha512-HHM3nxyUZ3zAylX8ZEyrDNd2XZOnQ0D5XfunJF5FLQnZbHHYq4UWvW1QfelQNXv1ICNkwYhfxjwfnqivYB6bFg=="], @@ -815,6 +819,8 @@ "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], + "pngjs": ["pngjs@7.0.0", "", {}, "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow=="], + "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], "pretty-format": ["pretty-format@27.5.1", "", { "dependencies": { "ansi-regex": "^5.0.1", "ansi-styles": "^5.0.0", "react-is": "^17.0.1" } }, "sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ=="], diff --git a/examples/dynamic-image.test.tsx b/examples/dynamic-image.test.tsx new file mode 100644 index 00000000..882b331b --- /dev/null +++ b/examples/dynamic-image.test.tsx @@ -0,0 +1,30 @@ +import fs from 'node:fs' +import React from 'react' +import { afterEach, expect, it, vi } from 'vitest' +import { createTestRoot, hasNativeTestRenderer } from '@gpuix/react/testing' +import { DynamicImage } from './dynamic-image' + +const itNative = hasNativeTestRenderer ? it : it.skip + +afterEach(() => vi.useRealTimers()) + +itNative('animates one img without React commits and stops on unmount', () => { + vi.useFakeTimers() + const root = createTestRoot({ width: 304, height: 304 }) + const uploads = vi.spyOn(root.renderer, 'updateImage') + const commits = vi.spyOn(root.renderer, 'applyBatch') + root.render() + const committed = commits.mock.calls.length + fs.mkdirSync('screenshots', { recursive: true }) + root.renderer.captureScreenshot('screenshots/dynamic-image-first.png') + const first = fs.readFileSync('screenshots/dynamic-image-first.png') + vi.advanceTimersByTime(200) + root.renderer.captureScreenshot('screenshots/dynamic-image-animated.png') + expect(fs.readFileSync('screenshots/dynamic-image-animated.png')).not.toEqual(first) + expect(root.renderer.findByType('img')).toHaveLength(1) + expect(uploads).toHaveBeenCalledTimes(6) + expect(commits).toHaveBeenCalledTimes(committed) + root.unmount() + vi.advanceTimersByTime(200) + expect(uploads).toHaveBeenCalledTimes(6) +}) diff --git a/examples/dynamic-image.tsx b/examples/dynamic-image.tsx new file mode 100644 index 00000000..33c3e7e5 --- /dev/null +++ b/examples/dynamic-image.tsx @@ -0,0 +1,44 @@ +import { useEffect, useRef } from 'react' +import { render, useGpuixRequired } from '@gpuix/react' + +export function DynamicImage() { + const renderer = useGpuixRequired() + const image = useRef<{ id: number }>(null) + + useEffect(() => { + const id = image.current!.id + const pixels = new Uint8Array(32 * 32 * 4) + let frame = 0 + const paint = () => { + for (let y = 0; y < 32; y++) { + for (let x = 0; x < 32; x++) { + const i = (y * 32 + x) * 4 + pixels[i] = (x * 8 + frame) % 256 + pixels[i + 1] = (y * 8 + frame) % 256 + pixels[i + 2] = 180 + pixels[i + 3] = 255 + } + } + renderer.updateImage!(id, 32, 32, pixels) + frame += 4 + } + paint() + const timer = setInterval(paint, 40) + // Unmount releases the image; no explicit clearImage is needed here. + return () => clearInterval(timer) + }, [renderer]) + + return ( +
+ Animated color field +
+ ) +} + +if (import.meta.main) { + render(, { + title: 'Dynamic image', width: 304, height: 304, + focus: process.env.GPUIX_BACKGROUND !== '1', + }) +} diff --git a/packages/native/Cargo.lock b/packages/native/Cargo.lock index 687667e4..e94e9d98 100644 --- a/packages/native/Cargo.lock +++ b/packages/native/Cargo.lock @@ -2607,6 +2607,7 @@ dependencies = [ "core-graphics 0.24.0", "core-text", "csscolorparser", + "ctor 0.6.3", "env_logger", "futures", "gpui", diff --git a/packages/native/Cargo.toml b/packages/native/Cargo.toml index f812df47..2846a953 100644 --- a/packages/native/Cargo.toml +++ b/packages/native/Cargo.toml @@ -92,6 +92,7 @@ napi-build = "2" # shipped addon: the wire format is still JSON until the bench says otherwise. [dev-dependencies] rmp-serde = "1.3" +ctor = "0.6" [[example]] name = "hello" diff --git a/packages/native/index.d.ts b/packages/native/index.d.ts index 0f2a4b95..eadd7625 100644 --- a/packages/native/index.d.ts +++ b/packages/native/index.d.ts @@ -26,6 +26,10 @@ export declare class GpuixRenderer { * Acquires the tree mutex ONCE for the entire batch. */ applyBatch(json: string): Array + /** Copy tightly packed BGRA pixels into an existing img. Paints on the next frame. */ + updateImage(elementId: number, width: number, height: number, bgra: Uint8Array): void + /** Release an img's pixel override on the next frame and return to its src. */ + clearImage(elementId: number): void /** Pump the native event loop. Returns false after the last window closes. */ tick(): boolean isInitialized(): boolean @@ -149,6 +153,10 @@ export declare class GpuixRenderer { */ export declare class TestGpuixRenderer { constructor(width?: number | undefined | null, height?: number | undefined | null) + /** Copy tightly packed BGRA pixels into an existing img. Call flush to paint. */ + updateImage(elementId: number, width: number, height: number, bgra: Uint8Array): void + /** Release an img's pixel override on the next flush and return to its src. */ + clearImage(elementId: number): void /** * How many elements the retained tree holds, reachable from the root or * not. `getTreeJson` walks from the root, so it cannot see a node that was diff --git a/packages/native/src/custom_elements/img.rs b/packages/native/src/custom_elements/img.rs index d298aa9f..13675fed 100644 --- a/packages/native/src/custom_elements/img.rs +++ b/packages/native/src/custom_elements/img.rs @@ -184,12 +184,16 @@ impl CustomElement for ImgElement { ) -> gpui::AnyElement { use gpui::prelude::*; - let el = match &self.source { - ImgSource::Path(path) => gpui::img(path.clone()), - ImgSource::Uri(uri) => gpui::img(uri.clone()), - ImgSource::Data(image) => gpui::img(image.clone()), - ImgSource::Empty => return img_fallback(&ctx, &self.alt, "img: no src"), - ImgSource::Invalid => return img_fallback(&ctx, &self.alt, "img: load failed"), + let el = if let Some(image) = &ctx.image { + gpui::img(image.clone()) + } else { + match &self.source { + ImgSource::Path(path) => gpui::img(path.clone()), + ImgSource::Uri(uri) => gpui::img(uri.clone()), + ImgSource::Data(image) => gpui::img(image.clone()), + ImgSource::Empty => return img_fallback(&ctx, &self.alt, "img: no src"), + ImgSource::Invalid => return img_fallback(&ctx, &self.alt, "img: load failed"), + } }; // The id is what makes gpui's `ImgState` persist. Without it `Img` has no // `GlobalElementId`, so the animated-GIF frame index and the delayed diff --git a/packages/native/src/custom_elements/mod.rs b/packages/native/src/custom_elements/mod.rs index fd4b81f3..6b626794 100644 --- a/packages/native/src/custom_elements/mod.rs +++ b/packages/native/src/custom_elements/mod.rs @@ -54,6 +54,8 @@ pub struct CustomRenderContext<'a> { pub highlight_set: Option>, /// Retained custom props, including `role` and `aria-*`. pub props: &'a HashMap, + /// Direct pixel override for an img, never serialized through custom props. + pub image: Option>, } impl CustomRenderContext<'_> { diff --git a/packages/native/src/dynamic_image.rs b/packages/native/src/dynamic_image.rs new file mode 100644 index 00000000..d9c3f372 --- /dev/null +++ b/packages/native/src/dynamic_image.rs @@ -0,0 +1,242 @@ +//! Direct pixel overrides for retained `` nodes. No JS storage crosses a frame. +use std::{collections::HashMap, sync::Arc}; + +use gpui::{RenderImage, Window}; + +use crate::retained_tree::RetainedTree; + +pub(crate) type Images = HashMap>; + +// Bound ingress memory and stay below desktop atlas texture limits. This is a +// resource limit, not layout geometry or a promise that GPU allocation succeeds. +const MAX_IMAGE_DIMENSION: u32 = 4096; + +fn image_element(tree: &RetainedTree, id: f64) -> Result { + if !id.is_finite() || id < 0.0 || id.fract() != 0.0 || id > 9_007_199_254_740_991.0 { + return Err("elementId must be a non-negative safe integer".into()); + } + let id = id as u64; + if !tree + .elements + .get(&id) + .is_some_and(|el| el.element_type == "img") + { + return Err("elementId must identify an existing img in this renderer".into()); + } + Ok(id) +} + +pub(crate) fn update( + tree: &mut RetainedTree, + id: f64, + width: f64, + height: f64, + bgra: &[u8], +) -> Result<(), String> { + let id = image_element(tree, id)?; + let dimension = |value: f64| { + if !value.is_finite() || value < 1.0 || value.fract() != 0.0 || value > u32::MAX as f64 { + Err("image dimensions must be positive 32-bit integers".to_string()) + } else { + Ok(value as u32) + } + }; + let (width, height) = (dimension(width)?, dimension(height)?); + let len = (width as usize) + .checked_mul(height as usize) + .and_then(|pixels| pixels.checked_mul(4)) + .ok_or("image byte length overflow")?; + if width > MAX_IMAGE_DIMENSION || height > MAX_IMAGE_DIMENSION { + return Err(format!( + "image dimensions must not exceed {MAX_IMAGE_DIMENSION} pixels per axis" + )); + } + if bgra.len() != len { + return Err(format!( + "BGRA byte length must be exactly {len}, got {}", + bgra.len() + )); + } + // from_bgra accepts an owned Vec. Copy the typed-array view now, before NAPI returns. + let mut image = + RenderImage::from_bgra(width, height, bgra.to_vec()).ok_or("invalid BGRA image")?; + if let Some(previous) = tree + .images + .get(&id) + .filter(|previous| previous.size(0) == image.size(0)) + { + // Window::update_image explicitly uses RenderImage::id to preserve atlas identity. + image.id = previous.id; + } + tree.images.insert(id, Arc::new(image)); + tree.mark_render_changed(id); + Ok(()) +} + +pub(crate) fn clear(tree: &mut RetainedTree, id: f64) -> Result<(), String> { + let id = image_element(tree, id)?; + if tree.images.remove(&id).is_some() { + tree.mark_render_changed(id); + } + Ok(()) +} + +/// Upload only changed images and evict overrides cleared or destroyed since the last frame. +/// All maps are renderer-owned; uploaded and failed versions live with their GPUI window. +pub(crate) fn sync( + images: &Images, + uploaded: &mut Images, + failed: &mut Images, + window: &mut Window, +) { + sync_with(images, uploaded, failed, window, Window::update_image); +} + +pub(crate) fn sync_with( + images: &Images, + uploaded: &mut Images, + failed: &mut Images, + window: &mut Window, + mut upload: impl FnMut(&mut Window, Arc) -> anyhow::Result, +) { + failed.retain(|id, image| images.get(id).is_some_and(|next| Arc::ptr_eq(next, image))); + uploaded.retain(|id, image| { + let keep = images.get(id).is_some_and(|next| next.id == image.id); + if !keep { + // GPUI owns atlas reclamation; dropping the CPU Arc alone does not evict a tile. + if let Err(error) = window.drop_image(image.clone()) { + log::error!("Failed to drop dynamic image: {error}"); + } + } + keep + }); + for (&id, image) in images { + if uploaded.get(&id).is_some_and(|old| Arc::ptr_eq(old, image)) || failed.contains_key(&id) + { + continue; + } + if let Err(error) = upload(window, image.clone()) { + log::error!("Failed to upload dynamic image: {error}"); + // GPUI may have removed the previous tile before an allocation failed. + // drop_image safely removes any remaining entry. Do not pass this + // version to Img, whose paint would otherwise retry the allocation. + if let Err(error) = window.drop_image(image.clone()) { + log::error!("Failed to drop dynamic image: {error}"); + } + uploaded.remove(&id); + failed.insert(id, image.clone()); + continue; + } + uploaded.insert(id, image.clone()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tree() -> RetainedTree { + let mut tree = RetainedTree::new(); + tree.create_element(1, "img".into()); + tree + } + + #[test] + fn copies_pixels_preserves_same_size_identity_and_replaces_on_resize() { + let mut tree = tree(); + let mut bytes = vec![3, 2, 1, 255]; + update(&mut tree, 1.0, 1.0, 1.0, &bytes).unwrap(); + let first = tree.images[&1].clone(); + bytes.fill(0); + assert_eq!(first.as_bytes(0), Some([3, 2, 1, 255].as_slice())); + update(&mut tree, 1.0, 1.0, 1.0, &[255; 4]).unwrap(); + assert_eq!(tree.images[&1].id, first.id); + update(&mut tree, 1.0, 2.0, 1.0, &[255; 8]).unwrap(); + assert_ne!(tree.images[&1].id, first.id); + assert_eq!(tree.images[&1].size(0), gpui::size(2.into(), 1.into())); + assert!(!Arc::ptr_eq(&first, &tree.images[&1])); + clear(&mut tree, 1.0).unwrap(); + clear(&mut tree, 1.0).unwrap(); + update(&mut tree, 1.0, 1.0, 1.0, &[0; 4]).unwrap(); + assert_ne!(tree.images[&1].id, first.id); + } + + #[test] + fn rejects_invalid_input_without_replacing_pixels() { + let mut tree = tree(); + update(&mut tree, 1.0, 1.0, 1.0, &[0; 4]).unwrap(); + let first = tree.images[&1].clone(); + for dimension in [ + 0.0, + -1.0, + 1.5, + f64::NAN, + f64::INFINITY, + u32::MAX as f64 + 1.0, + ] { + assert!(update(&mut tree, 1.0, dimension, 1.0, &[0; 4]).is_err()); + assert!(update(&mut tree, 1.0, 1.0, dimension, &[0; 4]).is_err()); + } + for bytes in [vec![], vec![0; 3], vec![0; 5]] { + assert!(update(&mut tree, 1.0, 1.0, 1.0, &bytes).is_err()); + } + assert!(update(&mut tree, 1.0, u32::MAX as f64, u32::MAX as f64, &[]).is_err()); + for id in [ + -1.0, + 1.5, + f64::NAN, + f64::INFINITY, + 9_007_199_254_740_992.0, + 2.0, + ] { + assert!(update(&mut tree, id, 1.0, 1.0, &[0; 4]).is_err()); + assert!(clear(&mut tree, id).is_err()); + } + assert!(Arc::ptr_eq(&first, &tree.images[&1])); + tree.create_element(2, "div".into()); + assert!(update(&mut tree, 2.0, 1.0, 1.0, &[0; 4]).is_err()); + } + + #[test] + fn rejects_oversized_dimensions_before_copying_or_length_validation() { + let mut tree = tree(); + update(&mut tree, 1.0, 1.0, 1.0, &[0; 4]).unwrap(); + let first = tree.images[&1].clone(); + for (width, height) in [(4097.0, 1.0), (1.0, 4097.0), (1_000_000.0, 1.0)] { + assert!(update(&mut tree, 1.0, width, height, &[]) + .unwrap_err() + .contains("4096")); + assert!(Arc::ptr_eq(&first, &tree.images[&1])); + } + update(&mut tree, 1.0, 4096.0, 1.0, &vec![0; 4096 * 4]).unwrap(); + update(&mut tree, 1.0, 1.0, 4096.0, &vec![0; 4096 * 4]).unwrap(); + } + + #[test] + fn actual_src_mutations_clear_overrides_but_identical_values_do_not() { + let mut tree = tree(); + tree.set_custom_prop(1, "src".into(), "one.png".into()); + update(&mut tree, 1.0, 1.0, 1.0, &[0; 4]).unwrap(); + tree.set_custom_prop(1, "src".into(), "one.png".into()); + assert_eq!(tree.images.len(), 1); + tree.set_custom_prop(1, "src".into(), "two.png".into()); + assert!(tree.images.is_empty()); + update(&mut tree, 1.0, 1.0, 1.0, &[0; 4]).unwrap(); + tree.set_custom_prop(1, "src".into(), serde_json::Value::Null); + assert!(tree.images.is_empty()); + } + + #[test] + fn destruction_and_id_reuse_release_staged_images() { + let mut tree = tree(); + update(&mut tree, 1.0, 1.0, 1.0, &[0; 4]).unwrap(); + let image = Arc::downgrade(&tree.images[&1]); + tree.destroy_element(1); + assert!(tree.images.is_empty()); + assert!(image.upgrade().is_none()); + tree.create_element(1, "img".into()); + update(&mut tree, 1.0, 1.0, 1.0, &[0; 4]).unwrap(); + tree.create_element(1, "div".into()); + assert!(tree.images.is_empty()); + } +} diff --git a/packages/native/src/dynamic_image_tests.rs b/packages/native/src/dynamic_image_tests.rs new file mode 100644 index 00000000..7071c00b --- /dev/null +++ b/packages/native/src/dynamic_image_tests.rs @@ -0,0 +1,275 @@ +use super::*; +use gpui::{RenderImage, Window}; + +const MAIN_THREAD_TEST: &str = "GPUIX_DYNAMIC_IMAGE_TEST"; + +// libtest spawns a worker even with --test-threads=1, but AppKit requires +// NSWindow creation on main. Re-exec only these GPU tests and enter their +// bodies before the child harness starts, without changing production code. +#[ctor::ctor] +fn main_thread_image_test() { + let Ok(name) = std::env::var(MAIN_THREAD_TEST) else { + return; + }; + let result = std::panic::catch_unwind(|| match name.as_str() { + "failure" => dynamic_image_failed_upload_is_not_retried_and_does_not_block_other_images(), + "lifecycle" => dynamic_image_gpu_pixels_atlas_and_lifecycle(), + _ => panic!("unknown dynamic image test: {name}"), + }); + std::process::exit(if result.is_ok() { 0 } else { 1 }); +} + +fn run_on_main_thread(name: &str) -> bool { + if std::env::var(MAIN_THREAD_TEST).as_deref() == Ok(name) { + return false; + } + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .env(MAIN_THREAD_TEST, name) + .output() + .unwrap(); + assert!( + output.status.success(), + "GPU test {name}: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + true +} + +fn with_window(f: impl FnOnce(&mut Window) -> R) -> R { + with_test_state(|cx, window, _| { + cx.update_window(window, |_, window, _| f(window)) + .map_err(|error| Error::from_reason(error.to_string())) + }) + .unwrap() +} + +fn has_atlas_entry(image: &Arc) -> bool { + with_window(|window| { + // PlatformAtlas::contains defaults to false on Metal at this GPUI pin. + // update_image reports real tile retention. Undo an absent-tile probe + // immediately so it cannot repopulate an entry that cleanup removed. + let retained = window.update_image(image.clone()).unwrap(); + if !retained { + window.drop_image(image.clone()).unwrap(); + } + retained + }) +} + +fn staged(renderer: &TestGpuixRenderer) -> Arc { + renderer.tree.lock().unwrap().images[&2].clone() +} + +fn upload(renderer: &TestGpuixRenderer, width: u32, height: u32, pixel: [u8; 4]) { + renderer + .update_image( + 2.0, + width as f64, + height as f64, + Uint8Array::new(pixel.repeat((width * height) as usize)), + ) + .unwrap(); +} + +fn assert_pixel(renderer: &TestGpuixRenderer, x: f32, y: f32, expected: [u8; 4]) { + renderer.flush().unwrap(); + let actual = with_test_state(|cx, window, _| { + let scale = cx + .update_window(window, |_, window, _| window.scale_factor()) + .unwrap(); + let image = cx.capture_screenshot(window).unwrap(); + Ok(image.get_pixel((x * scale) as u32, (y * scale) as u32).0) + }) + .unwrap(); + for (actual, expected) in actual.into_iter().zip(expected) { + assert!( + actual.abs_diff(expected) <= 2, + "pixel ({x}, {y}): {actual} != {expected}" + ); + } +} + +#[test] +fn dynamic_image_failed_upload_is_not_retried_and_does_not_block_other_images() { + if run_on_main_thread("failure") { + return; + } + use crate::dynamic_image::{self, Images}; + + let _renderer = TestGpuixRenderer::new(Some(64.0), Some(64.0)).unwrap(); + let mut tree = RetainedTree::new(); + tree.create_element(1, "img".into()); + tree.create_element(2, "img".into()); + for id in [1.0, 2.0] { + dynamic_image::update(&mut tree, id, 4.0, 4.0, &[255; 64]).unwrap(); + } + let mut uploaded = Images::new(); + let mut failed = Images::new(); + with_window(|window| dynamic_image::sync(&tree.images, &mut uploaded, &mut failed, window)); + let first = uploaded[&1].clone(); + for id in [1.0, 2.0] { + dynamic_image::update(&mut tree, id, 4.0, 4.0, &[0; 64]).unwrap(); + } + let mut attempts = 0; + for _ in 0..3 { + with_window(|window| { + dynamic_image::sync_with( + &tree.images, + &mut uploaded, + &mut failed, + window, + |window, image| { + attempts += 1; + if image.id == first.id { + anyhow::bail!("injected allocation failure"); + } + window.update_image(image) + }, + ); + }); + } + assert_eq!(attempts, 2); + assert!(!uploaded.contains_key(&1)); + assert!(Arc::ptr_eq(&uploaded[&2], &tree.images[&2])); + assert!(Arc::ptr_eq(&failed[&1], &tree.images[&1])); + assert!(!has_atlas_entry(&first)); + + dynamic_image::update(&mut tree, 1.0, 4.0, 4.0, &[255; 64]).unwrap(); + with_window(|window| dynamic_image::sync(&tree.images, &mut uploaded, &mut failed, window)); + assert!(failed.is_empty()); + assert!(Arc::ptr_eq(&uploaded[&1], &tree.images[&1])); + assert!(has_atlas_entry(&first)); + + dynamic_image::update(&mut tree, 1.0, 4.0, 4.0, &[0; 64]).unwrap(); + with_window(|window| { + dynamic_image::sync_with(&tree.images, &mut uploaded, &mut failed, window, |_, _| { + anyhow::bail!("injected allocation failure") + }); + }); + let weak = Arc::downgrade(&failed[&1]); + dynamic_image::clear(&mut tree, 1.0).unwrap(); + tree.destroy_element(2); + with_window(|window| dynamic_image::sync(&tree.images, &mut uploaded, &mut failed, window)); + assert!(uploaded.is_empty()); + assert!(failed.is_empty()); + assert!(weak.upgrade().is_none()); +} + +#[test] +fn dynamic_image_gpu_pixels_atlas_and_lifecycle() { + if run_on_main_thread("lifecycle") { + return; + } + let renderer = TestGpuixRenderer::new(Some(64.0), Some(64.0)).unwrap(); + renderer + .apply_batch( + r##"[ + ["createElement",1,"div"], + ["setStyle",1,{"width":64,"height":64,"backgroundColor":"#00ff00"}], + ["createElement",2,"img"], + ["setStyle",2,{"width":64,"height":64}], + ["appendChild",1,2],["setRoot",1] + ]"## + .into(), + ) + .unwrap(); + + let top = [ + [0, 0, 255, 255], + [0, 0, 255, 255], + [255, 0, 0, 255], + [255, 0, 0, 255], + ] + .concat(); + let bottom = [ + [0, 255, 0, 255], + [0, 255, 0, 255], + [255, 255, 255, 255], + [255, 255, 255, 255], + ] + .concat(); + let pixels = [top.repeat(2), bottom.repeat(2)].concat(); + renderer + .update_image(2.0, 4.0, 4.0, Uint8Array::new(pixels)) + .unwrap(); + assert_pixel(&renderer, 16.0, 16.0, [255, 0, 0, 255]); + assert_pixel(&renderer, 48.0, 16.0, [0, 0, 255, 255]); + assert_pixel(&renderer, 16.0, 48.0, [0, 255, 0, 255]); + assert_pixel(&renderer, 48.0, 48.0, [255, 255, 255, 255]); + + upload(&renderer, 4, 4, [0, 0, 255, 255]); + let first = staged(&renderer); + renderer.flush().unwrap(); + assert!(has_atlas_entry(&first)); + assert_pixel(&renderer, 32.0, 32.0, [255, 0, 0, 255]); + + upload(&renderer, 4, 4, [255, 0, 0, 255]); + let second = staged(&renderer); + assert_eq!(second.id, first.id); + assert!(!Arc::ptr_eq(&first, &second)); + // Assert GPUI's real atlas retention result, not just the host's stable id. + assert!(with_window(|window| window + .update_image(second.clone()) + .unwrap())); + assert_pixel(&renderer, 32.0, 32.0, [0, 0, 255, 255]); + + upload(&renderer, 8, 4, [0, 0, 255, 128]); + let resized = staged(&renderer); + assert_ne!(resized.id, second.id); + assert!(!has_atlas_entry(&resized)); + assert_pixel(&renderer, 32.0, 32.0, [128, 127, 0, 255]); + assert!(!has_atlas_entry(&second)); + assert!(has_atlas_entry(&resized)); + // A wide bitmap is contained, leaving the parent's green fill above/below. + assert_pixel(&renderer, 32.0, 4.0, [0, 255, 0, 255]); + + renderer.apply_batch(r#"[["setCustomProp",2,"objectFit","fill"],["setStyle",2,{"width":64,"height":64,"borderRadius":32}]]"#.into()).unwrap(); + upload(&renderer, 8, 4, [0, 0, 255, 255]); + assert_pixel(&renderer, 32.0, 32.0, [255, 0, 0, 255]); + assert_pixel(&renderer, 2.0, 2.0, [0, 255, 0, 255]); + upload(&renderer, 8, 4, [255, 0, 255, 0]); + assert_pixel(&renderer, 32.0, 32.0, [0, 255, 0, 255]); + + // A real src change takes over; retransmitting the same src does not. + renderer + .apply_batch(r#"[["setCustomProp",2,"src","https://example.test/image.svg"]]"#.into()) + .unwrap(); + renderer.flush().unwrap(); + assert!(renderer.tree.lock().unwrap().images.is_empty()); + assert!(!has_atlas_entry(&resized)); + upload(&renderer, 4, 4, [255, 0, 0, 255]); + let overridden = staged(&renderer); + renderer + .apply_batch(r#"[["setCustomProp",2,"src","https://example.test/image.svg"]]"#.into()) + .unwrap(); + assert_eq!(staged(&renderer).id, overridden.id); + assert_pixel(&renderer, 32.0, 32.0, [0, 0, 255, 255]); + renderer.clear_image(2.0).unwrap(); + renderer.clear_image(2.0).unwrap(); + renderer.flush().unwrap(); + assert!(!has_atlas_entry(&overridden)); + assert!(renderer.tree.lock().unwrap().images.is_empty()); + + // Clear + replace before painting must still retire the old atlas identity. + upload(&renderer, 4, 4, [0, 0, 255, 255]); + let before_clear = staged(&renderer); + renderer.flush().unwrap(); + renderer.clear_image(2.0).unwrap(); + upload(&renderer, 4, 4, [255, 0, 0, 255]); + let last = staged(&renderer); + assert_ne!(last.id, before_clear.id); + renderer.flush().unwrap(); + assert!(!has_atlas_entry(&before_clear)); + renderer + .apply_batch(r#"[["destroyElement",1]]"#.into()) + .unwrap(); + renderer.flush().unwrap(); + assert!(!has_atlas_entry(&last)); + assert!(renderer.tree.lock().unwrap().images.is_empty()); + assert!(renderer.clear_image(2.0).is_err()); + drop((first, second, resized, overridden, before_clear)); + let weak = Arc::downgrade(&last); + drop(last); + assert!(weak.upgrade().is_none()); +} diff --git a/packages/native/src/lib.rs b/packages/native/src/lib.rs index 36e7daeb..46e45f67 100644 --- a/packages/native/src/lib.rs +++ b/packages/native/src/lib.rs @@ -18,6 +18,7 @@ mod automation; mod color; mod custom_elements; mod diff; +mod dynamic_image; mod element_tree; mod markdown; mod motion; diff --git a/packages/native/src/renderer.rs b/packages/native/src/renderer.rs index 2ff9f51e..3116988c 100644 --- a/packages/native/src/renderer.rs +++ b/packages/native/src/renderer.rs @@ -1212,6 +1212,53 @@ impl GpuixRenderer { Ok(destroyed) } + fn check_image_owner(&self) -> Result<()> { + if !*self.initialized.lock().unwrap() { + return Err(Error::from_reason( + "Renderer not initialized. Call init() first.", + )); + } + #[cfg(target_os = "macos")] + if !update_window(|view, _, _| Arc::ptr_eq(&view.tree, &self.tree))? { + return Err(Error::from_reason("Renderer no longer owns this window")); + } + #[cfg(any(target_os = "windows", target_os = "linux", target_os = "freebsd"))] + if !self.ui_running.load(Ordering::SeqCst) { + return Err(Error::from_reason("The GPUI UI thread is not running")); + } + Ok(()) + } + + /// Copy tightly packed BGRA pixels into an existing img. Paints on the next frame. + #[napi] + pub fn update_image( + &self, + element_id: f64, + width: f64, + height: f64, + bgra: Uint8Array, + ) -> Result<()> { + self.check_image_owner()?; + crate::dynamic_image::update( + &mut self.tree.lock().unwrap(), + element_id, + width, + height, + bgra.as_ref(), + ) + .map_err(Error::from_reason)?; + self.request_invalidate() + } + + /// Release an img's pixel override on the next frame and return to its src. + #[napi] + pub fn clear_image(&self, element_id: f64) -> Result<()> { + self.check_image_owner()?; + crate::dynamic_image::clear(&mut self.tree.lock().unwrap(), element_id) + .map_err(Error::from_reason)?; + self.request_invalidate() + } + // ── Frame loop ─────────────────────────────────────────────────── /// Pump the native event loop. Returns false after the last window closes. @@ -2847,6 +2894,9 @@ pub(crate) struct GpuixView { /// Registry for custom element types (input, editor, diff, etc.). /// Stores factories (one per type) and live instances (one per element ID). pub(crate) custom_registry: CustomElementRegistry, + /// Atlas entries uploaded for this window, released on override removal. + images: crate::dynamic_image::Images, + failed_images: crate::dynamic_image::Images, /// Persistent ScrollHandles keyed by element ID. /// Created lazily for elements with overflow: "scroll" (or per-axis scroll). /// Handles persist across renders so GPUI maintains scroll offset state. @@ -3046,6 +3096,8 @@ impl GpuixView { focus_handles: HashMap::new(), focus_subscriptions: HashMap::new(), custom_registry: CustomElementRegistry::with_defaults(), + images: crate::dynamic_image::Images::new(), + failed_images: crate::dynamic_image::Images::new(), scroll_handles: HashMap::new(), motion_states: HashMap::new(), selection, @@ -3133,6 +3185,7 @@ impl GpuixView { let mut build_ctx = BuildCtx { tree: &tree, + images: &self.images, event_callback: &callback, focus_handles: &self.focus_handles, scroll_handles: &mut self.scroll_handles, @@ -3257,6 +3310,9 @@ impl GpuixView { /// `cx` stay separate parameters: they are `&mut` and gpui reborrows them. pub(crate) struct BuildCtx<'a> { pub tree: &'a RetainedTree, + /// Uploaded at root render. Deferred rows must use this frame's images, + /// not newer JS writes that could introduce untracked atlas entries mid-frame. + images: &'a crate::dynamic_image::Images, pub event_callback: &'a Option, pub focus_handles: &'a HashMap, pub scroll_handles: &'a mut HashMap, @@ -3854,6 +3910,13 @@ impl gpui::Render for GpuixView { let tree = tree_arc.lock().unwrap(); let callback = self.event_callback.clone(); + crate::dynamic_image::sync( + &tree.images, + &mut self.images, + &mut self.failed_images, + window, + ); + // Sync focus handles before building elements. self.sync_focus_handles(&tree, &callback, window, cx); @@ -3889,6 +3952,7 @@ impl gpui::Render for GpuixView { Some(root_id) => { let mut ctx = BuildCtx { tree: &tree, + images: &self.images, event_callback: &callback, focus_handles: &self.focus_handles, scroll_handles: &mut self.scroll_handles, @@ -4086,6 +4150,7 @@ pub(crate) fn build_element( selection_wash: inherited.selection_wash, highlight_set: inherited.highlight.clone(), props: &element.custom_props, + image: ctx.images.get(&id).cloned(), }; ctx.custom_registry .render(custom_type, &element.custom_props, render_ctx, window, cx) diff --git a/packages/native/src/retained_tree.rs b/packages/native/src/retained_tree.rs index 62092f17..b795cf1c 100644 --- a/packages/native/src/retained_tree.rs +++ b/packages/native/src/retained_tree.rs @@ -185,6 +185,8 @@ impl StyleTable { pub struct RetainedTree { pub elements: ElementMap, pub styles: StyleTable, + /// Copied pixel overrides, owned by their retained img nodes. + pub(crate) images: crate::dynamic_image::Images, /// The root element ID set by appendChildToContainer. pub root_id: Option, next_revision: u64, @@ -195,12 +197,14 @@ impl RetainedTree { Self { elements: ElementMap::default(), styles: StyleTable::default(), + images: crate::dynamic_image::Images::new(), root_id: None, next_revision: 1, } } pub fn create_element(&mut self, id: u64, element_type: String) { + self.images.remove(&id); let revision = self.take_revision(); self.elements .insert(id, RetainedElement::new(id, element_type, revision)); @@ -220,7 +224,7 @@ impl RetainedTree { /// Invalidate for rendering only. Use for changes that cannot move a glyph /// into or out of the searchable text: style, and a native element's own /// props, whose text is matched at paint and never enters a `GroupList`. - fn mark_render_changed(&mut self, id: u64) { + pub(crate) fn mark_render_changed(&mut self, id: u64) { self.mark_changed_detail(id, false); } @@ -263,6 +267,7 @@ impl RetainedTree { } fn destroy_element_recursive(&mut self, id: u64, destroyed: &mut Vec) { + self.images.remove(&id); if let Some(element) = self.elements.remove(&id) { destroyed.push(id); for child_id in element.children { @@ -375,6 +380,7 @@ impl RetainedTree { pub fn set_custom_prop(&mut self, id: u64, key: String, value: serde_json::Value) { let mut changed = false; let is_highlight = key == "highlight"; + let is_src = key == "src"; let was_declaration = self .elements .get(&id) @@ -402,6 +408,9 @@ impl RetainedTree { if !changed { return; } + if is_src { + self.images.remove(&id); + } self.mark_render_changed(id); let is_declaration = self .elements diff --git a/packages/native/src/test_renderer.rs b/packages/native/src/test_renderer.rs index d638ce81..c4d653af 100644 --- a/packages/native/src/test_renderer.rs +++ b/packages/native/src/test_renderer.rs @@ -27,6 +27,10 @@ use crate::renderer::{ }; use crate::retained_tree::RetainedTree; +#[cfg(test)] +#[path = "dynamic_image_tests.rs"] +mod dynamic_image_tests; + // ── Thread-local storage for !Send GPUI types ──────────────────────── /// Bundles VisualTestAppContext + window handle + view entity. @@ -64,6 +68,7 @@ impl Drop for VisualTestState { view.update(cx, |view, cx| { if let Ok(mut tree) = view.tree.lock() { tree.root_id = None; + tree.images.clear(); } view.custom_registry.destroy_all(); view.focus_subscriptions.clear(); @@ -262,6 +267,44 @@ impl TestGpuixRenderer { }) } + fn check_image_owner(&self) -> Result<()> { + with_test_state(|cx, _, view| { + let owned = cx.update(|cx| Arc::ptr_eq(&view.read(cx).tree, &self.tree)); + if !owned { + return Err(Error::from_reason("Renderer no longer owns this window")); + } + Ok(()) + }) + } + + /// Copy tightly packed BGRA pixels into an existing img. Call flush to paint. + #[napi] + pub fn update_image( + &self, + element_id: f64, + width: f64, + height: f64, + bgra: Uint8Array, + ) -> Result<()> { + self.check_image_owner()?; + crate::dynamic_image::update( + &mut self.tree.lock().unwrap(), + element_id, + width, + height, + bgra.as_ref(), + ) + .map_err(Error::from_reason) + } + + /// Release an img's pixel override on the next flush and return to its src. + #[napi] + pub fn clear_image(&self, element_id: f64) -> Result<()> { + self.check_image_owner()?; + crate::dynamic_image::clear(&mut self.tree.lock().unwrap(), element_id) + .map_err(Error::from_reason) + } + /// How many elements the retained tree holds, reachable from the root or /// not. `getTreeJson` walks from the root, so it cannot see a node that was /// detached and never destroyed. This is the only way a test can prove a diff --git a/packages/react/package.json b/packages/react/package.json index 5ad166ea..cb998da0 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -83,8 +83,10 @@ "devDependencies": { "@testing-library/react": "^16.3.2", "@types/node": "^25.3.3", + "@types/pngjs": "6.0.5", "@types/react": "^19.2.0", "@types/react-reconciler": "^0.28.0", + "pngjs": "7.0.0", "react-refresh": "^0.18.0", "typescript": "^5.3.0", "vitest": "^4.0.18" diff --git a/packages/react/src/__tests__/dynamic-image.test.tsx b/packages/react/src/__tests__/dynamic-image.test.tsx new file mode 100644 index 00000000..35cfdeaa --- /dev/null +++ b/packages/react/src/__tests__/dynamic-image.test.tsx @@ -0,0 +1,165 @@ +import fs from "node:fs" +import { PNG } from "pngjs" +import React, { useEffect, useLayoutEffect, useRef } from "react" +import { describe, expect, it, vi } from "vitest" +import { GpuixRenderer } from "@gpuix/native" +import { useGpuix, type NativeRenderer } from "../index" +import { createTestRoot, hasNativeTestRenderer } from "../testing" +import { SHOTS_DIR } from "./test-utils" + +const describeNative = hasNativeTestRenderer ? describe : describe.skip +const red = () => new Uint8Array([0, 0, 255, 255]) +const imageSource = (color: string) => `data:image/svg+xml,${encodeURIComponent( + ``, +)}` +const src = imageSource("#0000ff") + +function shot(renderer: ReturnType["renderer"], name: string) { + const path = `${SHOTS_DIR}/dynamic-image-${name}.png` + renderer.captureScreenshot(path) + const image = PNG.sync.read(fs.readFileSync(path)) + // Sample well inside each texel field, away from GPUI's linear atlas edges. + return [0.375, 0.5, 0.625].flatMap(y => [0.375, 0.5, 0.625].map(x => { + const offset = (Math.floor(y * image.height) * image.width + Math.floor(x * image.width)) * 4 + return Array.from(image.data.subarray(offset, offset + 4)) + })) +} + +function expectColor(samples: number[][], rgba: number[]) { + for (const pixel of samples) { + pixel.forEach((channel, i) => expect(Math.abs(channel - rgba[i]!)).toBeLessThanOrEqual(2)) + } +} + +describeNative("dynamic image public path", () => { + it.each(["layout", "passive"] as const)("uploads from a %s effect after create/src commit", (timing) => { + const red = () => new Uint8Array(Array.from({ length: 16 }, () => [0, 0, 255, 255]).flat()) + const blue = () => new Uint8Array(Array.from({ length: 16 }, () => [255, 0, 0, 255]).flat()) + const root = createTestRoot({ width: 64, height: 64 }) + const commits = vi.spyOn(root.renderer, "applyBatch") + let renderer!: NativeRenderer + let id = -1 + let renders = 0 + const useUploadEffect = timing === "layout" ? useLayoutEffect : useEffect + function App({ source = src, upload = true }: { source?: string; upload?: boolean }) { + renders++ + renderer = useGpuix().renderer! + const ref = useRef<{ id: number }>(null) + useUploadEffect(() => { + id = ref.current!.id + if (upload) renderer.updateImage!(id, 4, 4, red()) + }, [source, upload]) + return + } + root.render() + expect(id).toBeGreaterThanOrEqual(0) + expect(root.renderer.findByType("img")).toHaveLength(1) + const redShot = shot(root.renderer, `${timing}-red`) + expectColor(redShot, [255, 0, 0, 255]) + const commitCount = commits.mock.calls.length + const renderCount = renders + const tree = root.renderer.toJSON() + + // Only the offset typed-array view is copied; mutating it after return is safe. + const backing = new Uint8Array([99, 99, ...red(), 77, 77]) + renderer.updateImage!(id, 4, 4, backing.subarray(2, 66)) + backing.fill(0) + expectColor(shot(root.renderer, `${timing}-copied`), [255, 0, 0, 255]) + renderer.updateImage!(id, 4, 4, blue()) + expectColor(shot(root.renderer, `${timing}-blue`), [0, 0, 255, 255]) + renderer.updateImage!(id, 4, 4, blue()) + renderer.updateImage!(id, 4, 4, Buffer.from(red())) + expectColor(shot(root.renderer, `${timing}-latest`), [255, 0, 0, 255]) + expect(commits).toHaveBeenCalledTimes(commitCount) + expect(renders).toBe(renderCount) + expect(root.renderer.toJSON()).toEqual(tree) + + // Same src across a React commit preserves pixels; a real src change takes over. + root.render() + expectColor(shot(root.renderer, `${timing}-same-src`), [255, 0, 0, 255]) + root.render() + root.renderer.flush() + root.renderer.flush() + const sourceShot = shot(root.renderer, `${timing}-src`) + expectColor(sourceShot, [0, 255, 0, 255]) + renderer.updateImage!(id, 4, 4, red()) + renderer.clearImage!(id) + renderer.clearImage!(id) + expectColor(shot(root.renderer, `${timing}-cleared`), [0, 255, 0, 255]) + + // An effect on the next commit wins after that commit's src mutation. + root.render() + const nextRed = shot(root.renderer, `${timing}-next-effect`) + expect(nextRed).not.toEqual(sourceShot) + expectColor(nextRed, [255, 0, 0, 255]) + root.unmount() + root.renderer.flush() + expect(root.renderer.findByType("img")).toHaveLength(0) + expect(() => renderer.updateImage!(id, 4, 4, red())).toThrow(/existing img/) + expect(() => renderer.clearImage!(id)).toThrow(/existing img/) + }) + + it("remeasures intrinsic image height inside a virtualized row on resize", () => { + const root = createTestRoot({ width: 64, height: 64 }) + root.render( + +
+
+ , + ) + const renderer = root.renderer + const id = renderer.findByTestId("resizing")!.id + const following = renderer.findByTestId("following")!.id + renderer.updateImage(id, 1, 1, red()) + renderer.flush() + expect(renderer.getElementBounds(id)![3]).toBeCloseTo(32) + const before = renderer.getElementBounds(following)![1] + renderer.updateImage(id, 2, 1, new Uint8Array([...red(), ...red()])) + renderer.flush() + expect(renderer.getElementBounds(id)![3]).toBeCloseTo(16) + expect(renderer.getElementBounds(following)![1]).toBeCloseTo(before - 16) + root.unmount() + }) + + it("rejects invalid native-boundary input without changing the image", () => { + const root = createTestRoot({ width: 64, height: 64 }) + root.render(
) + const renderer = root.renderer + const id = renderer.findByTestId("image")!.id + renderer.updateImage(id, 1, 1, red()) + const before = shot(renderer, "errors-before") + for (const dimension of [0, -1, 1.5, NaN, Infinity, 2 ** 32]) { + expect(() => renderer.updateImage(id, dimension, 1, red())).toThrow(/dimensions/) + expect(() => renderer.updateImage(id, 1, dimension, red())).toThrow(/dimensions/) + } + for (const [width, height] of [[4097, 1], [1, 4097], [1_000_000, 1]]) { + expect(() => renderer.updateImage(id, width!, height!, new Uint8Array())).toThrow(/4096/) + } + for (const length of [0, 3, 5, 8]) { + expect(() => renderer.updateImage(id, 1, 1, new Uint8Array(length))).toThrow(/byte length/) + } + expect(() => renderer.updateImage(id, 2 ** 32 - 1, 2 ** 32 - 1, red())).toThrow(/overflow/) + for (const bad of [-1, 0.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => renderer.updateImage(bad, 1, 1, red())).toThrow(/safe integer/) + expect(() => renderer.clearImage(bad)).toThrow(/safe integer/) + } + expect(() => renderer.updateImage(renderer.findByType("div")[0]!.id, 1, 1, red())).toThrow(/existing img/) + expect(() => renderer.updateImage(id, 1, 1, new Uint16Array(2) as unknown as Uint8Array)).toThrow() + expect(shot(renderer, "errors-after")).toEqual(before) + root.unmount() + }) + + it("rejects stale renderer ownership and uninitialized production renderers", () => { + const old = createTestRoot() + old.render() + const id = old.renderer.findByTestId("old")!.id + const current = createTestRoot() + current.render() + expect(() => old.renderer.updateImage(id, 1, 1, red())).toThrow(/no longer owns/) + expect(() => old.renderer.clearImage(id)).toThrow(/no longer owns/) + const production = new GpuixRenderer() + expect(() => production.updateImage(id, 1, 1, red())).toThrow(/not initialized/) + expect(() => production.clearImage(id)).toThrow(/not initialized/) + current.unmount() + }) +}) diff --git a/packages/react/src/testing.ts b/packages/react/src/testing.ts index ef690429..c91dd091 100644 --- a/packages/react/src/testing.ts +++ b/packages/react/src/testing.ts @@ -30,6 +30,8 @@ export { export type { MacCpuThrottle } from "./cpu-throttle.js" interface NativeTestRendererApi extends NativeRenderer { + updateImage(elementId: number, width: number, height: number, bgra: Uint8Array): void + clearImage(elementId: number): void flush(): void drainEvents(): EventPayload[] simulateKeystrokes(keystrokes: string): void @@ -191,6 +193,16 @@ export class TestRenderer implements NativeRenderer { this.native.flush() } + /** Copy a pixel override; flush() paints it through ordinary GPUI img. */ + updateImage(elementId: number, width: number, height: number, bgra: Uint8Array): void { + this.native.updateImage(elementId, width, height, bgra) + } + + /** Remove a pixel override; flush() releases its atlas entry. */ + clearImage(elementId: number): void { + this.native.clearImage(elementId) + } + /** Drain events collected by the native GPUI event handlers. */ drainEvents(): EventPayload[] { return this.native.drainEvents() diff --git a/packages/react/src/types/host.ts b/packages/react/src/types/host.ts index b7b3098f..73810248 100644 --- a/packages/react/src/types/host.ts +++ b/packages/react/src/types/host.ts @@ -608,6 +608,16 @@ export interface NativeRenderer { /** Apply one React commit. Returns every element id destroyed by the batch. */ applyBatch(json: string): Array + /** Desktop only. Copy exactly width * height * 4 tightly packed, top-to-bottom + * BGRA bytes into an existing img (1–4096 pixels per axis, at most 64 MiB). + * Call after commit (for example in an effect). + * Pixels override src until clearImage, an actual src change, or unmount. + * The input view may be reused after return; painting happens on the next frame. */ + updateImage?(elementId: number, width: number, height: number, bgra: Uint8Array): void + /** Desktop only. Release the pixel override on the next frame and return to src. + * Idempotent for a live img; unknown, destroyed, or non-img IDs throw. */ + clearImage?(elementId: number): void + // ── Focus API ────────────────────────────────────────────────── focusElement?(elementId: number): void focusNext?(): void diff --git a/zed b/zed index 1f9d1cd8..81c99f81 160000 --- a/zed +++ b/zed @@ -1 +1 @@ -Subproject commit 1f9d1cd88656cf1759b0bdad32fa3e2df3c4b0b9 +Subproject commit 81c99f816b4a5f69d3c014774068034c24d1d7af From 5375b6bd93136788cb1c2b7adeb331b86dc513ce Mon Sep 17 00:00:00 2001 From: Tom X Nguyen Date: Wed, 9 Sep 2026 12:14:56 +0700 Subject: [PATCH 2/2] test: sample dynamic images independently of native window size Decode file URLs for Windows subprocess imports. Sample decoded screenshots inside actual image bounds and cover larger windows with offset images. --- .../src/__tests__/dynamic-image.test.tsx | 27 +++++++++++++++---- packages/react/src/__tests__/events.test.tsx | 4 +-- packages/react/src/__tests__/render.test.tsx | 5 ++-- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/packages/react/src/__tests__/dynamic-image.test.tsx b/packages/react/src/__tests__/dynamic-image.test.tsx index 35cfdeaa..dbd4d8dc 100644 --- a/packages/react/src/__tests__/dynamic-image.test.tsx +++ b/packages/react/src/__tests__/dynamic-image.test.tsx @@ -18,9 +18,21 @@ function shot(renderer: ReturnType["renderer"], name: str const path = `${SHOTS_DIR}/dynamic-image-${name}.png` renderer.captureScreenshot(path) const image = PNG.sync.read(fs.readFileSync(path)) - // Sample well inside each texel field, away from GPUI's linear atlas edges. + const target = renderer.findByTestId("image")! + const [left, top, width, height] = renderer.getElementBounds(target.id)! + const viewport = renderer.getWindowSize() + const scaleX = image.width / viewport.width + const scaleY = image.height / viewport.height + // Sample the element, not the window: platforms can clamp the requested window size. + // Stay inside each texel field, away from GPUI's linear atlas edges. return [0.375, 0.5, 0.625].flatMap(y => [0.375, 0.5, 0.625].map(x => { - const offset = (Math.floor(y * image.height) * image.width + Math.floor(x * image.width)) * 4 + const pixelX = Math.floor((left! + x * width!) * scaleX) + const pixelY = Math.floor((top! + y * height!) * scaleY) + expect(pixelX).toBeGreaterThanOrEqual(0) + expect(pixelX).toBeLessThan(image.width) + expect(pixelY).toBeGreaterThanOrEqual(0) + expect(pixelY).toBeLessThan(image.height) + const offset = (pixelY * image.width + pixelX) * 4 return Array.from(image.data.subarray(offset, offset + 4)) })) } @@ -32,10 +44,15 @@ function expectColor(samples: number[][], rgba: number[]) { } describeNative("dynamic image public path", () => { - it.each(["layout", "passive"] as const)("uploads from a %s effect after create/src commit", (timing) => { + it.each([ + ["layout", 64, 0], + ["passive", 64, 0], + ["layout", 256, 24], + ["passive", 256, 24], + ] as const)("uploads from a %s effect in a %ipx window at offset %i", (timing, windowSize, offset) => { const red = () => new Uint8Array(Array.from({ length: 16 }, () => [0, 0, 255, 255]).flat()) const blue = () => new Uint8Array(Array.from({ length: 16 }, () => [255, 0, 0, 255]).flat()) - const root = createTestRoot({ width: 64, height: 64 }) + const root = createTestRoot({ width: windowSize, height: windowSize }) const commits = vi.spyOn(root.renderer, "applyBatch") let renderer!: NativeRenderer let id = -1 @@ -49,7 +66,7 @@ describeNative("dynamic image public path", () => { id = ref.current!.id if (upload) renderer.updateImage!(id, 4, 4, red()) }, [source, upload]) - return + return } root.render() expect(id).toBeGreaterThanOrEqual(0) diff --git a/packages/react/src/__tests__/events.test.tsx b/packages/react/src/__tests__/events.test.tsx index 4ffe32a3..7a08a9cd 100644 --- a/packages/react/src/__tests__/events.test.tsx +++ b/packages/react/src/__tests__/events.test.tsx @@ -12,6 +12,7 @@ import fs from "fs" import { spawnSync } from "node:child_process" +import { fileURLToPath } from "node:url" import { describe, it, expect, beforeEach } from "vitest" import React, { useState, useRef } from "react" import { createTestRoot, hasNativeTestRenderer } from "../testing" @@ -173,8 +174,7 @@ describe("frame loop", () => { }) it("keeps the process alive after an uncaught exception", () => { - const rendererPath = new URL("../reconciler/renderer.ts", import.meta.url) - .pathname + const rendererPath = fileURLToPath(new URL("../reconciler/renderer.ts", import.meta.url)) const script = [ `import { installRuntimeErrorHandlers, startFrameLoop } from ${JSON.stringify(rendererPath)}`, "installRuntimeErrorHandlers()", diff --git a/packages/react/src/__tests__/render.test.tsx b/packages/react/src/__tests__/render.test.tsx index 5cc0a617..fd9c0992 100644 --- a/packages/react/src/__tests__/render.test.tsx +++ b/packages/react/src/__tests__/render.test.tsx @@ -183,9 +183,8 @@ describeNative("render()", () => { }) it("shows a process-level unhandled rejection on the overlay", () => { - const testingPath = new URL("../testing.ts", import.meta.url).pathname - const rendererPath = new URL("../reconciler/renderer.ts", import.meta.url) - .pathname + const testingPath = fileURLToPath(new URL("../testing.ts", import.meta.url)) + const rendererPath = fileURLToPath(new URL("../reconciler/renderer.ts", import.meta.url)) const script = [ 'import React from "react"', `import { TestRenderer } from ${JSON.stringify(testingPath)}`,