diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..bbd3af4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,142 @@ +# AGENTS.md — rumcodecs + +## Overview + +rumcodecs is a pnpm workspace providing **buffer compression codecs for +JavaScript** — a Rust/WASM reimplementation of +[numcodecs.js](https://github.com/manzt/numcodecs.js) built for Zarr. Five +codec classes are exported from one npm package, +`@fideus-labs/rumcodecs`: + +- **`Blosc`** — the Blosc meta-compressor via the pure-Rust + [blusc](https://crates.io/crates/blusc) crate, compiled to WASM (SIMD128). +- **`LZ4`** — [lz4_flex](https://crates.io/crates/lz4_flex) via WASM, using + numcodecs framing (4-byte little-endian original size + LZ4 block). +- **`Zstd`** — the [zstd](https://crates.io/crates/zstd) crate via WASM, + standard Zstd frames. +- **`GZip`** / **`Zlib`** — pure JS via + [fflate](https://github.com/101arrowz/fflate) (a peer dependency); no WASM + involved. + +The contract is **drop-in compatibility**: identical API surface to +numcodecs.js and identical byte formats to Python numcodecs, so chunks are +interchangeable across ecosystems. Three layers hold that up: +[`test/compat.test.ts`](./test/compat.test.ts) asserts the API surface, +[`test/interop.test.ts`](./test/interop.test.ts) decodes buffers actually +written by Python `numcodecs`, and the Rust unit tests in +[`lib.rs`](./crates/rumcodecs-wasm/src/lib.rs) pin the framing we emit. Note +what the per-codec suites do _not_ prove: they only show encode/decode symmetry, +which would survive both directions drifting off the format together. The +interop fixtures are the ones that fail when that happens. + +## Modules + +| Module | Read | +| ---------------------------------------------- | ------------------------------------------------------------------------ | +| TypeScript codec classes | [./src/](./src/) | +| Shared `Codec` / `CodecConstructor` interfaces | [./src/types.ts](./src/types.ts) | +| Rust WASM bindings (blusc, lz4_flex, zstd) | [./crates/rumcodecs-wasm/src/lib.rs](./crates/rumcodecs-wasm/src/lib.rs) | +| Test suites (round-trip, compat, interop) | [./test/](./test/) | +| Benchmark package (vs numcodecs.js) | [./benchmark/](./benchmark/) | +| CI / release workflows | [./.github/workflows/](./.github/workflows/) | + +## Context to load on demand + +| Task | Read | +| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Add or change a codec class | [./src/types.ts](./src/types.ts), the sibling codec in [./src/](./src/), its test in [./test/](./test/) | +| numcodecs API-compatibility rules | [./test/compat.test.ts](./test/compat.test.ts) | +| Byte-format / interop questions | [./test/interop.test.ts](./test/interop.test.ts), [./test/fixtures/README.md](./test/fixtures/README.md) | +| Change or add WASM exports | [./crates/rumcodecs-wasm/src/lib.rs](./crates/rumcodecs-wasm/src/lib.rs), the `build:wasm*` scripts in [./package.json](./package.json) | +| Run or extend benchmarks | [./benchmark/src/run.ts](./benchmark/src/run.ts), [./benchmark/src/data.ts](./benchmark/src/data.ts), [./benchmark/src/results.ts](./benchmark/src/results.ts) | +| CI matrix and gates | [./.github/workflows/ci.yml](./.github/workflows/ci.yml), [./.github/workflows/rust-ci.yml](./.github/workflows/rust-ci.yml) | +| Release / publish flow | [./.github/workflows/release.yml](./.github/workflows/release.yml) | + +## Conventions + +- Conventional Commits with optional scopes: `feat: …`, `fix(blosc): …`, + `ci: …`, `build: …` — match the existing `git log` style. +- Tooling is [vite-plus](https://www.npmjs.com/package/vite-plus) (`vp`): + `pnpm build` / `pnpm test` / `pnpm check` wrap `vp build` / `vp test` / + `vp check` (oxlint + oxfmt + typecheck). Keep `pnpm check` clean before + committing — CI runs it as a gate. +- **Every codec class is a drop-in for numcodecs.js**: static `codecId`, + static `fromConfig(config)`, the same constructor signature, and the same + static members (`Blosc.SHUFFLE`, `LZ4.max_buffer_size`, + `Zstd.MAX_CLEVEL`, …). Any new surface must be asserted in + [./test/compat.test.ts](./test/compat.test.ts). +- **Byte formats are Python-numcodecs compatible** and must never change + silently: LZ4 uses a 4-byte little-endian original-size header before the + block; Zstd emits standard frames; Blosc emits Blosc1-compatible frames + (`BLOSC_FORWARD_COMPAT_SPLIT`). A new codec or option combination needs a + fixture in [./test/fixtures/](./test/fixtures/) — regenerate with its + `generate.py`, and do not refresh existing fixtures to make a failure go + away, since that discards the evidence of the break. +- WASM-backed codecs load the module lazily through a cached dynamic import + (the `getModule()` pattern in [./src/blosc.ts](./src/blosc.ts)); new + WASM-backed codecs must reuse it so pure-JS codecs never pay the WASM load. +- `GZip` and `Zlib` stay pure-JS on fflate, and fflate stays a + `peerDependency` — do not move it to `dependencies` or reimplement it in + the WASM crate. +- The Rust functions return an empty `Vec` on failure; the TypeScript layer + converts that into a thrown `Error`. Keep error handling on that boundary — + **no panics across `wasm_bindgen`**, because a panic traps the WASM instance + and every later call fails with it, so one corrupt chunk breaks every + subsequent read. +- **An empty `Vec` alone is not a sufficient error signal**, because a frame + that legitimately holds zero bytes decodes to exactly the same thing. Each + wrapper compares the result against the length the frame itself declares: + `blosc_declared_size`, the 4-byte LE header `src/lz4.ts` reads directly, and + `zstd_declared_content_size`. Only a mismatch is an error. Getting this wrong + is silent: with `out` supplied, a "failed" decode returns the caller's own + untouched buffer, which reads as data rather than as an error. Note the limit + of this check — it catches truncation and a short decode, but a Blosc1 frame + carries no checksum over its payload, so corruption that still parses can + decode to the declared length and be returned as valid. That is pinned in + [./test/compat.test.ts](./test/compat.test.ts) so it stays a known fact. +- **Allocations sized from a header must be fallible.** The plausibility guards + below bound what a header may claim, not what the allocator can supply; an + infallible `vec![0; n]` turns a shortfall into `handle_alloc_error`, which + aborts the module exactly like a panic. Reserve through `try_reserve_exact` + (see `try_zeroed`) so it degrades to an ordinary decode failure. +- **Sizes read out of a compressed header are untrusted input.** Every decoder + we wrap trusts them to index or to size an allocation up front — `blusc` + indexes the source with them, `lz4_flex` zero-fills the declared size before + reading a byte, `zstd` reserves the declared content size — so a corrupt + header turns into an out-of-bounds panic or a multi-gigabyte allocation that + aborts the module. Validate before handing anything over: + `blosc_header_is_sane`, `lz4_declared_size_is_plausible`, + `zstd_declared_size_is_plausible`. The expansion bounds come from each + format's own ceiling (LZ4 255:1, zstd 32768:1) and are pinned by tests that + compress a constant buffer, which sits right on that ceiling — tighten them + and real data starts failing to decode. +- `decode(data, out?)` on the WASM-backed codecs must honor the optional `out` + buffer (write into it and return it) — zarr consumers rely on this. `GZip` + and `Zlib` ignore `out` and return fflate's own buffer, matching numcodecs.js + exactly; both behaviors are pinned in + [./test/compat.test.ts](./test/compat.test.ts). +- WASM builds require `RUSTFLAGS="-C target-feature=+simd128"` and build both + `web` and `nodejs` targets into `pkg/` (see the `build:wasm*` scripts); + `pkg/` is generated output, never hand-edited. +- A new codec touches five places: `src/.ts`, `src/index.ts`, the export + map in `package.json` (subpath export), the `build.lib.entry` list in + [./vite.config.ts](./vite.config.ts), and a `test/.test.ts` — plus the + codec table in [./README.md](./README.md). Miss the vite entry and the + advertised subpath resolves to a file that was never built; + [./test/index.test.ts](./test/index.test.ts) checks the three lists agree. +- Node ≥ 18 is supported (`engines`); CI tests Node 22 and 24. + +## Key dependencies + +| Dependency | Version | Notes | +| ------------------------- | --------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `blusc` | 0.0.6 | Pure-Rust Blosc (Blosc2 API, Blosc1-compatible output) — the `Blosc` codec | +| `lz4_flex` | ~0.12 | Pure-Rust LZ4 block compression — the `LZ4` codec | +| `zstd` | ~0.13 (no default features) | Zstd — the `Zstd` codec | +| `wasm-bindgen` / `js-sys` | 0.2 / 0.3 | WASM boundary | +| `fflate` | ^0.8 (peer) | Pure-JS gzip/zlib — the `GZip` and `Zlib` codecs | +| `vite-plus` (`vp`) | 0.2 | Build / test / check driver | +| `vitest` | 4 | Test runner | +| `tinybench` | 6 | Benchmark timing (benchmark package) | +| `numcodecs` (npm) | ^0.3.2 (dev) | **numcodecs.js** — the head-to-head baseline in [./benchmark/](./benchmark/) | +| `numcodecs` (PyPI) | 0.16.5 | **Python numcodecs** — wrote [./test/fixtures/](./test/fixtures/); never installed, `uv` fetches it pinned | diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..7a2597a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,40 @@ +# Builder's Code v1.0 + +A Code of Conduct for people who build things. + +--- + +## The Rule + +**Stay professional. Stay technical.** + +## Expected + +- Contribute constructively. +- Respect others' time and work. +- Focus on the work and its technical merit. + +## Not Welcome + +- Harassment, name-calling, or personal attacks. +- Trolling, spamming, or derailing discussions. +- Discussions about contributors rather than their contributions. + +## Enforcement + +Violations result in: + +1. **Warning** - First offense. +2. **Temporary suspension** - Repeated or serious violations. +3. **Permanent ban** - Continued violations. + +Maintainers can remove, block, or ban anyone who disrupts the project. + +--- + +## License + +This Code of Conduct — the text of this file — is dedicated to the public domain +under [CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/), so +other projects may reuse it freely. It does not change the license of the rest of +the repository, which is [MIT](LICENSE.txt). diff --git a/README.md b/README.md new file mode 100644 index 0000000..843d32d --- /dev/null +++ b/README.md @@ -0,0 +1,147 @@ +

+ + + rumcodecs + +

+ +

rumcodecs

+ +

+ CI + Rust CI + npm + MIT License +

+ +

+ Buffer compression codecs for the browser and Node.js — numcodecs, reimplemented in Rust and WebAssembly. +

+ +

+ A drop-in replacement for + numcodecs.js: the same + codec classes with the same API and the same byte formats, backed by Rust + compiled to WASM (SIMD128) via + blusc. Built for + Zarr — chunks written by Python + numcodecs decode in the browser, pinned by fixtures Python + actually wrote. +

+ +## 📦 The codecs + +| Codec | Backend | Format | +| ----------- | ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| **`Blosc`** | [blusc](https://crates.io/crates/blusc) (Rust → WASM) | Blosc meta-compressor: `blosclz` / `lz4` / `lz4hc` / `snappy` / `zlib` / `zstd` with shuffle and bitshuffle filters | +| **`LZ4`** | [lz4_flex](https://crates.io/crates/lz4_flex) (Rust → WASM) | numcodecs framing — 4-byte little-endian original size + LZ4 block | +| **`Zstd`** | [zstd](https://crates.io/crates/zstd) (Rust → WASM) | Standard Zstd frames | +| **`GZip`** | [fflate](https://github.com/101arrowz/fflate) (pure JS) | gzip | +| **`Zlib`** | [fflate](https://github.com/101arrowz/fflate) (pure JS) | zlib | + +Every class carries the numcodecs.js surface — static `codecId`, static +`fromConfig`, the same constructor signatures and static members +(`Blosc.SHUFFLE`, `Zstd.MAX_CLEVEL`, …) — and +[`test/compat.test.ts`](test/compat.test.ts) asserts it stays that way. +Compatibility with the _Python_ side is checked against real chunks: the +buffers in [`test/fixtures/`](test/fixtures/) were written by Python +`numcodecs`, and [`test/interop.test.ts`](test/interop.test.ts) decodes every +one of them. That pins the Python → JavaScript direction; the reverse is the +same shared byte format but is not exercised here, since asserting it would +mean running Python in CI. The WASM module loads lazily on first +encode/decode, so the pure-JS codecs never pay for it. + +## 🚀 Quick start + +```sh +npm install @fideus-labs/rumcodecs +# fflate is a peer dependency, needed only for GZip and Zlib +npm install fflate +``` + +```ts +import { Blosc } from "@fideus-labs/rumcodecs"; + +const codec = new Blosc(5, "zstd", Blosc.SHUFFLE); +const compressed = await codec.encode(data); +const restored = await codec.decode(compressed); +``` + +Each codec is also a subpath export, so bundlers ship only what you use: + +```ts +import Blosc from "@fideus-labs/rumcodecs/blosc"; + +// The numcodecs config shape works unchanged — e.g. straight from Zarr metadata. +const codec = Blosc.fromConfig({ id: "blosc", cname: "lz4", clevel: 5, shuffle: 1 }); +``` + +## 📊 Benchmarks + +The [`benchmark/`](benchmark/) workspace package runs rumcodecs head-to-head +against numcodecs.js on the same generated arrays +([tinybench](https://github.com/tinylibs/tinybench), multiple sizes and +dtypes): + +```sh +pnpm bench # console table with speedups +pnpm bench:report # write JSON + markdown report +``` + +## 🛠️ Development + +### Prerequisites + +- Rust 1.91+ with [wasm-pack](https://rustwasm.github.io/wasm-pack/) +- Node 18+ (CI tests 22 and 24) and [pnpm](https://pnpm.io/) + +### Setup + +```sh +git clone https://github.com/fideus-labs/rumcodecs +cd rumcodecs +pnpm install # runs prepare: builds WASM (web + node) and the bundle +pnpm test +``` + +### Repository structure + +```text +rumcodecs/ +├── src/ # TypeScript codec classes (Blosc, GZip, Zlib, LZ4, Zstd) +├── crates/rumcodecs-wasm/ # Rust WASM bindings: blusc, lz4_flex, zstd +├── pkg/ # wasm-pack output (generated) +├── test/ # vitest suites: round-trip, numcodecs API, interop fixtures +├── benchmark/ # head-to-head benchmark vs numcodecs.js +└── .github/workflows/ # ci.yml, rust-ci.yml, release.yml +``` + +### Commands + +| Command | Purpose | +| ------------------------------------------ | ------------------------------------------------------------------------------------------- | +| `pnpm test` | Run all vitest suites (round-trip, per-codec, compat, interop) | +| `pnpm check` | Lint, format, and typecheck (oxlint / oxfmt / tsc via `vp`) | +| `pnpm build` | Bundle with [vite-plus](https://www.npmjs.com/package/vite-plus) and emit type declarations | +| `pnpm build:wasm` / `pnpm build:wasm:node` | Rebuild the WASM package for web / Node (SIMD128) | +| `pnpm bench` | Benchmark against numcodecs.js | +| `cargo test -- --test-threads=1` | Rust unit tests for the WASM crate | + +## 🤝 Contributing + +Start with [AGENTS.md](AGENTS.md) for the repository map and conventions, and +the [open issues](https://github.com/fideus-labs/rumcodecs/issues) for what to +pick up next. All participation is governed by our +[Code of Conduct](CODE_OF_CONDUCT.md). + +## 📄 License + +MIT — see [LICENSE.txt](LICENSE.txt). Copyright (c) Fideus Labs LLC. + +rumcodecs reimplements the API of +[numcodecs.js](https://github.com/manzt/numcodecs.js) (MIT, © Trevor Manz) so +existing Zarr tooling works unchanged. Blosc support comes from the pure-Rust +[blusc](https://crates.io/crates/blusc) implementation of the +[Blosc](https://www.blosc.org/) meta-compressor; LZ4 from +[lz4_flex](https://crates.io/crates/lz4_flex); Zstd from the +[zstd](https://crates.io/crates/zstd) crate. diff --git a/crates/rumcodecs-wasm/src/lib.rs b/crates/rumcodecs-wasm/src/lib.rs index 4b2b619..34d0e3c 100644 --- a/crates/rumcodecs-wasm/src/lib.rs +++ b/crates/rumcodecs-wasm/src/lib.rs @@ -4,8 +4,8 @@ use blusc::api::{ blosc2_cbuffer_sizes as blusc_cbuffer_sizes, blosc2_compress_ctx, blosc2_create_cctx, Blosc2Cparams, BLOSC2_CPARAMS_DEFAULTS, }; -use blusc::internal::constants::*; use blusc::convenience::blosc1_decompress as blusc_decompress; +use blusc::internal::constants::*; fn cname_to_compcode(cname: &str) -> u8 { match cname { @@ -59,8 +59,60 @@ pub fn blosc_compress( dest } +/// A Blosc header is untrusted input: `blusc` reads the buffer sizes straight out +/// of it and then indexes the source with them, so a corrupt chunk can panic +/// inside the decompressor. A panic traps the whole WASM instance and takes every +/// later call down with it, so implausible headers are rejected here instead and +/// `src/blosc.ts` turns the empty result into a thrown Error. +fn blosc_header_is_sane(data: &[u8]) -> bool { + if data.len() < BLOSC_MIN_HEADER_LENGTH { + return false; + } + + let version = data[0]; + if version == 0 || version > BLOSC2_VERSION_FORMAT_STABLE { + return false; + } + + let header_len = if version == BLOSC2_VERSION_FORMAT_STABLE + || version == BLOSC2_VERSION_FORMAT_BETA1 + || version == BLOSC2_VERSION_FORMAT_ALPHA + { + BLOSC_EXTENDED_HEADER_LENGTH + } else { + BLOSC_MIN_HEADER_LENGTH + }; + if data.len() < header_len { + return false; + } + + let (nbytes, cbytes, _) = blusc_cbuffer_sizes(data); + + // The frame records its own compressed length; it cannot exceed what we hold. + if cbytes < header_len || cbytes > data.len() { + return false; + } + + // A header claiming more than Blosc allows only gets us an allocation + // failure, which aborts the instance just like a panic does. + if nbytes > BLOSC2_MAX_BUFFERSIZE { + return false; + } + + // An uncompressed (memcpy'd) frame stores nbytes verbatim after the header. + // This is the bound blusc itself omits before copying out of the source. + if data[2] & BLOSC_MEMCPYED != 0 && header_len.saturating_add(nbytes) > data.len() { + return false; + } + + true +} + #[wasm_bindgen] pub fn blosc_decompress(data: &[u8]) -> Vec { + if !blosc_header_is_sane(data) { + return vec![]; + } match blusc_decompress(data) { Ok(result) => result, Err(_) => vec![], @@ -73,6 +125,24 @@ pub fn blosc_cbuffer_sizes(data: &[u8]) -> Vec { vec![nb as u32, cb as u32, bs as u32] } +/// The uncompressed size a well-formed Blosc frame declares, or `undefined` if +/// the header is not one we would decode. +/// +/// `blosc_decompress` signals failure with an empty `Vec`, and a frame that +/// really does hold zero bytes decodes to exactly the same thing — +/// `blosc_compress(&[])` emits a valid 32-byte frame declaring `nbytes = 0`. +/// `src/blosc.ts` uses this to tell them apart. Reading `blosc_cbuffer_sizes` +/// directly would not do: it reports zeroes for garbage too, which is the +/// answer that makes a corrupt chunk look like an empty one. +#[wasm_bindgen] +pub fn blosc_declared_size(data: &[u8]) -> Option { + if !blosc_header_is_sane(data) { + return None; + } + let (nbytes, _, _) = blusc_cbuffer_sizes(data); + Some(nbytes) +} + // Standalone LZ4 codec — numcodecs format: 4-byte LE original size + LZ4 block #[wasm_bindgen] pub fn lz4_compress(data: &[u8], _acceleration: i32) -> Vec { @@ -84,14 +154,59 @@ pub fn lz4_compress(data: &[u8], _acceleration: i32) -> Vec { result } +/// numcodecs' ceiling on a single buffer, mirrored in `src/lz4.ts`. +const MAX_DECODED_SIZE: usize = 0x7e00_0000; + +/// Max output bytes an LZ4 block can produce per input byte: each match-length +/// extension byte adds 255 to the copy. A constant buffer measures 255.0:1. +const LZ4_MAX_EXPANSION: usize = 256; + +/// The size in an LZ4 header is untrusted, and `lz4_flex` zero-fills a buffer +/// that large before it reads a single byte of the block — so a corrupt header +/// asking for 4 GB aborts the WASM instance rather than failing cleanly. Reject +/// sizes no LZ4 block could actually expand to. +fn lz4_declared_size_is_plausible(orig_size: usize, block_len: usize) -> bool { + orig_size <= MAX_DECODED_SIZE + && orig_size + <= block_len + .saturating_mul(LZ4_MAX_EXPANSION) + .saturating_add(1024) +} + +/// A zeroed buffer of `len` bytes, or `None` if the allocator cannot supply one. +/// +/// The plausibility guards above reject sizes no real block could produce, but +/// what they cannot bound is how much memory this instance still has. An +/// infallible `vec![0; n]` turns that shortfall into `handle_alloc_error`, which +/// aborts the module exactly like a panic and takes every later decode with it — +/// so the reservation is made fallibly and a shortfall becomes an ordinary +/// decode failure. +fn try_zeroed(len: usize) -> Option> { + let mut buf = Vec::new(); + buf.try_reserve_exact(len).ok()?; + buf.resize(len, 0); + Some(buf) +} + #[wasm_bindgen] pub fn lz4_decompress(data: &[u8]) -> Vec { if data.len() < 4 { return vec![]; } - let orig_size = - u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; - lz4_flex::block::decompress(&data[4..], orig_size).unwrap_or_default() + let orig_size = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize; + let block = &data[4..]; + if !lz4_declared_size_is_plausible(orig_size, block.len()) { + return vec![]; + } + let Some(mut out) = try_zeroed(orig_size) else { + return vec![]; + }; + match lz4_flex::block::decompress_into(block, &mut out) { + // A block that decodes short of its declared size is corrupt; returning + // the zero-padded remainder would hand back silent garbage. + Ok(n) if n == orig_size => out, + _ => vec![], + } } // Standalone Zstd codec — standard Zstd frame format @@ -100,10 +215,56 @@ pub fn zstd_compress(data: &[u8], level: i32) -> Vec { zstd::bulk::compress(data, level).unwrap_or_default() } +/// Max output bytes a zstd frame can produce per input byte: a 4-byte RLE block +/// regenerates a full 128 KB block. A constant buffer measures ~32500:1. +const ZSTD_MAX_EXPANSION: usize = 131_072 / 4; + +/// Same hazard as the LZ4 header, one size class up: `zstd::bulk::decompress` +/// reserves the capacity we hand it up front, and that capacity comes straight +/// out of the frame's declared content size. +fn zstd_declared_size_is_plausible(declared: usize, frame_len: usize) -> bool { + declared <= MAX_DECODED_SIZE + && declared + <= frame_len + .saturating_mul(ZSTD_MAX_EXPANSION) + .saturating_add(131_072) +} + #[wasm_bindgen] pub fn zstd_decompress(data: &[u8]) -> Vec { - let capacity = zstd_frame_content_size(data).unwrap_or(data.len() * 4); - zstd::bulk::decompress(data, capacity).unwrap_or_default() + // Frames written by `zstd_compress` always declare their content size; the + // multiple-of-input fallback only covers frames from other producers that + // omit it, and will fail cleanly if it guesses low. + let capacity = zstd_frame_content_size(data).unwrap_or_else(|| data.len().saturating_mul(4)); + if !zstd_declared_size_is_plausible(capacity, data.len()) { + return vec![]; + } + // `zstd::bulk::decompress` would reserve `capacity` infallibly; reserve it + // ourselves so a request this instance cannot satisfy fails as a decode + // error rather than aborting the module. See `try_zeroed`. + let mut out = Vec::new(); + if out.try_reserve_exact(capacity).is_err() { + return vec![]; + } + match zstd::bulk::Decompressor::new() { + Ok(mut decoder) => match decoder.decompress_to_buffer(data, &mut out) { + Ok(_) => out, + Err(_) => vec![], + }, + Err(_) => vec![], + } +} + +/// The content size a zstd frame declares in its header, or `undefined` when it +/// declares none (or is not a zstd frame at all). +/// +/// `zstd_decompress` reports failure the same way every decoder here does — an +/// empty `Vec` — which on its own cannot be told apart from a frame that really +/// does hold nothing. `src/zstd.ts` uses this to make that call before deciding +/// whether to throw. Exposed for that reason, not as part of the codec API. +#[wasm_bindgen] +pub fn zstd_declared_content_size(data: &[u8]) -> Option { + zstd_frame_content_size(data) } fn zstd_frame_content_size(data: &[u8]) -> Option { @@ -179,8 +340,465 @@ fn zstd_frame_content_size(data: &[u8]) -> Option { data[offset + 6], data[offset + 7], ]); - Some(val as usize) + // `usize` is 32-bit on wasm32, so a declared size past 4 GiB would + // wrap to a small plausible-looking one rather than being rejected. + usize::try_from(val).ok() } _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + + const COMPRESSORS: [&str; 6] = ["blosclz", "lz4", "lz4hc", "snappy", "zlib", "zstd"]; + + /// Mildly compressible bytes, so a round trip exercises a real codec path + /// rather than an all-zero fast path. + fn sample() -> Vec { + (0..8192u32) + .flat_map(|i| (i as u16).to_le_bytes()) + .collect() + } + + // The TypeScript layer always passes typesize 1; see `src/blosc.ts`. + fn compress_blosc(data: &[u8], cname: &str, clevel: u8, shuffle: i32) -> Vec { + blosc_compress(data, cname, clevel, shuffle, 1, 0) + } + + #[test] + fn cname_maps_to_blosc_compcode() { + assert_eq!(cname_to_compcode("lz4"), BLOSC_LZ4); + assert_eq!(cname_to_compcode("lz4hc"), BLOSC_LZ4HC); + assert_eq!(cname_to_compcode("snappy"), BLOSC_SNAPPY); + assert_eq!(cname_to_compcode("zlib"), BLOSC_ZLIB); + assert_eq!(cname_to_compcode("zstd"), BLOSC_ZSTD); + assert_eq!(cname_to_compcode("blosclz"), BLOSC_BLOSCLZ); + // numcodecs validates the name in JS; an unknown one still must not panic. + assert_eq!(cname_to_compcode("not-a-compressor"), BLOSC_BLOSCLZ); + } + + #[test] + fn blosc_round_trips_every_compressor() { + let data = sample(); + for cname in COMPRESSORS { + let compressed = compress_blosc(&data, cname, 5, BLOSC_SHUFFLE as i32); + assert!(!compressed.is_empty(), "{cname} produced no output"); + assert_eq!(blosc_decompress(&compressed), data, "{cname} round trip"); + } + } + + #[test] + fn blosc_round_trips_every_shuffle_mode() { + let data = sample(); + let modes = [ + BLOSC_NOSHUFFLE as i32, + BLOSC_SHUFFLE as i32, + BLOSC_BITSHUFFLE as i32, + ]; + for shuffle in modes { + let compressed = compress_blosc(&data, "lz4", 5, shuffle); + assert!( + !compressed.is_empty(), + "shuffle {shuffle} produced no output" + ); + assert_eq!(blosc_decompress(&compressed), data, "shuffle {shuffle}"); + } + } + + #[test] + fn blosc_round_trips_at_clevel_zero() { + // clevel 0 stores the buffer uncompressed but still emits a Blosc frame. + let data = sample(); + let compressed = compress_blosc(&data, "lz4", 0, BLOSC_NOSHUFFLE as i32); + assert!(!compressed.is_empty()); + assert_eq!(blosc_decompress(&compressed), data); + } + + #[test] + fn blosc_round_trips_an_empty_buffer() { + let compressed = compress_blosc(&[], "lz4", 5, BLOSC_SHUFFLE as i32); + // An empty input may legitimately yield no output; what must not happen + // is a panic across the wasm_bindgen boundary. + assert!(blosc_decompress(&compressed).is_empty()); + } + + #[test] + fn blosc_header_is_self_describing() { + // Blosc1-compatible framing: the header carries the sizes back out, which + // is what lets Python numcodecs read a chunk we wrote. + let data = sample(); + let compressed = compress_blosc(&data, "zstd", 5, BLOSC_SHUFFLE as i32); + let sizes = blosc_cbuffer_sizes(&compressed); + assert_eq!(sizes.len(), 3); + assert_eq!(sizes[0] as usize, data.len(), "nbytes"); + assert_eq!(sizes[1] as usize, compressed.len(), "cbytes"); + assert!(sizes[2] > 0, "blocksize"); + } + + #[test] + fn blosc_decompress_returns_empty_on_garbage() { + // The error convention across the boundary is an empty Vec, never a panic; + // src/blosc.ts turns that into a thrown Error. A panic here would trap the + // WASM instance, so a single corrupt Zarr chunk would break every later read. + assert!(blosc_decompress(&[]).is_empty(), "empty"); + assert!( + blosc_decompress(&[0u8; 3]).is_empty(), + "shorter than a header" + ); + assert!(blosc_decompress(&[0u8; 64]).is_empty(), "zeroed header"); + assert!(blosc_decompress(&[0xff; 64]).is_empty(), "all-ones header"); + assert!( + blosc_decompress(&[0x02; 64]).is_empty(), + "plausible version byte" + ); + } + + #[test] + fn blosc_decompress_rejects_a_lying_header() { + let data = sample(); + let compressed = compress_blosc(&data, "lz4", 5, BLOSC_SHUFFLE as i32); + + // cbytes larger than the buffer we were actually handed. + let mut overlong = compressed.clone(); + overlong[12..16].copy_from_slice(&(compressed.len() as u32 + 1).to_le_bytes()); + assert!( + blosc_decompress(&overlong).is_empty(), + "cbytes past the buffer" + ); + + // A memcpy'd frame promising far more payload than it carries. This is the + // bound blusc skips before copying, and the shape that used to panic. + let mut memcpyed = compressed.clone(); + memcpyed[2] |= BLOSC_MEMCPYED; + memcpyed[4..8].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!( + blosc_decompress(&memcpyed).is_empty(), + "memcpy past the buffer" + ); + + // Truncation must not be mistaken for a decodable frame. + assert!( + blosc_decompress(&compressed[..compressed.len() / 2]).is_empty(), + "truncated" + ); + } + + #[test] + fn blosc_header_check_accepts_what_we_emit() { + // The guard must not reject our own frames, in any of the shapes we emit: + // compressed, stored at clevel 0 (which sets BLOSC_MEMCPYED), and empty. + let data = sample(); + for cname in COMPRESSORS { + let compressed = compress_blosc(&data, cname, 5, BLOSC_SHUFFLE as i32); + assert!(blosc_header_is_sane(&compressed), "{cname}"); + } + let stored = compress_blosc(&data, "lz4", 0, BLOSC_NOSHUFFLE as i32); + assert!(blosc_header_is_sane(&stored), "clevel 0"); + assert_ne!(stored[2] & BLOSC_MEMCPYED, 0, "clevel 0 should be memcpy'd"); + } + + #[test] + fn lz4_uses_numcodecs_framing() { + // 4-byte little-endian original size, then a raw LZ4 block. Changing this + // silently would break interchange with Python numcodecs. + let data = sample(); + let compressed = lz4_compress(&data, 1); + assert!(compressed.len() > 4); + assert_eq!(&compressed[..4], &(data.len() as u32).to_le_bytes()); + + let block = lz4_flex::block::decompress(&compressed[4..], data.len()).unwrap(); + assert_eq!(block, data); + } + + #[test] + fn lz4_round_trips() { + let data = sample(); + assert_eq!(lz4_decompress(&lz4_compress(&data, 1)), data); + } + + #[test] + fn lz4_round_trips_an_empty_buffer() { + let compressed = lz4_compress(&[], 1); + assert_eq!(&compressed[..4], &0u32.to_le_bytes()); + assert!(lz4_decompress(&compressed).is_empty()); + } + + #[test] + fn lz4_decompress_returns_empty_on_garbage() { + assert!(lz4_decompress(&[]).is_empty(), "no header"); + assert!(lz4_decompress(&[0, 0, 0]).is_empty(), "truncated header"); + // Well-formed header, unreadable block. + let mut bad = 32u32.to_le_bytes().to_vec(); + bad.extend_from_slice(&[0xff; 16]); + assert!(lz4_decompress(&bad).is_empty(), "corrupt block"); + } + + #[test] + fn lz4_rejects_an_implausible_declared_size() { + // A declared size is a request to zero-fill that many bytes before the + // block is even read, so it has to be bounded by what the block could + // possibly expand to. 4 GB from an 8-byte chunk is an OOM, not an error. + assert!( + !lz4_declared_size_is_plausible(u32::MAX as usize, 8), + "u32::MAX" + ); + assert!( + !lz4_declared_size_is_plausible(MAX_DECODED_SIZE + 1, usize::MAX), + "over cap" + ); + assert!(!lz4_declared_size_is_plausible(16 << 20, 16), "past 255:1"); + + let mut hostile = u32::MAX.to_le_bytes().to_vec(); + hostile.extend_from_slice(&[0u8; 8]); + assert!(lz4_decompress(&hostile).is_empty()); + } + + #[test] + fn lz4_accepts_the_maximum_real_expansion() { + // The guard must not reject genuinely well-compressed data. A constant + // buffer sits right on the format's 255:1 ceiling, so if anything real + // trips this bound, it is this. + let data = vec![0u8; 1 << 20]; + let compressed = lz4_compress(&data, 1); + let ratio = data.len() / (compressed.len() - 4); + assert!(ratio >= 250, "expected a near-ceiling ratio, got {ratio}:1"); + assert!(lz4_declared_size_is_plausible( + data.len(), + compressed.len() - 4 + )); + assert_eq!(lz4_decompress(&compressed), data); + } + + #[test] + fn zstd_emits_a_standard_frame() { + let data = sample(); + let compressed = zstd_compress(&data, 1); + assert!(compressed.len() > 4); + assert_eq!( + u32::from_le_bytes([compressed[0], compressed[1], compressed[2], compressed[3]]), + 0xFD2FB528, + "zstd frame magic" + ); + } + + #[test] + fn zstd_round_trips_across_levels() { + let data = sample(); + for level in [1, 10, 22] { + let compressed = zstd_compress(&data, level); + assert!(!compressed.is_empty(), "level {level} produced no output"); + assert_eq!(zstd_decompress(&compressed), data, "level {level}"); + } + } + + #[test] + fn zstd_round_trips_highly_compressible_data() { + // Guards the `data.len() * 4` capacity fallback in zstd_decompress: this + // buffer compresses far past 4:1, so decoding only fits if the frame's + // declared content size is used instead. + let data = vec![7u8; 1 << 16]; + assert_eq!(zstd_decompress(&zstd_compress(&data, 1)), data); + } + + #[test] + fn zstd_decompress_returns_empty_on_garbage() { + assert!(zstd_decompress(&[]).is_empty()); + assert!(zstd_decompress(&[0xff; 64]).is_empty()); + } + + #[test] + fn zstd_rejects_an_implausible_declared_size() { + assert!( + !zstd_declared_size_is_plausible(usize::MAX, 64), + "usize::MAX" + ); + assert!( + !zstd_declared_size_is_plausible(MAX_DECODED_SIZE + 1, usize::MAX), + "over cap" + ); + assert!( + !zstd_declared_size_is_plausible(1 << 30, 16), + "past 32768:1" + ); + + // A frame header is enough to trigger the allocation; the block never + // has to be readable, so the guard has to run before decompression. + let mut hostile = frame_header(0xE0, &[0u8; 8]); + hostile[5..13].copy_from_slice(&u64::MAX.to_le_bytes()); + assert!(zstd_decompress(&hostile).is_empty()); + } + + #[test] + fn zstd_accepts_the_maximum_real_expansion() { + // 16 MB of one byte lands near the format's 32768:1 RLE ceiling, so this + // is the shape most likely to be rejected by an over-tight bound. + let data = vec![0u8; 16 << 20]; + let compressed = zstd_compress(&data, 1); + let ratio = data.len() / compressed.len(); + assert!( + ratio >= 30_000, + "expected a near-ceiling ratio, got {ratio}:1" + ); + assert!(zstd_declared_size_is_plausible( + data.len(), + compressed.len() + )); + assert_eq!(zstd_decompress(&compressed), data); + } + + /// Builds a synthetic zstd frame header: magic, frame descriptor, then the + /// trailing bytes the descriptor says to expect. + fn frame_header(descriptor: u8, rest: &[u8]) -> Vec { + let mut frame = vec![0x28, 0xB5, 0x2F, 0xFD, descriptor]; + frame.extend_from_slice(rest); + frame + } + + #[test] + fn frame_content_size_rejects_non_zstd_input() { + assert_eq!(zstd_frame_content_size(&[]), None, "empty"); + assert_eq!( + zstd_frame_content_size(&[0x28, 0xB5, 0x2F, 0xFD]), + None, + "no descriptor" + ); + assert_eq!(zstd_frame_content_size(&[0u8; 16]), None, "wrong magic"); + } + + #[test] + fn frame_content_size_reads_single_byte_field() { + // fcs_flag 0 + single_segment: one byte of size, no window descriptor. + assert_eq!( + zstd_frame_content_size(&frame_header(0x20, &[0x42])), + Some(0x42) + ); + // fcs_flag 0 without single_segment means the field is absent entirely. + assert_eq!(zstd_frame_content_size(&frame_header(0x00, &[0x42])), None); + } + + #[test] + fn frame_content_size_reads_two_byte_field() { + // fcs_flag 1: u16 biased by 256, and a window descriptor byte to skip. + let frame = frame_header(0x40, &[0x00, 0x10, 0x00]); + assert_eq!(zstd_frame_content_size(&frame), Some(16 + 256)); + } + + #[test] + fn frame_content_size_skips_the_dictionary_id() { + // fcs_flag 2 + single_segment + 1-byte dict id. + let frame = frame_header(0xA1, &[0x07, 0x00, 0x10, 0x00, 0x00]); + assert_eq!(zstd_frame_content_size(&frame), Some(0x1000)); + // fcs_flag 3 + single_segment + 4-byte dict id. + let mut rest = vec![0xDE, 0xAD, 0xBE, 0xEF]; + rest.extend_from_slice(&1_000_000u64.to_le_bytes()); + assert_eq!( + zstd_frame_content_size(&frame_header(0xE3, &rest)), + Some(1_000_000) + ); + } + + #[test] + fn frame_content_size_does_not_wrap_a_64_bit_field() { + // `usize` is 32-bit on wasm32, where an `as usize` cast would turn a + // declared size past 4 GiB into a small, plausible-looking one. + let frame = frame_header(0xE0, &u64::MAX.to_le_bytes()); + assert_eq!( + zstd_frame_content_size(&frame), + usize::try_from(u64::MAX).ok() + ); + } + + #[test] + fn frame_content_size_rejects_a_truncated_field() { + assert_eq!( + zstd_frame_content_size(&frame_header(0x20, &[])), + None, + "1-byte" + ); + assert_eq!( + zstd_frame_content_size(&frame_header(0x40, &[0x00, 0x10])), + None, + "2-byte" + ); + assert_eq!( + zstd_frame_content_size(&frame_header(0x80, &[0x00, 0x10])), + None, + "4-byte" + ); + assert_eq!( + zstd_frame_content_size(&frame_header(0xE0, &[0x00; 4])), + None, + "8-byte" + ); + } + + #[test] + fn frame_content_size_matches_a_real_frame() { + let data = sample(); + let compressed = zstd_compress(&data, 1); + assert_eq!(zstd_frame_content_size(&compressed), Some(data.len())); + } + + #[test] + fn blosc_declared_size_separates_an_empty_frame_from_a_bad_one() { + // The empty frame is a real frame: 32 bytes of header declaring nothing. + // Without this distinction src/blosc.ts reports it as a decode failure. + let empty = compress_blosc(&[], "lz4", 5, BLOSC_SHUFFLE as i32); + assert!(!empty.is_empty(), "empty input still yields a frame"); + assert_eq!(blosc_declared_size(&empty), Some(0)); + assert!(blosc_decompress(&empty).is_empty()); + + let data = sample(); + let compressed = compress_blosc(&data, "lz4", 5, BLOSC_SHUFFLE as i32); + assert_eq!(blosc_declared_size(&compressed), Some(data.len())); + + // Garbage must not read as "a frame holding zero bytes" — which is + // exactly what blosc_cbuffer_sizes reports for a zeroed buffer, and why + // the raw sizes cannot be used as the empty-vs-failed signal. + assert_eq!(blusc_cbuffer_sizes(&[0u8; 64]).0, 0, "the trap"); + assert_eq!(blosc_declared_size(&[0u8; 64]), None); + assert_eq!(blosc_declared_size(&[0xff; 64]), None); + assert_eq!(blosc_declared_size(&[]), None); + assert_eq!( + blosc_declared_size(&compressed[..compressed.len() / 2]), + None, + "truncated" + ); + } + + #[test] + fn declared_content_size_is_visible_to_the_wrapper() { + // src/zstd.ts needs this to tell a frame that holds nothing from one that + // failed to decode — both come back as an empty Vec. + let data = sample(); + assert_eq!( + zstd_declared_content_size(&zstd_compress(&data, 1)), + Some(data.len()) + ); + assert_eq!(zstd_declared_content_size(&zstd_compress(&[], 1)), Some(0)); + assert_eq!(zstd_declared_content_size(&[0xff; 64]), None, "not a frame"); + } + + #[test] + fn lz4_rejects_a_block_that_stops_short_of_its_declared_size() { + // Decoding into a preallocated buffer means a block that ends early + // leaves the tail zeroed. Handing that back would be silent corruption, + // so a short decode is a failure like any other. + let data = sample(); + let compressed = lz4_compress(&data, 1); + let truncated = &compressed[..compressed.len() - 8]; + assert!(lz4_decompress(truncated).is_empty()); + } + + #[test] + fn try_zeroed_reports_a_reservation_it_cannot_make() { + // The plausibility guards bound what a header may claim; this bounds what + // the allocator can actually supply. Without it the shortfall is an abort, + // which on wasm32 takes the whole instance down. + assert_eq!(try_zeroed(0).map(|b| b.len()), Some(0)); + assert_eq!(try_zeroed(64).map(|b| b.len()), Some(64)); + assert!(try_zeroed(usize::MAX).is_none()); + } +} diff --git a/docs/assets/rumcodecs-logo-dark.svg b/docs/assets/rumcodecs-logo-dark.svg new file mode 100644 index 0000000..aacefee --- /dev/null +++ b/docs/assets/rumcodecs-logo-dark.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/rumcodecs-logo.svg b/docs/assets/rumcodecs-logo.svg new file mode 100644 index 0000000..2e3b303 --- /dev/null +++ b/docs/assets/rumcodecs-logo.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/src/blosc.ts b/src/blosc.ts index 2e50fba..a1593a7 100644 --- a/src/blosc.ts +++ b/src/blosc.ts @@ -83,7 +83,13 @@ const Blosc: CodecConstructor = class Blosc implements Codec { async decode(data: Uint8Array, out?: Uint8Array): Promise { const m = await getModule(); const result = m.blosc_decompress(data); - if (result.length === 0) { + // The frame header states how many bytes it holds, so compare against that + // rather than treating any empty result as failure: `encode` of an empty + // buffer produces a valid frame declaring zero bytes, which decodes to + // nothing and is not an error. Checking the full length also catches a + // frame that decoded short of what it promised. + const declared = m.blosc_declared_size(data); + if (declared === undefined || result.length !== declared) { throw new Error("Blosc decompression failed"); } if (out !== undefined) { diff --git a/src/lz4.ts b/src/lz4.ts index 345f7c0..5133e73 100644 --- a/src/lz4.ts +++ b/src/lz4.ts @@ -48,8 +48,20 @@ const LZ4: CodecConstructor = class LZ4 implements Codec { if (data.length > MAX_BUFFER_SIZE) { throw Error(`Codec does not support buffers of > ${MAX_BUFFER_SIZE} bytes.`); } + // The header is the numcodecs framing: a 4-byte LE original size. It gives + // us the exact length a successful decode has to produce, which is what + // separates a genuine empty payload from the empty Vec the WASM side + // returns on failure. Without this check a corrupt chunk decodes to a + // zero-filled `out` and reaches the caller as data. + if (data.length < 4) { + throw new Error("LZ4 decompression failed: input is shorter than the size header"); + } + const declared = new DataView(data.buffer, data.byteOffset, 4).getUint32(0, true); const m = await getModule(); const result = m.lz4_decompress(data); + if (result.length !== declared) { + throw new Error("LZ4 decompression failed"); + } if (out !== undefined) { out.set(result); return out; diff --git a/src/zstd.ts b/src/zstd.ts index d008b1b..0653336 100644 --- a/src/zstd.ts +++ b/src/zstd.ts @@ -50,6 +50,14 @@ const Zstd: CodecConstructor = class Zstd implements Codec { async decode(data: Uint8Array, out?: Uint8Array): Promise { const m = await getModule(); const result = m.zstd_decompress(data); + // An empty result is how the WASM side reports failure, and a frame that + // really does hold nothing looks identical. The frame header settles it: + // only a frame declaring zero bytes may legitimately decode to nothing. + // A frame carrying no declared size at all (`undefined`) cannot be told + // apart, and erring toward an Error beats returning an untouched `out`. + if (result.length === 0 && m.zstd_declared_content_size(data) !== 0) { + throw new Error("Zstd decompression failed"); + } if (out !== undefined) { out.set(result); return out; diff --git a/test/compat.test.ts b/test/compat.test.ts index fccfa44..ae9558a 100644 --- a/test/compat.test.ts +++ b/test/compat.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect } from "vitest"; import { Blosc, GZip, Zlib, LZ4, Zstd } from "../src/index.js"; +import { range } from "./helpers.js"; describe("numcodecs compatibility", () => { it("Blosc has all numcodecs static members", () => { @@ -39,4 +40,130 @@ describe("numcodecs compatibility", () => { expect(Zstd.DEFAULT_CLEVEL).toBe(1); expect(Zstd.MAX_CLEVEL).toBe(22); }); + + it("LZ4 and Zstd fromConfig round-trip their options", () => { + const l = LZ4.fromConfig({ id: "lz4", acceleration: 10 }); + expect((l as any).acceleration).toBe(10); + const z = Zstd.fromConfig({ id: "zstd", level: 9 }); + expect((z as any).level).toBe(9); + }); + + describe("decode(data, out)", () => { + const arr = range(1000, " { + const encoded = await codec.encode(bytes); + const out = new Uint8Array(bytes.length); + const decoded = await codec.decode(encoded, out); + expect(decoded).toBe(out); + expect(Array.from(out)).toEqual(Array.from(bytes)); + }); + } + + // fflate hands back its own buffer; numcodecs.js does the same, so consumers + // that pass `out` to GZip/Zlib must keep using the return value. + for (const [name, codec] of [ + ["GZip", GZip.fromConfig({ id: "gzip" })], + ["Zlib", Zlib.fromConfig({ id: "zlib" })], + ] as const) { + it(`${name} still returns the decoded bytes when given an out buffer`, async () => { + const encoded = await codec.encode(bytes); + const out = new Uint8Array(bytes.length); + const decoded = await codec.decode(encoded, out); + // Asserting the bytes alone would pass under either behavior. Pinning + // the identity is what documents which one callers actually get: the + // divergence from Blosc/LZ4/Zstd above is deliberate parity with + // numcodecs.js, so it should fail here if it ever changes silently. + expect(decoded).not.toBe(out); + expect(Array.from(decoded)).toEqual(Array.from(bytes)); + expect(Array.from(out)).toEqual(Array.from(new Uint8Array(bytes.length))); + }); + } + }); + + // A decoder that reports failure by returning nothing is indistinguishable + // from one that succeeded on an empty payload — and with `out` supplied, the + // caller gets their own untouched buffer back and reads it as data. Corrupt + // chunks have to throw, in both call shapes. + describe("corrupt input", () => { + const corrupt = (encoded: Uint8Array) => { + const bad = new Uint8Array(encoded); + // Damage the payload, not the header: a header the guards reject is the + // easy case. This is the one that reaches the decompressor. + for (let i = Math.floor(bad.length / 2); i < bad.length; i++) { + bad[i] ^= 0xff; + } + return bad; + }; + + const arr = range(1000, " { + const encoded = await codec.encode(bytes); + const truncated = encoded.slice(0, Math.floor(encoded.length / 2)); + await expect(codec.decode(truncated)).rejects.toThrow(); + }); + } + + // Corruption *within* the payload is a different matter, and only LZ4 and + // Zstd are asserted here. A Blosc1 frame carries no checksum over its + // compressed body, so flipped bytes that still parse can decode to the + // declared length and be returned as valid data — see the note below. That + // is a property of the format, not something this layer can detect. + for (const [name, codec] of codecs.filter(([n]) => n !== "Blosc")) { + it(`${name} throws on a corrupt chunk`, async () => { + const bad = corrupt(await codec.encode(bytes)); + await expect(codec.decode(bad)).rejects.toThrow(); + }); + + it(`${name} throws on a corrupt chunk rather than returning an untouched out`, async () => { + const bad = corrupt(await codec.encode(bytes)); + const out = new Uint8Array(bytes.length); + await expect(codec.decode(bad, out)).rejects.toThrow(); + }); + } + + // Pinned so the limitation above is a recorded fact rather than an + // assumption: if blusc ever does start rejecting this, that is a behavior + // change worth noticing here rather than discovering in the field. + it("Blosc cannot detect payload corruption its format does not checksum", async () => { + const codec = Blosc.fromConfig({ id: "blosc", cname: "lz4", clevel: 5, shuffle: 1 }); + const bad = corrupt(await codec.encode(bytes)); + const decoded = await codec.decode(bad).catch(() => null); + if (decoded !== null) { + expect(decoded.length).toBe(bytes.length); + expect(Array.from(decoded)).not.toEqual(Array.from(bytes)); + } + }); + + // The flip side: an empty payload is a legitimate thing to encode, and must + // not be mistaken for the empty buffer a failed decode returns. + for (const [name, codec] of codecs) { + it(`${name} round-trips an empty buffer without reporting failure`, async () => { + const encoded = await codec.encode(new Uint8Array(0)); + const decoded = await codec.decode(encoded); + expect(decoded.length).toBe(0); + }); + } + }); }); diff --git a/test/fixtures/README.md b/test/fixtures/README.md new file mode 100644 index 0000000..c69dbef --- /dev/null +++ b/test/fixtures/README.md @@ -0,0 +1,46 @@ +# numcodecs interop fixtures + +Compressed buffers produced by **Python** `numcodecs`, used by +[`../interop.test.ts`](../interop.test.ts) to prove that rumcodecs decodes what +the reference implementation writes. The round-trip suites cannot show this: +they would keep passing if the encoder and decoder drifted off the numcodecs +format together. + +- `source.u2.bin` — the uncompressed input every fixture decodes back to: + `numpy.arange(256, dtype=".bin` — that input encoded by the codec described under `` in + `fixtures.json`. +- `fixtures.json` — the exact `get_config()` for each codec, plus the + `numcodecs` version that wrote them. The test builds each codec from this + config, the way a Zarr reader builds one from stored chunk metadata. + +Only the decode direction is pinned here. Encoded output is not compared byte +for byte, because a compressor is free to emit different bytes across versions +while staying readable — what has to hold is that both sides read each other. + +## Regenerating + +Requires no local install; `uv` fetches `numcodecs` on demand, pinned to the +version that wrote the checked-in buffers: + +```sh +cd test/fixtures +uv run --with numcodecs==0.16.5 python generate.py +``` + +Regenerate only to add a codec or a new option combination. Refreshing the +existing files against a newer `numcodecs` weakens the test — it would hide a +format break by replacing the fixture that should have caught it. That is why +the version is pinned rather than resolved: unpinned, adding a single fixture +rewrites all of them against whatever release happens to be current. The +generator checks the pin and refuses to run under a different version, so +bumping it takes editing `EXPECTED_NUMCODECS` in +[`generate.py`](./generate.py) and this line together. + +Under that pin the output is byte-for-byte reproducible, so re-running the +generator on an unchanged tree leaves `git status` clean and any diff is a real +one. The single exception is handled in `normalize()`: a gzip member stores the +current time in its header, which otherwise made `gzip.bin` differ on every run +regardless of whether anything meaningful had changed. That field is advisory +and decoders ignore it, so it is zeroed the way `gzip -n` does; the compressed +payload is untouched Python output. diff --git a/test/fixtures/blosc-blosclz-noshuffle.bin b/test/fixtures/blosc-blosclz-noshuffle.bin new file mode 100644 index 0000000..d6835f8 Binary files /dev/null and b/test/fixtures/blosc-blosclz-noshuffle.bin differ diff --git a/test/fixtures/blosc-lz4-shuffle.bin b/test/fixtures/blosc-lz4-shuffle.bin new file mode 100644 index 0000000..6d51abb Binary files /dev/null and b/test/fixtures/blosc-lz4-shuffle.bin differ diff --git a/test/fixtures/blosc-zlib-shuffle.bin b/test/fixtures/blosc-zlib-shuffle.bin new file mode 100644 index 0000000..d5f32cc Binary files /dev/null and b/test/fixtures/blosc-zlib-shuffle.bin differ diff --git a/test/fixtures/blosc-zstd-bitshuffle.bin b/test/fixtures/blosc-zstd-bitshuffle.bin new file mode 100644 index 0000000..5bdcdec Binary files /dev/null and b/test/fixtures/blosc-zstd-bitshuffle.bin differ diff --git a/test/fixtures/fixtures.json b/test/fixtures/fixtures.json new file mode 100644 index 0000000..69588ca --- /dev/null +++ b/test/fixtures/fixtures.json @@ -0,0 +1,79 @@ +{ + "numcodecsVersion": "0.16.5", + "source": { + "dtype": " bytes: + """Strip the one field that makes a fixture differ from run to run. + + A gzip member carries the modification time in header bytes 4..8, so + numcodecs stamps the current clock into every `gzip.bin` it writes. That + made the fixture unreproducible: regenerating it changed the file even + under an identical numcodecs, which buries a real format change in noise + and means the buffer cannot be re-derived to check it. The field is + advisory — decoders ignore it — so zeroing it is what `gzip -n` does, and + leaves the compressed payload untouched. + """ + if not name.startswith("gzip"): + return encoded + assert encoded[:2] == b"\x1f\x8b", f"{name} is not a gzip member" + return encoded[:4] + b"\x00\x00\x00\x00" + encoded[8:] + + +def main() -> None: + if numcodecs.__version__ != EXPECTED_NUMCODECS: + raise SystemExit( + f"numcodecs {numcodecs.__version__} is installed, but the fixtures were " + f"written by {EXPECTED_NUMCODECS}. Run with " + f"`uv run --with numcodecs=={EXPECTED_NUMCODECS} python generate.py`, or " + f"update EXPECTED_NUMCODECS here and in README.md if the bump is intended." + ) + + here = pathlib.Path(__file__).parent + raw = SOURCE.tobytes() + (here / "source.u2.bin").write_bytes(raw) + + manifest = {} + for name, codec in CODECS.items(): + encoded = normalize(name, bytes(codec.encode(raw))) + (here / f"{name}.bin").write_bytes(encoded) + manifest[name] = {"config": codec.get_config(), "bytes": len(encoded)} + print(f"{name:26} {len(encoded):5} bytes") + + (here / "fixtures.json").write_text( + json.dumps( + { + "numcodecsVersion": numcodecs.__version__, + "source": { + "dtype": " + typeof v === "function" && typeof (v as any).codecId === "string", +); describe("rumcodecs exports", () => { it("exports all codec classes", () => { @@ -25,4 +34,47 @@ describe("rumcodecs exports", () => { expect(typeof rumcodecs.LZ4.fromConfig).toBe("function"); expect(typeof rumcodecs.Zstd.fromConfig).toBe("function"); }); + + // Adding a codec means touching src/index.ts, the package.json export map and + // the vite entry list together. Miss either one and the advertised subpath + // resolves to a file that was never built, which only shows up once published. + // + // The three lists are compared as sets, in both directions. Walking only the + // source list catches a codec that was added and never exported, but not the + // reverse: a `./old` export and an `old` vite entry left behind after a codec + // is removed keep advertising a subpath nothing backs, and every one-way + // check still passes. + const sourceIds = CODECS.map((c) => c.codecId).sort(); + + // "." is the package root, not a codec, and `index` is its vite entry. + const exportedIds = Object.keys(pkg.exports) + .filter((k) => k !== ".") + .map((k) => k.replace(/^\.\//, "")) + .sort(); + const viteIds = Object.keys((viteConfig as any).build.lib.entry) + .filter((k) => k !== "index") + .sort(); + + it("the source, package export and vite entry lists name the same codecs", () => { + expect(sourceIds).toEqual(["blosc", "gzip", "lz4", "zlib", "zstd"]); + expect(exportedIds).toEqual(sourceIds); + expect(viteIds).toEqual(sourceIds); + }); + + it("every codec has a matching subpath export", () => { + for (const codecId of sourceIds) { + expect(pkg.exports[`./${codecId}`]).toEqual({ + types: `./dist/${codecId}.d.ts`, + import: `./dist/${codecId}.js`, + }); + } + }); + + it("every subpath export is built by vite", () => { + const entries = (viteConfig as any).build.lib.entry; + for (const codecId of sourceIds) { + expect(entries[codecId]).toMatch(new RegExp(`src/${codecId}\\.ts$`)); + } + expect(entries.index).toMatch(/src\/index\.ts$/); + }); }); diff --git a/test/interop.test.ts b/test/interop.test.ts new file mode 100644 index 0000000..8165e46 --- /dev/null +++ b/test/interop.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import * as rumcodecs from "../src/index.js"; +import type { Codec, CodecConstructor } from "../src/types.js"; + +// Buffers in ./fixtures were written by Python numcodecs, not by this package. +// They are the only thing standing behind the claim that a Zarr chunk written by +// Python decodes here byte for byte — a round-trip suite would keep passing even +// if both directions drifted off the numcodecs format together. +// See ./fixtures/README.md to regenerate them. +const fixture = (name: string) => + new Uint8Array(readFileSync(new URL(`./fixtures/${name}`, import.meta.url))); + +const manifest = JSON.parse( + readFileSync(new URL("./fixtures/fixtures.json", import.meta.url), "utf8"), +) as { + numcodecsVersion: string; + codecs: Record }>; +}; + +// Derived from the package's own exports rather than written out by hand. A +// hand-kept list is only ever as complete as the last person to edit it: add a +// codec, forget both this list and a fixture, and the coverage check below +// compares two lists that are wrong in the same way and passes. Reading the +// exports means a new codec has to be given a fixture or fail here. +const REGISTRY: Record> = Object.fromEntries( + Object.values(rumcodecs) + .filter( + (v): v is CodecConstructor => + typeof v === "function" && typeof (v as any).codecId === "string", + ) + .map((c) => [(c as any).codecId as string, c]), +); + +describe(`decodes Python numcodecs ${manifest.numcodecsVersion} output`, () => { + const expected = Array.from(fixture("source.u2.bin")); + + for (const [name, { config }] of Object.entries(manifest.codecs)) { + it(name, async () => { + const constructor = REGISTRY[config.id]; + expect(constructor, `no codec registered for id '${config.id}'`).toBeDefined(); + + // Built from the config numcodecs itself recorded, the way a Zarr reader + // builds a codec out of the stored chunk metadata. + const codec: Codec = constructor.fromConfig(config as any); + const decoded = await codec.decode(fixture(`${name}.bin`)); + expect(Array.from(decoded)).toEqual(expected); + }); + } + + it("covers every codec this package exports", () => { + const ids = new Set(Object.values(manifest.codecs).map((c) => c.config.id)); + const exported = Object.keys(REGISTRY).sort(); + // Guards the derivation itself: if the export scan ever silently found + // nothing, every other assertion here would vacuously agree with it. + expect(exported).toEqual(["blosc", "gzip", "lz4", "zlib", "zstd"]); + expect([...ids].sort()).toEqual(exported); + }); +});