diff --git a/bare-metal/esp32s3/.cargo/config.toml b/bare-metal/esp32s3/.cargo/config.toml index eee1c44..6d7a0d7 100644 --- a/bare-metal/esp32s3/.cargo/config.toml +++ b/bare-metal/esp32s3/.cargo/config.toml @@ -1,8 +1,14 @@ [target.xtensa-esp32s3-none-elf] -runner = "espflash flash --monitor " +runner = "espflash flash --monitor" rustflags = [ "-C", "link-arg=-nostartfiles", "-C", "link-arg=-Wl,-Tlinkall.x", + # rusty_alloc refuses to build without the first (loudly), and SILENTLY yields + # zero segments on a chip-scale region without the second: SEGMENT_SIZE stays + # at 32 MiB and every allocation fails. Harmless under the esp-alloc arm, + # where nothing reads them. + "--cfg", "ra_single_threaded", + "--cfg", "ra_small_profile", ] [build] diff --git a/bare-metal/esp32s3/Cargo.lock b/bare-metal/esp32s3/Cargo.lock index 5168efe..e162787 100644 --- a/bare-metal/esp32s3/Cargo.lock +++ b/bare-metal/esp32s3/Cargo.lock @@ -1222,6 +1222,25 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rusty_alloc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59f4e6793d58ca76f11c5138939761fbabb6978b1fe404e1c097fa6bd45e706" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "rusty_alloc-api" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66795d559e48d97cc9e5df15471c98ffd6a95870110fec8ca49ad22318e546c0" +dependencies = [ + "rusty_alloc", +] + [[package]] name = "rusty_zstd" version = "0.2.5" @@ -1241,6 +1260,8 @@ dependencies = [ "esp-bootloader-esp-idf", "esp-hal", "esp-println", + "rusty_alloc", + "rusty_alloc-api", "rusty_zstd", ] @@ -1513,7 +1534,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1522,6 +1543,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1531,6 +1561,71 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "1.0.4" diff --git a/bare-metal/esp32s3/Cargo.toml b/bare-metal/esp32s3/Cargo.toml index 9ec2e6a..3a8e422 100644 --- a/bare-metal/esp32s3/Cargo.toml +++ b/bare-metal/esp32s3/Cargo.toml @@ -5,17 +5,40 @@ edition = "2021" # On-board proof for rusty_zstd's bare-metal support: an ESP32-S3 (Xtensa LX7, # 32-bit, NO 64-bit atomics) compressing and decompressing in `no_std + alloc`. -# Deliberately outside the codec repo: it needs the `esp` toolchain, which CI -# does not have, so it is a hand-run instrument, not a gate. +# Deliberately outside the codec repo's workspace: it needs the `esp` toolchain, +# which CI does not have, so it is a hand-run instrument, not a gate. +# +# TWO ARMS, ONE SOURCE. The allocator is a cargo feature, so the only difference +# between the builds is which allocator serves the codec: +# +# cargo build --release # esp-alloc +# cargo build --release --no-default-features -F arm-rusty-alloc +# +# rusty_alloc additionally needs `--cfg ra_single_threaded --cfg ra_small_profile` +# (both are in .cargo/config.toml). Without the second, SEGMENT_SIZE stays at +# 32 MiB, a chip-scale region yields zero segments, and every allocation fails. + +[features] +default = ["arm-esp-alloc"] +arm-esp-alloc = ["dep:esp-alloc"] +arm-rusty-alloc = ["dep:rusty_alloc", "dep:rusty_alloc-api"] [dependencies] esp-hal = { version = "1.2.1", features = ["esp32s3", "unstable"] } -esp-alloc = { version = "0.11.0", features = ["esp32s3"] } -esp-bootloader-esp-idf = { version = "0.6.0", features = ["esp32s3"] } esp-println = { version = "0.18.0", default-features = false, features = ["esp32s3", "jtag-serial"] } esp-backtrace = { version = "0.20.0", features = ["esp32s3", "println", "panic-handler"] } +esp-bootloader-esp-idf = { version = "0.6.0", features = ["esp32s3"] } rusty_zstd = { path = "../../crates/rusty_zstd", default-features = false, features = ["alloc"] } +# `internal-heap-stats` is what makes `max_usage` readable, which is the whole +# point of the footprint arm. +esp-alloc = { version = "0.11.0", features = ["esp32s3", "internal-heap-stats"], optional = true } + +# The core crate carries `prim::fixed::Region` and `good_region_size`; the api +# crate carries the `GlobalAlloc`. Both pinned, both default-features off. +rusty_alloc = { version = "=2.0.5", default-features = false, optional = true } +rusty_alloc-api = { version = "=2.0.5", default-features = false, optional = true } + [profile.release] opt-level = "s" lto = true diff --git a/bare-metal/esp32s3/README.md b/bare-metal/esp32s3/README.md index 321052e..61d9921 100644 --- a/bare-metal/esp32s3/README.md +++ b/bare-metal/esp32s3/README.md @@ -1,26 +1,26 @@ -# rusty_zstd on an ESP32-S3 — the on-board proof +# rusty_zstd on an ESP32-S3 — the on-board proof, and the heap budget "Builds for a bare-metal target" and "runs on the part" are different claims. CI makes the first one every push (`cargo check --target thumbv7em-none-eabihf` and `--target riscv32imac-unknown-none-elf`). This directory is the second one, and it is hand-run: it needs Espressif's Rust fork, which CI does not have. -**Result, 2026-09-09, ESP32-S3 revision v0.2, 8 MB flash, 192 KiB heap:** +**Result, 2026-09-09, ESP32-S3 revision v0.2, 8 MB flash:** ``` === rusty_zstd on ESP32-S3 (xtensa, no_std + alloc) === +allocator esp-alloc 0.11.0 census64::CENSUS_LIVE = false (false is expected here: no 64-bit atomics) source 8260 bytes L1 8260 -> 468 bytes (17.65x) round trip OK L3 8260 -> 426 bytes (19.39x) round trip OK L5 8260 -> 263 bytes (31.41x) round trip OK - RESULT: PASS -- compressed and decompressed on the board ``` Three levels because they are three different match finders: L1 is Fast, L3 is -DFast, L5 is Greedy. Each compressed and then decompressed back to a buffer -compared byte for byte against the source. +DFast, L5 is Greedy. Each was decompressed and compared byte for byte against +the source, on the part. ## Why this part is the right test @@ -30,9 +30,60 @@ the same reason Cortex-M4F and RV32 could not build the crate at all until why `CENSUS_LIVE` prints `false`. **A zero from a counter on this part means "not measurable on this target", never "measured zero".** -The board therefore exercises the exact configuration the fix created, and the -`PASS` line is the evidence that stubbing the instrument did not disturb the -codec. +## The heap budget, measured + +The number a firmware author actually needs. Peak is what the allocator reports +having in use; the floor is the smallest heap that still round-trips, found by +shrinking until it fails. + +| workload | peak heap | smallest heap that works | +|---|---|---| +| L1 only | 112,468 B | **112 KiB** (110 KiB fails) | +| L1 + L3 + L5 | 175,832 B | **176 KiB** (172 KiB fails) | + +**The jump is DFast, not depth.** L1 peaks at 112,468 and L3 at 175,832; L5 adds +nothing over L3. So the second hash table DFast introduces costs ~62 KiB here, +and a firmware that can live with L1 saves that. Compression is barely worse: +17.65x against 19.39x on this corpus. + +Everything is transient: `current` after a round trip is 41,292 bytes, so the +tables are freed and the budget is a peak, not a resident cost. + +## Two allocator arms, one source + +```sh +cargo run --release # esp-alloc +cargo run --release --no-default-features -F arm-rusty-alloc # rusty_alloc +``` + +`rusty_alloc` additionally needs `--cfg ra_single_threaded --cfg +ra_small_profile`, both already in `.cargo/config.toml`. Without the second, +`SEGMENT_SIZE` stays at 32 MiB, a chip-scale region yields zero segments, and +every allocation fails with nothing to tell you why. + +**The rusty_alloc arm does not currently work on this part**, and the reason is +not the codec: + +| | esp-alloc 0.11 | rusty_alloc 2.0.5 | +|---|---|---| +| round trip at 192 KiB | **PASS** | OOM on a 1,536 B allocation | +| round trip at 256 KiB | n/a | OOM on a 656 B allocation | +| 320 KiB region | n/a | does not link, "Main stack is smaller than 8192 bytes" | + +Reduced to a two-line reproduction with no zstd in it: in a **256 KiB region, +rusty_alloc serves exactly one 64 KiB allocation**, and the second fails with +192 KiB of the region still free. Blocks of 32 KiB pack fine (four of them, half +the region). The cliff is at the segment size. + +RAM on this part is a fixed map, so a larger region comes straight out of +`.stack` until the linker refuses, which means there is no region size on an S3 +where this consumer can use it. Written up for the allocator's maintainers in +`rusty_alloc/docs/plans/esp32-large-alloc-ceiling.md`. + +Static cost of the arm, both at the same 192 KiB budget, from the linked ELF: +`.bss` +2,060, `.data` −76, `.stack` −1,996, flash +7,536. `.data + .bss + +.stack` sums to a constant within 12 bytes, so the stack column is the one that +moves — quote the sum, never a `.bss` delta alone. ## Running it @@ -47,12 +98,17 @@ Passes when the last line reads `RESULT: PASS`. It is not in the workspace (`exclude` in the root manifest), so a normal `cargo build` at the repo root never sees it and never needs the Xtensa toolchain. +To re-measure the floor, `heapsweep.py` in the session scratchpad patches +`HEAP_BYTES`, rebuilds, flashes and classifies PASS / OOM per point. Note that +`espflash --monitor` never exits: the timeout is the stop signal, and the log is +complete by then. Redirect it to a FILE — a `| tail` loses everything when the +timeout kills the pipe. + ## What it does not claim - **No timing.** There is no cycle count here, so nothing in this directory is a - performance claim. It answers "does it work", not "how fast". -- **Heap, not measured to a floor.** 192 KiB was enough for an 8 KiB source at - levels 1 to 5. The crate sizes its tables from the SOURCE length, so a bigger - input needs more; finding the minimum heap per level is a separate exercise. + performance claim. It answers "does it work" and "how much RAM", not "how + fast". Buffer placement alone moves compute kernels on this part by up to 20%, + so a throughput comparison across allocator arms would measure placement. - **One part.** The S3 is Xtensa. The two targets CI compiles are ARM and RISC-V, and neither has been run on silicon here. diff --git a/bare-metal/esp32s3/src/main.rs b/bare-metal/esp32s3/src/main.rs index 5756e63..455eba1 100644 --- a/bare-metal/esp32s3/src/main.rs +++ b/bare-metal/esp32s3/src/main.rs @@ -1,12 +1,23 @@ -//! rusty_zstd on an ESP32-S3, `no_std + alloc`. +//! rusty_zstd on an ESP32-S3, `no_std + alloc`, with the allocator as an arm. //! -//! The point of this firmware is one line of output: a round trip that -//! happened on a part with no 64-bit atomics. Until this ran, "bare metal" -//! meant "the compiler accepted it". +//! The point of this firmware is one line of output: a round trip that happened +//! on a part with no 64-bit atomics. Until this ran, "bare metal" meant "the +//! compiler accepted it". //! //! Xtensa LX7 is 32-bit, so `core::sync::atomic::AtomicU64` does not exist and //! every census counter in the codec is the `census64` stub. `CENSUS_LIVE` is //! printed so the log says which build this was. +//! +//! TWO ARMS, one source, selected by a cargo feature: +//! +//! - `arm-esp-alloc` (default): a linked-list heap. Floor is bytes live plus a +//! header per allocation. +//! - `arm-rusty-alloc`: the house allocator, a size-class page allocator. Floor +//! is (classes touched) x (page size), independent of bytes requested, so it +//! is roughly fixed and amortises as the working set grows. +//! +//! `HEAP_BYTES` is patched by `heapsweep.py` to find the smallest heap that +//! still round-trips, which is the number a firmware author actually budgets. #![no_std] #![no_main] @@ -15,10 +26,43 @@ extern crate alloc; use alloc::vec::Vec; use esp_backtrace as _; +use esp_println::println; // The image header espflash refuses to flash without. esp_bootloader_esp_idf::esp_app_desc!(); -use esp_println::println; + +/// Patched by the sweep script. Both arms are given the same budget. +/// `LEVELS` is patched too, so each level's own heap requirement is readable +/// rather than only the maximum across all three. +const HEAP_BYTES: usize = 192 * 1024; + +/// Which levels this build exercises; patched by the sweep script. +const LEVELS: &[i32] = &[1, 3, 5]; + +#[cfg(all(feature = "arm-esp-alloc", feature = "arm-rusty-alloc"))] +compile_error!("pick ONE allocator arm: two global allocators cannot link"); +#[cfg(not(any(feature = "arm-esp-alloc", feature = "arm-rusty-alloc")))] +compile_error!("pick an allocator arm: arm-esp-alloc or arm-rusty-alloc"); + +// ---------------------------------------------------------------- rusty_alloc +#[cfg(feature = "arm-rusty-alloc")] +use rusty_alloc::prim::fixed::{good_region_size, Region}; + +/// The region is handed to the backend once. Sized with `good_region_size` so +/// it is whole segments and unpadded: the README warns that a round number +/// strands the remainder, and that wrapping it in an aligned container of your +/// own costs stack. +#[cfg(feature = "arm-rusty-alloc")] +static REGION: Region<{ good_region_size(HEAP_BYTES) }> = Region::new(); + +#[cfg(feature = "arm-rusty-alloc")] +#[global_allocator] +static ALLOC: rusty_alloc_api::RustyAlloc = rusty_alloc_api::RustyAlloc; + +#[cfg(feature = "arm-rusty-alloc")] +const ARM: &str = "rusty_alloc 2.0.5"; +#[cfg(feature = "arm-esp-alloc")] +const ARM: &str = "esp-alloc 0.11.0"; /// Compressible but not trivial: repeated phrases with varying joins, so the /// match finder actually walks chains instead of emitting one long RLE. @@ -43,13 +87,54 @@ fn corpus() -> Vec { v } +/// Peak bytes the allocator reports having in use, where it can report one. +fn peak_report() { + #[cfg(feature = "arm-esp-alloc")] + { + let s = esp_alloc::HEAP.stats(); + println!( + "heap size {} current {} PEAK {}", + s.size, s.current_usage, s.max_usage + ); + } + #[cfg(feature = "arm-rusty-alloc")] + { + // (elapsed_ms, user_ms, system_ms, current_rss, peak_rss, + // current_commit, peak_commit, page_faults); best effort per platform. + let (_, _, _, cur, peak, ccommit, pcommit, _) = rusty_alloc::stats::process_info(); + println!( + "heap current_rss {cur} PEAK_rss {peak} commit {ccommit} peak_commit {pcommit}" + ); + } +} + #[esp_hal::main] fn main() -> ! { let _p = esp_hal::init(esp_hal::Config::default()); - esp_alloc::heap_allocator!(size: 192 * 1024); + + #[cfg(feature = "arm-esp-alloc")] + esp_alloc::heap_allocator!(size: HEAP_BYTES); + + #[cfg(feature = "arm-rusty-alloc")] + let usable = match REGION.give() { + Ok(u) => u, + Err(e) => { + println!("REGION.give FAILED: {e:#x}"); + loop { + core::hint::spin_loop() + } + } + }; println!(); println!("=== rusty_zstd on ESP32-S3 (xtensa, no_std + alloc) ==="); + println!("allocator {ARM}"); + println!("heap budget {HEAP_BYTES} bytes"); + #[cfg(feature = "arm-rusty-alloc")] + println!( + "region {} bytes declared, {usable} usable", + REGION.len() + ); println!( "census64::CENSUS_LIVE = {} (false is expected here: no 64-bit atomics)", rusty_zstd::census64::CENSUS_LIVE @@ -59,14 +144,27 @@ fn main() -> ! { println!("source {} bytes", src.len()); let mut all_ok = true; - for level in [1i32, 3, 5] { + for level in LEVELS { + let level = *level; match rusty_zstd::compress(&src, level) { Ok(z) => { let ratio = src.len() as f32 / z.len() as f32; + #[cfg(feature = "arm-esp-alloc")] + println!( + " after compress peak {} current {}", + esp_alloc::HEAP.stats().max_usage, + esp_alloc::HEAP.stats().current_usage + ); match rusty_zstd::decompress(&z) { Ok(back) => { let ok = back.len() == src.len() && back == src; all_ok &= ok; + #[cfg(feature = "arm-esp-alloc")] + println!( + " after decompress peak {} current {}", + esp_alloc::HEAP.stats().max_usage, + esp_alloc::HEAP.stats().current_usage + ); println!( "L{level} {} -> {} bytes ({:.2}x) round trip {}", src.len(), @@ -88,6 +186,8 @@ fn main() -> ! { } } + println!(); + peak_report(); println!(); if all_ok { println!("RESULT: PASS -- compressed and decompressed on the board"); @@ -96,6 +196,6 @@ fn main() -> ! { } loop { - core::hint::spin_loop(); + core::hint::spin_loop() } }