diff --git a/.github/workflows/bl616-tangcore-build.yml b/.github/workflows/bl616-tangcore-build.yml index d4ff7311..efe7442c 100644 --- a/.github/workflows/bl616-tangcore-build.yml +++ b/.github/workflows/bl616-tangcore-build.yml @@ -47,6 +47,10 @@ env: TOOLCHAIN_SHA: c4afe91cbd01bf7dce525e0d23b4219c8691e8f0 BOUFFALO_SDK_SHA: 7f44f9ea6b4ccf96db8c5236c8024b68e2a76df7 FIRMWARE_BL616_SHA: a5a6ea1cf7c81f32c1c3f0f91ea9d913be5ba078 + # micro-flac is fetched, never vendored: it is Apache-2.0 and libchdr stays + # BSD-3 for its other consumers. Pinned, because the backend compiles + # against its internals. + MICROFLAC_SHA: ffbe8a9ba5e78c16e535a4695a8b2418b0c091ee jobs: bl616-tangcore-build: @@ -116,6 +120,11 @@ jobs: mkdir -p "$F/chd" cp contrib/tangcore-bl616/chd/chd_fatfs.h contrib/tangcore-bl616/chd/chd_fatfs.c "$F/chd/" + # micro-flac is the default backend for this integration; fetched at a + # pinned commit, never vendored into libchdr's own tree. + git clone -q https://github.com/esphome-libs/micro-flac.git "$F/thirdparty/micro-flac" + git -C "$F/thirdparty/micro-flac" checkout -q "$MICROFLAC_SHA" + - name: Build (TANG_BOARD=console60k) run: | export PATH="${{ github.workspace }}/tangcore-work/toolchain_gcc_t-head_linux/bin:$PATH" @@ -124,3 +133,19 @@ jobs: make 2>&1 | tee build.log echo "---- memory region summary ----" grep -A 7 "Memory region" build.log + + # The dr_flac fallback has to keep working for anyone Apache-2.0 does not + # suit, and nothing else here would notice if the revert patch rotted. + - name: Build again with the dr_flac fallback + run: | + set -euo pipefail + export PATH="${{ github.workspace }}/tangcore-work/toolchain_gcc_t-head_linux/bin:$PATH" + F=tangcore-work/firmware-bl616 + + git -C "$F" apply "${{ github.workspace }}/contrib/tangcore-bl616/patches/firmware-bl616-libchdr-drflac.patch" + + cd "$F" + rm -rf build + make 2>&1 | tee build-drflac.log + echo "---- memory region summary (dr_flac) ----" + grep -A 7 "Memory region" build-drflac.log diff --git a/contrib/esp32p4/README.md b/contrib/esp32p4/README.md index 99033c9e..4f71fae1 100644 --- a/contrib/esp32p4/README.md +++ b/contrib/esp32p4/README.md @@ -36,6 +36,43 @@ directory provides instead: `esp_vfs_fat_sdmmc_mount()` for SD/MMC), then call `chd_esp_vfs_open("/sdcard/game.chd", &chd)`. +## Options worth setting, and what they measured + +`chd_esp_vfs_open()` already calls `chd_set_cache_budget()` with 32KB. +Compressed hunks are a few KB and laid out sequentially, so one larger read +serves many and the per-read fixed cost is paid far less often: **1.11x on an +ESP32-S3** over SPI, **1.05-1.12x on an RP2350** with 32KB the knee there. The +P4's own number has not been taken. Set `CHD_ESP_VFS_CACHE_BUDGET` to 0 to turn +it off. + +**micro-flac is the default backend here**, unlike libchdr itself. The +component fetches it at a pinned commit when `CHDR_MICROFLAC_SOURCE_DIR` is not +given - point that at your own checkout, or at a managed component under +`managed_components/esphome__micro-flac`, if you would rather not fetch. +`CHDR_FLAC_BACKEND=drflac` goes back to dr_flac. + +Output is byte-identical either way. On an ESP32-S3 with I/O excluded: +**1.198x** on a CD-FLAC hunk, **1.233x** on raw FLAC. Across eleven real discs: +**1.072x** overall, 1.21x where the image is FLAC-heavy, **0.988x** on one +profile where FLAC barely appears. Peak heap is lower than dr_flac's on most +images. + +libchdr's own default stays dr_flac: a desktop consumer vendoring `src/` must +not have to fetch anything, and micro-flac is C++ and Apache-2.0. An MCU +integrator is already cloning an SDK and a toolchain, so one more pinned +checkout costs nothing - hence the different default on this side. + +An earlier revision of this file quoted 1.41x for the P4. That predates the +STREAMINFO block-size fix, which removed an oversized decoded-sample buffer +from dr_flac and took most of micro-flac's lead with it. **The P4 has not been +re-measured since**; treat the S3 numbers above as the estimate until it is. + +Two things not to try, both with their numbers in +`../../docs/perf-esp32p4-findings.md`: **`-Os`** is 1.12x *slower* than the +`PERF` (-O2) default on an S3, and **`Z7_LZMA_PROB32`** costs 15,980 bytes per +LZMA instance for a speedup the LZMA SDK only claims for "some CPUs" and that +was never measured on any target. + ## CI Two workflows: diff --git a/contrib/esp32p4/chd/chd_esp_vfs.c b/contrib/esp32p4/chd/chd_esp_vfs.c index f65dc40f..814becc1 100644 --- a/contrib/esp32p4/chd/chd_esp_vfs.c +++ b/contrib/esp32p4/chd/chd_esp_vfs.c @@ -67,7 +67,22 @@ chd_error chd_esp_vfs_open(const char *path, chd_file **chd) err = chd_open_core_file_callbacks(&chd_esp_vfs_callbacks, f, CHD_OPEN_READ, NULL, chd); if (err != CHDERR_NONE) + { fclose(f); + return err; + } - return err; + /* Compressed hunks are small - a few KB - and laid out strictly + * sequentially, so one larger read serves many of them and the fixed cost + * of each read (VFS dispatch, FATFS bookkeeping, controller command setup, + * DMA, interrupt) is paid far less often. Measured 1.11x on an ESP32-S3 + * reading over SPI, and 1.05-1.12x on an RP2350 with 32KB the knee there. + * + * A ceiling, not an allocation request: an image whose hunks exceed it + * leaves caching off rather than over-allocating, and failing to set it is + * not fatal to the open. */ + if (CHD_ESP_VFS_CACHE_BUDGET != 0) + (void)chd_set_cache_budget(*chd, CHD_ESP_VFS_CACHE_BUDGET); + + return CHDERR_NONE; } diff --git a/contrib/esp32p4/chd/chd_esp_vfs.h b/contrib/esp32p4/chd/chd_esp_vfs.h index 52911afa..bf38dfda 100644 --- a/contrib/esp32p4/chd/chd_esp_vfs.h +++ b/contrib/esp32p4/chd/chd_esp_vfs.h @@ -32,6 +32,13 @@ extern const core_file_callbacks chd_esp_vfs_callbacks; /* Opens path via fopen("rb"), then hands it to libchdr as the argp for * chd_esp_vfs_callbacks. On any failure, the FILE* is closed if it was * opened and *chd is left untouched. */ +/* Read-ahead window handed to chd_set_cache_budget() on every open. Set to 0 + * to leave it off. 32KB is the knee measured on the boards where this was + * tried; the ESP32-P4's own number has not been taken. */ +#ifndef CHD_ESP_VFS_CACHE_BUDGET +#define CHD_ESP_VFS_CACHE_BUDGET (32 * 1024) +#endif + chd_error chd_esp_vfs_open(const char *path, chd_file **chd); #ifdef __cplusplus diff --git a/contrib/esp32p4/idf-benchmark/components/libchdr/CMakeLists.txt b/contrib/esp32p4/idf-benchmark/components/libchdr/CMakeLists.txt index f44794c2..bb075ba4 100644 --- a/contrib/esp32p4/idf-benchmark/components/libchdr/CMakeLists.txt +++ b/contrib/esp32p4/idf-benchmark/components/libchdr/CMakeLists.txt @@ -20,18 +20,51 @@ endif() set(LIBCHDR_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../../../..") -# FLAC backend, mirroring libchdr's own CHDR_FLAC_BACKEND. dr_flac is the -# default. microflac is measurably faster here - 1.46x on the S3 and 1.41x on -# the P4 decoding CD audio - but it is C++ and Apache-2.0, so it is not -# vendored: point CHDR_MICROFLAC_SOURCE_DIR at a checkout, or add -# esphome/micro-flac to the project's idf_component.yml and point this at +# FLAC backend. micro-flac is the default here, unlike libchdr itself: on an +# ESP32-S3 with I/O excluded it is 1.198x on a CD-FLAC hunk and 1.233x on raw +# FLAC, its peak heap is lower on most images, and an MCU integrator already +# clones an SDK and a toolchain, so one more pinned checkout costs nothing. +# libchdr's own default stays dr_flac: a desktop consumer vendoring src/ must +# not have to fetch anything. +# +# The ESP32-P4 has not been re-measured since the STREAMINFO block-size fix, +# which took an oversized buffer out of dr_flac and most of micro-flac's lead +# with it - the older 1.41x quoted here is not valid. +# +# Set CHDR_FLAC_BACKEND=drflac to go back, e.g. if Apache-2.0 does not suit +# (micro-flac is Apache-2.0; libchdr stays BSD-3 because this is fetched, never +# vendored). Or point CHDR_MICROFLAC_SOURCE_DIR at your own checkout, including +# a managed component at # ${CMAKE_BINARY_DIR}/../managed_components/esphome__micro-flac. if(NOT DEFINED CHDR_FLAC_BACKEND) + set(CHDR_FLAC_BACKEND microflac) +endif() + +# ESP-IDF runs every component CMakeLists in script mode first, just to collect +# REQUIRES, and FetchContent is not scriptable there (define_property fails). +# The backend choice does not affect requirements, so that pass takes the +# in-tree source and the fetch happens once, in the real configure. +if(CMAKE_SCRIPT_MODE_FILE) set(CHDR_FLAC_BACKEND drflac) endif() +set(CHDR_MICROFLAC_GIT_TAG "ffbe8a9ba5e78c16e535a4695a8b2418b0c091ee" + CACHE STRING "micro-flac commit to fetch when no local checkout is given") if(CHDR_FLAC_BACKEND STREQUAL "microflac") if(NOT CHDR_MICROFLAC_SOURCE_DIR) - message(FATAL_ERROR "CHDR_FLAC_BACKEND=microflac needs CHDR_MICROFLAC_SOURCE_DIR") + # Pinned SHA, because the backend compiles against micro-flac's + # internals. Fetched at configure time rather than through + # idf_component.yml so that this component stands alone. + include(FetchContent) + # SOURCE_SUBDIR names a directory that does not exist on purpose: + # it downloads the sources without running micro-flac's own + # CMakeLists, whose targets would otherwise be configured for the + # host and joined to this cross build. + FetchContent_Declare(microflac + GIT_REPOSITORY https://github.com/esphome-libs/micro-flac.git + GIT_TAG ${CHDR_MICROFLAC_GIT_TAG} + SOURCE_SUBDIR chdr-sources-only) + FetchContent_MakeAvailable(microflac) + set(CHDR_MICROFLAC_SOURCE_DIR "${microflac_SOURCE_DIR}") endif() # micro-flac enables its Xtensa LPC assembly whenever the core has the # required features, so the .S files must be compiled in on esp32/esp32s3 diff --git a/contrib/rp2350/CMakeLists.txt b/contrib/rp2350/CMakeLists.txt index b4f6b871..8a9bff55 100644 --- a/contrib/rp2350/CMakeLists.txt +++ b/contrib/rp2350/CMakeLists.txt @@ -35,22 +35,40 @@ if(NOT DEFINED SD_BAUD_HZ) set(SD_BAUD_HZ 25000000) endif() -# The LZMA SDK ships decoder assembly for x86-64 and AArch64 only; Cortex-M33 -# is ARMv8-M Thumb, so neither applies. Z7_LZMA_PROB32 is the one portable -# knob: it widens the probability model from UInt16 to UInt32, which the SDK -# notes "can increase the speed on some CPUs" at double the probs memory -# (+15,980 bytes per LZMA instance at CHD's lc=3/lp=0). Off by default. -option(CHDR_LZMA_PROB32 "Use 32-bit LZMA probability model" OFF) -# FLAC backend, mirroring libchdr's own CHDR_FLAC_BACKEND option. dr_flac is -# the default; microflac needs CHDR_MICROFLAC_SOURCE_DIR pointing at a checkout. +# FLAC backend. micro-flac is the default here, unlike libchdr itself: it is +# measurably faster on every board this was tried on (1.198x on a CD-FLAC hunk +# with I/O excluded on an ESP32-S3, 1.12-1.14x on two real discs here), its +# peak heap is lower on most images, and its worst-case largest-free-block is +# better - and an MCU integrator is already cloning an SDK and a toolchain, so +# one more pinned checkout costs them nothing. libchdr's own default stays +# dr_flac, because a desktop consumer vendoring src/ must not have to fetch +# anything. +# +# Set CHDR_FLAC_BACKEND=drflac to go back, e.g. if Apache-2.0 does not suit +# (micro-flac is Apache-2.0; libchdr stays BSD-3 because this is fetched, never +# vendored). if(NOT DEFINED CHDR_FLAC_BACKEND) - set(CHDR_FLAC_BACKEND drflac) + set(CHDR_FLAC_BACKEND microflac) endif() +set(CHDR_MICROFLAC_GIT_TAG "ffbe8a9ba5e78c16e535a4695a8b2418b0c091ee" + CACHE STRING "micro-flac commit to fetch when no local checkout is given") if(CHDR_FLAC_BACKEND STREQUAL "microflac") enable_language(CXX) if(NOT CHDR_MICROFLAC_SOURCE_DIR) - message(FATAL_ERROR "CHDR_FLAC_BACKEND=microflac needs CHDR_MICROFLAC_SOURCE_DIR") + # Same shape as libchdr's own root CMakeLists: pinned SHA, because the + # backend compiles against micro-flac's internals. + include(FetchContent) + # SOURCE_SUBDIR names a directory that does not exist on purpose: + # it downloads the sources without running micro-flac's own + # CMakeLists, whose targets would otherwise be configured for the + # host and joined to this cross build. + FetchContent_Declare(microflac + GIT_REPOSITORY https://github.com/esphome-libs/micro-flac.git + GIT_TAG ${CHDR_MICROFLAC_GIT_TAG} + SOURCE_SUBDIR chdr-sources-only) + FetchContent_MakeAvailable(microflac) + set(CHDR_MICROFLAC_SOURCE_DIR "${microflac_SOURCE_DIR}") endif() set(CHDR_FLAC_SRC ${LIBCHDR_ROOT}/src/libchdr_flac_microflac.cpp @@ -120,11 +138,6 @@ target_compile_definitions(rp2350-bench PRIVATE MINIZ_NO_TIME ) -if(CHDR_LZMA_PROB32) - set_source_files_properties(${LIBCHDR_ROOT}/deps/lzma-26.02/src/LzmaDec.c - PROPERTIES COMPILE_DEFINITIONS Z7_LZMA_PROB32) -endif() - target_compile_options(rp2350-bench PRIVATE -O3) target_link_libraries(rp2350-bench pico_stdlib diff --git a/contrib/rp2350/README.md b/contrib/rp2350/README.md index 5a4b9d6e..f50165bf 100644 --- a/contrib/rp2350/README.md +++ b/contrib/rp2350/README.md @@ -78,6 +78,22 @@ corpus. No RP2350 hardware in CI, so this can't prove `chd_open()`/ neither build regresses correctness or blows past budget on what CI *can* see. +## FLAC backend: micro-flac by default + +The benchmark here decodes FLAC through +[micro-flac](https://github.com/esphome-libs/micro-flac) rather than dr_flac, +fetched at a pinned commit unless `CHDR_MICROFLAC_SOURCE_DIR` points at a local +one. `-DCHDR_FLAC_BACKEND=drflac` goes back. + +Output is byte-identical. Measured on this board across seven real discs, +two runs each with a spread under 0.17%: **1.032x** overall, **1.12-1.14x** on +two of them, and a worst-case largest-free-block of 51 KB against dr_flac's +47 KB. On an ESP32-S3 with I/O excluded it is 1.198x on a CD-FLAC hunk. + +libchdr's own default stays dr_flac - a desktop consumer vendoring `src/` +should not have to fetch an Apache-2.0 C++ dependency. An MCU integrator is +already cloning an SDK, so the trade is different here. + ## Real-hardware benchmark (this directory) The CI workflows above run under QEMU and measure RAM only; as their note diff --git a/contrib/tangcore-bl616/README.md b/contrib/tangcore-bl616/README.md index f4482ee6..5cf17b95 100644 --- a/contrib/tangcore-bl616/README.md +++ b/contrib/tangcore-bl616/README.md @@ -93,126 +93,62 @@ cp /path/to/libchdr/contrib/tangcore-bl616/chd/*.{h,c} chd/ make # BL_SDK_BASE defaults to ../bouffalo_sdk, TANG_BOARD defaults to console60k ``` -## Optional: the micro-flac backend - -libchdr can decode FLAC through [micro-flac](https://github.com/esphome-libs/micro-flac) -instead of dr_flac (`CHDR_FLAC_BACKEND=microflac`). Measured on hardware it is -1.46x faster on an ESP32-S3 and 1.41x on an ESP32-P4 with byte-identical -output, and most of that comes from its C rather than its Xtensa assembly, so -RV32 sees it too. It is not wired into the patch above, because the BL616 -integration selects sources with a glob and the backend needs three changes -that a glob cannot express. - -**Licensing.** micro-flac is Apache-2.0, including its `.S` files. That is -permissive and does not relicense libchdr, and TangCore's firmware is already -Apache-2.0 end to end (Bouffalo SDK and firmware-bl616 both), so it adds no -obligation here. Do not vendor it into libchdr's own tree - fetch it, so -libchdr stays BSD-3 for its other consumers. - -**1. Source selection.** `thirdparty/libchdr/src/*.c` globs `libchdr_flac.c`, -which is the dr_flac backend; compiling both is a duplicate-symbol error. List -the sources explicitly, or exclude that one file, and add: - - thirdparty/libchdr/src/libchdr_flac_microflac.cpp - thirdparty/micro-flac/src/{flac_decoder,decorrelation,frame_header,pcm_packing,crc,lpc}.cpp - -with `CHDR_FLAC_BACKEND_MICROFLAC` and `MICRO_FLAC_DISABLE_OGG` defined, and -`-fno-exceptions -fno-rtti -fno-threadsafe-statics -fno-use-cxa-atexit` on the -C++ sources (the last one is what ESP-IDF does; without it the objects also -reference `__cxa_atexit`, which newlib does provide, so it links either way). Put -only micro-flac's `include/` on the include path - its `src/` holds a `crc.h` -that will shadow another component's own header of that name. - -**2. Do not link with `g++`.** A plain C++ link pulls ~82 KB of libstdc++ even -with exceptions off: `std::__throw_length_error` drags in `cow-stdexcept`, -`tinfo`, `cp-demangle` and around fifteen `eh_*.o` objects. ESP-IDF hides this -by wrapping `__cxa_throw`; the BL616 toolchain does not. Link with the **`gcc` -driver** (the T-Head GCC is 10.2, which predates `-nostdlib++`) and supply the -handful of symbols micro-flac actually needs: - -```c -/* everything micro-flac wants from the C++ runtime */ -#include -void *operator new(unsigned sz) { return malloc(sz); } -void *operator new[](unsigned sz) { return malloc(sz); } -void operator delete(void *p) noexcept { free(p); } -void operator delete[](void *p) noexcept { free(p); } -void operator delete(void *p, unsigned) noexcept { free(p); } -void operator delete[](void *p, unsigned) noexcept { free(p); } -namespace std { void __throw_length_error(const char *) { abort(); } } -``` - -With those flags the only C++ runtime symbols left undefined are the three the -shim defines - verified by `nm -u` on the compiled objects at the real ABI. -Measured effect on a test image: 154,514 B of text down to 54,569 B, with zero -`libstdc++.a` members linked. - -**The shim exists because the toolchain is GCC 10.2, not because of anything -about this chip.** `-nostdlib++` does the same job in one flag and has been -available since GCC 11, so if this ever moves to a newer toolchain, delete the -shim rather than carrying it forward. Do not copy it into a project that is -already on a modern compiler. - -**3. Use the vendor toolchain.** The T-Head GCC at -`toolchain_gcc_t-head_linux` and Debian's `gcc-riscv64-unknown-elf` share the -`riscv64-unknown-elf-` prefix, so PATH order decides which one you get, and -`CROSS_COMPILE ?= riscv64-unknown-elf-` does not disambiguate. Debian's ships -the `g++` driver but **no libstdc++ at all** - no `cstddef`, no `libstdc++.a` - -so micro-flac will not compile against it, for reasons that say nothing about -the BL616. Check with `riscv64-unknown-elf-gcc --version`: the vendor one -reports "Xuantie-900". - -Upstream GCC is not an option here yet, and it is worth writing down why so -nobody re-derives it. The T-Head vendor extensions themselves are not the -obstacle - GCC has had the XThead* collection since GCC 13. Three other things -are, and all three were still missing when checked against the GCC 15.2 and -16.1 manuals (2026-09): - -- `-mtune=e907`, which `bouffalo_sdk` sets, is rejected as an unknown cpu. - GCC 16 did grow the Xuantie application cores - `xt-c908`, `xt-c910`, - `xt-c920` and their variants - but not the small embedded E907. -- the `p` (packed SIMD) extension in the ABI string below is not in GCC's - `-march` table at all; it is still unratified, and the implementations that - exist live in vendor forks. -- `zpsfoperand` and `xtheade` likewise have no upstream spelling. - -`-mtune=size` is the documented substitute for the first, at the cost of the -core-specific tuning. The other two have no substitute. The vendor toolchain is -also frozen: its last commit is from October 2022. So this is a real constraint -rather than an upgrade nobody got round to. - -Clang does not unblock it either, checked at the same time against LLVM main. -It carries the same XThead* extensions, knows no E907 either (its only Xuantie -processors are `xt-c910v2` and `xt-c920v2`), and rejects `xtheade` and -`zpsfoperand` outright. It does have a `p` extension where GCC has none - but -as `experimental-p` behind `-menable-experimental-extensions`, implementing -draft 0.21, whereas `zpsfoperand` belongs to the older 0.9.x drafts this core -was built to. So they are not the same instruction set, and P being ratified -some day would not by itself make an upstream compiler target this chip. - -The vendor fork is the only route, and it has moved since the pin above. -[XUANTIE-RV/gcc](https://github.com/XUANTIE-RV/gcc) carries three branches -(checked 2026-09): - -| branch | last commit | declares `e907` | -|---|---|---| -| `xuantie-gcc-10.2.0` | 2024-07 | yes - c906, c908, c910, c920, e902, e906, e907 | -| `xuantie-gcc-10.4.0` | 2024-12 | yes, plus the c907 family | -| `xuantie-gcc-14.1.1` | 2025-03 | **no** - `riscv-cores.def` is upstream's, no Xuantie cores at all | - -So the GCC 14 branch cannot build this chip yet; it looks like a rebase in -progress rather than a finished port. `xuantie-gcc-10.4.0` can, and is two -years of GCC fixes newer than the GCC 10.2 blob pinned in -`bl616-tangcore-build.yml` - but it is still below GCC 11, so it does not -retire the shim above. Moving to it is `firmware-bl616`'s call, not ours. - -The community forks are not an alternative: `openbouffalo/xuantie-gnu-toolchain` -was last pushed in 2023 and `revyos/xuantie-gnu-toolchain` in 2024, both behind -the upstream they forked. - -micro-flac itself compiles clean at the real BL616 ABI -(`-march=rv32imafcpzpsfoperand_xtheade -mabi=ilp32f`, zero warnings) and its -object code is smaller than dr_flac's there: 36,304 B against 45,129 B. +## Read-ahead budget + +`chd_fatfs_open()` calls `chd_set_cache_budget()` with 32KB on every open. +Compressed hunks are small - a few KB - and laid out strictly sequentially, so +one larger read serves many of them and the fixed cost of each `f_read` (FatFs +bookkeeping, SPI command setup, DMA, interrupt) is paid far less often. + +Measured **1.11x on an ESP32-S3** and **1.05-1.12x on an RP2350**, both reading +over SPI, with 32KB the knee on the RP2350: 64KB doubled the cost for under +0.7% more. Not measured on BL616. + +The budget is a ceiling, not an allocation request: an image whose hunks exceed +it leaves caching off rather than over-allocating. Set `CHD_FATFS_CACHE_BUDGET` +to 0 to turn it off, or to another size to trade RAM for fewer reads. + +## Two things not to try + +Both are recorded with their numbers in `docs/perf-esp32p4-findings.md`: + +- **`-Os`** is 1.12x *slower* than -O2 on an ESP32-S3. The decoders are + loop-heavy and lose more to reduced unrolling than the smaller text wins back. +- **`Z7_LZMA_PROB32`** costs 15,980 bytes per LZMA instance for a speedup the + LZMA SDK only claims for "some CPUs", and was never measured on any target. + The option has been removed rather than left as a trap. + +## FLAC backend: micro-flac by default + +The integration patch decodes FLAC through +[micro-flac](https://github.com/esphome-libs/micro-flac) rather than dr_flac, +with byte-identical output. Clone it before applying the patch: + + git -C thirdparty clone https://github.com/esphome-libs/micro-flac.git + git -C thirdparty/micro-flac checkout ffbe8a9ba5e78c16e535a4695a8b2418b0c091ee + +Pinned, because the backend compiles against micro-flac's internals. Fetched, +never vendored: it is Apache-2.0 and libchdr stays BSD-3 for its other +consumers. TangCore's firmware is already Apache-2.0 end to end, so it adds no +obligation here - but if it does not suit you, apply +`patches/firmware-bl616-libchdr-drflac.patch` on top and you are back on +dr_flac. CI builds both, so neither path rots. + +**What is actually known.** Nothing has been measured on BL616 - there is no +hardware in CI, and this board has never run a decode. On an ESP32-S3 with I/O +excluded it is **1.198x** on a CD-FLAC hunk and **1.233x** on raw FLAC; across +eleven real discs **1.072x** overall, 1.21x where the image is FLAC-heavy, and +**0.988x** on one profile where FLAC barely appears. An RP2350 Cortex-M33 gives +1.032x overall. Peak heap is lower than dr_flac's on most images, and the +worst-case largest-free-block is better. + +Earlier revisions of this file quoted 1.46x and 1.41x. Those predate the +STREAMINFO block-size fix, which removed an oversized decoded-sample buffer +from dr_flac and took most of micro-flac's lead with it. Do not use them. + +The gain is on the FLAC part of the decode only. On a board reading over SPI, +storage is usually the larger share of wall time - see the read-ahead budget +above, which attacks that side. ## Status (2026-08-25) diff --git a/contrib/tangcore-bl616/chd/chd_fatfs.c b/contrib/tangcore-bl616/chd/chd_fatfs.c index afb983cf..27666711 100644 --- a/contrib/tangcore-bl616/chd/chd_fatfs.c +++ b/contrib/tangcore-bl616/chd/chd_fatfs.c @@ -77,7 +77,23 @@ chd_error chd_fatfs_open(const char *path, FIL *fil, chd_file **chd) err = chd_open_core_file_callbacks(&chd_fatfs_callbacks, fil, CHD_OPEN_READ, NULL, chd); if (err != CHDERR_NONE) + { f_close(fil); + return err; + } - return err; + /* Compressed hunks are small - a few KB - and laid out strictly + * sequentially, so one larger read serves many of them and the fixed cost + * of each f_read (FatFs bookkeeping, SPI command setup, DMA, interrupt) + * is paid far less often. Measured 1.11x on an ESP32-S3 and 1.05-1.12x on + * an RP2350, both reading over SPI, with 32KB the knee on the RP2350: 64KB + * doubled the cost for under 0.7% more. + * + * Not measured on BL616. The budget is a ceiling, not an allocation + * request: an image whose hunks exceed it simply leaves caching off rather + * than over-allocating, and a failure here is not fatal to the open. */ + if (CHD_FATFS_CACHE_BUDGET != 0) + (void)chd_set_cache_budget(*chd, CHD_FATFS_CACHE_BUDGET); + + return CHDERR_NONE; } diff --git a/contrib/tangcore-bl616/chd/chd_fatfs.h b/contrib/tangcore-bl616/chd/chd_fatfs.h index 4c75acf1..c41f9738 100644 --- a/contrib/tangcore-bl616/chd/chd_fatfs.h +++ b/contrib/tangcore-bl616/chd/chd_fatfs.h @@ -27,6 +27,13 @@ extern const core_file_callbacks chd_fatfs_callbacks; /* Opens path via f_open(FA_READ) into *fil, then hands it to libchdr as the * argp for chd_fatfs_callbacks. On any failure, *fil is closed if it was * opened and *chd is left untouched. */ +/* Read-ahead window handed to chd_set_cache_budget() on every open. Set to 0 + * to leave it off. 32KB is the value measured as the knee on the boards where + * this was tried; BL616 has not been measured. */ +#ifndef CHD_FATFS_CACHE_BUDGET +#define CHD_FATFS_CACHE_BUDGET (32 * 1024) +#endif + chd_error chd_fatfs_open(const char *path, FIL *fil, chd_file **chd); #ifdef __cplusplus diff --git a/contrib/tangcore-bl616/patches/firmware-bl616-libchdr-drflac.patch b/contrib/tangcore-bl616/patches/firmware-bl616-libchdr-drflac.patch new file mode 100644 index 00000000..b84a7eb8 --- /dev/null +++ b/contrib/tangcore-bl616/patches/firmware-bl616-libchdr-drflac.patch @@ -0,0 +1,52 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 83ffe15..b748fb3 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -33,8 +33,6 @@ add_definitions( + -DWANT_RAW_DATA_SECTOR=1 + -DWANT_SUBCODE=1 + -DVERIFY_BLOCK_CRC=1 +- -DCHDR_FLAC_BACKEND_MICROFLAC +- -DMICRO_FLAC_DISABLE_OGG + -DMINIZ_NO_ARCHIVE_APIS + -DMINIZ_NO_DEFLATE_APIS + -DMINIZ_NO_STDIO +@@ -48,7 +46,6 @@ sdk_add_include_directories( + . usb fpga ui core utils chd + thirdparty/libchdr/include + thirdparty/libchdr/src +- thirdparty/micro-flac/include + ) + + # Add source files +@@ -68,29 +65,7 @@ file(GLOB LIBCHDR_SOURCES + "thirdparty/libchdr/deps/zstd-1.5.7/zstddeclib.c" + ) + +-# libchdr_flac.c is the dr_flac backend; exactly one backend is compiled, so +-# it goes when micro-flac comes in. +-list(REMOVE_ITEM LIBCHDR_SOURCES +- "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/libchdr/src/libchdr_flac.c") +- +-set(LIBCHDR_MICROFLAC_SOURCES +- "thirdparty/libchdr/src/libchdr_flac_microflac.cpp" +- "thirdparty/micro-flac/src/flac_decoder.cpp" +- "thirdparty/micro-flac/src/decorrelation.cpp" +- "thirdparty/micro-flac/src/frame_header.cpp" +- "thirdparty/micro-flac/src/pcm_packing.cpp" +- "thirdparty/micro-flac/src/crc.cpp" +- "thirdparty/micro-flac/src/lpc.cpp" +-) +- +-# -fno-exceptions/-fno-rtti keep libstdc++'s exception and RTTI machinery out +-# of a bare-metal link, -fno-threadsafe-statics drops the __cxa_guard_* +-# dependency, and -fno-use-cxa-atexit is what ESP-IDF does here - newlib does +-# provide __cxa_atexit, so it links either way. +-set_source_files_properties(${LIBCHDR_MICROFLAC_SOURCES} PROPERTIES +- COMPILE_OPTIONS "-fno-exceptions;-fno-rtti;-fno-threadsafe-statics;-fno-use-cxa-atexit") +- +-target_sources(app PRIVATE ${MY_SOURCES} ${LIBCHDR_SOURCES} ${LIBCHDR_MICROFLAC_SOURCES}) ++target_sources(app PRIVATE ${MY_SOURCES} ${LIBCHDR_SOURCES}) + sdk_set_main_file(main.cpp) + + project(tangcore CXX) diff --git a/contrib/tangcore-bl616/patches/firmware-bl616-libchdr-integration.patch b/contrib/tangcore-bl616/patches/firmware-bl616-libchdr-integration.patch index 19528dfb..7f93fe0b 100644 --- a/contrib/tangcore-bl616/patches/firmware-bl616-libchdr-integration.patch +++ b/contrib/tangcore-bl616/patches/firmware-bl616-libchdr-integration.patch @@ -1,8 +1,8 @@ diff --git a/CMakeLists.txt b/CMakeLists.txt -index 8e7d0fe..b748fb3 100644 +index 8e7d0fe..83ffe15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -26,11 +26,26 @@ elseif(TANG_BOARD STREQUAL "nano20k") +@@ -26,11 +26,29 @@ elseif(TANG_BOARD STREQUAL "nano20k") add_definitions(-DTANG_NANO20K) endif() @@ -13,6 +13,8 @@ index 8e7d0fe..b748fb3 100644 + -DWANT_RAW_DATA_SECTOR=1 + -DWANT_SUBCODE=1 + -DVERIFY_BLOCK_CRC=1 ++ -DCHDR_FLAC_BACKEND_MICROFLAC ++ -DMICRO_FLAC_DISABLE_OGG + -DMINIZ_NO_ARCHIVE_APIS + -DMINIZ_NO_DEFLATE_APIS + -DMINIZ_NO_STDIO @@ -27,10 +29,11 @@ index 8e7d0fe..b748fb3 100644 + . usb fpga ui core utils chd + thirdparty/libchdr/include + thirdparty/libchdr/src ++ thirdparty/micro-flac/include ) # Add source files -@@ -42,7 +57,15 @@ file(GLOB_RECURSE MY_SOURCES +@@ -42,7 +60,37 @@ file(GLOB_RECURSE MY_SOURCES "utils/*.cpp" ) @@ -43,7 +46,29 @@ index 8e7d0fe..b748fb3 100644 + "thirdparty/libchdr/deps/zstd-1.5.7/zstddeclib.c" +) + -+target_sources(app PRIVATE ${MY_SOURCES} ${LIBCHDR_SOURCES}) ++# libchdr_flac.c is the dr_flac backend; exactly one backend is compiled, so ++# it goes when micro-flac comes in. ++list(REMOVE_ITEM LIBCHDR_SOURCES ++ "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/libchdr/src/libchdr_flac.c") ++ ++set(LIBCHDR_MICROFLAC_SOURCES ++ "thirdparty/libchdr/src/libchdr_flac_microflac.cpp" ++ "thirdparty/micro-flac/src/flac_decoder.cpp" ++ "thirdparty/micro-flac/src/decorrelation.cpp" ++ "thirdparty/micro-flac/src/frame_header.cpp" ++ "thirdparty/micro-flac/src/pcm_packing.cpp" ++ "thirdparty/micro-flac/src/crc.cpp" ++ "thirdparty/micro-flac/src/lpc.cpp" ++) ++ ++# -fno-exceptions/-fno-rtti keep libstdc++'s exception and RTTI machinery out ++# of a bare-metal link, -fno-threadsafe-statics drops the __cxa_guard_* ++# dependency, and -fno-use-cxa-atexit is what ESP-IDF does here - newlib does ++# provide __cxa_atexit, so it links either way. ++set_source_files_properties(${LIBCHDR_MICROFLAC_SOURCES} PROPERTIES ++ COMPILE_OPTIONS "-fno-exceptions;-fno-rtti;-fno-threadsafe-statics;-fno-use-cxa-atexit") ++ ++target_sources(app PRIVATE ${MY_SOURCES} ${LIBCHDR_SOURCES} ${LIBCHDR_MICROFLAC_SOURCES}) sdk_set_main_file(main.cpp) project(tangcore CXX) diff --git a/docs/perf-esp32p4-findings.md b/docs/perf-esp32p4-findings.md index 1c243a31..9f70cda8 100644 --- a/docs/perf-esp32p4-findings.md +++ b/docs/perf-esp32p4-findings.md @@ -208,6 +208,33 @@ a stack array. SWAR wins everywhere and needs no gating. chdman's precision keeps `bps + precision + ilog2(order)` <= 32. dr_flac already uses the 32-bit path exclusively. +### -Os: slower, not smaller where it counts + +Measured on an ESP32-S3: **1.12x slower** than the `PERF` (-O2) default across +the decode corpus. The decoders are loop-heavy and their hot paths lose more to +the smaller unrolling than the smaller text buys back on a cached part. The +contrib benchmarks build -O2 (ESP-IDF `PERF`) and -O3 (RP2350 Release) +deliberately. + +Do not reach for -Os as an MCU default. It is a plausible-sounding change with +a measured negative. + +### Z7_LZMA_PROB32: dropped, never justified + +The LZMA SDK ships decoder assembly for x86-64 and AArch64 only, so neither +applies to Cortex-M33, RV32 or Xtensa. `Z7_LZMA_PROB32` was the one portable +knob left: it widens the probability model from `UInt16` to `UInt32`, which the +SDK says "can increase the speed on some CPUs", at **+15,980 bytes per LZMA +instance** at CHD's lc=3/lp=0. + +It was exposed as an option on RP2350 and **never measured on any target**. On +parts where the whole point is fitting in a few hundred KB, a 16KB-per-instance +cost against an unquantified "some CPUs" is not a trade worth carrying. The +option is removed rather than left as a trap. + +If anyone revisits it: measure it, on hardware, against the RAM it costs - +"the SDK says it might help" is not a result. + ### PIE (ESP32-P4 vector extension): not worth it Four independent reasons. The LPC recurrence is serial across samples; only the