From 84936c9faefdd6bf646f86df7db96bda305cd671 Mon Sep 17 00:00:00 2001 From: Romain TISSERAND Date: Sun, 6 Sep 2026 14:03:21 +0200 Subject: [PATCH 1/2] CI: fail on a RAM regression, not only on running out of RAM The per-codec ceilings were sized as platform budgets - what the target can afford - so they sit two to eight times above what the code actually uses. That catches a catastrophe and nothing smaller. cd_cdlz used 99014 bytes before the in-place CD spread and uses 57905 now; the whole 41% win could be handed back and the job would stay green, because 99014 is still under the 200000 ceiling. So there are two thresholds per codec now, because there are two questions. REGRESSION sits about 10% over what the code uses today and answers "did we just lose ground". PLATFORM stays where it was and answers "does it still fit". Both have to hold; the output names which one failed. The 10% is not a guess. All seven codecs were measured on all three targets: rv32imafc, cortex-m33 and hazard3 agree exactly on five of them and differ by 8 bytes on the other two, so peak heap here is a property of the allocation sizes rather than of the ABI. The headroom is for allocator and toolchain drift, not for design changes - raise one deliberately, in the commit that spends the memory, and say why. Checked against the real qemu logs from all three targets, and against a log edited to put cd_cdlz back at its pre-spread 99014: the job now fails with REGRESSION where it previously passed. --- tests/rp2350-arm/check_budget.py | 56 +++++++++++++++++++++++------- tests/rp2350-riscv/check_budget.py | 56 +++++++++++++++++++++++------- tests/rv32/check_budget.py | 53 ++++++++++++++++++++++------ 3 files changed, 131 insertions(+), 34 deletions(-) diff --git a/tests/rp2350-arm/check_budget.py b/tests/rp2350-arm/check_budget.py index 1e908253..0206f926 100755 --- a/tests/rp2350-arm/check_budget.py +++ b/tests/rp2350-arm/check_budget.py @@ -4,11 +4,13 @@ Usage: check_budget.py Fails (exit 1) if any codec's decode failed (OPEN/READ FAILED - a -correctness regression, e.g. the kind the LZMA dict-size bug caught by -tests/rv32's identical check would have caused) or if any codec's peak heap -exceeds its budget (a memory regression). Budgets leave headroom over the -measured cortex-m33 baseline; see the libchdr issue/PR that added this check -for the measured numbers. +correctness regression, e.g. the kind the LZMA dict-size bug caused) or if +any codec's peak heap crosses either of two thresholds. + +REGRESSION is about 10% over what the code uses today: it fails when a memory +win is given back, which a platform ceiling sized for headroom cannot see. +PLATFORM is what RP2350 can afford at all. They answer different questions - +"did we just lose ground" and "does it still fit" - and both have to hold. """ import re import sys @@ -16,7 +18,27 @@ # bytes; measured baseline (2026-08-31, cortex-m33 hard-float, # qemu-system-arm mps2-an505) rounded up with headroom for the RP2350's # 520KB SRAM budget. -BUDGETS = { +# Regression thresholds: about 10% over the measured baseline, which is what +# fails the job when a memory win is silently given back. Measured 2026-09-06 +# on all three targets - rv32imafc, cortex-m33 and hazard3 agree to within +# 8 bytes on every codec, so these are not architecture-specific and the +# headroom is for allocator and toolchain drift, not for design changes. +# +# Raise one deliberately, in the commit that spends the memory, and say why. +REGRESSION = { + "hd_zlib": 22_000, # measured 19_766 + "hd_zstd": 118_000, # measured 107_205 + "hd_lzma": 31_000, # measured 27_858 + "hd_huff": 26_000, # measured 23_219 + "cd_cdzl": 55_000, # measured 49_786 + "cd_cdzs": 142_000, # measured 128_795 + "cd_cdlz": 64_000, # measured 57_905 +} + +# Platform ceilings: what this target can actually afford, independent of what +# the code happens to use today. These answer "does it still fit", the +# thresholds above answer "did we just lose ground". Both have to hold. +PLATFORM = { "hd_zlib": 100_000, "hd_zstd": 200_000, "hd_lzma": 80_000, @@ -54,16 +76,26 @@ def main(): name, hunkbytes, hunks, peak = m.groups() peak = int(peak) seen[name] = peak - budget = BUDGETS.get(name) - if budget is None: + plat = PLATFORM.get(name) + reg = REGRESSION.get(name) + if plat is None and reg is None: print(f"WARN {name}: no budget defined, skipping (peak={peak})") continue - status = "ok" if peak <= budget else "OVER BUDGET" - print(f"{status:12s} {name:10s} peak={peak:>7d} budget={budget:>7d}") - if peak > budget: + over_reg = reg is not None and peak > reg + over_plat = plat is not None and peak > plat + if over_plat: + status = "OVER PLATFORM" + elif over_reg: + status = "REGRESSION" + else: + status = "ok" + print(f"{status:14s} {name:10s} peak={peak:>7d} " + f"regression={reg if reg is not None else '-':>7} " + f"platform={plat if plat is not None else '-':>7}") + if over_reg or over_plat: failed = True - missing = set(BUDGETS) - set(seen) + missing = (set(PLATFORM) | set(REGRESSION)) - set(seen) if missing: print(f"FAIL: expected codecs missing from output: {sorted(missing)}") failed = True diff --git a/tests/rp2350-riscv/check_budget.py b/tests/rp2350-riscv/check_budget.py index f74865f6..b436d098 100755 --- a/tests/rp2350-riscv/check_budget.py +++ b/tests/rp2350-riscv/check_budget.py @@ -4,11 +4,13 @@ Usage: check_budget.py Fails (exit 1) if any codec's decode failed (OPEN/READ FAILED - a -correctness regression, e.g. the kind the LZMA dict-size bug caught by -tests/rv32's identical check would have caused) or if any codec's peak heap -exceeds its budget (a memory regression). Budgets leave headroom over the -measured rv32imac/ilp32 (Hazard3) baseline; see the libchdr issue/PR that -added this check for the measured numbers. +correctness regression, e.g. the kind the LZMA dict-size bug caused) or if +any codec's peak heap crosses either of two thresholds. + +REGRESSION is about 10% over what the code uses today: it fails when a memory +win is given back, which a platform ceiling sized for headroom cannot see. +PLATFORM is what RP2350 can afford at all. They answer different questions - +"did we just lose ground" and "does it still fit" - and both have to hold. """ import re import sys @@ -19,7 +21,27 @@ # heap here is driven by codec allocation sizes, not the core ISA/ABI) - # kept as its own measured baseline rather than assumed equal, same as # tests/rp2350-arm not assuming tests/rv32's numbers. -BUDGETS = { +# Regression thresholds: about 10% over the measured baseline, which is what +# fails the job when a memory win is silently given back. Measured 2026-09-06 +# on all three targets - rv32imafc, cortex-m33 and hazard3 agree to within +# 8 bytes on every codec, so these are not architecture-specific and the +# headroom is for allocator and toolchain drift, not for design changes. +# +# Raise one deliberately, in the commit that spends the memory, and say why. +REGRESSION = { + "hd_zlib": 22_000, # measured 19_766 + "hd_zstd": 118_000, # measured 107_205 + "hd_lzma": 31_000, # measured 27_858 + "hd_huff": 26_000, # measured 23_219 + "cd_cdzl": 55_000, # measured 49_786 + "cd_cdzs": 142_000, # measured 128_795 + "cd_cdlz": 64_000, # measured 57_905 +} + +# Platform ceilings: what this target can actually afford, independent of what +# the code happens to use today. These answer "does it still fit", the +# thresholds above answer "did we just lose ground". Both have to hold. +PLATFORM = { "hd_zlib": 100_000, "hd_zstd": 200_000, "hd_lzma": 80_000, @@ -57,16 +79,26 @@ def main(): name, hunkbytes, hunks, peak = m.groups() peak = int(peak) seen[name] = peak - budget = BUDGETS.get(name) - if budget is None: + plat = PLATFORM.get(name) + reg = REGRESSION.get(name) + if plat is None and reg is None: print(f"WARN {name}: no budget defined, skipping (peak={peak})") continue - status = "ok" if peak <= budget else "OVER BUDGET" - print(f"{status:12s} {name:10s} peak={peak:>7d} budget={budget:>7d}") - if peak > budget: + over_reg = reg is not None and peak > reg + over_plat = plat is not None and peak > plat + if over_plat: + status = "OVER PLATFORM" + elif over_reg: + status = "REGRESSION" + else: + status = "ok" + print(f"{status:14s} {name:10s} peak={peak:>7d} " + f"regression={reg if reg is not None else '-':>7} " + f"platform={plat if plat is not None else '-':>7}") + if over_reg or over_plat: failed = True - missing = set(BUDGETS) - set(seen) + missing = (set(PLATFORM) | set(REGRESSION)) - set(seen) if missing: print(f"FAIL: expected codecs missing from output: {sorted(missing)}") failed = True diff --git a/tests/rv32/check_budget.py b/tests/rv32/check_budget.py index 2319ae81..627cdb78 100755 --- a/tests/rv32/check_budget.py +++ b/tests/rv32/check_budget.py @@ -5,16 +5,39 @@ Fails (exit 1) if any codec's decode failed (OPEN/READ FAILED - a correctness regression, e.g. the kind the LZMA dict-size bug caused) or if -any codec's peak heap exceeds its budget (a memory regression). Budgets -leave headroom over the measured rv32imafc/ilp32f baseline; see the libchdr -issue/PR that added this check for the measured numbers. +any codec's peak heap crosses either of two thresholds. + +REGRESSION is about 10% over what the code uses today: it fails when a memory +win is given back, which a platform ceiling sized for headroom cannot see. +PLATFORM is what BL616 can afford at all. They answer different questions - +"did we just lose ground" and "does it still fit" - and both have to hold. """ import re import sys # bytes; measured baseline (2026-08-20, rv32imafc/ilp32f, qemu-system-riscv32 # virt) rounded up with headroom for BL616's 480KB SRAM budget. -BUDGETS = { +# Regression thresholds: about 10% over the measured baseline, which is what +# fails the job when a memory win is silently given back. Measured 2026-09-06 +# on all three targets - rv32imafc, cortex-m33 and hazard3 agree to within +# 8 bytes on every codec, so these are not architecture-specific and the +# headroom is for allocator and toolchain drift, not for design changes. +# +# Raise one deliberately, in the commit that spends the memory, and say why. +REGRESSION = { + "hd_zlib": 22_000, # measured 19_766 + "hd_zstd": 118_000, # measured 107_205 + "hd_lzma": 31_000, # measured 27_858 + "hd_huff": 26_000, # measured 23_219 + "cd_cdzl": 55_000, # measured 49_786 + "cd_cdzs": 142_000, # measured 128_795 + "cd_cdlz": 64_000, # measured 57_905 +} + +# Platform ceilings: what this target can actually afford, independent of what +# the code happens to use today. These answer "does it still fit", the +# thresholds above answer "did we just lose ground". Both have to hold. +PLATFORM = { "hd_zlib": 100_000, "hd_zstd": 150_000, "hd_lzma": 80_000, @@ -52,16 +75,26 @@ def main(): name, hunkbytes, hunks, peak = m.groups() peak = int(peak) seen[name] = peak - budget = BUDGETS.get(name) - if budget is None: + plat = PLATFORM.get(name) + reg = REGRESSION.get(name) + if plat is None and reg is None: print(f"WARN {name}: no budget defined, skipping (peak={peak})") continue - status = "ok" if peak <= budget else "OVER BUDGET" - print(f"{status:12s} {name:10s} peak={peak:>7d} budget={budget:>7d}") - if peak > budget: + over_reg = reg is not None and peak > reg + over_plat = plat is not None and peak > plat + if over_plat: + status = "OVER PLATFORM" + elif over_reg: + status = "REGRESSION" + else: + status = "ok" + print(f"{status:14s} {name:10s} peak={peak:>7d} " + f"regression={reg if reg is not None else '-':>7} " + f"platform={plat if plat is not None else '-':>7}") + if over_reg or over_plat: failed = True - missing = set(BUDGETS) - set(seen) + missing = (set(PLATFORM) | set(REGRESSION)) - set(seen) if missing: print(f"FAIL: expected codecs missing from output: {sorted(missing)}") failed = True From c116d4925e66c6dfbf224c8e26d3b02672bca460 Mon Sep 17 00:00:00 2001 From: Romain TISSERAND Date: Thu, 10 Sep 2026 09:31:08 +0200 Subject: [PATCH 2/2] Cover the two FLAC codecs in the regression thresholds They joined the budget firmware with the STREAMINFO block-size fix and had platform ceilings but no regression threshold, so giving those bytes back would not have failed the job - which is the whole point of this change. Baselines taken on rv32imafc; the three targets agree to within 8 bytes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01846EhHuAFk5qvxwEA5Gq6y --- tests/rp2350-arm/check_budget.py | 4 ++++ tests/rp2350-riscv/check_budget.py | 4 ++++ tests/rv32/check_budget.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/tests/rp2350-arm/check_budget.py b/tests/rp2350-arm/check_budget.py index d723a171..a0456b58 100755 --- a/tests/rp2350-arm/check_budget.py +++ b/tests/rp2350-arm/check_budget.py @@ -23,6 +23,8 @@ # on all three targets - rv32imafc, cortex-m33 and hazard3 agree to within # 8 bytes on every codec, so these are not architecture-specific and the # headroom is for allocator and toolchain drift, not for design changes. +# hd_flac and cd_cdfl joined the firmware later and their baselines were taken +# on rv32imafc alone (2026-09-10); the agreement above is why that is enough. # # Raise one deliberately, in the commit that spends the memory, and say why. REGRESSION = { @@ -30,9 +32,11 @@ "hd_zstd": 118_000, # measured 107_205 "hd_lzma": 31_000, # measured 27_858 "hd_huff": 26_000, # measured 23_219 + "hd_flac": 27_000, # measured 24_338 "cd_cdzl": 55_000, # measured 49_786 "cd_cdzs": 142_000, # measured 128_795 "cd_cdlz": 64_000, # measured 57_905 + "cd_cdfl": 84_000, # measured 76_329 } # Platform ceilings: what this target can actually afford, independent of what diff --git a/tests/rp2350-riscv/check_budget.py b/tests/rp2350-riscv/check_budget.py index 94c73e73..9005b4c5 100755 --- a/tests/rp2350-riscv/check_budget.py +++ b/tests/rp2350-riscv/check_budget.py @@ -26,6 +26,8 @@ # on all three targets - rv32imafc, cortex-m33 and hazard3 agree to within # 8 bytes on every codec, so these are not architecture-specific and the # headroom is for allocator and toolchain drift, not for design changes. +# hd_flac and cd_cdfl joined the firmware later and their baselines were taken +# on rv32imafc alone (2026-09-10); the agreement above is why that is enough. # # Raise one deliberately, in the commit that spends the memory, and say why. REGRESSION = { @@ -33,9 +35,11 @@ "hd_zstd": 118_000, # measured 107_205 "hd_lzma": 31_000, # measured 27_858 "hd_huff": 26_000, # measured 23_219 + "hd_flac": 27_000, # measured 24_338 "cd_cdzl": 55_000, # measured 49_786 "cd_cdzs": 142_000, # measured 128_795 "cd_cdlz": 64_000, # measured 57_905 + "cd_cdfl": 84_000, # measured 76_329 } # Platform ceilings: what this target can actually afford, independent of what diff --git a/tests/rv32/check_budget.py b/tests/rv32/check_budget.py index 9a1d1eec..60dccac2 100755 --- a/tests/rv32/check_budget.py +++ b/tests/rv32/check_budget.py @@ -22,6 +22,8 @@ # on all three targets - rv32imafc, cortex-m33 and hazard3 agree to within # 8 bytes on every codec, so these are not architecture-specific and the # headroom is for allocator and toolchain drift, not for design changes. +# hd_flac and cd_cdfl joined the firmware later and their baselines were taken +# on rv32imafc alone (2026-09-10); the agreement above is why that is enough. # # Raise one deliberately, in the commit that spends the memory, and say why. REGRESSION = { @@ -29,9 +31,11 @@ "hd_zstd": 118_000, # measured 107_205 "hd_lzma": 31_000, # measured 27_858 "hd_huff": 26_000, # measured 23_219 + "hd_flac": 27_000, # measured 24_338 "cd_cdzl": 55_000, # measured 49_786 "cd_cdzs": 142_000, # measured 128_795 "cd_cdlz": 64_000, # measured 57_905 + "cd_cdfl": 84_000, # measured 76_329 } # Platform ceilings: what this target can actually afford, independent of what