diff --git a/tests/rp2350-arm/check_budget.py b/tests/rp2350-arm/check_budget.py index dee1b6d..a0456b5 100755 --- a/tests/rp2350-arm/check_budget.py +++ b/tests/rp2350-arm/check_budget.py @@ -4,11 +4,13 @@ Usage: check_budget.py Fails (exit 1) if any codec's decode failed (OPEN/READ FAILED - a -correctness regression, e.g. the kind the LZMA dict-size bug caught by -tests/rv32's identical check would have caused) or if any codec's peak heap -exceeds its budget (a memory regression). Budgets leave headroom over the -measured cortex-m33 baseline; see the libchdr issue/PR that added this check -for the measured numbers. +correctness regression, e.g. the kind the LZMA dict-size bug caused) or if +any codec's peak heap crosses either of two thresholds. + +REGRESSION is about 10% over what the code uses today: it fails when a memory +win is given back, which a platform ceiling sized for headroom cannot see. +PLATFORM is what RP2350 can afford at all. They answer different questions - +"did we just lose ground" and "does it still fit" - and both have to hold. """ import re import sys @@ -16,7 +18,31 @@ # bytes; measured baseline (2026-08-31, cortex-m33 hard-float, # qemu-system-arm mps2-an505) rounded up with headroom for the RP2350's # 520KB SRAM budget. -BUDGETS = { +# Regression thresholds: about 10% over the measured baseline, which is what +# fails the job when a memory win is silently given back. Measured 2026-09-06 +# on all three targets - rv32imafc, cortex-m33 and hazard3 agree to within +# 8 bytes on every codec, so these are not architecture-specific and the +# headroom is for allocator and toolchain drift, not for design changes. +# hd_flac and cd_cdfl joined the firmware later and their baselines were taken +# on rv32imafc alone (2026-09-10); the agreement above is why that is enough. +# +# Raise one deliberately, in the commit that spends the memory, and say why. +REGRESSION = { + "hd_zlib": 22_000, # measured 19_766 + "hd_zstd": 118_000, # measured 107_205 + "hd_lzma": 31_000, # measured 27_858 + "hd_huff": 26_000, # measured 23_219 + "hd_flac": 27_000, # measured 24_338 + "cd_cdzl": 55_000, # measured 49_786 + "cd_cdzs": 142_000, # measured 128_795 + "cd_cdlz": 64_000, # measured 57_905 + "cd_cdfl": 84_000, # measured 76_329 +} + +# Platform ceilings: what this target can actually afford, independent of what +# the code happens to use today. These answer "does it still fit", the +# thresholds above answer "did we just lose ground". Both have to hold. +PLATFORM = { "hd_zlib": 100_000, "hd_zstd": 200_000, "hd_lzma": 80_000, @@ -56,16 +82,26 @@ def main(): name, hunkbytes, hunks, peak = m.groups() peak = int(peak) seen[name] = peak - budget = BUDGETS.get(name) - if budget is None: + plat = PLATFORM.get(name) + reg = REGRESSION.get(name) + if plat is None and reg is None: print(f"WARN {name}: no budget defined, skipping (peak={peak})") continue - status = "ok" if peak <= budget else "OVER BUDGET" - print(f"{status:12s} {name:10s} peak={peak:>7d} budget={budget:>7d}") - if peak > budget: + over_reg = reg is not None and peak > reg + over_plat = plat is not None and peak > plat + if over_plat: + status = "OVER PLATFORM" + elif over_reg: + status = "REGRESSION" + else: + status = "ok" + print(f"{status:14s} {name:10s} peak={peak:>7d} " + f"regression={reg if reg is not None else '-':>7} " + f"platform={plat if plat is not None else '-':>7}") + if over_reg or over_plat: failed = True - missing = set(BUDGETS) - set(seen) + missing = (set(PLATFORM) | set(REGRESSION)) - set(seen) if missing: print(f"FAIL: expected codecs missing from output: {sorted(missing)}") failed = True diff --git a/tests/rp2350-riscv/check_budget.py b/tests/rp2350-riscv/check_budget.py index 7fe0cbd..9005b4c 100755 --- a/tests/rp2350-riscv/check_budget.py +++ b/tests/rp2350-riscv/check_budget.py @@ -4,11 +4,13 @@ Usage: check_budget.py Fails (exit 1) if any codec's decode failed (OPEN/READ FAILED - a -correctness regression, e.g. the kind the LZMA dict-size bug caught by -tests/rv32's identical check would have caused) or if any codec's peak heap -exceeds its budget (a memory regression). Budgets leave headroom over the -measured rv32imac/ilp32 (Hazard3) baseline; see the libchdr issue/PR that -added this check for the measured numbers. +correctness regression, e.g. the kind the LZMA dict-size bug caused) or if +any codec's peak heap crosses either of two thresholds. + +REGRESSION is about 10% over what the code uses today: it fails when a memory +win is given back, which a platform ceiling sized for headroom cannot see. +PLATFORM is what RP2350 can afford at all. They answer different questions - +"did we just lose ground" and "does it still fit" - and both have to hold. """ import re import sys @@ -19,7 +21,31 @@ # heap here is driven by codec allocation sizes, not the core ISA/ABI) - # kept as its own measured baseline rather than assumed equal, same as # tests/rp2350-arm not assuming tests/rv32's numbers. -BUDGETS = { +# Regression thresholds: about 10% over the measured baseline, which is what +# fails the job when a memory win is silently given back. Measured 2026-09-06 +# on all three targets - rv32imafc, cortex-m33 and hazard3 agree to within +# 8 bytes on every codec, so these are not architecture-specific and the +# headroom is for allocator and toolchain drift, not for design changes. +# hd_flac and cd_cdfl joined the firmware later and their baselines were taken +# on rv32imafc alone (2026-09-10); the agreement above is why that is enough. +# +# Raise one deliberately, in the commit that spends the memory, and say why. +REGRESSION = { + "hd_zlib": 22_000, # measured 19_766 + "hd_zstd": 118_000, # measured 107_205 + "hd_lzma": 31_000, # measured 27_858 + "hd_huff": 26_000, # measured 23_219 + "hd_flac": 27_000, # measured 24_338 + "cd_cdzl": 55_000, # measured 49_786 + "cd_cdzs": 142_000, # measured 128_795 + "cd_cdlz": 64_000, # measured 57_905 + "cd_cdfl": 84_000, # measured 76_329 +} + +# Platform ceilings: what this target can actually afford, independent of what +# the code happens to use today. These answer "does it still fit", the +# thresholds above answer "did we just lose ground". Both have to hold. +PLATFORM = { "hd_zlib": 100_000, "hd_zstd": 200_000, "hd_lzma": 80_000, @@ -59,16 +85,26 @@ def main(): name, hunkbytes, hunks, peak = m.groups() peak = int(peak) seen[name] = peak - budget = BUDGETS.get(name) - if budget is None: + plat = PLATFORM.get(name) + reg = REGRESSION.get(name) + if plat is None and reg is None: print(f"WARN {name}: no budget defined, skipping (peak={peak})") continue - status = "ok" if peak <= budget else "OVER BUDGET" - print(f"{status:12s} {name:10s} peak={peak:>7d} budget={budget:>7d}") - if peak > budget: + over_reg = reg is not None and peak > reg + over_plat = plat is not None and peak > plat + if over_plat: + status = "OVER PLATFORM" + elif over_reg: + status = "REGRESSION" + else: + status = "ok" + print(f"{status:14s} {name:10s} peak={peak:>7d} " + f"regression={reg if reg is not None else '-':>7} " + f"platform={plat if plat is not None else '-':>7}") + if over_reg or over_plat: failed = True - missing = set(BUDGETS) - set(seen) + missing = (set(PLATFORM) | set(REGRESSION)) - set(seen) if missing: print(f"FAIL: expected codecs missing from output: {sorted(missing)}") failed = True diff --git a/tests/rv32/check_budget.py b/tests/rv32/check_budget.py index c7d5808..60dccac 100755 --- a/tests/rv32/check_budget.py +++ b/tests/rv32/check_budget.py @@ -5,16 +5,43 @@ Fails (exit 1) if any codec's decode failed (OPEN/READ FAILED - a correctness regression, e.g. the kind the LZMA dict-size bug caused) or if -any codec's peak heap exceeds its budget (a memory regression). Budgets -leave headroom over the measured rv32imafc/ilp32f baseline; see the libchdr -issue/PR that added this check for the measured numbers. +any codec's peak heap crosses either of two thresholds. + +REGRESSION is about 10% over what the code uses today: it fails when a memory +win is given back, which a platform ceiling sized for headroom cannot see. +PLATFORM is what BL616 can afford at all. They answer different questions - +"did we just lose ground" and "does it still fit" - and both have to hold. """ import re import sys # bytes; measured baseline (2026-08-20, rv32imafc/ilp32f, qemu-system-riscv32 # virt) rounded up with headroom for BL616's 480KB SRAM budget. -BUDGETS = { +# Regression thresholds: about 10% over the measured baseline, which is what +# fails the job when a memory win is silently given back. Measured 2026-09-06 +# on all three targets - rv32imafc, cortex-m33 and hazard3 agree to within +# 8 bytes on every codec, so these are not architecture-specific and the +# headroom is for allocator and toolchain drift, not for design changes. +# hd_flac and cd_cdfl joined the firmware later and their baselines were taken +# on rv32imafc alone (2026-09-10); the agreement above is why that is enough. +# +# Raise one deliberately, in the commit that spends the memory, and say why. +REGRESSION = { + "hd_zlib": 22_000, # measured 19_766 + "hd_zstd": 118_000, # measured 107_205 + "hd_lzma": 31_000, # measured 27_858 + "hd_huff": 26_000, # measured 23_219 + "hd_flac": 27_000, # measured 24_338 + "cd_cdzl": 55_000, # measured 49_786 + "cd_cdzs": 142_000, # measured 128_795 + "cd_cdlz": 64_000, # measured 57_905 + "cd_cdfl": 84_000, # measured 76_329 +} + +# Platform ceilings: what this target can actually afford, independent of what +# the code happens to use today. These answer "does it still fit", the +# thresholds above answer "did we just lose ground". Both have to hold. +PLATFORM = { "hd_zlib": 100_000, "hd_zstd": 150_000, "hd_lzma": 80_000, @@ -54,16 +81,26 @@ def main(): name, hunkbytes, hunks, peak = m.groups() peak = int(peak) seen[name] = peak - budget = BUDGETS.get(name) - if budget is None: + plat = PLATFORM.get(name) + reg = REGRESSION.get(name) + if plat is None and reg is None: print(f"WARN {name}: no budget defined, skipping (peak={peak})") continue - status = "ok" if peak <= budget else "OVER BUDGET" - print(f"{status:12s} {name:10s} peak={peak:>7d} budget={budget:>7d}") - if peak > budget: + over_reg = reg is not None and peak > reg + over_plat = plat is not None and peak > plat + if over_plat: + status = "OVER PLATFORM" + elif over_reg: + status = "REGRESSION" + else: + status = "ok" + print(f"{status:14s} {name:10s} peak={peak:>7d} " + f"regression={reg if reg is not None else '-':>7} " + f"platform={plat if plat is not None else '-':>7}") + if over_reg or over_plat: failed = True - missing = set(BUDGETS) - set(seen) + missing = (set(PLATFORM) | set(REGRESSION)) - set(seen) if missing: print(f"FAIL: expected codecs missing from output: {sorted(missing)}") failed = True