From 0ef4ab304a4e7833014756cb5a999add128613e7 Mon Sep 17 00:00:00 2001 From: Rafael Vuijk Date: Wed, 28 Oct 2026 21:30:44 +0000 Subject: [PATCH] The benchmark is measured after the library's set-up The Kernel Benchmark gate failed twice today on unchanged library code: on #2187's first run, and on #2188 after its rebase. Both times it was SimplifyEasy, SolveEasy and EvalTrigPrecise, at 3-12x their time, and SimplifyEasy at 3% more allocation. Their re-runs matched the baseline to the byte. BenchmarkDotNet times one call before it decides how to measure, and where that call takes more than two thirds of its 500 ms iteration, it times one call per iteration from then on. The first call of those three took 349-362 ms on the failing runs and 252-300 ms on the passing ones. For a call of tens of microseconds, one call per iteration is another measurement. That first call paid for parsing every static expression of the class and for the library's own set-up: EvalEasy, a cache lookup, took 133 ms the first time on one machine. A [GlobalSetup] now touches the class and runs one simplification, one solve and one evaluation before the timed call. Measured on one machine, master against this, the first call drops from 172 to 8 ms (SimplifyEasy), 164 to 5 ms (SolveEasy) and 158 to 45 ms (EvalTrigPrecise). What the gated benchmarks allocate is unchanged: SimplifyEasy is 79,682 B/op on both, and the largest move is ParseEasy's +0.11% (21 B/op). The two compilation benchmarks, which the gate leaves out for their own spread, moved -0.5% and +2.0%. SolveHard, a call of 70 ms, is now timed one call per iteration rather than piloted, and allocates 9,832,912 B/op against 9,831,370. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012sonx8iAspMiwRwokT1Ura --- .../CommonFunctionsInterVersion.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Sources/Tests/DotnetBenchmark/CommonFunctionsInterVersion.cs b/Sources/Tests/DotnetBenchmark/CommonFunctionsInterVersion.cs index 6c63c030f..0de6c9ef5 100644 --- a/Sources/Tests/DotnetBenchmark/CommonFunctionsInterVersion.cs +++ b/Sources/Tests/DotnetBenchmark/CommonFunctionsInterVersion.cs @@ -21,6 +21,24 @@ namespace DotnetBenchmark [MemoryDiagnoser] public class CommonFunctionsInterVersion { + // BenchmarkDotNet times one call before it decides how to measure, and where that call + // takes more than two thirds of its 500 ms iteration it times one call per iteration from + // then on. For a call of tens of microseconds that is another measurement: on CI, with the + // first call at 349-362 ms, SimplifyEasy read 12x its time and 3% more allocation and failed + // the gate, while at 300 ms it was piloted and matched the baseline. The first call paid for + // this class's static fields, parsed when the class is first touched, and for the library's + // own set-up, which is why EvalEasy's took 133 ms. Done here, before the timed call, + // SimplifyEasy's first call is 8 ms where it was 172 ms, on one machine, and what each + // benchmark allocates is unchanged. + [GlobalSetup] + public void SetUpBeforeTheFirstTimedCall() + { + GC.KeepAlive(simplifyEasy); + "x + 1".ToEntity().Simplify(); + "x2 + 1".ToEntity().SolveEquation("x"); + MathS.Sin(2).EvalNumerical(); + } + // Testing parsing [Benchmark] public void ParseEasy() => MathS.FromString("1 + 2 / x + 2 / (y + x)", useCache: false); [Benchmark] public void ParseHard() => MathS.FromString("x ^ (x + y ^ 2 ^ sin(3 / z + i)) - log(-i, 3) ^ (x + x * y) - sqrt(y) / (i + sqrt(-1))", useCache: false);