diff --git a/CLAUDE.md b/CLAUDE.md index 0470eaa..fee5ad8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ Tally benchmarks PHP code by counting instructions, cycles, and memory under Cod - `src/Expectation` – where expected metrics are stored. `ExpectationSource` is the interface: `read()` the expectation, `file()` the file it lives in, the pure `rewrite(code, metrics)` that returns the file's code with new expectations, and `location()`, which throws `ExpectationException` when the source cannot be rewritten. `AttributeExpectations` implements it for `#[Expected]` and `#[ExpectedFor]` through `ExpectedAttributeRewriter`, the pure code-to-code rewrite, and `Tokens`, the token list it walks. `ExpectationChanges` plans the rewrites of many cases (pure): one `FileChange` per file with every case in it, in a `ChangePlan` together with the cases that failed. `ExpectationFiles` reads and writes the files. - `src/Discovery` – `DirectorySuiteLoader` finds and loads `*Bench.php` files (I/O), `CaseExpander` turns one method into its cases (pure, works on reflection); `hooks()` finds a class's `#[Before]` and `#[After]` methods once per class, because scanning them for every benchmark method made loading a class quadratic in its size. `AttributeFactory` creates every attribute Tally reads: it checks the arguments against the attribute's constructor first (names, count, types through `ValueTypeCheck`, as under strict_types), so a mistake is an `InvalidBenchmarkException` naming the argument, not a PHP error. Everything a benchmark author can get wrong, including a file that does not load and a data provider that throws, ends as an `InvalidBenchmarkException` that names the benchmark or file; PHP errors with stack traces are bugs. `ArgumentResolver` checks that a method takes the arguments of a data set (a hook the leading ones) and `ConstructorArguments` that a constructor takes only `Meter` and `ScratchDirectory`; discovery checks both, so a mismatch fails `tally list` instead of the worker. - `src/Selection` – `CaseSelector` implementations: `FilterSelector`, `ShardSelector` (longest-processing-time sharding by expected instructions). Both keep a case compared by its growth with its base: the filter adds the base, the shards keep a method's cases together. -- `src/Runner` – running one case in this process. `CaseRunner` calls the hooks uncounted, prepares the counter, calls the warm-up uncounted, then counts a `#warmup` span, an empty `#baseline` call, and one call of the benchmark, and subtracts the baseline's memory. The `#warmup` span exists because the first counted span costs more than later ones. `CodSpeedCounter` starts Callgrind's instrumentation once, when it is prepared, and only toggles collecting for each span: starting the instrumentation empties the simulated caches, and doing it for every span made the cycles of small benchmarks depend on the cache lines the hooks touched. `MeterSession` is the `Meter` the benchmark object gets in its constructor, one per case: every call `CaseRunner` makes is a span of it with its own `Counter` (`CodSpeedCounter` over `InstrumentHooks` for the counted ones, `IdleCounter` for hooks and warm-up), and it measures memory above the start of the span through `Memory`. `ExtensionInstrumentHooks` calls the `tally_hooks` extension, the only C that Tally calls. +- `src/Runner` – running one case in this process. `CaseRunner` calls the hooks uncounted, prepares the counter, calls the warm-up uncounted, then counts a `#warmup` span, an empty `#baseline` call, and one call of the benchmark, and subtracts the baseline's memory. The `#warmup` span exists because the first counted span costs more than later ones. `CodSpeedCounter` starts Callgrind's instrumentation once, when it is prepared, stops it when it is finished after the benchmark's span, so the after methods and the end of the process run at Valgrind's quickest, and only toggles collecting for each span: starting the instrumentation empties the simulated caches, and doing it for every span made the cycles of small benchmarks depend on the cache lines the hooks touched. `MeterSession` is the `Meter` the benchmark object gets in its constructor, one per case: every call `CaseRunner` makes is a span of it with its own `Counter` (`CodSpeedCounter` over `InstrumentHooks` for the counted ones, `IdleCounter` for hooks and warm-up), and it measures memory above the start of the span through `Memory`. `ExtensionInstrumentHooks` calls the `tally_hooks` extension, the only C that Tally calls. - `src/Comparison` – `Comparator` turns a case's expectations and measurement into a `CaseComparison` of `MetricComparison`s with a `Status`. `TolerancePolicy` is implemented by the tolerance attributes; `CaseExpander` picks the method's, else the class's, else that of the `Configuration`, which has one for expected numbers and one for growth. `Percent` formats shares. - `src/Report` – `Report` groups comparisons into `Section`s by their worst status. Renderers are pure and return strings: `ConsoleReport` (summary and full table, with or without colors), `MarkdownReport` (GitHub job summary), `GitHubAnnotations` (workflow commands). `Rows` and `TextTable` lay out the tables. Cases measured without an expectation become `NewExpectations`: `ExpectationsPatch` renders the planned file changes as one patch with `UnifiedDiff` (the hunks GNU `diff -u` prints, applicable with `git apply`), so a contributor adds a benchmark without numbers and copies the expectations from the CI output. `RelativePath` makes paths relative to the working directory. The golden files in `tests/Unit/Report/golden` are the expected output; regenerate them only when the output changes on purpose, and read the diff. - `src/Result` – `MeasurementJson`, the format of a measured case in results files; `ResultsDirectory` reads and writes a directory of them; `CallgrindProfile` reads the instructions and estimated cycles of each named part of a profile CodSpeed wrote. diff --git a/README.md b/README.md index 68c14ef..600a26d 100644 --- a/README.md +++ b/README.md @@ -581,9 +581,10 @@ In each of the two processes of a case, Tally: 6. counts a warm-up span, because the first counted span costs more than later ones, and throws it away, 7. counts an empty call through the same path as the benchmark, the baseline, 8. counts one call of the benchmark, -9. calls the `#[After]` methods, not counted. +9. stops the simulation, also when the benchmark throws, +10. calls the `#[After]` methods, not counted. -The simulation starts once, and only collecting the counts is turned on for each span, so the simulated caches stay warm from one span to the next. `codspeed run` starts it for each benchmark, which empties the caches, so the cycles of a benchmark of a few thousand instructions depend on which cache lines the code just before it happens to touch. Tally counts the same instructions as CodSpeed, and fewer cycles for small benchmarks. +The simulation runs from step 4 to step 9, and only collecting the counts is turned on for each span, so the simulated caches stay warm from one span to the next. `codspeed run` starts it for each benchmark, which empties the caches, so the cycles of a benchmark of a few thousand instructions depend on which cache lines the code just before it happens to touch. Tally counts the same instructions as CodSpeed, and fewer cycles for small benchmarks. The native process runs the same steps without the simulation, so hooks and warm-up calls run twice per case, once in each process. The scratch directory is shared between the two, so a `#[Before]` method that writes a file finds it already there in the second process. diff --git a/build/stubs/TallyHooks.stub b/build/stubs/TallyHooks.stub index c087bdb..ccca669 100644 --- a/build/stubs/TallyHooks.stub +++ b/build/stubs/TallyHooks.stub @@ -15,6 +15,10 @@ function start_instrumentation(): void { } +function stop_instrumentation(): void +{ +} + function toggle_collect(): void { } diff --git a/ext/tally_hooks/tally_hooks.c b/ext/tally_hooks/tally_hooks.c index 9c1bbe3..89a5bbd 100644 --- a/ext/tally_hooks/tally_hooks.c +++ b/ext/tally_hooks/tally_hooks.c @@ -14,7 +14,7 @@ #include "php.h" #include "ext/standard/info.h" -#define TALLY_HOOKS_VERSION "1.1.0" +#define TALLY_HOOKS_VERSION "1.2.0" #define TALLY_HOOKS_NAMESPACE "Tally\\Hooks" #ifdef TALLY_HOOKS_RECORDING @@ -25,6 +25,7 @@ static smart_str recorded; # define TALLY_INSTRUMENTED() (getenv("TALLY_HOOKS_OUTSIDE") == NULL) # define TALLY_START_INSTRUMENTATION() smart_str_appends(&recorded, "instrument\n") +# define TALLY_STOP_INSTRUMENTATION() smart_str_appends(&recorded, "stop\n") # define TALLY_TOGGLE_COLLECT() smart_str_appends(&recorded, "toggle\n") # define TALLY_DUMP(name) do { \ smart_str_appends(&recorded, "dump "); \ @@ -36,6 +37,7 @@ static smart_str recorded; # define TALLY_INSTRUMENTED() (RUNNING_ON_VALGRIND > 0) # define TALLY_START_INSTRUMENTATION() CALLGRIND_START_INSTRUMENTATION +# define TALLY_STOP_INSTRUMENTATION() CALLGRIND_STOP_INSTRUMENTATION # define TALLY_TOGGLE_COLLECT() CALLGRIND_TOGGLE_COLLECT # define TALLY_DUMP(name) CALLGRIND_DUMP_STATS_AT(name) #endif @@ -66,6 +68,14 @@ PHP_FUNCTION(tally_hooks_start_instrumentation) TALLY_START_INSTRUMENTATION(); } +/* Stops simulating the process, which then runs at Valgrind's quickest */ +PHP_FUNCTION(tally_hooks_stop_instrumentation) +{ + ZEND_PARSE_PARAMETERS_NONE(); + + TALLY_STOP_INSTRUMENTATION(); +} + /* Turns collecting the counts off or on, keeping the simulated caches */ PHP_FUNCTION(tally_hooks_toggle_collect) { @@ -107,6 +117,7 @@ PHP_FUNCTION(tally_hooks_recorded_calls) static const zend_function_entry tally_hooks_functions[] = { ZEND_NS_NAMED_FE(TALLY_HOOKS_NAMESPACE, instrumented, ZEND_FN(tally_hooks_instrumented), arginfo_bool) ZEND_NS_NAMED_FE(TALLY_HOOKS_NAMESPACE, start_instrumentation, ZEND_FN(tally_hooks_start_instrumentation), arginfo_void) + ZEND_NS_NAMED_FE(TALLY_HOOKS_NAMESPACE, stop_instrumentation, ZEND_FN(tally_hooks_stop_instrumentation), arginfo_void) ZEND_NS_NAMED_FE(TALLY_HOOKS_NAMESPACE, toggle_collect, ZEND_FN(tally_hooks_toggle_collect), arginfo_void) ZEND_NS_NAMED_FE(TALLY_HOOKS_NAMESPACE, dump, ZEND_FN(tally_hooks_dump), arginfo_dump) #ifdef TALLY_HOOKS_RECORDING diff --git a/src/Runner/CaseRunner.php b/src/Runner/CaseRunner.php index 1e8e9b0..f2e6a8b 100644 --- a/src/Runner/CaseRunner.php +++ b/src/Runner/CaseRunner.php @@ -24,7 +24,8 @@ * CPU simulation the warm-up calls fill the simulated caches. The first counted span still costs * more than later ones, because PHP does work once on the first call of the counting path. So the * baseline is counted twice, and only the second one, named after the case with #baseline, is - * subtracted. + * subtracted. The counter is finished after the benchmark's span, also when it throws, so the + * after methods and the end of the process do not run under the simulation. * * The baseline is a separate part of the profile. It goes * through the same steps as the counted call: the return from starting the counter, a closure @@ -75,15 +76,19 @@ public function run(BenchmarkCase $case, ScratchDirectory $scratch, Counter $cou $this->callHook($meter, $object, $class->getMethod($name), $case); } $counter->prepare(); - for ($i = 0; $i < $case->warmup; $i++) { - $this->span($meter, $this->uncounted, $call, $arguments, $benchmark->getName()); - } + try { + for ($i = 0; $i < $case->warmup; $i++) { + $this->span($meter, $this->uncounted, $call, $arguments, $benchmark->getName()); + } - $baseline = $this->baseline(...); - $this->span($meter, $counter, $baseline, $arguments, $id . self::WARMUP_SUFFIX); - $baselineMemory = $this->span($meter, $counter, $baseline, $arguments, $id . self::BASELINE_SUFFIX); + $baseline = $this->baseline(...); + $this->span($meter, $counter, $baseline, $arguments, $id . self::WARMUP_SUFFIX); + $baselineMemory = $this->span($meter, $counter, $baseline, $arguments, $id . self::BASELINE_SUFFIX); - return $this->span($meter, $counter, $call, $arguments, $id)->minus($baselineMemory); + return $this->span($meter, $counter, $call, $arguments, $id)->minus($baselineMemory); + } finally { + $counter->finish(); + } } finally { foreach ($case->afterMethods as $name) { $this->callHook($meter, $object, $class->getMethod($name), $case); diff --git a/src/Runner/CodSpeedCounter.php b/src/Runner/CodSpeedCounter.php index 9728ced..b63fafd 100644 --- a/src/Runner/CodSpeedCounter.php +++ b/src/Runner/CodSpeedCounter.php @@ -11,7 +11,8 @@ * Callgrind empties its simulated caches when it starts the instrumentation, so the counter starts * it once, before the warm-up calls, and only turns collecting on and off for each span. The caches * stay warm from one span to the next, and the cycles of a small benchmark do not depend on which - * cache lines the code between the spans happens to touch. + * cache lines the code between the spans happens to touch. It stops the instrumentation after the + * last span, so the after methods and the end of the process run at Valgrind's quickest. */ final class CodSpeedCounter implements Counter { @@ -26,6 +27,12 @@ public function prepare(): void $this->hooks->startInstrumentation(); } + #[Override] + public function finish(): void + { + $this->hooks->stopInstrumentation(); + } + #[Override] public function start(): void { diff --git a/src/Runner/Counter.php b/src/Runner/Counter.php index 499df92..a78b964 100644 --- a/src/Runner/Counter.php +++ b/src/Runner/Counter.php @@ -12,6 +12,9 @@ interface Counter /** Called once for each case before its warm-up calls, so that they warm up what the counter's tool simulates */ public function prepare(): void; + /** Called once for each case after its last span, also when the benchmark throws */ + public function finish(): void; + public function start(): void; public function pause(): void; diff --git a/src/Runner/ExtensionInstrumentHooks.php b/src/Runner/ExtensionInstrumentHooks.php index 2f34996..e3c10fc 100644 --- a/src/Runner/ExtensionInstrumentHooks.php +++ b/src/Runner/ExtensionInstrumentHooks.php @@ -7,6 +7,7 @@ use function Tally\Hooks\dump; use function Tally\Hooks\instrumented; use function Tally\Hooks\start_instrumentation; +use function Tally\Hooks\stop_instrumentation; use function Tally\Hooks\toggle_collect; /** @@ -44,6 +45,12 @@ public function startInstrumentation(): void start_instrumentation(); } + #[Override] + public function stopInstrumentation(): void + { + stop_instrumentation(); + } + #[Override] public function toggleCollect(): void { diff --git a/src/Runner/IdleCounter.php b/src/Runner/IdleCounter.php index 3b138a5..be35b2a 100644 --- a/src/Runner/IdleCounter.php +++ b/src/Runner/IdleCounter.php @@ -16,6 +16,11 @@ public function prepare(): void { } + #[Override] + public function finish(): void + { + } + #[Override] public function start(): void { diff --git a/src/Runner/InstrumentHooks.php b/src/Runner/InstrumentHooks.php index dbf5eef..ac068a6 100644 --- a/src/Runner/InstrumentHooks.php +++ b/src/Runner/InstrumentHooks.php @@ -9,6 +9,9 @@ interface InstrumentHooks /** Starts simulating the process with empty caches; it collects counts only while collecting is on */ public function startInstrumentation(): void; + /** Stops simulating the process, which then runs at Valgrind's quickest */ + public function stopInstrumentation(): void; + /** Turns collecting the counts on or off, keeping the simulated caches */ public function toggleCollect(): void; diff --git a/tests/AllocatingCounter.php b/tests/AllocatingCounter.php index 7647508..18570bf 100644 --- a/tests/AllocatingCounter.php +++ b/tests/AllocatingCounter.php @@ -18,6 +18,11 @@ public function prepare(): void { } + #[Override] + public function finish(): void + { + } + #[Override] public function start(): void { diff --git a/tests/Integration/Cli/WorkerCommandTest.php b/tests/Integration/Cli/WorkerCommandTest.php index 15aa954..4538880 100644 --- a/tests/Integration/Cli/WorkerCommandTest.php +++ b/tests/Integration/Cli/WorkerCommandTest.php @@ -70,7 +70,7 @@ public function testCountsTheCaseAndItsBaselineWithTheLibrary(): void $this->assertSame(0, $exitCode, $errorOutput); $this->assertSame( - sprintf("instrument\ntoggle\ntoggle\ndump %1\$s::benchAllocate#warmup\ntoggle\ntoggle\ndump %1\$s::benchAllocate#baseline\ntoggle\ntoggle\ndump %1\$s::benchAllocate\n", self::BENCH), + sprintf("instrument\ntoggle\ntoggle\ndump %1\$s::benchAllocate#warmup\ntoggle\ntoggle\ndump %1\$s::benchAllocate#baseline\ntoggle\ntoggle\ndump %1\$s::benchAllocate\nstop\n", self::BENCH), recorded_calls(), ); $this->assertSame(['memory'], array_keys($this->writtenResult()->metrics->toArray())); diff --git a/tests/Integration/Runner/ExtensionInstrumentHooksTest.php b/tests/Integration/Runner/ExtensionInstrumentHooksTest.php index 8775d2f..5af63db 100644 --- a/tests/Integration/Runner/ExtensionInstrumentHooksTest.php +++ b/tests/Integration/Runner/ExtensionInstrumentHooksTest.php @@ -35,8 +35,9 @@ public function testCallsTheExtension(): void $hooks->toggleCollect(); $hooks->toggleCollect(); $hooks->dump('App\FooBench::benchFoo@a'); + $hooks->stopInstrumentation(); - $this->assertSame(implode("\n", ['instrument', 'toggle', 'toggle', 'dump App\FooBench::benchFoo@a']) . "\n", recorded_calls()); + $this->assertSame(implode("\n", ['instrument', 'toggle', 'toggle', 'dump App\FooBench::benchFoo@a', 'stop']) . "\n", recorded_calls()); } /** diff --git a/tests/JournalCounter.php b/tests/JournalCounter.php index 0c436ce..cce0548 100644 --- a/tests/JournalCounter.php +++ b/tests/JournalCounter.php @@ -16,6 +16,12 @@ public function prepare(): void Journal::$entries[] = 'counter prepare'; } + #[Override] + public function finish(): void + { + Journal::$entries[] = 'counter finish'; + } + #[Override] public function start(): void { diff --git a/tests/RecordingCounter.php b/tests/RecordingCounter.php index 979b1d3..8fe2a8b 100644 --- a/tests/RecordingCounter.php +++ b/tests/RecordingCounter.php @@ -23,6 +23,12 @@ public function prepare(): void $this->calls[] = 'prepare'; } + #[Override] + public function finish(): void + { + $this->calls[] = 'finish'; + } + #[Override] public function start(): void { diff --git a/tests/RecordingInstrumentHooks.php b/tests/RecordingInstrumentHooks.php index 4e0de28..7688535 100644 --- a/tests/RecordingInstrumentHooks.php +++ b/tests/RecordingInstrumentHooks.php @@ -23,6 +23,12 @@ public function startInstrumentation(): void $this->calls[] = 'instrument'; } + #[Override] + public function stopInstrumentation(): void + { + $this->calls[] = 'stop'; + } + #[Override] public function toggleCollect(): void { diff --git a/tests/Unit/Runner/CaseRunnerTest.php b/tests/Unit/Runner/CaseRunnerTest.php index 3a54bc2..0257190 100644 --- a/tests/Unit/Runner/CaseRunnerTest.php +++ b/tests/Unit/Runner/CaseRunnerTest.php @@ -67,6 +67,7 @@ public function testRunsTheHooksAndWarmUpUncountedAndCountsOneCallMinusTheBaseli 'excluded', 'counter resume', 'counter stop ' . RunnerBench::class . '::benchName@x', + 'counter finish', 'tear down x', ], Journal::$entries); } @@ -88,6 +89,7 @@ public function testWithoutWarmUpTheCountedCallIsTheFirst(): void 'excluded', 'counter resume', 'counter stop ' . RunnerBench::class . '::benchName@y', + 'counter finish', ], Journal::$entries); } @@ -138,6 +140,7 @@ public function testRunsTheAfterMethodsWhenTheBenchmarkThrows(): void 'counter stop ' . RunnerBench::class . '::benchThrows@z#baseline', 'counter start', 'bench throws', + 'counter finish', 'tear down z', ], Journal::$entries); } diff --git a/tests/Unit/Runner/CodSpeedCounterTest.php b/tests/Unit/Runner/CodSpeedCounterTest.php index c304ebc..0beb64b 100644 --- a/tests/Unit/Runner/CodSpeedCounterTest.php +++ b/tests/Unit/Runner/CodSpeedCounterTest.php @@ -9,7 +9,7 @@ final class CodSpeedCounterTest extends TestCase { - public function testInstrumentsOnceAndCollectsOnlyInTheSpans(): void + public function testInstrumentsFromPreparingToFinishingAndCollectsOnlyInTheSpans(): void { $hooks = new RecordingInstrumentHooks(); $counter = new CodSpeedCounter($hooks); @@ -21,6 +21,7 @@ public function testInstrumentsOnceAndCollectsOnlyInTheSpans(): void $counter->pause(); $counter->resume(); $counter->stop('App\FooBench::benchFoo@a'); + $counter->finish(); $this->assertSame([ 'instrument', @@ -32,6 +33,7 @@ public function testInstrumentsOnceAndCollectsOnlyInTheSpans(): void 'toggle', 'toggle', 'dump App\FooBench::benchFoo@a', + 'stop', ], $hooks->calls); }