diff --git a/.gitattributes b/.gitattributes index 9bdf4e6..394aede 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,13 +1,22 @@ *.php text eol=lf +.agents export-ignore .github export-ignore bench export-ignore examples export-ignore tests export-ignore tmp export-ignore +build/PHPStan export-ignore +build/stubs export-ignore +build/build-release.sh export-ignore +build/check-coverage.php export-ignore +build/check-release.sh export-ignore +build/tag-release.sh export-ignore .editorconfig export-ignore .gitattributes export-ignore .gitignore export-ignore +CLAUDE.md export-ignore +infection.json5 export-ignore Makefile export-ignore phpcs.xml export-ignore phpstan.neon export-ignore diff --git a/CLAUDE.md b/CLAUDE.md index c873aa4..05a77bd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ Tally benchmarks PHP code by counting instructions, cycles, and memory under Cod - `src/Expression` – the expression language of complexity classes, all pure: `Lexer` (tokens with offsets), `Parser` (recursive descent into the `Node` tree, sealed), `Printer` (the canonical form, parsing it gives the same tree), `TypeChecker` (finds what the parameter types reveal, through `ValueType`), `Evaluator`, and the `MathFunction`s of the `FunctionLibrary`. Every mistake is an `ExpressionException` that quotes the expression and underlines the part that is wrong; `ExpressionErrors` words them the same for the checker and the evaluator. - `src/Complexity` – `ComplexityClass`, the interface of `#[Complexity]` and its presets (`#[Linear]` and the others, which compose a `Complexity`), and `GrowthPoint`, where a case is on its growth curve. `Discovery\GrowthPoints` computes them at discovery, `Comparator::compareGrowth()` compares a case with its base. These cases have no numbers to accept. - `src/Quantity` – `QuantityFormat` formats and parses numbers with units (`3.119B`, `44.49MiB`). -- `src/Expectation` – where expected metrics are stored. `ExpectationSource` is the interface: `read()` the expectation, `file()` the file it lives in, the pure `rewrite(code, metrics)` that returns the file's code with new expectations, and `location()`, which throws `ExpectationException` when the source cannot be rewritten. `AttributeExpectations` implements it for `#[Expected]` and `#[ExpectedFor]` through `ExpectedAttributeRewriter`, the pure code-to-code rewrite, and `Tokens`, the token list it walks. `ExpectationChanges` plans the rewrites of many cases (pure): one `FileChange` per file with every case in it, in a `ChangePlan` together with the cases that failed. `ExpectationFiles` reads and writes the files. +- `src/Expectation` – where expected metrics are stored. `ExpectationSource` is the interface: `read()` the expectation, `file()` the file it lives in, the pure `edits(code, metrics)` that returns the `CodeEdit`s that write new expectations into the file's code, and `location()`, which throws `ExpectationException` when the source cannot be rewritten. `AttributeExpectations` implements it for `#[Expected]` and `#[ExpectedFor]` through `ExpectedAttributeRewriter`, which reads a file once into an `AttributeIndex` of every method's attributes, with `Tokens`, the token list it walks, and locates and edits each case in the index; `DirectorySuiteLoader` indexes each file once for all its cases, because tokenizing it for every case made locating and rewriting quadratic in the cases of a file. `ExpectationChanges` plans the rewrites of many cases (pure): it applies the edits of all cases of a file in one pass, one `FileChange` per file with every case in it, in a `ChangePlan` together with the cases that failed. `ExpectationFiles` reads and writes the files. - `src/Discovery` – `DirectorySuiteLoader` finds and loads `*Bench.php` files (I/O), `CaseExpander` turns one method into its cases (pure, works on reflection); `hooks()` finds a class's `#[Before]` and `#[After]` methods once per class, because scanning them for every benchmark method made loading a class quadratic in its size. `AttributeFactory` creates every attribute Tally reads: it checks the arguments against the attribute's constructor first (names, count, types through `ValueTypeCheck`, as under strict_types), so a mistake is an `InvalidBenchmarkException` naming the argument, not a PHP error. Everything a benchmark author can get wrong, including a file that does not load and a data provider that throws, ends as an `InvalidBenchmarkException` that names the benchmark or file; PHP errors with stack traces are bugs. `ArgumentResolver` checks that a method takes the arguments of a data set (a hook the leading ones) and `ConstructorArguments` that a constructor takes only `Meter` and `ScratchDirectory`; discovery checks both, so a mismatch fails `tally list` instead of the worker. - `src/Selection` – `CaseSelector` implementations: `FilterSelector`, `ShardSelector` (longest-processing-time sharding by expected instructions). Both keep a case compared by its growth with its base: the filter adds the base, the shards keep a method's cases together. - `src/Runner` – running one case in this process. `CaseRunner` calls the hooks uncounted, prepares the counter, calls the warm-up uncounted, then counts a `#warmup` span, an empty `#baseline` call, and one call of the benchmark, and subtracts the baseline's memory. The `#warmup` span exists because the first counted span costs more than later ones. `CodSpeedCounter` starts Callgrind's instrumentation once, when it is prepared, stops it when it is finished after the benchmark's span, so the after methods and the end of the process run at Valgrind's quickest, and only toggles collecting for each span: starting the instrumentation empties the simulated caches, and doing it for every span made the cycles of small benchmarks depend on the cache lines the hooks touched. `MeterSession` is the `Meter` the benchmark object gets in its constructor, one per case: every call `CaseRunner` makes is a span of it with its own `Counter` (`CodSpeedCounter` over `InstrumentHooks` for the counted ones, `IdleCounter` for hooks and warm-up), and it measures memory above the start of the span through `Memory`. `ExtensionInstrumentHooks` calls the `tally_hooks` extension, the only C that Tally calls. @@ -24,7 +24,7 @@ Tally benchmarks PHP code by counting instructions, cycles, and memory under Cod - `ext/tally_hooks` – the PHP extension the counted worker calls: functions in `Tally\Hooks` that make Callgrind's client requests through Valgrind's `valgrind.h` and `callgrind.h`. Built with `-DTALLY_HOOKS_RECORDING`, it records the calls instead, for the tests. Keep it to these few functions; the logic belongs in PHP. - `bench/` – Tally's benchmarks of itself, listed in `tally.php`. They cover the code whose cost grows with the suite, a benchmark file, or a profile, and that runs once per case or once per worker: locating and rewriting expectations, planning and diffing new expectations, expanding cases, sharding, reading profiles, and rendering reports. `SuiteSource` generates the code and cases of large benchmark classes as their input. Build inputs in data providers and before methods, which are not counted, and keep the counted call to the one method being measured. `#[Warmup(1)]` keeps autoloading out of the count. - `examples/` – short, artificial benchmarks for README.md, one feature each. `examples/Scripts` shows a data provider with its own `ExpectationSource`, which keeps the numbers in a comment at the top of each script. Keep them readable over realistic; real measurements of Tally belong in `bench/`. CI runs them under CodSpeed next to `bench/`, so they stay valid. -- `build/` – scripts for the build, such as the coverage check. `build-codspeed.sh` builds CodSpeed's Valgrind (Callgrind only, static launcher) at a pinned tag into `codspeed/linux-`, which `1.x` ignores, and `build-tally-hooks.sh` builds `ext/tally_hooks` with `phpize` next to it; `make codspeed` runs both for this machine and PHP, CI builds `tally_hooks` for PHP 8.2 to 8.6, NTS and ZTS, in the official `php` Docker images. `build-release.sh` turns `1.x` into the `release/1.x` tree by committing the builds of both platforms; CI pushes it on every push to `1.x`, and releases are tagged there. `check-release.sh` installs a release tree into a new project with Composer and counts a benchmark, which CI runs on every pull request on x86_64 and aarch64, with PHP 8.2 and 8.5, and 8.5 ZTS. `build/PHPStan` holds Tally's own PHPStan rules and `BenchmarkUsageProvider`, which tells the dead code detector about the methods Tally calls through reflection. `build/stubs` holds the stubs for what PHPStan cannot see, such as the functions of `tally_hooks`. +- `build/` – scripts for the build, such as the coverage check. `build-codspeed.sh` builds CodSpeed's Valgrind (Callgrind only, static launcher) at a pinned tag into `codspeed/linux-`, which `1.x` ignores, and `build-tally-hooks.sh` builds `ext/tally_hooks` with `phpize` next to it; `make codspeed` runs both for this machine and PHP, CI builds `tally_hooks` for PHP 8.2 to 8.6, NTS and ZTS, in the official `php` Docker images. `build-release.sh` turns `1.x` into the `release/1.x` tree by committing the builds of both platforms; CI pushes it on every push to `1.x`, and releases are tagged there with `tag-release.sh `, which checks that the generated commit sits on the `1.x` commit to release. `check-release.sh` installs a release tree into a new project with Composer and counts a benchmark, which CI runs on every pull request on x86_64 and aarch64, with PHP 8.2 and 8.5, and 8.5 ZTS. `build/PHPStan` holds Tally's own PHPStan rules and `BenchmarkUsageProvider`, which tells the dead code detector about the methods Tally calls through reflection. `build/stubs` holds the stubs for what PHPStan cannot see, such as the functions of `tally_hooks`. - `tests/Unit` – pure logic, no file system, processes, or network. `tests/Integration` – adapters and the CLI. `tests/Fixtures` – benchmark classes and configuration files the tests load; excluded from PHPStan and the coding standard because some are invalid on purpose. ## Design rules diff --git a/README.md b/README.md index b013084..342b2c3 100644 --- a/README.md +++ b/README.md @@ -224,7 +224,7 @@ Without `#[Warmup]`, the counted call is the first one. The difference can be la ```php public function benchCold(): int|false { - // 73.16K instructions, mostly compiling the regular expression + // 74.63K instructions, mostly compiling the regular expression return preg_match('~^(?[a-z]+)@(?[a-z]+)\.cold$~', 'tally@example.cold'); } @@ -562,7 +562,7 @@ An `ExpectationSource` has four methods: | --- | --- | | `read(): ?Metrics` | The expected numbers, `null` for a new case. | | `file(): string` | The file that keeps them. | -| `rewrite(string $code, Metrics $measured): string` | The code of the file with the measured numbers instead. Takes and returns code without reading or writing the file, so `tally accept` writes it and `tally run` shows it as a patch. | +| `edits(string $code, Metrics $measured): list` | The edits of the file's code that write the measured numbers instead, each a `Tally\Expectation\CodeEdit` of an offset, a length, and the text that replaces those bytes. Takes code and returns edits without reading or writing the file, so `tally accept` writes the edited code and `tally run` shows it as a patch. Tally applies the edits of all cases of a file in one pass: they must not overlap, equal edits, such as the same import, are made once, and insertions at one offset go in the natural order of their text. | | `location(): SourceLocation` | Where reports point to. Throws `ExpectationException` when the case's numbers cannot be rewritten, so `tally run` fails before counting. | [examples/Scripts](examples/Scripts) is the whole example. @@ -571,11 +571,11 @@ An `ExpectationSource` has four methods: `tally run` loads the benchmarks, then runs each case in two processes of its own: first natively, which fails quickly when the case is broken, then under CodSpeed's Valgrind, which counts it. Several cases run at once, see `--jobs`. -Every process loads the whole suite, so **data providers run in every process**: in `tally run` itself, and twice for each case. Every `tally` command runs them too. Keep them cheap and do expensive preparation in a `#[Before]` method, which runs only for the case that needs it. +`tally run` loads the whole suite, and each process of a case loads only the file of the case's class, so **the data providers of a file run twice for each of its cases**, and once in `tally run`. Every `tally` command runs them too. Keep them cheap and do expensive preparation in a `#[Before]` method, which runs only for the case that needs it. A benchmark class finds the other classes it uses through the autoloader, not because another `*Bench.php` file declared them. In each of the two processes of a case, Tally: -1. loads the benchmarks and calls every data provider, not counted, +1. loads the file of the case's class and calls the data providers of its benchmarks, not counted, 2. creates the benchmark class, once, with its `Meter` and `ScratchDirectory`, 3. calls the `#[Before]` methods, not counted, 4. starts CodSpeed's CPU simulation, with empty caches, diff --git a/bench/Discovery/CaseExpanderBench.php b/bench/Discovery/CaseExpanderBench.php index 3fdaf1d..3ee2042 100644 --- a/bench/Discovery/CaseExpanderBench.php +++ b/bench/Discovery/CaseExpanderBench.php @@ -17,6 +17,7 @@ use Tally\Discovery\GrowthPoints; use Tally\Discovery\InvalidBenchmarkException; use Tally\Discovery\ValueTypeCheck; +use Tally\Expectation\AttributeIndex; use Tally\Expectation\ExpectedAttributeRewriter; use Tally\Expression\Evaluator; use Tally\Expression\ExpressionErrors; @@ -35,20 +36,23 @@ use function sprintf; /** - * Every tally command, and every worker tally run starts for a case, loads the whole suite, so - * the cases of each benchmark method are expanded once per case of the run. + * Every tally command loads the whole suite, and every worker tally run starts for a case loads + * the file of its class, so the cases of each benchmark method are expanded once per case of + * their file. */ #[Warmup(1)] final class CaseExpanderBench { /** - * @return iterable the methods and data sets of the class + * @return iterable the methods and data sets of the class, + * and the attributes of its file, which the loader indexes once before expanding its methods */ public static function classes(): iterable { + $source = new SuiteSource(); foreach ([10, 100] as $size) { - yield sprintf('size-%d', $size) => [$size, $size]; + yield sprintf('size-%d', $size) => [$size, $size, (new ExpectedAttributeRewriter())->index($source->code($size, $size, true))]; } } @@ -73,12 +77,13 @@ public function __construct(private ScratchDirectory $scratch) public function declareClass(int $methods, int $dataSets): void { $source = new SuiteSource(); + $code = $source->code($methods, $dataSets, true); if (class_exists($source->className($methods, $dataSets), false)) { return; } $file = sprintf('%s/Suite%dx%dBench.php', $this->scratch->path(), $methods, $dataSets); - file_put_contents($file, $source->code($methods, $dataSets, true)); + file_put_contents($file, $code); require $file; } @@ -88,7 +93,7 @@ public function declareClass(int $methods, int $dataSets): void #[DataProvider('classes')] #[ExpectedFor('size-10', instructions: '1.835M', cycles: '2.451M', memory: '19.59KiB')] #[ExpectedFor('size-100', instructions: '19.02M', cycles: '25.07M', memory: '174.3KiB')] - public function benchExpand(int $methods, int $dataSets): void + public function benchExpand(int $methods, int $dataSets, AttributeIndex $attributes): void { $name = (new SuiteSource())->className($methods, $dataSets); if (!class_exists($name, false)) { @@ -102,7 +107,7 @@ public function benchExpand(int $methods, int $dataSets): void continue; } - $this->expander->expand($class, $method, $hooks, $this->configuration); + $this->expander->expand($class, $method, $hooks, $attributes, $this->configuration); } } diff --git a/bench/Expectation/ExpectationChangesBench.php b/bench/Expectation/ExpectationChangesBench.php index 8a3f8d0..5575aec 100644 --- a/bench/Expectation/ExpectationChangesBench.php +++ b/bench/Expectation/ExpectationChangesBench.php @@ -39,8 +39,8 @@ public static function newCases(): iterable * @param array $code */ #[DataProvider('newCases')] - #[ExpectedFor('cases-2', instructions: '641.4K', cycles: '749.8K', memory: '35.08KiB')] - #[ExpectedFor('cases-200', instructions: '1.895B', cycles: '2.071B', memory: '1.043MiB')] + #[ExpectedFor('cases-2', instructions: '143.8K', cycles: '194.3K', memory: '5.148KiB')] + #[ExpectedFor('cases-200', instructions: '21.63M', cycles: '26.19M', memory: '291.0KiB')] public function benchPlan(array $accepted, array $code): void { (new ExpectationChanges())->plan($accepted, $code); diff --git a/bench/Expectation/ExpectedAttributeRewriterBench.php b/bench/Expectation/ExpectedAttributeRewriterBench.php index 49efd46..ee1a8a9 100644 --- a/bench/Expectation/ExpectedAttributeRewriterBench.php +++ b/bench/Expectation/ExpectedAttributeRewriterBench.php @@ -6,6 +6,7 @@ use Tally\Attribute\ExpectedFor; use Tally\Attribute\Warmup; use Tally\Bench\SuiteSource; +use Tally\Expectation\AttributeIndex; use Tally\Expectation\ExpectationException; use Tally\Expectation\ExpectedAttributeRewriter; use Tally\Expectation\MethodNotFoundException; @@ -13,47 +14,45 @@ use function sprintf; /** - * tally run and tally compare locate the expectation of every case in its file, and tally accept - * rewrites it, so a file with many cases is tokenized once per case. + * Every tally command indexes the attributes of each benchmark file once, and tally run, compare, + * and accept then locate and rewrite the expectation of each case in the index. */ #[Warmup(1)] final class ExpectedAttributeRewriterBench { /** - * @return iterable the code, the class, and the data set whose attribute to find + * @return iterable */ public static function files(): iterable { $source = new SuiteSource(); foreach ([10, 100] as $size) { - yield sprintf('size-%d', $size) => [ - $source->code($size, $size, true), - $source->className($size, $size), - $source->dataSet($size), - ]; + yield sprintf('size-%d', $size) => [$source->code($size, $size, true)]; } } - /** - * @throws ExpectationException - * @throws MethodNotFoundException - */ #[DataProvider('files')] - #[ExpectedFor('size-10', instructions: '1.906M', cycles: '2.077M', memory: '128.6KiB')] - #[ExpectedFor('size-100', instructions: '16.84M', cycles: '18.60M', memory: '1.002MiB')] - public function benchLocate(string $code, string $class, string $dataSet): void + #[ExpectedFor('size-10', instructions: '2.007M', cycles: '2.209M', memory: '135.9KiB')] + #[ExpectedFor('size-100', instructions: '17.36M', cycles: '19.11M', memory: '1.093MiB')] + public function benchIndex(string $code): void { - (new ExpectedAttributeRewriter())->locate($code, $class, SuiteSource::SIZED_METHOD, $dataSet); + (new ExpectedAttributeRewriter())->index($code); } /** - * @return iterable files() with the measured metrics + * @return iterable the index, the class, the data set whose attribute to rewrite, and its metrics */ - public static function rewrites(): iterable + public static function indexes(): iterable { - foreach (self::files() as $name => $arguments) { - yield $name => [...$arguments, (new SuiteSource())->measured(1)]; + $source = new SuiteSource(); + foreach ([10, 100] as $size) { + yield sprintf('size-%d', $size) => [ + (new ExpectedAttributeRewriter())->index($source->code($size, $size, true)), + $source->className($size, $size), + $source->dataSet($size), + $source->measured(1), + ]; } } @@ -61,12 +60,12 @@ public static function rewrites(): iterable * @throws ExpectationException * @throws MethodNotFoundException */ - #[DataProvider('rewrites')] - #[ExpectedFor('size-10', instructions: '1.947M', cycles: '2.133M', memory: '128.7KiB')] - #[ExpectedFor('size-100', instructions: '16.91M', cycles: '18.68M', memory: '1.007MiB')] - public function benchRewrite(string $code, string $class, string $dataSet, Metrics $measured): void + #[DataProvider('indexes')] + #[ExpectedFor('size-10', instructions: '47.67K', cycles: '65.38K', memory: '1.445KiB')] + #[ExpectedFor('size-100', instructions: '80.65K', cycles: '103.6K', memory: '1.445KiB')] + public function benchEdits(AttributeIndex $index, string $class, string $dataSet, Metrics $measured): void { - (new ExpectedAttributeRewriter())->rewrite($code, $class, SuiteSource::SIZED_METHOD, $dataSet, $measured, false); + (new ExpectedAttributeRewriter())->edits($index, $class, SuiteSource::SIZED_METHOD, $dataSet, $measured, false); } } diff --git a/bench/Report/UnifiedDiffBench.php b/bench/Report/UnifiedDiffBench.php index 903c099..152307b 100644 --- a/bench/Report/UnifiedDiffBench.php +++ b/bench/Report/UnifiedDiffBench.php @@ -13,8 +13,8 @@ use function sprintf; /** - * tally run prints the expectations of new cases as a patch. The diff compares every line between - * the first and the last change with every other, so its cost grows with the distance between them. + * tally run prints the expectations of new cases as a patch. The diff's search takes time for each + * changed line, so changes far apart in a long file must cost no more than changes close together. */ #[Warmup(1)] final class UnifiedDiffBench @@ -42,9 +42,9 @@ public static function changes(): iterable } #[DataProvider('changes')] - #[ExpectedFor('first-and-last-line-changed', instructions: '2.186B', cycles: '2.341B', memory: '39.59MiB')] + #[ExpectedFor('first-and-last-line-changed', instructions: '6.776M', cycles: '7.125M', memory: '405.8KiB')] #[ExpectedFor('middle-line-changed', instructions: '5.292M', cycles: '5.540M', memory: '375.6KiB')] - #[ExpectedFor('new-expectations-200', instructions: '787.4M', cycles: '848.6M', memory: '20.46MiB')] + #[ExpectedFor('new-expectations-200', instructions: '71.04M', cycles: '73.48M', memory: '1.816MiB')] public function benchRender(string $before, string $after): void { (new UnifiedDiff())->render('bench/SuiteBench.php', $before, $after); diff --git a/bench/Selection/ShardSelectorBench.php b/bench/Selection/ShardSelectorBench.php index 25d311f..50aa7c5 100644 --- a/bench/Selection/ShardSelectorBench.php +++ b/bench/Selection/ShardSelectorBench.php @@ -15,7 +15,7 @@ use function sprintf; /** - * Every shard of a CI run, and every worker it starts, selects its cases from the whole suite. + * Every shard of a CI run selects its cases from the whole suite. */ #[Warmup(1)] final class ShardSelectorBench @@ -49,17 +49,20 @@ private static function cases(int $count): array { $source = new SuiteSource(); $rewriter = new ExpectedAttributeRewriter(); + // sharding reads only the expected numbers + $index = $rewriter->index(''); $class = $source->className($count, 0); $cases = []; for ($i = 1; $i <= $count; $i++) { $method = sprintf('benchMethod%d', $i); $cases[] = new BenchmarkCase( new CaseId($class, $method, null), + SuiteSource::FILE, [], 0, [], [], - new AttributeExpectations($rewriter, SuiteSource::FILE, $class, $method, null, $i % 10 === 0 ? null : $source->measured($i * 7_919 % 1_000), false), + new AttributeExpectations($rewriter, SuiteSource::FILE, $index, $class, $method, null, $i % 10 === 0 ? null : $source->measured($i * 7_919 % 1_000), false), new RelativeTolerance(0.1), null, ); diff --git a/bench/SuiteSource.php b/bench/SuiteSource.php index afc3914..27fd791 100644 --- a/bench/SuiteSource.php +++ b/bench/SuiteSource.php @@ -6,6 +6,7 @@ use Tally\BenchmarkCase; use Tally\CaseId; use Tally\Expectation\AttributeExpectations; +use Tally\Expectation\AttributeIndex; use Tally\Expectation\ExpectedAttributeRewriter; use Tally\Metric; use Tally\Metrics; @@ -38,13 +39,14 @@ public function className(int $methods, int $dataSets): string public function measuredCases(int $methods, int $dataSets): array { $rewriter = new ExpectedAttributeRewriter(); + $index = $rewriter->index($this->code($methods, $dataSets, false)); $class = $this->className($methods, $dataSets); $cases = []; for ($i = 1; $i <= $methods; $i++) { - $cases[] = [$this->case($rewriter, $class, sprintf('benchMethod%d', $i), null), $this->measured($i)]; + $cases[] = [$this->case($rewriter, $index, $class, sprintf('benchMethod%d', $i), null), $this->measured($i)]; } for ($i = 1; $i <= $dataSets; $i++) { - $cases[] = [$this->case($rewriter, $class, self::SIZED_METHOD, $this->dataSet($i)), $this->measured($i)]; + $cases[] = [$this->case($rewriter, $index, $class, self::SIZED_METHOD, $this->dataSet($i)), $this->measured($i)]; } return $cases; @@ -59,15 +61,16 @@ public function measured(int $index): Metrics ->with(Metric::Memory, 1_024 * $index); } - private function case(ExpectedAttributeRewriter $rewriter, string $class, string $method, ?string $dataSet): BenchmarkCase + private function case(ExpectedAttributeRewriter $rewriter, AttributeIndex $index, string $class, string $method, ?string $dataSet): BenchmarkCase { return new BenchmarkCase( new CaseId($class, $method, $dataSet), + self::FILE, [], 0, [], [], - new AttributeExpectations($rewriter, self::FILE, $class, $method, $dataSet, null, false), + new AttributeExpectations($rewriter, self::FILE, $index, $class, $method, $dataSet, null, false), new RelativeTolerance(0.1), null, ); diff --git a/build/tag-release.sh b/build/tag-release.sh new file mode 100755 index 0000000..dd85623 --- /dev/null +++ b/build/tag-release.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Tags a release on release/1.x, which CI force-pushes after every push to 1.x: the tag goes on the +# generated commit only when it sits directly on the 1.x commit to release, so it ships that code +# with its builds. Pushes nothing; prints the command that does. +# Usage: build/tag-release.sh [<1.x commit>, origin/1.x by default] +set -euo pipefail + +version="${1:?Usage: build/tag-release.sh [<1.x commit>]}" +git fetch -q origin 1.x release/1.x +source_commit="$(git rev-parse "${2:-origin/1.x}^{commit}")" +release_commit="$(git rev-parse origin/release/1.x)" + +if [ "$(git rev-parse "$release_commit^")" != "$source_commit" ]; then + echo "origin/release/1.x is not built from $source_commit yet: wait for CI on 1.x to push it." >&2 + exit 1 +fi +if ! git log -1 --format=%s "$release_commit" | grep -q '^\[GENERATED\]'; then + echo "origin/release/1.x does not end with the generated commit of the builds." >&2 + exit 1 +fi + +git tag -a "$version" -m "$version" "$release_commit" +echo "Tagged $version on $release_commit, built from $source_commit. Push it with:" +echo " git push origin $version" diff --git a/composer.json b/composer.json index 52184de..246eae1 100644 --- a/composer.json +++ b/composer.json @@ -6,7 +6,8 @@ ], "keywords": ["benchmark", "performance", "codspeed"], "require": { - "php": "^8.2" + "php": "^8.2", + "ext-posix": "*" }, "require-dev": { "infection/infection": "^0.35.10", diff --git a/examples/Scripts/ScriptExpectations.php b/examples/Scripts/ScriptExpectations.php index f7a43b4..d7bef78 100644 --- a/examples/Scripts/ScriptExpectations.php +++ b/examples/Scripts/ScriptExpectations.php @@ -30,9 +30,9 @@ public function file(): string } #[Override] - public function rewrite(string $code, Metrics $measured): string + public function edits(string $code, Metrics $measured): array { - return $this->header->write($code, $measured); + return [$this->header->write($code, $measured)]; } #[Override] diff --git a/examples/Scripts/ScriptHeader.php b/examples/Scripts/ScriptHeader.php index ae0dca2..538a14e 100644 --- a/examples/Scripts/ScriptHeader.php +++ b/examples/Scripts/ScriptHeader.php @@ -2,16 +2,14 @@ namespace Tally\Examples\Scripts; +use Tally\Expectation\CodeEdit; use Tally\Expectation\ExpectationException; use Tally\Metric; use Tally\Metrics; use Tally\Quantity\InvalidQuantityException; -use function addcslashes; use function preg_match; -use function preg_replace; use function sprintf; use function strlen; -use function substr; /** * Reads and writes the expected numbers in the comment at the top of a script: @@ -24,7 +22,7 @@ * * memory: 12.00KiB * *\/ * - * Pure: it takes code and returns code, ScriptExpectations does the rest. + * Pure: it takes code and returns the edit of it, ScriptExpectations does the rest. */ final class ScriptHeader { @@ -47,9 +45,11 @@ public function read(string $code): ?Metrics } /** + * Replaces the comment, or adds it after the format($metrics, Metric::Cycles), $this->format($metrics, Metric::Memory), ); - if (preg_match(self::PATTERN, $code) === 1) { - return (string) preg_replace(self::PATTERN, '$1' . "\n" . $this->escape($header), $code); + if (preg_match(self::PATTERN, $code, $matches) === 1) { + return new CodeEdit(strlen($matches[1]), strlen($matches[0]) - strlen($matches[1]), "\n" . $header); } if (preg_match('~\A<\?php[^\n]*\n~', $code, $matches) !== 1) { throw new ExpectationException('The script does not start with a format()->format($metrics->get($metric) ?? 0); } - /** The replacement of preg_replace() reads $ and \ */ - private function escape(string $replacement): string - { - return addcslashes($replacement, '$\\'); - } - } diff --git a/examples/WarmupBench.php b/examples/WarmupBench.php index 89bba46..4df9ec9 100644 --- a/examples/WarmupBench.php +++ b/examples/WarmupBench.php @@ -14,7 +14,7 @@ final class WarmupBench { - #[Expected(instructions: '73.16K', cycles: '236.6K', memory: '48B')] + #[Expected(instructions: '74.63K', cycles: '256.1K', memory: '48B')] public function benchCold(): int|false { return preg_match('~^(?[a-z]+)@(?[a-z]+)\.cold$~', 'tally@example.cold'); diff --git a/src/BenchmarkCase.php b/src/BenchmarkCase.php index c0278fb..452af99 100644 --- a/src/BenchmarkCase.php +++ b/src/BenchmarkCase.php @@ -13,6 +13,7 @@ final class BenchmarkCase { /** + * @param string $file the file that declares the benchmark's class, the one the workers load * @param list $arguments * @param list $beforeMethods * @param list $afterMethods @@ -20,6 +21,7 @@ final class BenchmarkCase */ public function __construct( public readonly CaseId $id, + public readonly string $file, public readonly array $arguments, public readonly int $warmup, public readonly array $beforeMethods, @@ -34,7 +36,7 @@ public function __construct( /** The case compared by how its instructions grow, instead of by expected numbers */ public function withGrowth(GrowthPoint $growth): self { - return new self($this->id, $this->arguments, $this->warmup, $this->beforeMethods, $this->afterMethods, $this->expectations, $this->tolerance, $growth); + return new self($this->id, $this->file, $this->arguments, $this->warmup, $this->beforeMethods, $this->afterMethods, $this->expectations, $this->tolerance, $growth); } } diff --git a/src/Cli/ApplicationFactory.php b/src/Cli/ApplicationFactory.php index cd3d2b7..c83d06f 100644 --- a/src/Cli/ApplicationFactory.php +++ b/src/Cli/ApplicationFactory.php @@ -74,11 +74,12 @@ public function create($stdout, $stderr, string $workingDirectory, array $enviro $closestName = new ClosestName(); $argumentResolver = new ArgumentResolver(); $constructorArguments = new ConstructorArguments(); + $rewriter = new ExpectedAttributeRewriter(); $functions = new FunctionLibrary([new Log(), new SquareRoot(), new Exponential(), new Factorial()]); $expressionErrors = new ExpressionErrors($closestName); $growthPoints = new GrowthPoints(new Parser(new Lexer()), new TypeChecker($functions, $expressionErrors), new Evaluator($functions, $expressionErrors)); $selectedCases = new SelectedCases( - new DirectorySuiteLoader(new CaseExpander(new ExpectedAttributeRewriter(), new AttributeFactory(new ValueTypeCheck()), $argumentResolver, $growthPoints), $constructorArguments, $relativePath), + new DirectorySuiteLoader(new CaseExpander($rewriter, new AttributeFactory(new ValueTypeCheck()), $argumentResolver, $growthPoints), $constructorArguments, $rewriter, $relativePath), $workingDirectory, ); $idle = new IdleCounter(); diff --git a/src/Cli/RunCommand.php b/src/Cli/RunCommand.php index e242613..b004451 100644 --- a/src/Cli/RunCommand.php +++ b/src/Cli/RunCommand.php @@ -118,7 +118,7 @@ public function execute(Input $input, Output $output): int $workDirectory = sys_get_temp_dir() . '/tally-' . getmypid(); $settings = new WorkerSettings( $configuration->iniEntries(), - $this->suiteArguments($input), + $this->suiteOptions($input), $valgrind, $extension, $workDirectory, @@ -164,7 +164,7 @@ private function measure(array $cases, WorkerSettings $settings, array $environm { $jobsOfCases = []; foreach ($cases as $case) { - $jobsOfCases[] = new Job($case, $this->caseJobs->steps($settings, $case[0]->id)); + $jobsOfCases[] = new Job($case, $this->caseJobs->steps($settings, $case[0])); } $measurements = []; @@ -233,31 +233,20 @@ private function jobs(Input $input): int } /** - * The options and paths with which the workers load the same cases, with absolute paths, so - * they do not depend on the working directory. + * The options with which the workers load the configuration, with an absolute path, so they do + * not depend on the working directory. * * @return list */ - private function suiteArguments(Input $input): array + private function suiteOptions(Input $input): array { - $arguments = []; $config = $input->option('config'); - if ($config !== null) { - $arguments[] = '--config=' . $this->absolute($config); + if ($config === null) { + return []; } - $paths = $input->arguments(); - if ($paths === []) { - return $arguments; - } - - return [...$arguments, '--', ...array_map($this->absolute(...), $paths)]; - } - - private function absolute(string $path): string - { - $realPath = realpath($path); + $realPath = realpath($config); - return $realPath === false ? $path : $realPath; + return ['--config=' . ($realPath === false ? $config : $realPath)]; } } diff --git a/src/Discovery/CaseExpander.php b/src/Discovery/CaseExpander.php index 6eb4dcf..1160fa5 100644 --- a/src/Discovery/CaseExpander.php +++ b/src/Discovery/CaseExpander.php @@ -19,6 +19,7 @@ use Tally\Configuration; use Tally\DataSet; use Tally\Expectation\AttributeExpectations; +use Tally\Expectation\AttributeIndex; use Tally\Expectation\ExpectationSource; use Tally\Expectation\ExpectedAttributeRewriter; use Tally\Metrics; @@ -77,10 +78,11 @@ public function hooks(ReflectionClass $class): array /** * @param ReflectionClass $class * @param array{before: list, after: list} $hooks what hooks() returns for the class + * @param AttributeIndex $attributes of the file that declares the method * @return list in the order the data provider yields them * @throws InvalidBenchmarkException */ - public function expand(ReflectionClass $class, ReflectionMethod $method, array $hooks, Configuration $configuration): array + public function expand(ReflectionClass $class, ReflectionMethod $method, array $hooks, AttributeIndex $attributes, Configuration $configuration): array { $where = $class->getName() . '::' . $method->getName() . '()'; foreach ([Before::class => 'Before', After::class => 'After'] as $hook => $name) { @@ -103,7 +105,7 @@ public function expand(ReflectionClass $class, ReflectionMethod $method, array $ $arguments, $warmup, $hooks, - new AttributeExpectations($this->rewriter, $file, $declaringClass, $method->getName(), $dataSet, $metrics, $configuration->namedArguments()), + new AttributeExpectations($this->rewriter, $file, $attributes, $declaringClass, $method->getName(), $dataSet, $metrics, $configuration->namedArguments()), $tolerance, ); @@ -186,7 +188,7 @@ private function case( $this->resolver->resolveHook($class->getMethod($hook), $arguments); } - return new BenchmarkCase(new CaseId($class->getName(), $method->getName(), $dataSet), $arguments, $warmup, $hooks['before'], $hooks['after'], $expectations, $tolerance, null); + return new BenchmarkCase(new CaseId($class->getName(), $method->getName(), $dataSet), (string) $class->getFileName(), $arguments, $warmup, $hooks['before'], $hooks['after'], $expectations, $tolerance, null); } /** diff --git a/src/Discovery/DirectorySuiteLoader.php b/src/Discovery/DirectorySuiteLoader.php index d5c94b0..41b02e1 100644 --- a/src/Discovery/DirectorySuiteLoader.php +++ b/src/Discovery/DirectorySuiteLoader.php @@ -11,12 +11,15 @@ use Tally\BenchmarkCase; use Tally\CaseId; use Tally\Configuration; +use Tally\Expectation\ExpectedAttributeRewriter; use Tally\Report\RelativePath; use Throwable; +use function array_fill_keys; use function array_filter; use function array_map; +use function array_merge; +use function file_get_contents; use function get_declared_classes; -use function in_array; use function is_file; use function iterator_to_array; use function realpath; @@ -33,7 +36,12 @@ final class DirectorySuiteLoader { - public function __construct(private CaseExpander $expander, private ConstructorArguments $constructor, private RelativePath $relativePath) + public function __construct( + private CaseExpander $expander, + private ConstructorArguments $constructor, + private ExpectedAttributeRewriter $rewriter, + private RelativePath $relativePath + ) { } @@ -50,13 +58,16 @@ public function load(array $paths, string $workingDirectory, Configuration $conf $this->require($file, $this->relativePath->of($file, $workingDirectory)); } - $cases = []; + $expanded = []; + // the attributes of each file that declares a benchmark method, read once for all its cases + $attributes = []; + $isLoaded = array_fill_keys($files, true); $classNames = get_declared_classes(); // the order PHP declared them in depends on what was autoloaded before, so the first invalid benchmark would too sort($classNames); foreach ($classNames as $className) { $class = new ReflectionClass($className); - if (!in_array($class->getFileName(), $files, true) || !$class->isInstantiable()) { + if (!isset($isLoaded[(string) $class->getFileName()]) || !$class->isInstantiable()) { continue; } $this->constructor->check($class); @@ -69,10 +80,13 @@ public function load(array $paths, string $workingDirectory, Configuration $conf throw new InvalidBenchmarkException($className . '::' . $method->getName() . '()', 'a bench* method must be public and not static. Rename it if it is not a benchmark.'); } - $cases = [...$cases, ...$this->expander->expand($class, $method, $hooks, $configuration)]; + $file = (string) $method->getDeclaringClass()->getFileName(); + $attributes[$file] ??= $this->rewriter->index((string) file_get_contents($file)); + $expanded[] = $this->expander->expand($class, $method, $hooks, $attributes[$file], $configuration); } } + $cases = array_merge(...$expanded); usort($cases, static fn (BenchmarkCase $a, BenchmarkCase $b): int => CaseId::compare($a->id, $b->id)); return $cases; diff --git a/src/Execution/CaseJobs.php b/src/Execution/CaseJobs.php index 5287a33..97e4408 100644 --- a/src/Execution/CaseJobs.php +++ b/src/Execution/CaseJobs.php @@ -3,6 +3,7 @@ namespace Tally\Execution; use RuntimeException; +use Tally\BenchmarkCase; use Tally\CaseId; use Tally\Measurement; use Tally\Process\ProcessCommand; @@ -38,9 +39,9 @@ public function __construct( * @return non-empty-list * @throws RuntimeException when the case's directories cannot be created */ - public function steps(WorkerSettings $settings, CaseId $id): array + public function steps(WorkerSettings $settings, BenchmarkCase $case): array { - $directory = $this->directory($settings, $id); + $directory = $this->directory($settings, $case->id); foreach (['/scratch', '/profile'] as $subdirectory) { if (!is_dir($directory . $subdirectory) && !@mkdir($directory . $subdirectory, 0777, true)) { throw new RuntimeException(sprintf('Cannot create %s.', $directory . $subdirectory)); @@ -49,12 +50,12 @@ public function steps(WorkerSettings $settings, CaseId $id): array return [ new ProcessCommand( - $this->commandLine->native($settings, $id, $directory . '/scratch', $directory . '/native.json'), + $this->commandLine->native($settings, $case, $directory . '/scratch', $directory . '/native.json'), $directory . '/native.log', $settings->nativeTimeLimitSeconds, ), new ProcessCommand( - $this->commandLine->counted($settings, $id, $directory . '/scratch', $directory . '/counted.json', $directory . '/profile'), + $this->commandLine->counted($settings, $case, $directory . '/scratch', $directory . '/counted.json', $directory . '/profile'), $directory . '/counted.log', $settings->countedTimeLimitSeconds, ), diff --git a/src/Execution/WorkerCommandLine.php b/src/Execution/WorkerCommandLine.php index b7d0681..b55e6e3 100644 --- a/src/Execution/WorkerCommandLine.php +++ b/src/Execution/WorkerCommandLine.php @@ -2,7 +2,7 @@ namespace Tally\Execution; -use Tally\CaseId; +use Tally\BenchmarkCase; /** * The commands that run one case with tally worker: natively, and under CodSpeed's CPU simulation. @@ -51,12 +51,12 @@ public function __construct(private string $phpBinary, private string $tallyBina /** * @return non-empty-list */ - public function native(WorkerSettings $settings, CaseId $id, string $scratch, string $result): array + public function native(WorkerSettings $settings, BenchmarkCase $case, string $scratch, string $result): array { return [ $this->phpBinary, ...$this->ini($settings->iniEntries), - ...$this->worker($settings, $id, $scratch, $result, []), + ...$this->worker($settings, $case, $scratch, $result, []), ]; } @@ -66,7 +66,7 @@ public function native(WorkerSettings $settings, CaseId $id, string $scratch, st * * @return non-empty-list */ - public function counted(WorkerSettings $settings, CaseId $id, string $scratch, string $result, string $profileFolder): array + public function counted(WorkerSettings $settings, BenchmarkCase $case, string $scratch, string $result, string $profileFolder): array { return [ $settings->valgrind, @@ -78,7 +78,7 @@ public function counted(WorkerSettings $settings, CaseId $id, string $scratch, s ...($settings->extension === null ? [] : ['extension=' . $settings->extension]), ...$settings->iniEntries, ]), - ...$this->worker($settings, $id, $scratch, $result, ['--counted']), + ...$this->worker($settings, $case, $scratch, $result, ['--counted']), ]; } @@ -98,20 +98,24 @@ private function ini(array $entries): array } /** + * The worker loads only the file of the case's class, so the cost of starting it does not grow + * with the suite, and the data providers of other files do not run. + * * @param list $options * @return list */ - private function worker(WorkerSettings $settings, CaseId $id, string $scratch, string $result, array $options): array + private function worker(WorkerSettings $settings, BenchmarkCase $case, string $scratch, string $result, array $options): array { return [ $this->tallyBinary, 'worker', - '--case=' . $id->toString(), + '--case=' . $case->id->toString(), '--scratch=' . $scratch, '--result=' . $result, ...$options, - // last, they can end with paths after -- - ...$settings->suiteArguments, + ...$settings->suiteOptions, + '--', + $case->file, ]; } diff --git a/src/Execution/WorkerSettings.php b/src/Execution/WorkerSettings.php index 2db06a8..ffff73f 100644 --- a/src/Execution/WorkerSettings.php +++ b/src/Execution/WorkerSettings.php @@ -10,7 +10,7 @@ final class WorkerSettings /** * @param list $iniEntries PHP settings of the workers, like memory_limit=-1 - * @param list $suiteArguments the options and paths with which the workers load the suite + * @param list $suiteOptions the options with which the workers load the benchmark file of their case * @param string $valgrind the launcher of CodSpeed's Valgrind * @param string|null $extension the tally_hooks extension the counted worker loads, null when PHP * loads it already @@ -18,7 +18,7 @@ final class WorkerSettings */ public function __construct( public readonly array $iniEntries, - public readonly array $suiteArguments, + public readonly array $suiteOptions, public readonly string $valgrind, public readonly ?string $extension, public readonly string $workDirectory, diff --git a/src/Expectation/AttributeExpectations.php b/src/Expectation/AttributeExpectations.php index 9a26d83..1364cff 100644 --- a/src/Expectation/AttributeExpectations.php +++ b/src/Expectation/AttributeExpectations.php @@ -4,18 +4,22 @@ use Override; use Tally\Metrics; -use function file_get_contents; use function sprintf; /** - * The #[Expected] attribute of a benchmark method, or the #[ExpectedFor] of one of its data sets. + * The #[Expected] attribute of a benchmark method, or the #[ExpectedFor] of one of its data sets, + * found in the index of its file, which all cases of the file share. */ final class AttributeExpectations implements ExpectationSource { + /** + * @param AttributeIndex $index of the file's code when the benchmarks were loaded + */ public function __construct( private ExpectedAttributeRewriter $rewriter, private string $file, + private AttributeIndex $index, private string $class, private string $method, private ?string $dataSet, @@ -38,20 +42,19 @@ public function file(): string } #[Override] - public function rewrite(string $code, Metrics $measured): string + public function edits(string $code, Metrics $measured): array { - return $this->rewriter->rewrite($code, $this->class, $this->method, $this->dataSet, $measured, $this->namedArguments); + if ($code !== $this->index->code) { + throw new ExpectationException(sprintf('%s changed after Tally loaded it, run the command again.', $this->file)); + } + + return $this->rewriter->edits($this->index, $this->class, $this->method, $this->dataSet, $measured, $this->namedArguments); } #[Override] public function location(): SourceLocation { - $code = @file_get_contents($this->file); - if ($code === false) { - throw new ExpectationException(sprintf('Cannot read %s.', $this->file)); - } - - return new SourceLocation($this->file, $this->rewriter->locate($code, $this->class, $this->method, $this->dataSet)); + return new SourceLocation($this->file, $this->rewriter->locate($this->index, $this->class, $this->method, $this->dataSet)); } } diff --git a/src/Expectation/AttributeIndex.php b/src/Expectation/AttributeIndex.php new file mode 100644 index 0000000..856d852 --- /dev/null +++ b/src/Expectation/AttributeIndex.php @@ -0,0 +1,36 @@ +, insertAt: int, line: int, indent: string} + * @phpstan-type Import array{name: string, alias: string, start: int, end: int} + */ +final class AttributeIndex +{ + + /** + * @param string $code what the index describes + * @param int|null $namespaceEnd after the semicolon of the namespace declaration + * @param list $imports the class imports, from the use keyword to after the semicolon + * @param int|null $otherUse the first import of a function or a constant + * @param array $members the methods by lowercase Class::method, the attributes + * before each, and where an attribute goes after them, with its line and indentation + */ + public function __construct( + public readonly string $code, + public readonly ?string $namespace, + public readonly ?int $namespaceEnd, + public readonly array $imports, + public readonly ?int $otherUse, + public readonly array $members + ) + { + } + +} diff --git a/src/Expectation/CodeEdit.php b/src/Expectation/CodeEdit.php new file mode 100644 index 0000000..12fd190 --- /dev/null +++ b/src/Expectation/CodeEdit.php @@ -0,0 +1,22 @@ + $files */ - $files = []; + /** @var array>}> $editsByFile the code and the edits by case */ + $editsByFile = []; $failures = []; foreach ($accepted as [$case, $measured]) { $id = $case->id->toString(); @@ -33,17 +37,99 @@ public function plan(array $accepted, array $code): ChangePlan continue; } - $change = $files[$file] ?? new FileChange($file, $before, $before, []); try { - $after = $case->expectations->rewrite($change->after, $measured); + $edits = $case->expectations->edits($before, $measured); } catch (ExpectationException $e) { $failures[$id] = $e->getMessage(); continue; } - $files[$file] = new FileChange($file, $before, $after, [...$change->cases, $id]); + $editsByFile[$file] ??= [$before, []]; + $editsByFile[$file][1][$id] = $edits; + } + + $files = []; + foreach ($editsByFile as $file => [$before, $editsByCase]) { + $sorted = $this->sorted($editsByCase); + $overlapping = $this->overlapping($sorted); + $failures = [...$failures, ...$overlapping]; + $files[] = new FileChange($file, $before, $this->apply($before, $sorted, $overlapping), array_keys(array_diff_key($editsByCase, $overlapping))); + } + + return new ChangePlan($files, $failures); + } + + /** + * @param array> $editsByCase + * @return list the edits with their cases, by offset, insertions before + * replacements, and insertions at one offset in the natural order of their text + */ + private function sorted(array $editsByCase): array + { + $edits = []; + foreach ($editsByCase as $id => $caseEdits) { + foreach ($caseEdits as $edit) { + $edits[] = [$edit, $id]; + } + } + usort($edits, static function (array $a, array $b): int { + $order = [$a[0]->offset, $a[0]->length] <=> [$b[0]->offset, $b[0]->length]; + + return $order !== 0 ? $order : strnatcmp($a[0]->text, $b[0]->text); + }); + + return $edits; + } + + /** + * @param list $edits sorted + * @return array the cases with an edit that starts inside an earlier one, with why + */ + private function overlapping(array $edits): array + { + $overlapping = []; + $previous = null; + $previousCase = ''; + foreach ($edits as [$edit, $id]) { + if ($this->equal($edit, $previous)) { + continue; + } + if ($previous !== null && $edit->offset < $previous->offset + $previous->length) { + $overlapping[$id] = sprintf('Its expectations overlap those of %s.', $previousCase); + continue; + } + + $previous = $edit; + $previousCase = $id; } - return new ChangePlan(array_values($files), $failures); + return $overlapping; + } + + /** + * @param list $edits sorted + * @param array $overlapping the cases whose edits are left out + */ + private function apply(string $code, array $edits, array $overlapping): string + { + $after = ''; + $position = 0; + $previous = null; + foreach ($edits as [$edit, $id]) { + if (isset($overlapping[$id]) || $this->equal($edit, $previous)) { + continue; + } + + $after .= substr($code, $position, $edit->offset - $position) . $edit->text; + $position = $edit->offset + $edit->length; + $previous = $edit; + } + + return $after . substr($code, $position); + } + + private function equal(CodeEdit $edit, ?CodeEdit $other): bool + { + return $other !== null && $edit->offset === $other->offset && $edit->length === $other->length && $edit->text === $other->text; } } diff --git a/src/Expectation/ExpectationSource.php b/src/Expectation/ExpectationSource.php index 5d1dedb..ba39dba 100644 --- a/src/Expectation/ExpectationSource.php +++ b/src/Expectation/ExpectationSource.php @@ -9,8 +9,10 @@ * provider can attach its own source to a data set with DataSet::withExpectations(), such as one * that keeps the numbers in a comment of the file the data set benchmarks. * - * Writing is split from reading the file: tally accept writes what rewrite() returns, and a run - * shows it as a diff for cases without expectations, which a contributor can apply. + * Writing is split from reading the file: tally accept writes the code with the edits of edits() + * applied, and a run shows them as a diff for cases without expectations, which a contributor can + * apply. Tally applies the edits of all cases that keep their expectations in one file at once, + * so many cases in a long file cost one pass over it. * * @api */ @@ -24,12 +26,15 @@ public function read(): ?Metrics; public function file(): string; /** - * The code of the file with these expectations replaced by the measured metrics, or added. When - * several cases keep their expectations in one file, the code has the others' already replaced. + * The edits of the code of the file that replace these expectations with the measured metrics, + * or add them. The edits of the cases of one file must not overlap, except that equal edits, + * such as the same import, are made once. Edits that insert at the same offset are applied in + * the natural order of their text. * + * @return list * @throws ExpectationException */ - public function rewrite(string $code, Metrics $measured): string; + public function edits(string $code, Metrics $measured): array; /** * Where reports point to diff --git a/src/Expectation/ExpectedAttributeRewriter.php b/src/Expectation/ExpectedAttributeRewriter.php index b720e82..36a2385 100644 --- a/src/Expectation/ExpectedAttributeRewriter.php +++ b/src/Expectation/ExpectedAttributeRewriter.php @@ -21,8 +21,6 @@ use function strtolower; use function strtr; use function substr; -use function substr_replace; -use function usort; use function var_export; use const T_AS; use const T_ATTRIBUTE; @@ -39,15 +37,15 @@ use const T_WHITESPACE; /** - * Finds and rewrites the #[Expected] attribute of one benchmark method, or its #[ExpectedFor] of - * one data set, in PHP code, using PHP's tokenizer. Everything else in the code stays byte for byte - * the same. A missing attribute is added after the method's other attributes, and imported when - * the file does not import it yet. + * Finds the #[Expected] attribute of each benchmark method, or its #[ExpectedFor] of each data set, + * in PHP code, using PHP's tokenizer, and the edits that rewrite one of them. index() reads the + * whole code once, so a file with many cases is tokenized once, not once per case. Everything else + * in the code stays byte for byte the same. A missing attribute is added after the method's other + * attributes, and imported when the file does not import it yet. * - * @phpstan-type Attribute array{start: int, end: int, name: string, dataSet: string|null, line: int, literal: bool} - * @phpstan-type Member array{attributes: list, insertAt: int} - * @phpstan-type Import array{name: string, alias: string, start: int, end: int} - * @phpstan-type Scan array{namespace: string|null, namespaceEnd: int|null, imports: list, otherUse: int|null, member: Member} + * @phpstan-import-type Attribute from AttributeIndex + * @phpstan-import-type Member from AttributeIndex + * @phpstan-import-type Import from AttributeIndex */ final class ExpectedAttributeRewriter { @@ -60,59 +58,68 @@ final class ExpectedAttributeRewriter * @throws MethodNotFoundException * @throws ExpectationException */ - public function locate(string $code, string $class, string $method, ?string $dataSet): int + public function locate(AttributeIndex $index, string $class, string $method, ?string $dataSet): int { - $tokens = new Tokens($code); - $member = $this->scan($tokens, $class, $method)['member']; + $member = $this->member($index, $class, $method); $attribute = $this->findAttribute($member, $dataSet); - return $attribute !== null ? $attribute['line'] : $tokens->at($member['insertAt'])->line; + return $attribute !== null ? $attribute['line'] : $member['line']; } /** + * The edits of the indexed code that replace the attribute with the metrics, or add it with its + * import. Cases that add the same import get the same edit. + * * @param Metrics $metrics every metric * @param bool $namedArguments whether to write the data set as dataSet: '…' + * @return non-empty-list * @throws MethodNotFoundException * @throws ExpectationException * @throws InvalidArgumentException when a metric is missing */ - public function rewrite(string $code, string $class, string $method, ?string $dataSet, Metrics $metrics, bool $namedArguments): string + public function edits(AttributeIndex $index, string $class, string $method, ?string $dataSet, Metrics $metrics, bool $namedArguments): array { $arguments = $this->arguments($dataSet, $metrics, $namedArguments); - $tokens = new Tokens($code); - $scan = $this->scan($tokens, $class, $method); - $attribute = $this->findAttribute($scan['member'], $dataSet); - $attributeClass = $dataSet === null ? Expected::class : ExpectedFor::class; - + $member = $this->member($index, $class, $method); + $attribute = $this->findAttribute($member, $dataSet); if ($attribute !== null) { - $start = $tokens->at($attribute['start'])->pos; - $end = $tokens->at($attribute['end'])->pos + strlen($tokens->at($attribute['end'])->text); - - return substr_replace($code, $attribute['name'] . '(' . $arguments . ')', $start, $end - $start); + return [new CodeEdit($attribute['start'], $attribute['end'] - $attribute['start'], $attribute['name'] . '(' . $arguments . ')')]; } - $edits = []; - $name = $this->localName($scan, $attributeClass); + $attributeClass = $dataSet === null ? Expected::class : ExpectedFor::class; + $name = $this->localName($index, $attributeClass); + $imports = []; if ($name === null) { $name = $this->shortName($attributeClass); - $edits[] = $this->importEdit($tokens, $scan, $attributeClass); + $imports = $this->importEdits($index, $attributeClass); } - $insertAt = $scan['member']['insertAt']; - $indent = ''; - $previous = $tokens->at($insertAt - 1); - if ($previous->is(T_WHITESPACE)) { - $newline = strrpos($previous->text, "\n"); - $indent = $newline === false ? $previous->text : substr($previous->text, $newline + 1); - } - $edits[] = [$tokens->at($insertAt)->pos, '#[' . $name . '(' . $arguments . ')]' . "\n" . $indent]; + return [...$imports, new CodeEdit($member['insertAt'], 0, '#[' . $name . '(' . $arguments . ')]' . "\n" . $member['indent'])]; + } - usort($edits, static fn (array $a, array $b): int => $b[0] <=> $a[0]); - foreach ($edits as [$offset, $text]) { - $code = substr_replace($code, $text, $offset, 0); + /** + * Reads every class of the code once. + */ + public function index(string $code): AttributeIndex + { + return $this->scan(new Tokens($code), $code); + } + + /** + * @return Member + * @throws MethodNotFoundException + * @throws ExpectationException + */ + private function member(AttributeIndex $index, string $class, string $method): array + { + $member = $index->members[strtolower($class . '::' . $method)] ?? throw new MethodNotFoundException($class, $method); + foreach ($member['attributes'] as $attribute) { + if (!$attribute['literal']) { + throw new ExpectationException(sprintf('#[ExpectedFor] on line %d: write the data set as a string literal, so tally accept can find the attribute.', $attribute['line'])); + } } - return $code; + return $member; } private function arguments(?string $dataSet, Metrics $metrics, bool $namedArguments): string @@ -147,24 +154,23 @@ private function findAttribute(array $member, ?string $dataSet): ?array /** * The name the code can use for the attribute, null when it needs an import. * - * @param Scan $scan * @param class-string $attributeClass */ - private function localName(array $scan, string $attributeClass): ?string + private function localName(AttributeIndex $index, string $attributeClass): ?string { - foreach ($scan['imports'] as $import) { + foreach ($index->imports as $import) { if (strcasecmp($import['name'], $attributeClass) === 0) { return $import['alias']; } } - foreach ($scan['imports'] as $import) { + foreach ($index->imports as $import) { if (strcasecmp($import['alias'], $this->shortName($attributeClass)) === 0) { return '\\' . $attributeClass; } } - return $scan['namespace'] === null ? '\\' . $attributeClass : null; + return $index->namespace === null ? '\\' . $attributeClass : null; } /** @@ -176,40 +182,37 @@ private function shortName(string $class): string } /** - * Keeps the imports sorted, before the imports of functions and constants. + * Keeps the imports sorted, before the imports of functions and constants. The edits of + * #[Expected] and #[ExpectedFor] can go to the same place; in the natural order of their text, + * which is how ExpectationChanges applies edits at one place, they stay sorted too. * - * @param Scan $scan * @param class-string $attributeClass - * @return array{int, string} + * @return non-empty-list */ - private function importEdit(Tokens $tokens, array $scan, string $attributeClass): array + private function importEdits(AttributeIndex $index, string $attributeClass): array { $statement = 'use ' . $attributeClass . ';'; - foreach ($scan['imports'] as $import) { + foreach ($index->imports as $import) { // 0 would be the attribute itself, which needs no import if ((strtolower($import['name']) <=> strtolower($attributeClass)) === 1) { - return [$tokens->at($import['start'])->pos, $statement . "\n"]; + return [new CodeEdit($import['start'], 0, $statement . "\n")]; } } - $imports = $scan['imports']; + $imports = $index->imports; if ($imports !== []) { - return [$tokens->at($imports[count($imports) - 1]['end'])->pos + 1, "\n" . $statement]; + return [new CodeEdit($imports[count($imports) - 1]['end'], 0, "\n" . $statement)]; } - if ($scan['otherUse'] !== null) { - return [$tokens->at($scan['otherUse'])->pos, $statement . "\n"]; + if ($index->otherUse !== null) { + return [new CodeEdit($index->otherUse, 0, $statement . "\n")]; } - // localName() returns a fully qualified name without a namespace - return [$tokens->at((int) $scan['namespaceEnd'])->pos + 1, "\n\n" . $statement]; + // localName() returns a fully qualified name without a namespace; the blank line is an edit + // of its own, so the edits of both attributes add it once + return [new CodeEdit((int) $index->namespaceEnd, 0, "\n"), new CodeEdit((int) $index->namespaceEnd, 0, "\n" . $statement)]; } - /** - * @return Scan - * @throws MethodNotFoundException - * @throws ExpectationException - */ - private function scan(Tokens $tokens, string $class, string $method): array + private function scan(Tokens $tokens, string $code): AttributeIndex { $namespace = null; $namespaceEnd = null; @@ -222,6 +225,7 @@ private function scan(Tokens $tokens, string $class, string $method): array $classDepth = null; $attributes = []; $insertAt = null; + $members = []; $count = $tokens->count(); for ($i = 0; $i < $count; $i++) { @@ -258,7 +262,7 @@ private function scan(Tokens $tokens, string $class, string $method): array // namespace { opens the global namespace if ($tokens->at($next)->is([T_STRING, T_NAME_QUALIFIED])) { $namespace = $tokens->at($next)->text; - $namespaceEnd = $tokens->skipTo($next, ';'); + $namespaceEnd = $tokens->at($tokens->skipTo($next, ';'))->pos + 1; } continue; } @@ -266,10 +270,10 @@ private function scan(Tokens $tokens, string $class, string $method): array if ($depth === 0 && $token->is(T_USE)) { $end = $tokens->skipTo($i, ';'); if ($tokens->at($tokens->next($i))->is([T_FUNCTION, T_CONST])) { - $otherUse ??= $i; + $otherUse ??= $token->pos; } else { foreach ($this->parseImports($tokens, $i, $end) as [$name, $alias]) { - $imports[] = ['name' => $name, 'alias' => $alias, 'start' => $i, 'end' => $end]; + $imports[] = ['name' => $name, 'alias' => $alias, 'start' => $token->pos, 'end' => $tokens->at($end)->pos + 1]; } } $i = $end; @@ -309,34 +313,29 @@ private function scan(Tokens $tokens, string $class, string $method): array continue; } - $name = $tokens->next($i); - if ( - strcasecmp($currentClass, $class) === 0 - && strcasecmp($tokens->at($name)->text, $method) === 0 - ) { - foreach ($attributes as $attribute) { - if (!$attribute['literal']) { - throw new ExpectationException(sprintf('#[ExpectedFor] on line %d: write the data set as a string literal, so tally accept can find the attribute.', $attribute['line'])); - } - } - - return [ - 'namespace' => $namespace, - 'namespaceEnd' => $namespaceEnd, - 'imports' => $imports, - 'otherUse' => $otherUse, - 'member' => ['attributes' => $attributes, 'insertAt' => $insertAt], - ]; + $indent = ''; + $previous = $tokens->at($insertAt - 1); + if ($previous->is(T_WHITESPACE)) { + $newline = strrpos($previous->text, "\n"); + $indent = $newline === false ? $previous->text : substr($previous->text, $newline + 1); } + // the first declaration of the method counts, like PHP's + $members[strtolower($currentClass . '::' . $tokens->at($tokens->next($i))->text)] ??= [ + 'attributes' => $attributes, + 'insertAt' => $tokens->at($insertAt)->pos, + 'line' => $tokens->at($insertAt)->line, + 'indent' => $indent, + ]; + $attributes = []; + $insertAt = null; } - throw new MethodNotFoundException($class, $method); + return new AttributeIndex($code, $namespace, $namespaceEnd, $imports, $otherUse, $members); } /** * @param list $imports * @return array{list, int} the #[Expected] and #[ExpectedFor] attributes of the group, and the index of its closing bracket - * @throws ExpectationException */ private function parseAttributeGroup(Tokens $tokens, int $start, ?string $namespace, array $imports): array { @@ -379,7 +378,6 @@ private function parseAttributeGroup(Tokens $tokens, int $start, ?string $namesp /** * @param array $aliases * @return Attribute|null - * @throws ExpectationException */ private function parseAttribute(Tokens $tokens, int $start, int $end, ?string $namespace, array $aliases): ?array { @@ -401,8 +399,8 @@ private function parseAttribute(Tokens $tokens, int $start, int $end, ?string $n } return [ - 'start' => $name, - 'end' => $attributeEnd, + 'start' => $tokens->at($name)->pos, + 'end' => $tokens->at($attributeEnd)->pos + strlen($tokens->at($attributeEnd)->text), 'name' => $tokens->at($name)->text, 'dataSet' => $dataSet, 'line' => $tokens->at($name)->line, diff --git a/src/Process/ProcessKiller.php b/src/Process/ProcessKiller.php index d8a537b..7cbd598 100644 --- a/src/Process/ProcessKiller.php +++ b/src/Process/ProcessKiller.php @@ -7,7 +7,6 @@ use function file_get_contents; use function glob; use function posix_kill; -use const SIGKILL; /** * Kills a process and all its descendants, which it finds in /proc before killing any of them, @@ -16,6 +15,9 @@ final class ProcessKiller { + /** SIGKILL, whose constant only the pcntl extension defines, which many PHP builds lack */ + private const SIGKILL = 9; + public function __construct(private ProcessTable $table) { } @@ -30,7 +32,7 @@ public function kill(int $pid): void } foreach ([$pid, ...$this->table->descendants($this->table->parents($stats), $pid)] as $process) { - posix_kill($process, SIGKILL); + posix_kill($process, self::SIGKILL); } } diff --git a/src/Report/UnifiedDiff.php b/src/Report/UnifiedDiff.php index bc5beec..6ce5180 100644 --- a/src/Report/UnifiedDiff.php +++ b/src/Report/UnifiedDiff.php @@ -11,6 +11,7 @@ use function preg_split; use function sprintf; use function str_ends_with; +use const PHP_INT_MAX; use const PREG_SPLIT_NO_EMPTY; /** @@ -36,8 +37,19 @@ public function render(string $path, string $before, string $after): string $operations = $this->operations($this->lines($before), $this->lines($after)); $diff = sprintf("--- a/%s\n+++ b/%1\$s\n", $path); + $oldBefore = 0; + $newBefore = 0; + $position = 0; foreach ($this->hunks($operations) as [$start, $end]) { - $diff .= $this->hunk($operations, $start, $end); + foreach (array_slice($operations, $position, $start - $position) as [$kind]) { + $oldBefore += $kind === '+' ? 0 : 1; + $newBefore += $kind === '-' ? 0 : 1; + } + [$hunk, $oldCount, $newCount] = $this->hunk($operations, $start, $end, $oldBefore, $newBefore); + $diff .= $hunk; + $oldBefore += $oldCount; + $newBefore += $newCount; + $position = $end + 1; } return $diff; @@ -88,9 +100,9 @@ private function operations(array $old, array $new): array } /** - * The shortest edit script of the changed middle, from the number of lines to remove and add - * to turn each suffix of the old lines into each suffix of the new ones. Removed lines come - * before added ones, like in GNU diff. + * The shortest edit script of the changed middle. From the start, a common line is kept, and + * of removing and adding a line, the one that leaves fewer edits for the rest, removing on a + * tie, so removed lines come before added ones, like in GNU diff. * * @param list $old * @param list $new @@ -98,24 +110,8 @@ private function operations(array $old, array $new): array */ private function middle(array $old, array $new): array { - /** @var array> $edits */ - $edits = []; - for ($i = count($old); $i >= 0; $i--) { - $row = []; - for ($j = count($new); $j >= 0; $j--) { - if ($i === count($old)) { - $row[$j] = count($new) - $j; - } elseif ($j === count($new)) { - $row[$j] = count($old) - $i; - } elseif ($this->at($old, $i) === $this->at($new, $j)) { - $row[$j] = $this->at($this->at($edits, $i + 1), $j + 1); - } else { - $row[$j] = 1 + min($this->at($this->at($edits, $i + 1), $j), $this->at($row, $j + 1)); - } - } - $edits[$i] = $row; - } - + $reach = $this->reach($old, $new); + $edits = count($reach) - 1; $operations = []; $i = 0; $j = 0; @@ -124,18 +120,81 @@ private function middle(array $old, array $new): array $operations[] = [' ', $this->at($old, $i)]; $i++; $j++; - } elseif ($i < count($old) && ($j === count($new) || $this->at($this->at($edits, $i + 1), $j) <= $this->at($this->at($edits, $i), $j + 1))) { + continue; + } + + // the rest after removing the line, on diagonal i + 1 - j, takes one edit less, or one more + if ($i < count($old) && ($j === count($new) || ($this->at($reach, $edits - 1)[$i + 1 - $j] ?? PHP_INT_MAX) <= $i + 1)) { $operations[] = ['-', $this->at($old, $i)]; $i++; } else { $operations[] = ['+', $this->at($new, $j)]; $j++; } + $edits--; } return $operations; } + /** + * Myers' search from the end: for each number of edits d, the first old line from which the + * rest of each diagonal k = i - j takes at most d edits. From a line i on diagonal k, the rest + * takes at most d edits exactly from that line on, because the edits do not grow toward the + * end of a diagonal. The search ends at d edits for everything, so it takes O((N + M) * d) + * time and O(d²) memory, nothing for the common lines. + * + * @param list $old + * @param list $new + * @return non-empty-list> by the number of edits, the first old line by diagonal + */ + private function reach(array $old, array $new): array + { + $end = count($old) - count($new); + $reach = []; + for ($edits = 0;; $edits++) { + $row = []; + for ($k = $end - $edits; $k <= $end + $edits; $k += 2) { + $i = $edits === 0 ? count($old) : $this->closest($this->at($reach, $edits - 1), $k); + if ($i === null) { + continue; + } + + $j = $i - $k; + while ($i > 0 && $j > 0 && $this->at($old, $i - 1) === $this->at($new, $j - 1)) { + $i--; + $j--; + } + $row[$k] = $i; + if ($k === 0 && $i === 0) { + return [...$reach, $row]; + } + } + $reach[] = $row; + } + } + + /** + * The first line of diagonal k one edit more reaches: by removing the line before the one diagonal + * k + 1 reaches, or by adding the line before the one diagonal k - 1 reaches. + * + * @param array $previous the lines one edit less reaches + */ + private function closest(array $previous, int $k): ?int + { + $candidates = []; + $removing = $previous[$k + 1] ?? 0; + if ($removing > 0) { + $candidates[] = $removing - 1; + } + $adding = $previous[$k - 1] ?? null; + if ($adding !== null && $adding - $k >= 0) { + $candidates[] = $adding; + } + + return $candidates === [] ? null : min($candidates); + } + /** * @param list $operations * @return list the first and the last operation of each hunk @@ -170,16 +229,12 @@ private function hunks(array $operations): array /** * @param list $operations + * @param int $oldBefore the old lines before the hunk + * @param int $newBefore the new lines before the hunk + * @return array{string, int, int} the hunk, and its old and new lines */ - private function hunk(array $operations, int $start, int $end): string + private function hunk(array $operations, int $start, int $end, int $oldBefore, int $newBefore): array { - $oldBefore = 0; - $newBefore = 0; - foreach (array_slice($operations, 0, $start) as [$kind]) { - $oldBefore += $kind === '+' ? 0 : 1; - $newBefore += $kind === '-' ? 0 : 1; - } - $oldCount = 0; $newCount = 0; $lines = []; @@ -189,7 +244,11 @@ private function hunk(array $operations, int $start, int $end): string $lines[] = $kind . $line . (str_ends_with($line, "\n") ? '' : "\n\\ No newline at end of file\n"); } - return sprintf('@@ -%s +%s @@', $this->range($oldBefore, $oldCount), $this->range($newBefore, $newCount)) . "\n" . implode('', $lines); + return [ + sprintf('@@ -%s +%s @@', $this->range($oldBefore, $oldCount), $this->range($newBefore, $newCount)) . "\n" . implode('', $lines), + $oldCount, + $newCount, + ]; } /** Like GNU diff: an empty range starts at the line before it, a range of one line has no count */ diff --git a/tests/FixedEdits.php b/tests/FixedEdits.php new file mode 100644 index 0000000..5c338ea --- /dev/null +++ b/tests/FixedEdits.php @@ -0,0 +1,48 @@ + $edits + */ + public function __construct(private string $file, private array $edits) + { + } + + #[Override] + public function read(): ?Metrics + { + return null; + } + + #[Override] + public function file(): string + { + return $this->file; + } + + #[Override] + public function edits(string $code, Metrics $measured): array + { + return $this->edits; + } + + #[Override] + public function location(): SourceLocation + { + return new SourceLocation($this->file, 1); + } + +} diff --git a/tests/Fixtures/Isolation/IsolatedBench.php b/tests/Fixtures/Isolation/IsolatedBench.php new file mode 100644 index 0000000..954b733 --- /dev/null +++ b/tests/Fixtures/Isolation/IsolatedBench.php @@ -0,0 +1,15 @@ + + */ + public static function dataSets(): iterable + { + if (in_array('worker', $_SERVER['argv'], true)) { + throw new RuntimeException('A worker of another file ran this data provider.'); + } + + yield 'only' => []; + } + + #[DataProvider('dataSets')] + public function benchOther(): void + { + } + +} diff --git a/tests/Fixtures/Refusing/RefusingBench.php b/tests/Fixtures/Refusing/RefusingBench.php index 5709179..671d0e2 100644 --- a/tests/Fixtures/Refusing/RefusingBench.php +++ b/tests/Fixtures/Refusing/RefusingBench.php @@ -23,7 +23,7 @@ public function file(): string return __FILE__; } - public function rewrite(string $code, Metrics $measured): string + public function edits(string $code, Metrics $measured): array { throw new ExpectationException('The data set is generated.'); } diff --git a/tests/InMemoryExpectations.php b/tests/InMemoryExpectations.php index acc06da..828f9a8 100644 --- a/tests/InMemoryExpectations.php +++ b/tests/InMemoryExpectations.php @@ -3,14 +3,16 @@ namespace Tally\Tests; use Override; +use Tally\Expectation\CodeEdit; use Tally\Expectation\ExpectationSource; use Tally\Expectation\SourceLocation; use Tally\Metrics; use function json_encode; +use function strlen; use const JSON_THROW_ON_ERROR; /** - * Keeps the expectations of a data set in memory, for tests. Rewriting appends a line with the + * Keeps the expectations of a data set in memory, for tests. Its edit appends a line with the * measured metrics to the code. */ final class InMemoryExpectations implements ExpectationSource @@ -33,9 +35,9 @@ public function file(): string } #[Override] - public function rewrite(string $code, Metrics $measured): string + public function edits(string $code, Metrics $measured): array { - return $code . json_encode($measured->toArray(), JSON_THROW_ON_ERROR) . "\n"; + return [new CodeEdit(strlen($code), 0, json_encode($measured->toArray(), JSON_THROW_ON_ERROR) . "\n")]; } #[Override] diff --git a/tests/Integration/Cli/RunCommandTest.php b/tests/Integration/Cli/RunCommandTest.php index 55f204e..f327d95 100644 --- a/tests/Integration/Cli/RunCommandTest.php +++ b/tests/Integration/Cli/RunCommandTest.php @@ -7,6 +7,7 @@ use PHPUnit\Framework\TestCase; use Tally\Execution\DirectoryRemover; use Tally\Measurement; +use Tally\Metric; use Tally\Result\InvalidResultException; use Tally\Result\MeasurementJson; use Tally\Result\ResultsDirectory; @@ -116,6 +117,26 @@ public function testPassesWhenEveryCaseMatches(): void $this->assertStringContainsString("Benchmarks passed: all 1 match their expectations\n", $output); } + /** + * @throws InvalidResultException + */ + public function testTheWorkersLoadOnlyTheFileOfTheirCase(): void + { + $id = 'Tally\\Tests\\Fixtures\\Isolation\\IsolatedBench::benchOk'; + $environment = $this->environment(); + $environment['TALLY_FAKE_COUNTS'] = json_encode([$id => [1100, 1650, 100, 150]], JSON_THROW_ON_ERROR); + [, , $errorOutput] = InProcessTally::run(__DIR__, $environment, ['run', '--config=' . self::RUN . '/tally.php', '--filter=IsolatedBench', '--output=' . $this->output . '-isolation', __DIR__ . '/../../Fixtures/Isolation']); + + $results = array_map( + // the memory depends on the PHP build + static fn (Measurement $measurement): int|string|null => $measurement->failure ?? $measurement->metrics->get(Metric::Instructions), + (new ResultsDirectory(new MeasurementJson()))->read($this->output . '-isolation'), + ); + (new DirectoryRemover())->remove($this->output . '-isolation'); + $this->assertSame('', $errorOutput); + $this->assertSame([$id => 1000], $results); + } + public function testAShardWithoutCases(): void { [$exitCode, $output, $errorOutput] = self::runWithFixtureConfiguration($this->environment(), ['run', '--shard=5/5', self::RUN]); diff --git a/tests/Integration/Discovery/DirectorySuiteLoaderTest.php b/tests/Integration/Discovery/DirectorySuiteLoaderTest.php index 26285c7..20bd205 100644 --- a/tests/Integration/Discovery/DirectorySuiteLoaderTest.php +++ b/tests/Integration/Discovery/DirectorySuiteLoaderTest.php @@ -139,7 +139,7 @@ public function testNoBenchmarks(): void */ private static function ids(array $paths): array { - $loader = new DirectorySuiteLoader(new CaseExpander(new ExpectedAttributeRewriter(), new AttributeFactory(new ValueTypeCheck()), new ArgumentResolver(), (new Expressions())->growthPoints()), new ConstructorArguments(), new RelativePath()); + $loader = new DirectorySuiteLoader(new CaseExpander(new ExpectedAttributeRewriter(), new AttributeFactory(new ValueTypeCheck()), new ArgumentResolver(), (new Expressions())->growthPoints()), new ConstructorArguments(), new ExpectedAttributeRewriter(), new RelativePath()); return array_map(static fn (BenchmarkCase $case): string => $case->id->toString(), $loader->load($paths, dirname(__DIR__, 3), Configuration::create())); } diff --git a/tests/Integration/Execution/CaseJobsTest.php b/tests/Integration/Execution/CaseJobsTest.php index 0bd9967..444e7a2 100644 --- a/tests/Integration/Execution/CaseJobsTest.php +++ b/tests/Integration/Execution/CaseJobsTest.php @@ -5,6 +5,8 @@ use Override; use PHPUnit\Framework\TestCase; use RuntimeException; +use Tally\Attribute\RelativeTolerance; +use Tally\BenchmarkCase; use Tally\CaseId; use Tally\Execution\CaseJobs; use Tally\Execution\CaseOutcome; @@ -18,6 +20,7 @@ use Tally\Process\ProcessResult; use Tally\Result\CallgrindProfile; use Tally\Result\MeasurementJson; +use Tally\Tests\InMemoryExpectations; use function array_map; use function file_put_contents; use function getmypid; @@ -47,18 +50,18 @@ protected function tearDown(): void public function testTheCaseRunsNativelyAndThenUnderCallgrind(): void { - $steps = self::jobs()->steps($this->settings($this->directory), CaseId::fromString(self::ID)); + $steps = self::jobs()->steps($this->settings($this->directory), self::case()); $this->assertDirectoryExists($this->caseDirectory . '/scratch'); $this->assertDirectoryExists($this->caseDirectory . '/profile'); $commandLine = new WorkerCommandLine('php', 'bin/tally'); $settings = $this->settings($this->directory); - $id = CaseId::fromString(self::ID); + $case = self::case(); $scratch = $this->caseDirectory . '/scratch'; $this->assertEquals( [ - new ProcessCommand($commandLine->native($settings, $id, $scratch, $this->caseDirectory . '/native.json'), $this->caseDirectory . '/native.log', 60), - new ProcessCommand($commandLine->counted($settings, $id, $scratch, $this->caseDirectory . '/counted.json', $this->caseDirectory . '/profile'), $this->caseDirectory . '/counted.log', 1800), + new ProcessCommand($commandLine->native($settings, $case, $scratch, $this->caseDirectory . '/native.json'), $this->caseDirectory . '/native.log', 60), + new ProcessCommand($commandLine->counted($settings, $case, $scratch, $this->caseDirectory . '/counted.json', $this->caseDirectory . '/profile'), $this->caseDirectory . '/counted.log', 1800), ], $steps, ); @@ -69,12 +72,12 @@ public function testReportsADirectoryThatCannotBeCreated(): void $this->expectException(RuntimeException::class); $this->expectExceptionMessage('Cannot create /nonexistent/tally/' . CaseId::fromString(self::ID)->fileName() . '/scratch.'); - self::jobs()->steps($this->settings('/nonexistent/tally'), CaseId::fromString(self::ID)); + self::jobs()->steps($this->settings('/nonexistent/tally'), self::case()); } public function testMeasuresTheCaseFromWhatItsRunsLeft(): void { - $steps = self::jobs()->steps($this->settings($this->directory), CaseId::fromString(self::ID)); + $steps = self::jobs()->steps($this->settings($this->directory), self::case()); $json = new MeasurementJson(); file_put_contents($this->caseDirectory . '/native.json', $json->encode(self::memory(1))); file_put_contents($this->caseDirectory . '/counted.json', $json->encode(self::memory(2048))); @@ -93,7 +96,7 @@ public function testMeasuresTheCaseFromWhatItsRunsLeft(): void public function testAFailedNativeRunWithItsOutput(): void { - $steps = self::jobs()->steps($this->settings($this->directory), CaseId::fromString(self::ID)); + $steps = self::jobs()->steps($this->settings($this->directory), self::case()); file_put_contents($this->caseDirectory . '/native.log', "PHP Fatal error: Out of memory\n"); $measurement = self::jobs()->measurement($this->settings($this->directory), CaseId::fromString(self::ID), [ProcessResult::exited($steps[0], 255)]); @@ -103,7 +106,7 @@ public function testAFailedNativeRunWithItsOutput(): void public function testAnInvalidResult(): void { - $steps = self::jobs()->steps($this->settings($this->directory), CaseId::fromString(self::ID)); + $steps = self::jobs()->steps($this->settings($this->directory), self::case()); file_put_contents($this->caseDirectory . '/native.json', (new MeasurementJson())->encode(self::memory(1))); file_put_contents($this->caseDirectory . '/counted.json', '{'); @@ -116,6 +119,11 @@ public function testAnInvalidResult(): void $this->assertSame('The counted run wrote an invalid result: it is not JSON.', $measurement->failure); } + private static function case(): BenchmarkCase + { + return new BenchmarkCase(CaseId::fromString(self::ID), '/benchmarks/FooBench.php', [], 0, [], [], new InMemoryExpectations(null, '/benchmarks/FooBench.php'), new RelativeTolerance(0.1), null); + } + private static function jobs(): CaseJobs { return new CaseJobs(new WorkerCommandLine('php', 'bin/tally'), new CaseOutcome(), new CallgrindProfile(), new MeasurementJson()); diff --git a/tests/Integration/Expectation/AttributeExpectationsTest.php b/tests/Integration/Expectation/AttributeExpectationsTest.php deleted file mode 100644 index c22dc85..0000000 --- a/tests/Integration/Expectation/AttributeExpectationsTest.php +++ /dev/null @@ -1,53 +0,0 @@ - 1000, 'cycles' => 1100, 'memory' => 0]), false); - $code = (string) file_get_contents(self::FILE); - - $this->assertSame(self::FILE, $source->file()); - $this->assertSame( - str_replace("#[Expected(instructions: '1.000K', cycles: '1.100K', memory: '0B')]", "#[Expected(instructions: '3.000K', cycles: '3.300K', memory: '1.000KiB')]", $code), - $source->rewrite($code, Metrics::fromArray(['instructions' => 3_000, 'cycles' => 3_300, 'memory' => 1024])), - ); - // rewriting writes nothing and changes nothing - $this->assertSame($code, file_get_contents(self::FILE)); - $this->assertSame(['instructions' => 1000, 'cycles' => 1100, 'memory' => 0], $source->read()?->toArray()); - $this->assertEquals(new SourceLocation(self::FILE, 13), $source->location()); - } - - /** - * @throws ExpectationException - */ - public function testLocationFailsForAMissingFile(): void - { - $file = __DIR__ . '/../../Fixtures/Rewriter/MissingBench.php'; - $source = new AttributeExpectations(new ExpectedAttributeRewriter(), $file, SortBench::class, 'benchSort', null, null, false); - - $this->expectException(ExpectationException::class); - $this->expectExceptionMessage(sprintf('Cannot read %s.', $file)); - $source->location(); - } - -} diff --git a/tests/Integration/Process/ProcessPoolTest.php b/tests/Integration/Process/ProcessPoolTest.php index 12490d0..22df3b3 100644 --- a/tests/Integration/Process/ProcessPoolTest.php +++ b/tests/Integration/Process/ProcessPoolTest.php @@ -83,7 +83,8 @@ public function testReportsTheSignalThatKilledAProcess(): void public function testKillsAStepThatRunsTooLongWithItsDescendants(): void { $pidFile = $this->directory . '/grandchild.pid'; - $code = '$child = proc_open([PHP_BINARY, "-r", "file_put_contents(\'' . $pidFile . '\', getmypid()); sleep(30);"], [], $pipes); sleep(30);'; + // the grandchild ignores the signals next to SIGKILL, so killing it with another one fails + $code = '$child = proc_open(["sh", "-c", "trap \'\' FPE USR1; echo \$\$ > ' . $pidFile . '; exec sleep 30"], [], $pipes); sleep(30);'; $job = new Job('job', [new ProcessCommand([PHP_BINARY, '-r', $code], $this->directory . '/slow.log', 1), $this->php('', 'never')]); $this->assertSame(['job: timed out'], $this->runJobs([$job], 1, [])); diff --git a/tests/Unit/Comparison/ComparatorTest.php b/tests/Unit/Comparison/ComparatorTest.php index aba1250..f94f13b 100644 --- a/tests/Unit/Comparison/ComparatorTest.php +++ b/tests/Unit/Comparison/ComparatorTest.php @@ -208,10 +208,10 @@ public function testGrowthRoundsTheExpectedInstructions(): void public function testCompareCaseComparesByExpectationsOrByGrowth(): void { $expectations = new InMemoryExpectations(Metrics::fromArray(['instructions' => 100]), '/bench/ABench.php'); - $small = new BenchmarkCase(CaseId::fromString('A::bench@small'), [], 0, [], [], $expectations, new RelativeTolerance(0.1), new GrowthPoint('O(n)', CaseId::fromString('A::bench@small'), 1.0)); + $small = new BenchmarkCase(CaseId::fromString('A::bench@small'), '/benchmarks/FooBench.php', [], 0, [], [], $expectations, new RelativeTolerance(0.1), new GrowthPoint('O(n)', CaseId::fromString('A::bench@small'), 1.0)); $large = $small->withGrowth(new GrowthPoint('O(n)', CaseId::fromString('A::bench@small'), 3.0)); - $large = new BenchmarkCase(CaseId::fromString('A::bench@large'), [], 0, [], [], $expectations, new RelativeTolerance(0.1), $large->growth); - $plain = new BenchmarkCase(CaseId::fromString('A::plain'), [], 0, [], [], $expectations, new RelativeTolerance(0.1), null); + $large = new BenchmarkCase(CaseId::fromString('A::bench@large'), '/benchmarks/FooBench.php', [], 0, [], [], $expectations, new RelativeTolerance(0.1), $large->growth); + $plain = new BenchmarkCase(CaseId::fromString('A::plain'), '/benchmarks/FooBench.php', [], 0, [], [], $expectations, new RelativeTolerance(0.1), null); $measurements = [ 'A::bench@small' => Measurement::measured(CaseId::fromString('A::bench@small'), Metrics::fromArray(['instructions' => 100])), 'A::bench@large' => Measurement::measured(CaseId::fromString('A::bench@large'), Metrics::fromArray(['instructions' => 400])), diff --git a/tests/Unit/Discovery/CaseExpanderTest.php b/tests/Unit/Discovery/CaseExpanderTest.php index 38184a5..2b2d678 100644 --- a/tests/Unit/Discovery/CaseExpanderTest.php +++ b/tests/Unit/Discovery/CaseExpanderTest.php @@ -14,8 +14,10 @@ use Tally\Discovery\InvalidBenchmarkException; use Tally\Discovery\ValueTypeCheck; use Tally\Expectation\AttributeExpectations; +use Tally\Expectation\ExpectationChanges; use Tally\Expectation\ExpectationException; use Tally\Expectation\ExpectedAttributeRewriter; +use Tally\Expectation\FileChange; use Tally\Metrics; use Tally\Tests\Expressions; use Tally\Tests\Fixtures\Growth\GrowthBench; @@ -155,12 +157,12 @@ public function testTheConfigurationChoosesHowTheDataSetIsWritten(): void self::assertCount(3, $named); $this->assertSame( - str_replace("#[ExpectedFor('size-10', instructions: '100', cycles: '110', memory: '0B')]", "#[ExpectedFor('size-10', instructions: '7', cycles: '8', memory: '0B')]", $code), - $positional[1]->expectations->rewrite($code, $metrics), + [str_replace("#[ExpectedFor('size-10', instructions: '100', cycles: '110', memory: '0B')]", "#[ExpectedFor('size-10', instructions: '7', cycles: '8', memory: '0B')]", $code)], + self::rewrite($code, $positional[1], $metrics), ); $this->assertSame( - str_replace("#[ExpectedFor('size-10', instructions: '100', cycles: '110', memory: '0B')]", "#[ExpectedFor(dataSet: 'size-10', instructions: '7', cycles: '8', memory: '0B')]", $code), - $named[1]->expectations->rewrite($code, $metrics), + [str_replace("#[ExpectedFor('size-10', instructions: '100', cycles: '110', memory: '0B')]", "#[ExpectedFor(dataSet: 'size-10', instructions: '7', cycles: '8', memory: '0B')]", $code)], + self::rewrite($code, $named[1], $metrics), ); } @@ -239,9 +241,22 @@ private static function expandOne(string $class, string $method, Configuration $ private static function expand(string $class, string $method, Configuration $configuration): array { $reflection = new ReflectionClass($class); - $expander = new CaseExpander(new ExpectedAttributeRewriter(), new AttributeFactory(new ValueTypeCheck()), new ArgumentResolver(), (new Expressions())->growthPoints()); + $rewriter = new ExpectedAttributeRewriter(); + $expander = new CaseExpander($rewriter, new AttributeFactory(new ValueTypeCheck()), new ArgumentResolver(), (new Expressions())->growthPoints()); + $method = $reflection->getMethod($method); - return $expander->expand($reflection, $reflection->getMethod($method), $expander->hooks($reflection), $configuration); + return $expander->expand($reflection, $method, $expander->hooks($reflection), $rewriter->index((string) file_get_contents((string) $method->getDeclaringClass()->getFileName())), $configuration); + } + + /** + * @return list the code of each changed file + */ + private static function rewrite(string $code, BenchmarkCase $case, Metrics $metrics): array + { + $plan = (new ExpectationChanges())->plan([[$case, $metrics]], [$case->expectations->file() => $code]); + self::assertSame([], $plan->failures); + + return array_map(static fn (FileChange $file): string => $file->after, $plan->files); } } diff --git a/tests/Unit/Discovery/GrowthPointsTest.php b/tests/Unit/Discovery/GrowthPointsTest.php index 9f4f731..2d599aa 100644 --- a/tests/Unit/Discovery/GrowthPointsTest.php +++ b/tests/Unit/Discovery/GrowthPointsTest.php @@ -15,6 +15,7 @@ use Tally\Expectation\ExpectedAttributeRewriter; use Tally\Tests\Expressions; use Tally\Tests\Fixtures\Complexity\ComplexityBench; +use function file_get_contents; use function implode; use function log; @@ -131,9 +132,10 @@ public function testInvalid(string $method, array $message): void private static function expand(string $method): array { $class = new ReflectionClass(ComplexityBench::class); - $expander = new CaseExpander(new ExpectedAttributeRewriter(), new AttributeFactory(new ValueTypeCheck()), new ArgumentResolver(), (new Expressions())->growthPoints()); + $rewriter = new ExpectedAttributeRewriter(); + $expander = new CaseExpander($rewriter, new AttributeFactory(new ValueTypeCheck()), new ArgumentResolver(), (new Expressions())->growthPoints()); - return $expander->expand($class, $class->getMethod($method), $expander->hooks($class), Configuration::create()); + return $expander->expand($class, $class->getMethod($method), $expander->hooks($class), $rewriter->index((string) file_get_contents((string) $class->getFileName())), Configuration::create()); } } diff --git a/tests/Unit/Execution/WorkerCommandLineTest.php b/tests/Unit/Execution/WorkerCommandLineTest.php index dfd5b5f..b2a5c66 100644 --- a/tests/Unit/Execution/WorkerCommandLineTest.php +++ b/tests/Unit/Execution/WorkerCommandLineTest.php @@ -3,9 +3,12 @@ namespace Tally\Tests\Unit\Execution; use PHPUnit\Framework\TestCase; +use Tally\Attribute\RelativeTolerance; +use Tally\BenchmarkCase; use Tally\CaseId; use Tally\Execution\WorkerCommandLine; use Tally\Execution\WorkerSettings; +use Tally\Tests\InMemoryExpectations; use function array_slice; final class WorkerCommandLineTest extends TestCase @@ -26,8 +29,8 @@ public function testNative(): void '--result=/work/native.json', '--config=tally.php', '--', - 'benchmarks', - ], self::commandLine()->native(self::settings('/opt/codspeed/tally_hooks-8.5.so'), self::id(), '/work/scratch', '/work/native.json')); + '/project/benchmarks/FooBench.php', + ], self::commandLine()->native(self::settings('/opt/codspeed/tally_hooks-8.5.so'), self::case(), '/work/scratch', '/work/native.json')); } public function testCountedRunsTheWorkerUnderCallgrindWithTheCachesOfCodSpeed(): void @@ -68,8 +71,8 @@ public function testCountedRunsTheWorkerUnderCallgrindWithTheCachesOfCodSpeed(): '--counted', '--config=tally.php', '--', - 'benchmarks', - ], self::commandLine()->counted(self::settings('/opt/codspeed/tally_hooks-8.5.so'), self::id(), '/work/scratch', '/work/counted.json', '/work/profile')); + '/project/benchmarks/FooBench.php', + ], self::commandLine()->counted(self::settings('/opt/codspeed/tally_hooks-8.5.so'), self::case(), '/work/scratch', '/work/counted.json', '/work/profile')); } private static function commandLine(): WorkerCommandLine @@ -81,7 +84,7 @@ public function testCountedWithoutLoadingTheExtensionPhpLoadsAlready(): void { $this->assertSame( ['-d', 'zend.max_allowed_stack_size=-1', '-d', 'memory_limit=-1', '-d', 'opcache.enable_cli=1', 'vendor/bin/tally'], - array_slice(self::commandLine()->counted(self::settings(null), self::id(), '/work/scratch', '/work/counted.json', '/work/profile'), 19, 7), + array_slice(self::commandLine()->counted(self::settings(null), self::case(), '/work/scratch', '/work/counted.json', '/work/profile'), 19, 7), ); } @@ -89,7 +92,7 @@ private static function settings(?string $extension): WorkerSettings { return new WorkerSettings( ['memory_limit=-1', 'opcache.enable_cli=1'], - ['--config=tally.php', '--', 'benchmarks'], + ['--config=tally.php'], '/opt/codspeed/valgrind', $extension, '/work', @@ -98,9 +101,19 @@ private static function settings(?string $extension): WorkerSettings ); } - private static function id(): CaseId + private static function case(): BenchmarkCase { - return new CaseId('App\FooBench', 'benchFoo', "it's"); + return new BenchmarkCase( + new CaseId('App\FooBench', 'benchFoo', "it's"), + '/project/benchmarks/FooBench.php', + [], + 0, + [], + [], + new InMemoryExpectations(null, '/project/benchmarks/FooBench.php'), + new RelativeTolerance(0.1), + null, + ); } } diff --git a/tests/Unit/Expectation/AttributeExpectationsTest.php b/tests/Unit/Expectation/AttributeExpectationsTest.php new file mode 100644 index 0000000..334b10b --- /dev/null +++ b/tests/Unit/Expectation/AttributeExpectationsTest.php @@ -0,0 +1,51 @@ +assertSame('/FooBench.php', $source->file()); + $this->assertSame(['instructions' => 1, 'cycles' => 1, 'memory' => 0], $source->read()?->toArray()); + $this->assertEquals( + [new CodeEdit(56, 54, "Expected(instructions: '3.000K', cycles: '3.300K', memory: '1.000KiB')")], + $source->edits(self::CODE, Metrics::fromArray(['instructions' => 3_000, 'cycles' => 3_300, 'memory' => 1024])), + ); + $this->assertEquals(new SourceLocation('/FooBench.php', 4), $source->location()); + } + + /** + * @throws ExpectationException + */ + public function testRefusesCodeThatChangedSinceTheIndex(): void + { + $this->expectException(ExpectationException::class); + $this->expectExceptionMessage('/FooBench.php changed after Tally loaded it, run the command again.'); + self::source()->edits(self::CODE . "\n", Metrics::fromArray(['instructions' => 3_000, 'cycles' => 3_300, 'memory' => 1024])); + } + + private static function source(): AttributeExpectations + { + $rewriter = new ExpectedAttributeRewriter(); + + return new AttributeExpectations($rewriter, '/FooBench.php', $rewriter->index(self::CODE), 'FooBench', 'bench', null, Metrics::fromArray(['instructions' => 1, 'cycles' => 1, 'memory' => 0]), false); + } + +} diff --git a/tests/Unit/Expectation/ExpectationChangesTest.php b/tests/Unit/Expectation/ExpectationChangesTest.php index 3de77d8..b43fe1e 100644 --- a/tests/Unit/Expectation/ExpectationChangesTest.php +++ b/tests/Unit/Expectation/ExpectationChangesTest.php @@ -7,11 +7,13 @@ use Tally\BenchmarkCase; use Tally\CaseId; use Tally\Expectation\AttributeExpectations; +use Tally\Expectation\CodeEdit; use Tally\Expectation\ExpectationChanges; use Tally\Expectation\ExpectationSource; use Tally\Expectation\ExpectedAttributeRewriter; use Tally\Expectation\FileChange; use Tally\Metrics; +use Tally\Tests\FixedEdits; use Tally\Tests\InMemoryExpectations; use function array_map; use function str_replace; @@ -44,12 +46,13 @@ public function testACaseThatCannotBeRewrittenDoesNotStopTheOthersInItsFile(): v { $code = "index($code); $plan = (new ExpectationChanges())->plan( [ [self::case('App\GoneBench::benchGone', new InMemoryExpectations(null, '/GoneBench.php')), Metrics::fromArray(['instructions' => 3])], - [self::case('App\SortBench::benchLarge@large', new AttributeExpectations($rewriter, '/SortBench.php', 'App\SortBench', 'benchLarge', 'large', null, false)), Metrics::fromArray(['instructions' => 1, 'cycles' => 1, 'memory' => 0])], - [self::case('App\SortBench::benchSort', new AttributeExpectations($rewriter, '/SortBench.php', 'App\SortBench', 'benchSort', null, null, false)), Metrics::fromArray(['instructions' => 2, 'cycles' => 2, 'memory' => 0])], + [self::case('App\SortBench::benchLarge@large', new AttributeExpectations($rewriter, '/SortBench.php', $index, 'App\SortBench', 'benchLarge', 'large', null, false)), Metrics::fromArray(['instructions' => 1, 'cycles' => 1, 'memory' => 0])], + [self::case('App\SortBench::benchSort', new AttributeExpectations($rewriter, '/SortBench.php', $index, 'App\SortBench', 'benchSort', null, null, false)), Metrics::fromArray(['instructions' => 2, 'cycles' => 2, 'memory' => 0])], ], ['/SortBench.php' => $code, '/GoneBench.php' => false], ); @@ -72,6 +75,68 @@ public function testACaseThatCannotBeRewrittenDoesNotStopTheOthersInItsFile(): v ); } + public function testCasesThatAddTheSameImportAddItOnceAndKeepTheImportsSorted(): void + { + $code = "index($code); + $metrics = Metrics::fromArray(['instructions' => 1, 'cycles' => 2, 'memory' => 0]); + + $plan = (new ExpectationChanges())->plan( + [ + [self::case('App\SortBench::benchA@x', new AttributeExpectations($rewriter, '/SortBench.php', $index, 'App\SortBench', 'benchA', 'x', null, false)), $metrics], + [self::case('App\SortBench::benchB', new AttributeExpectations($rewriter, '/SortBench.php', $index, 'App\SortBench', 'benchB', null, null, false)), $metrics], + [self::case('App\SortBench::benchC', new AttributeExpectations($rewriter, '/SortBench.php', $index, 'App\SortBench', 'benchC', null, null, false)), $metrics], + ], + ['/SortBench.php' => $code], + ); + + $this->assertSame( + [[ + '/SortBench.php', + $code, + "files), + ); + $this->assertSame([], $plan->failures); + } + + public function testEditsAtOneOffsetGoInTheNaturalOrderOfTheirTextAndInsertionsBeforeReplacements(): void + { + $plan = (new ExpectationChanges())->plan( + [ + [self::case('A::bench10', new FixedEdits('/a.php', [new CodeEdit(1, 2, '[10]')])), Metrics::none()], + [self::case('A::bench2', new FixedEdits('/a.php', [new CodeEdit(1, 0, '<2>')])), Metrics::none()], + [self::case('A::bench1', new FixedEdits('/a.php', [new CodeEdit(1, 0, '<10>'), new CodeEdit(3, 0, '<3>')])), Metrics::none()], + ], + ['/a.php' => 'abcd'], + ); + + $this->assertSame([['/a.php', 'abcd', 'a<2><10>[10]<3>d', ['A::bench10', 'A::bench2', 'A::bench1']]], self::files($plan->files)); + $this->assertSame([], $plan->failures); + } + + public function testACaseWhoseEditsOverlapAnotherCaseIsLeftOut(): void + { + $plan = (new ExpectationChanges())->plan( + [ + [self::case('A::benchA', new FixedEdits('/a.php', [new CodeEdit(1, 2, 'X')])), Metrics::none()], + [self::case('A::benchB', new FixedEdits('/a.php', [new CodeEdit(0, 1, 'Y'), new CodeEdit(2, 0, 'Z')])), Metrics::none()], + [self::case('A::benchC', new FixedEdits('/a.php', [new CodeEdit(1, 2, 'X'), new CodeEdit(3, 1, 'W')])), Metrics::none()], + [self::case('A::benchD', new FixedEdits('/a.php', [new CodeEdit(2, 1, 'V')])), Metrics::none()], + ], + ['/a.php' => 'abcd'], + ); + + $this->assertSame([['/a.php', 'abcd', 'aXW', ['A::benchA', 'A::benchC']]], self::files($plan->files)); + $this->assertSame(['A::benchB' => 'Its expectations overlap those of A::benchA.', 'A::benchD' => 'Its expectations overlap those of A::benchA.'], $plan->failures); + } + public function testAFileThatWasNotReadCannotBeRewritten(): void { $plan = (new ExpectationChanges())->plan([[self::case('A::bench', new InMemoryExpectations(null, '/a.php')), Metrics::none()]], []); @@ -99,7 +164,7 @@ private static function files(array $files): array private static function case(string $id, ExpectationSource $expectations): BenchmarkCase { - return new BenchmarkCase(CaseId::fromString($id), [], 0, [], [], $expectations, new RelativeTolerance(0.1), null); + return new BenchmarkCase(CaseId::fromString($id), '/benchmarks/FooBench.php', [], 0, [], [], $expectations, new RelativeTolerance(0.1), null); } } diff --git a/tests/Unit/Expectation/ExpectedAttributeRewriterTest.php b/tests/Unit/Expectation/ExpectedAttributeRewriterTest.php index f234a58..ae85210 100644 --- a/tests/Unit/Expectation/ExpectedAttributeRewriterTest.php +++ b/tests/Unit/Expectation/ExpectedAttributeRewriterTest.php @@ -12,7 +12,7 @@ use Tally\Tests\Fixtures\Rewriter\SortBench; use function file_get_contents; use function str_replace; -use function substr_count; +use function substr; final class ExpectedAttributeRewriterTest extends TestCase { @@ -26,7 +26,7 @@ public function testRewriteReplacesOnlyTheAttributeOfTheMethod(): void { $code = self::read(self::SORT_BENCH); - $rewritten = (new ExpectedAttributeRewriter())->rewrite( + $rewritten = self::rewrite( $code, SortBench::class, 'benchSort', @@ -53,7 +53,7 @@ public function testRewriteReplacesTheAttributeOfTheDataSet(): void { $code = self::read(self::SORT_BENCH); - $rewritten = (new ExpectedAttributeRewriter())->rewrite($code, SortBench::class, 'benchSizes', 'large', Metrics::fromArray(['instructions' => 20_000, 'cycles' => 22_000, 'memory' => 2048]), false); + $rewritten = self::rewrite($code, SortBench::class, 'benchSizes', 'large', Metrics::fromArray(['instructions' => 20_000, 'cycles' => 22_000, 'memory' => 2048]), false); $this->assertSame( str_replace("#[ExpectedFor(dataSet: 'large', instructions: '10.00K', cycles: '12.00K', memory: '0B')]", "#[ExpectedFor('large', instructions: '20.00K', cycles: '22.00K', memory: '2.000KiB')]", $code), @@ -68,7 +68,7 @@ public function testRewriteAddsAMissingAttributeAfterTheOthers(): void { $code = self::read(self::SORT_BENCH); - $rewritten = (new ExpectedAttributeRewriter())->rewrite($code, SortBench::class, 'benchWithoutExpectations', null, Metrics::fromArray(['instructions' => 5, 'cycles' => 6, 'memory' => 0]), false); + $rewritten = self::rewrite($code, SortBench::class, 'benchWithoutExpectations', null, Metrics::fromArray(['instructions' => 5, 'cycles' => 6, 'memory' => 0]), false); $this->assertSame( str_replace("\t#[Warmup(1)]\n", "\t#[Warmup(1)]\n\t#[Expected(instructions: '5', cycles: '6', memory: '0B')]\n", $code), @@ -83,7 +83,7 @@ public function testRewriteWritesTheDataSetAsANamedArgumentWhenAsked(): void { $code = self::read(self::SORT_BENCH); - $rewritten = (new ExpectedAttributeRewriter())->rewrite($code, SortBench::class, 'benchSizes', 'large', Metrics::fromArray(['instructions' => 20_000, 'cycles' => 22_000, 'memory' => 2048]), true); + $rewritten = self::rewrite($code, SortBench::class, 'benchSizes', 'large', Metrics::fromArray(['instructions' => 20_000, 'cycles' => 22_000, 'memory' => 2048]), true); $this->assertSame( str_replace("#[ExpectedFor(dataSet: 'large', instructions: '10.00K', cycles: '12.00K', memory: '0B')]", "#[ExpectedFor(dataSet: 'large', instructions: '20.00K', cycles: '22.00K', memory: '2.000KiB')]", $code), @@ -98,7 +98,7 @@ public function testRewriteAddsAMissingDataSetAttribute(): void { $code = self::read(self::SORT_BENCH); - $rewritten = (new ExpectedAttributeRewriter())->rewrite($code, SortBench::class, 'benchSizes', "it's new", Metrics::fromArray(['instructions' => 1, 'cycles' => 2, 'memory' => 0]), false); + $rewritten = self::rewrite($code, SortBench::class, 'benchSizes', "it's new", Metrics::fromArray(['instructions' => 1, 'cycles' => 2, 'memory' => 0]), false); $this->assertSame( str_replace( @@ -117,7 +117,7 @@ public function testRewriteImportsTheAttribute(): void { $code = self::read(__DIR__ . '/../../Fixtures/Rewriter/NoImportBench.php'); - $rewritten = (new ExpectedAttributeRewriter())->rewrite($code, 'Tally\\Tests\\Fixtures\\Rewriter\\NoImportBench', 'benchSprintf', null, Metrics::fromArray(['instructions' => 1, 'cycles' => 1, 'memory' => 0]), false); + $rewritten = self::rewrite($code, 'Tally\\Tests\\Fixtures\\Rewriter\\NoImportBench', 'benchSprintf', null, Metrics::fromArray(['instructions' => 1, 'cycles' => 1, 'memory' => 0]), false); $this->assertSame( str_replace( @@ -129,20 +129,6 @@ public function testRewriteImportsTheAttribute(): void ); } - /** - * @throws ExpectationException - */ - public function testRewriteTwiceAddsTheImportOnce(): void - { - $rewriter = new ExpectedAttributeRewriter(); - $class = 'Tally\\Tests\\Fixtures\\Rewriter\\NoImportBench'; - $code = $rewriter->rewrite(self::read(__DIR__ . '/../../Fixtures/Rewriter/NoImportBench.php'), $class, 'benchSprintf', null, Metrics::fromArray(['instructions' => 1, 'cycles' => 1, 'memory' => 0]), false); - $code = $rewriter->rewrite($code, $class, 'benchSprintf', null, Metrics::fromArray(['instructions' => 2, 'cycles' => 2, 'memory' => 0]), false); - - $this->assertSame(1, substr_count($code, 'use Tally\\Attribute\\Expected;')); - $this->assertSame(1, substr_count($code, "#[Expected(instructions: '2', cycles: '2', memory: '0B')]")); - } - /** * @return iterable */ @@ -241,7 +227,7 @@ public static function dataRewriteCode(): iterable #[DataProvider('dataRewriteCode')] public function testRewriteCode(string $class, string $code, string $expected): void { - $this->assertSame($expected, (new ExpectedAttributeRewriter())->rewrite($code, $class, 'bench', null, Metrics::fromArray(['instructions' => 7, 'cycles' => 8, 'memory' => 0]), false)); + $this->assertSame($expected, self::rewrite($code, $class, 'bench', null, Metrics::fromArray(['instructions' => 7, 'cycles' => 8, 'memory' => 0]), false)); } /** @@ -267,7 +253,7 @@ public function testLocateDataSet(string $dataSet, int $expected): void . "\t#[ExpectedFor(\"tab\\there\", instructions: '1', cycles: '1', memory: '0B')]\n" . "\tpublic function bench() {}\n}\n"; - $this->assertSame($expected, (new ExpectedAttributeRewriter())->locate($code, 'App\\FooBench', 'bench', $dataSet)); + $this->assertSame($expected, self::locate($code, 'App\\FooBench', 'bench', $dataSet)); } /** @@ -293,7 +279,7 @@ public function testRewriteRejectsANonLiteralDataSet(string $attribute): void $this->expectException(ExpectationException::class); $this->expectExceptionMessage('#[ExpectedFor] on line 5: write the data set as a string literal, so tally accept can find the attribute.'); - (new ExpectedAttributeRewriter())->rewrite($code, 'App\\FooBench', 'bench', 'large', Metrics::fromArray(['instructions' => 7, 'cycles' => 8, 'memory' => 0]), false); + self::rewrite($code, 'App\\FooBench', 'bench', 'large', Metrics::fromArray(['instructions' => 7, 'cycles' => 8, 'memory' => 0]), false); } /** @@ -305,9 +291,9 @@ public function testANonLiteralDataSetOfAnotherMethodDoesNotMatter(): void $this->assertSame( "rewrite($code, 'App\\FooBench', 'bench', null, Metrics::fromArray(['instructions' => 7, 'cycles' => 8, 'memory' => 0]), false), + self::rewrite($code, 'App\\FooBench', 'bench', null, Metrics::fromArray(['instructions' => 7, 'cycles' => 8, 'memory' => 0]), false), ); - $this->assertSame(8, (new ExpectedAttributeRewriter())->locate($code, 'App\\FooBench', 'bench', null)); + $this->assertSame(8, self::locate($code, 'App\\FooBench', 'bench', null)); } /** @@ -329,7 +315,7 @@ public function testRewriteNeedsEveryMetric(array $metrics, string $message): vo { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage($message); - (new ExpectedAttributeRewriter())->rewrite(self::read(self::SORT_BENCH), SortBench::class, 'benchSort', null, Metrics::fromArray($metrics), false); + self::rewrite(self::read(self::SORT_BENCH), SortBench::class, 'benchSort', null, Metrics::fromArray($metrics), false); } /** @@ -361,7 +347,7 @@ public static function dataRewriteDataSetCode(): iterable #[DataProvider('dataRewriteDataSetCode')] public function testRewriteDataSetCode(string $code, string $expected): void { - $this->assertSame($expected, (new ExpectedAttributeRewriter())->rewrite($code, 'App\\FooBench', 'bench', 'a', Metrics::fromArray(['instructions' => 7, 'cycles' => 8, 'memory' => 0]), false)); + $this->assertSame($expected, self::rewrite($code, 'App\\FooBench', 'bench', 'a', Metrics::fromArray(['instructions' => 7, 'cycles' => 8, 'memory' => 0]), false)); } /** @@ -371,7 +357,7 @@ public function testRewriteDataSetInTheGlobalNamespace(): void { $this->assertSame( "rewrite(" 7, 'cycles' => 8, 'memory' => 0]), false), + self::rewrite(" 7, 'cycles' => 8, 'memory' => 0]), false), ); } @@ -381,14 +367,13 @@ public function testRewriteDataSetInTheGlobalNamespace(): void public function testLocate(): void { $code = self::read(self::SORT_BENCH); - $rewriter = new ExpectedAttributeRewriter(); - $this->assertSame(13, $rewriter->locate($code, SortBench::class, 'benchSort', null)); - $this->assertSame(25, $rewriter->locate($code, SortBench::class, 'benchSizes', 'large')); - $this->assertSame(24, $rewriter->locate($code, SortBench::class, 'benchSizes', 'small')); - $this->assertSame(35, $rewriter->locate($code, 'Tally\\Tests\\Fixtures\\Rewriter\\OtherBench', 'benchSort', null)); + $this->assertSame(13, self::locate($code, SortBench::class, 'benchSort', null)); + $this->assertSame(25, self::locate($code, SortBench::class, 'benchSizes', 'large')); + $this->assertSame(24, self::locate($code, SortBench::class, 'benchSizes', 'small')); + $this->assertSame(35, self::locate($code, 'Tally\\Tests\\Fixtures\\Rewriter\\OtherBench', 'benchSort', null)); // where the attribute would go - $this->assertSame(19, $rewriter->locate($code, SortBench::class, 'benchWithoutExpectations', null)); + $this->assertSame(19, self::locate($code, SortBench::class, 'benchWithoutExpectations', null)); } /** @@ -398,7 +383,35 @@ public function testLocateFailsForAMissingMethod(): void { $this->expectException(MethodNotFoundException::class); $this->expectExceptionMessage('Tally\Tests\Fixtures\Rewriter\SortBench::benchMissing'); - (new ExpectedAttributeRewriter())->locate(self::read(self::SORT_BENCH), SortBench::class, 'benchMissing', null); + self::locate(self::read(self::SORT_BENCH), SortBench::class, 'benchMissing', null); + } + + /** + * The code with the edits of the case applied, which come in the order of their offsets. + * + * @throws ExpectationException + */ + private static function rewrite(string $code, string $class, string $method, ?string $dataSet, Metrics $metrics, bool $namedArguments): string + { + $rewriter = new ExpectedAttributeRewriter(); + $rewritten = ''; + $position = 0; + foreach ($rewriter->edits($rewriter->index($code), $class, $method, $dataSet, $metrics, $namedArguments) as $edit) { + $rewritten .= substr($code, $position, $edit->offset - $position) . $edit->text; + $position = $edit->offset + $edit->length; + } + + return $rewritten . substr($code, $position); + } + + /** + * @throws ExpectationException + */ + private static function locate(string $code, string $class, string $method, ?string $dataSet): int + { + $rewriter = new ExpectedAttributeRewriter(); + + return $rewriter->locate($rewriter->index($code), $class, $method, $dataSet); } private static function read(string $file): string diff --git a/tests/Unit/Runner/CaseRunnerTest.php b/tests/Unit/Runner/CaseRunnerTest.php index 0257190..2bb1cf9 100644 --- a/tests/Unit/Runner/CaseRunnerTest.php +++ b/tests/Unit/Runner/CaseRunnerTest.php @@ -97,7 +97,7 @@ public function testMeasuresTheMemoryOfTheCountedCallOnly(): void { $memory = new FakeMemory(1_000_000); Journal::$memory = $memory; - $case = new BenchmarkCase(new CaseId(RunnerBench::class, 'benchAllocate', null), [], 3, [], [], new InMemoryExpectations(null, '/benchmarks/InMemoryBench.php'), new RelativeTolerance(0.1), null); + $case = new BenchmarkCase(new CaseId(RunnerBench::class, 'benchAllocate', null), '/benchmarks/FooBench.php', [], 3, [], [], new InMemoryExpectations(null, '/benchmarks/InMemoryBench.php'), new RelativeTolerance(0.1), null); $metrics = (new CaseRunner(new ConstructorArguments(), new ArgumentResolver(), $memory, new IdleCounter()))->run($case, new ScratchDirectory('/scratch'), new JournalCounter()); @@ -112,7 +112,7 @@ public function testSubtractsTheMemoryOfTheBaseline(): void { $memory = new FakeMemory(1_000_000); Journal::$memory = $memory; - $case = new BenchmarkCase(new CaseId(RunnerBench::class, 'benchAllocate', null), [], 0, [], [], new InMemoryExpectations(null, '/benchmarks/InMemoryBench.php'), new RelativeTolerance(0.1), null); + $case = new BenchmarkCase(new CaseId(RunnerBench::class, 'benchAllocate', null), '/benchmarks/FooBench.php', [], 0, [], [], new InMemoryExpectations(null, '/benchmarks/InMemoryBench.php'), new RelativeTolerance(0.1), null); $metrics = (new CaseRunner(new ConstructorArguments(), new ArgumentResolver(), $memory, new IdleCounter()))->run($case, new ScratchDirectory('/scratch'), new AllocatingCounter($memory, 56)); @@ -186,7 +186,7 @@ public function testABeforeMethodThatReturnsPausedFails(): void public function testArgumentsThatDoNotFitFail(): void { - $case = new BenchmarkCase(new CaseId(RunnerBench::class, 'benchName', 'too-many'), ['x', 'y'], 0, [], [], new InMemoryExpectations(null, '/benchmarks/InMemoryBench.php'), new RelativeTolerance(0.1), null); + $case = new BenchmarkCase(new CaseId(RunnerBench::class, 'benchName', 'too-many'), '/benchmarks/FooBench.php', ['x', 'y'], 0, [], [], new InMemoryExpectations(null, '/benchmarks/InMemoryBench.php'), new RelativeTolerance(0.1), null); $this->expectException(InvalidBenchmarkException::class); $this->expectExceptionMessage(RunnerBench::class . '::benchName(): the data set has 2 arguments, but the method takes 1.'); @@ -199,7 +199,7 @@ public function testArgumentsThatDoNotFitFail(): void */ public function testAMissingClassFails(): void { - $case = new BenchmarkCase(new CaseId('App\MissingBench', 'benchFoo', null), [], 0, [], [], new InMemoryExpectations(null, '/benchmarks/InMemoryBench.php'), new RelativeTolerance(0.1), null); + $case = new BenchmarkCase(new CaseId('App\MissingBench', 'benchFoo', null), '/benchmarks/FooBench.php', [], 0, [], [], new InMemoryExpectations(null, '/benchmarks/InMemoryBench.php'), new RelativeTolerance(0.1), null); $this->expectException(InvalidBenchmarkException::class); $this->expectExceptionMessage('App\MissingBench::benchFoo: class App\MissingBench does not exist.'); @@ -218,7 +218,7 @@ private static function runner(): CaseRunner */ private static function case(string $method, string $name, int $warmup, array $before, array $after): BenchmarkCase { - return new BenchmarkCase(new CaseId(RunnerBench::class, $method, $name), [$name], $warmup, $before, $after, new InMemoryExpectations(null, '/benchmarks/InMemoryBench.php'), new RelativeTolerance(0.1), null); + return new BenchmarkCase(new CaseId(RunnerBench::class, $method, $name), '/benchmarks/FooBench.php', [$name], $warmup, $before, $after, new InMemoryExpectations(null, '/benchmarks/InMemoryBench.php'), new RelativeTolerance(0.1), null); } } diff --git a/tests/Unit/Selection/Cases.php b/tests/Unit/Selection/Cases.php index 541e8d5..95e4083 100644 --- a/tests/Unit/Selection/Cases.php +++ b/tests/Unit/Selection/Cases.php @@ -34,6 +34,7 @@ public static function withBases(array $costs, array $bases): array $base = $bases[$id] ?? null; $cases[] = new BenchmarkCase( CaseId::fromString($id), + '/benchmarks/FooBench.php', [], 0, [],