diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..0fde798 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,13 @@ +{ + "name": "phpstan", + "owner": { + "name": "PHPStan" + }, + "plugins": [ + { + "name": "tally", + "source": "./", + "description": "Agent Skills for benchmarking PHP code with Tally." + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..34a0ba8 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,7 @@ +{ + "name": "tally", + "description": "Agent Skills for benchmarking PHP code with Tally.", + "homepage": "https://github.com/phpstan/tally", + "repository": "https://github.com/phpstan/tally", + "license": "MIT" +} diff --git a/.gitattributes b/.gitattributes index 19cb0ae..439b7da 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,7 @@ *.php text eol=lf .agents export-ignore +.claude-plugin export-ignore .bashunitrc export-ignore .github export-ignore bench export-ignore diff --git a/CLAUDE.md b/CLAUDE.md index a980852..0b5ea2a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ Tally benchmarks PHP code by counting instructions, cycles, and memory under Cod - `src/Complexity` – `ComplexityClass`, the interface of `#[Complexity]` and its presets (`#[Linear]` and the others, which compose a `Complexity`), and `GrowthPoint`, where a case is on its growth curve. `Discovery\GrowthPoints` computes them at discovery, `Comparator::compareGrowth()` compares a case with its base. These cases have no numbers to accept: `PresetFit` (pure) finds the lowest preset the measured instructions of a method grow within, which `tally accept` writes instead of the method's preset through the `ComplexityAttribute` of its `GrowthPoint`. - `src/Quantity` – `QuantityFormat` formats and parses numbers with units (`3.119B`, `44.49MiB`). - `src/Expectation` – where expected metrics are stored. `ExpectationSource` is the interface: `read()` the expectation, `file()` the file it lives in, the pure `edits(code, metrics)` that returns the `CodeEdit`s that write new expectations into the file's code, and `location()`, which throws `ExpectationException` when the source cannot be rewritten. `AttributeExpectations` implements it for `#[Expected]` and `#[ExpectedFor]` through `ExpectedAttributeRewriter`, which reads a file once into an `AttributeIndex` of every method's attributes, with `Tokens`, the token list it walks, and locates and edits each case in the index; `ComplexityAttribute` replaces the complexity class of a method through `ExpectedAttributeRewriter::complexityEdits()`, in the same index; `DirectorySuiteLoader` indexes each file once for all its cases, because tokenizing it for every case made locating and rewriting quadratic in the cases of a file. `ExpectationChanges` plans the rewrites of many cases and complexity classes (pure): it applies the edits of all of them in a file in one pass, one `FileChange` per file with every change in it, in a `ChangePlan` together with the cases that failed. `ExpectationFiles` reads and writes the files. -- `src/Discovery` – `DirectorySuiteLoader` finds and loads `*Bench.php` files (I/O), `CaseExpander` turns one method into its cases (pure, works on reflection); `hooks()` finds a class's `#[Before]` and `#[After]` methods once per class, because scanning them for every benchmark method made loading a class quadratic in its size. `AttributeFactory` creates every attribute Tally reads: it checks the arguments against the attribute's constructor first (names, count, types through `ValueTypeCheck`, as under strict_types), so a mistake is an `InvalidBenchmarkException` naming the argument, not a PHP error. Everything a benchmark author can get wrong, including a file that does not load and a data provider that throws, ends as an `InvalidBenchmarkException` that names the benchmark or file; PHP errors with stack traces are bugs. `ArgumentResolver` checks that a method takes the arguments of a data set (a hook the leading ones) and `ConstructorArguments` that a constructor takes only `Meter` and `ScratchDirectory`; discovery checks both, so a mismatch fails `tally list` instead of the worker. +- `src/Discovery` – `DirectorySuiteLoader` finds and loads `*Bench.php` files and takes the class named like each file as its benchmark class, so helpers can live next to it (I/O), `CaseExpander` turns one method into its cases (pure, works on reflection); `hooks()` finds a class's `#[Before]` and `#[After]` methods once per class, because scanning them for every benchmark method made loading a class quadratic in its size. `AttributeFactory` creates every attribute Tally reads: it checks the arguments against the attribute's constructor first (names, count, types through `ValueTypeCheck`, as under strict_types), so a mistake is an `InvalidBenchmarkException` naming the argument, not a PHP error. Everything a benchmark author can get wrong, including a file that does not load and a data provider that throws, ends as an `InvalidBenchmarkException` that names the benchmark or file; PHP errors with stack traces are bugs. `ArgumentResolver` checks that a method takes the arguments of a data set (a hook the leading ones) and `ConstructorArguments` that a constructor takes only `Meter` and `ScratchDirectory`; discovery checks both, so a mismatch fails `tally list` instead of the worker. - `src/Selection` – `CaseSelector` implementations: `FilterSelector`, `ShardSelector` (longest-processing-time sharding by expected instructions). Both keep a case compared by its growth with its base: the filter adds the base, the shards keep a method's cases together. - `src/Runner` – running one case in this process. `CaseRunner` calls the hooks uncounted, prepares the counter, calls the warm-up uncounted, then counts a `#warmup` span, an empty `#baseline` call, and one call of the benchmark, and subtracts the baseline's memory. The `#warmup` span exists because the first counted span costs more than later ones. `CodSpeedCounter` starts Callgrind's instrumentation once, when it is prepared, stops it when it is finished after the benchmark's span, so the after methods and the end of the process run at Valgrind's quickest, and only toggles collecting for each span: starting the instrumentation empties the simulated caches, and doing it for every span made the cycles of small benchmarks depend on the cache lines the hooks touched. `MeterSession` is the `Meter` the benchmark object gets in its constructor, one per case: every call `CaseRunner` makes is a span of it with its own `Counter` (`CodSpeedCounter` over `InstrumentHooks` for the counted ones, `IdleCounter` for hooks and warm-up), and it measures memory above the start of the span through `Memory`. `ExtensionInstrumentHooks` calls the `tally_hooks` extension, the only C that Tally calls. - `src/Comparison` – `Comparator` turns a case's expectations and measurement into a `CaseComparison` of `MetricComparison`s with a `Status`. `TolerancePolicy` is implemented by the tolerance attributes; `CaseExpander` picks the method's, else the class's, else that of the `Configuration`, which has one for expected numbers and one for growth. `Percent` formats shares. @@ -26,6 +26,7 @@ Tally benchmarks PHP code by counting instructions, cycles, and memory under Cod - `examples/` – short, artificial benchmarks for README.md, one feature each. `examples/Scripts` shows a data provider with its own `ExpectationSource`, which keeps the numbers in a comment at the top of each script. Keep them readable over realistic; real measurements of Tally belong in `bench/`. CI runs them under CodSpeed next to `bench/`, so they stay valid. - `build/` – scripts for the build, such as the coverage check. `build-codspeed.sh` builds CodSpeed's Valgrind (Callgrind only, static launcher) at a pinned tag into `codspeed/linux-`, which `1.x` ignores, and `build-tally-hooks.sh` builds `ext/tally_hooks` with `phpize` next to it; `make codspeed` runs both for this machine and PHP, CI builds `tally_hooks` for PHP 8.2 to 8.6, NTS and ZTS, in the official `php` Docker images. `build-release.sh` turns `1.x` into the `release/1.x` tree by committing the builds of both platforms; CI pushes it on every push to `1.x`, and releases are tagged there with `tag-release.sh `, which checks that the generated commit sits on the `1.x` commit to release. `check-release.sh` installs a release tree into a new project with Composer and counts a benchmark, which CI runs on every pull request on x86_64 and aarch64, with PHP 8.2 and 8.5, and 8.5 ZTS. `build/PHPStan` holds Tally's own PHPStan rules and `BenchmarkUsageProvider`, which tells the dead code detector about the methods Tally calls through reflection. `build/stubs` holds the stubs for what PHPStan cannot see, such as the functions of `tally_hooks`. - `e2e/` – end-to-end tests with [bashunit](https://bashunit.com), `/_test.sh`; `e2e/bootstrap.sh`, set in `.bashunitrc`, gives each test a directory of its own, which is also its `TMPDIR`. `stripped-extensions` counts a benchmark with CodSpeed's Valgrind while PHP loads the extensions in `E2E_EXTENSIONS` in several ways (its scan directory, another one, the php.ini in `PHPRC`, `-c`): the case must not see them and must count what it counts on a PHP without them. CI installs bashunit into `e2e/` and runs it once per PHP setup (Xdebug, PCOV, both); locally, with the extensions loaded and bashunit installed the same way: `E2E_EXTENSIONS='xdebug pcov' e2e/bashunit test e2e`. Like `tests/Fixtures`, their PHP files are only linted. +- `skills/` – Agent Skills for projects that use Tally, shipped in the Composer package, where `extra.skills` declares them for `llm/skills`, and as the plugin `tally` of the marketplace in `.claude-plugin/`, which Claude Code and Codex both read. `tally-rewind` turns past performance fixes of a project into benchmarks, replays them over its history, and improves on the best numbers; its `scripts/measure.sh` measures bench files at commits of the history, each in a worktree, with the Tally version the project has installed. - `tests/Unit` – pure logic, no file system, processes, or network. `tests/Integration` – adapters and the CLI. `tests/Fixtures` – benchmark classes and configuration files the tests load; excluded from PHPStan and the coding standard because some are invalid on purpose. ## Design rules diff --git a/README.md b/README.md index 1bf883b..bcc5527 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Tally runs each benchmark once under [CodSpeed](https://codspeed.io)'s CPU simul - [Contributing a benchmark](#contributing-a-benchmark) - [Storing expectations elsewhere](#storing-expectations-elsewhere) - [Lifecycle of a case](#lifecycle-of-a-case) +- [Agent skills](#agent-skills) ## Installation @@ -60,7 +61,7 @@ The counts depend on the PHP build and the system libraries. Numbers measured on ## Writing benchmarks -A benchmark is a public, non-static method whose name starts with `bench`, in a class in a file whose name ends with `Bench.php`. Tally creates the class with the constructor arguments described [below](#meter-and-scratch-directory). A `bench*` method that is static or not public is an error, so is a path without any benchmarks. +A benchmark is a public, non-static method whose name starts with `bench`, in a file whose name ends with `Bench.php`, in the class named like the file, such as `StrBench` in `StrBench.php`. Other classes in the file, such as helpers, are not benchmarks. Tally creates the class with the constructor arguments described [below](#meter-and-scratch-directory). A `bench*` method that is static or not public is an error, so is a path without any benchmarks. ```php use Tally\Attribute\DataProvider; @@ -358,6 +359,8 @@ Accept the new numbers vendor/bin/tally accept tmp/tally ``` +The commands under "Reproduce locally" and "Accept the new numbers" run Tally the way you ran it, such as `tools/tally/vendor/bin/tally` when Tally has a Composer project of its own. + The progress bar shows one character per case: `.` unchanged, `S` slower, `F` faster, `N` new, `E` not measured. A case is **slower** when a metric exceeds its expected value by more than the tolerance, and **faster** when it is lower by more. A case that throws, exits, or times out is **not measured**. A case without expected numbers is **new**. Slower and not measured cases fail the run. Faster and new cases pass, but their numbers should be accepted, so a later slowdown back to the old numbers does not go unnoticed. @@ -607,6 +610,42 @@ Instructions and cycles come from Callgrind's profile of the benchmark's span. M Cycles are Valgrind's estimate: instructions, plus 10 for each first-level cache miss and 100 for each last-level cache miss of the simulated cache. The cache sizes are the same on every machine. +## Agent skills + +Tally comes with [Agent Skills](https://agentskills.io) for coding agents such as Claude Code and Codex, in [skills/](skills): + +- `tally-rewind` finds the commits of a project that fixed performance problems, turns each into a Tally benchmark locked at the fix, and replays it on every release since. It then makes today's code beat the best numbers the code ever had, keeping only changes that make a real workload faster in wall-clock time. It adds the benchmarks to the project, and sets up Tally and its CI job when the project has none. + +Install them as a plugin, for all your projects: + +```bash +claude plugin marketplace add phpstan/tally +claude plugin install tally@phpstan + +codex plugin marketplace add phpstan/tally +codex plugin add tally@phpstan +``` + +Or into one project, in `.agents/skills/`, with [skills](https://github.com/vercel-labs/skills), which links them into `.claude/skills/` for Claude Code: + +```bash +npx skills add phpstan/tally -a claude-code -a codex +``` + +Or with Composer and [llm/skills](https://github.com/roxblnfk/skills), which copies the skills of the packages a project requires into `.agents/skills/` after every `composer install` and `update`, so they stay in the version of Tally the project has: + +```bash +composer require --dev llm/skills +``` + +Claude Code reads `.claude/skills/`, so for it add a `skills.json` next to `composer.json`, then run `composer skills:update`: + +```json +{ + "aliases": [".claude/skills"] +} +``` + ## License MIT diff --git a/bin/tally b/bin/tally index 45b909b..f8b38db 100755 --- a/bin/tally +++ b/bin/tally @@ -57,5 +57,8 @@ use Tally\Cli\ApplicationFactory; $iniFiles[] = $iniFile; } - exit((new ApplicationFactory())->create(STDOUT, STDERR, $workingDirectory, $environment, $colors, $errorColors, PHP_BINARY, $iniFiles, __FILE__, $codspeed)->run($arguments)); + // the commands in the report run Tally the way the user did, such as tools/tally/vendor/bin/tally + $program = $arguments[0] ?? 'vendor/bin/tally'; + + exit((new ApplicationFactory())->create(STDOUT, STDERR, $workingDirectory, $environment, $colors, $errorColors, PHP_BINARY, $iniFiles, __FILE__, $program, $codspeed)->run($arguments)); })(); diff --git a/composer.json b/composer.json index 246eae1..0740531 100644 --- a/composer.json +++ b/composer.json @@ -40,5 +40,10 @@ }, "bin": [ "bin/tally" - ] + ], + "extra": { + "skills": { + "source": "skills" + } + } } diff --git a/skills/tally-rewind/SKILL.md b/skills/tally-rewind/SKILL.md new file mode 100644 index 0000000..a21931b --- /dev/null +++ b/skills/tally-rewind/SKILL.md @@ -0,0 +1,107 @@ +--- +name: tally-rewind +description: Revisits the performance fixes in a PHP project's git history with Tally. It finds the commits that fixed slowness, adds a Tally benchmark for each to the project (setting up Tally and its CI job when the project has none), locks its numbers at the fix, replays it on every release since to find the best numbers that code ever had, then makes the current code beat them with a wall-clock gain on a real workload. Use when asked to revisit or rewind past performance fixes, guard old speedups with benchmarks, check whether they regressed, hunt performance regressions in history, or find performance work backed by history. Not for benchmarking a single new change. +--- + +# Tally rewind + +Past performance fixes show where a project's hot paths are and how fast that code has already been. This workflow turns each fix into a Tally benchmark, finds the best numbers the code reached between the fix and today, and then improves today's code past them. + +It delivers two things to the project, each as its own commits: + +- The benchmarks, committed to the project with Tally set up and running in CI, so the fixes stay guarded after this work ends. +- Improvements of today's code, each one counting only when it makes a real workload measurably faster in wall-clock time. + +## Rules for every step + +- Every number you compare must come from the same machine, PHP build, and Tally version, because Tally's counts depend on all three. `scripts/measure.sh` in this skill's directory keeps the Tally version and configuration the same at every commit. +- Tally runs on Linux, x86_64 or aarch64, with PHP 8.2 or later and the POSIX extension. On another OS, do all the work in a Linux container with the newest PHP the project supports. +- Work on the line where new development lands, which is not always the default branch. Replay points and best numbers come only from that line's first-parent history. A fix made on another branch counts where it arrives on this line. +- Keep notes in `$TALLY_REWIND_DIR/notes.md`, by default under `${TMPDIR:-/tmp}/tally-rewind//`: the period, the workload and its profile, the fixes and their commits, the bench files, the replay points, the best numbers, and the decisions you made. `measure.sh` appends every measurement to `measurements.tsv` in the same directory. Update the notes after each phase; after a context reset, read both files and continue from there. + +## How to measure + +`bash scripts/measure.sh ... -- ...` copies the bench files as they are in the working copy, with the data files of a directory, into a git worktree at each commit. It installs the project's dependencies there and runs Tally, in the working copy's version and with its `tally.php`, and prints one line per case with instructions, cycles, and memory in bytes. The commits and files are fixed when it starts, so you can keep editing while a batch runs. Options such as `--filter=` go to `tally run`. Exit code 1 means a commit could not be measured, and the output says why. Set `TALLY_REWIND_PREPARE` to a command that makes old commits installable or bootable, such as `cp .env.example .env`. Run `measure.sh --help` for the rest. + +Differences within Tally's tolerance are noise: ±2 % instructions, ±5 % cycles but at least 2,000 cycles, ±5 % memory, unless the project's `tally.php` sets another tolerance. Cycles move by a few percent when unrelated code changes, while the instructions do not. + +The simulation runs PHP 50 to 200 times slower than native. Run each case only at the commits that contain its fix, with `--filter`, and estimate the total time from the first measurements before starting a long batch. + +## 1. Ask for the period + +Ask the user how far back to look, 12 months by default, unless they already said. + +## 2. Profile a real workload + +Pick the project's own main use, such as the command it ships run on a real input, a replayed request, or a script that uses the library on real data. Ask the user when the project has no obvious one. Profile it with the profiler the project uses, or one that is installed, such as Excimer, SPX, or Xdebug, and note which code paths take what share of the time. Load the profiler only for the profiling run, such as with `php -d extension=excimer.so`, because a loaded profiler or coverage extension slows down every other run. A fix on a path that takes 3 % of the workload can save at most 3 %. + +## 3. Find the performance fixes + +Read the subject of every non-merge commit of the period that reached the line, and the body of each one that could be about speed. Fixes are often worded as what the code does now, such as "Compute token offsets once", so a keyword search alone misses them. With more than a few hundred commits, start from a search and widen it: + +```bash +git log --no-merges --since='12 months ago' -i -E --grep='perf|speed|fast|slow|optimi[sz]|quadratic|O\(n|once|cache|memory|allocat|rescan|hot path|bottleneck' --format='%h %cs %s' +``` + +Also read the changelog entries of the period, and when `gh` works, merged pull requests labeled or titled as performance work; their descriptions often have the input and the numbers. + +Read the diff of each candidate. Keep it when production code does less work for the same result: a better algorithm, work moved out of a loop, a cache, fewer allocations, queries, or file reads. Drop speedups of tests, CI, builds, or dev tooling, and fixes that were later reverted. "After" is the fix commit and "before" its parent; when a fix spans several commits, "before" is the parent of the first. + +When the period has no fixes, tell the user and offer to look further back. Otherwise list the fixes for the user, grouped by code path, each with its commit, date, what got faster, and that path's share of the profile. Ask which ones to pursue, by default the ones on the paths with the largest share. + +## 4. Set up Tally in the project + +Do the parts the project does not have yet. `measure.sh` needs Tally installed in the main checkout. + +- **Install.** When the lowest PHP the project supports, in `require.php` and `config.platform.php` of `composer.json`, is 8.2 or later, run `composer require --dev phpstan/tally`. Otherwise requiring Tally would break the project's installs on its older PHP versions: install it as its own Composer project in the project's tools directory, like the other tools it keeps that way, commit its `composer.lock`, and ignore its `vendor/`. Until Tally has a tagged release, require `dev-release/1.x` with `"minimum-stability": "dev"`, because `1.x-dev` has no builds of Valgrind. +- **Configure.** Add `tally.php` at the project root, returning `Tally\Configuration::create()->withPaths([__DIR__ . '/benchmarks'])`. When Tally has its own Composer project, its autoloader knows only Tally: start `tally.php` with `require_once __DIR__ . '/vendor/autoload.php';`, which loads the project's own autoloader. +- **Package.** When `.gitattributes` export-ignores the tests, export-ignore `benchmarks` and `tally.php` too, and Tally's tools directory if it is not already. +- **Command.** Add a command to run the benchmarks next to the project's other check commands, in the same form, and one line about it where the project documents its checks: CONTRIBUTING.md, AGENTS.md, CLAUDE.md, or the README. Say there to take expected numbers from CI. +- **Checks.** Make the project's lint, coding standard, and static analysis cover `benchmarks/` like the tests, unless they run on a PHP older than 8.2 or cannot see Tally's classes; then leave `benchmarks/` out of them. +- **CI.** When the project has `.github/workflows`, add `.github/workflows/benchmarks.yml`, which runs `tally run --output=tmp/tally-results` and uploads `tmp/tally-results` as the artifact `tally-results` with `if: always()`. Run it on a fixed image such as `ubuntu-24.04`, never `ubuntu-latest`, with one fixed PHP version, the newest the project's CI tests, with the `posix` extension, because a new image or PHP release changes every count. Copy the conventions of the project's existing workflows: triggers, `permissions`, `concurrency`, hardening steps, and the same actions at the same pinned versions. Run the workflow linters the project runs. On another CI service, add the same job there. Without any CI, tell the user that the benchmarks guard nothing until something runs them. + +## 5. Write a benchmark for each fix and lock it at the fix + +Write the bench files in `benchmarks/`, one per code path, named after the behavior, such as `benchmarks/RouteMatchingBench.php`. One file can cover several fixes of the same path. The working-copy file is the master copy; `measure.sh` copies it to each commit. + +- Call the fixed code through the most stable public entry point that reaches it. The same file must run unchanged at every commit from the fix to today, so private methods and internal classes are the wrong target. +- The project's autoloader at an old commit does not know classes you add, so put helper classes in the bench file, next to the benchmark class. Keep large inputs as PHP files returning arrays in `benchmarks/data/`, each with a comment naming its source and license, and pass `measure.sh` the `benchmarks` directory so they go along. +- Use the workload the fix was for, at the size real users have, with real data such as a real config, file, or request from the commit, pull request, or issue. When those name only a synthetic input, find a real file that takes the same code path, for example with `gh search code`, because a benchmark of a made-up input guards a case nobody has. +- Add `#[Warmup(1)]` so autoloading and first-call caches stay out of the count, unless the fix was about the first call. + +Measure "before" and "after" without expected numbers. The fix must show up: "after" counts fewer instructions, cycles, or memory than "before" by more than the tolerance. When it does not, the benchmark misses the fixed path or the input does not trigger the problem; change the benchmark and measure again. A "fix" that makes its own input slower is a finding to report. Drop a fix when you cannot make its benchmark show it. The numbers at "after" are the locked numbers; record them in the notes. + +## 6. Replay on every release since the fix + +The replay points are the tags on the line's first-parent history after the commit where the fix arrived, oldest first, plus the commit `HEAD` pointed to when you started: + +```bash +git log --first-parent --reverse --decorate-refs=refs/tags/ --format='%h %cs %D' ..HEAD | grep 'tag:' +``` + +With fewer than 6 tags, add the last first-parent commit of each month: `git rev-list -1 --first-parent --before= HEAD`. With more than 30 points, keep 30 evenly spaced, always keeping the first and the last. Run `measure.sh` on the points with the locked files. Separate `measure.sh` runs on different commits can run in parallel. + +When a file stops running because the entry point changed, adapt the master copy to the new API. Measure the previous point again with it when it runs there, and note where the benchmark changed, because numbers on the two sides of a change compare less exactly. When the fixed code was removed or rewritten, so that the benchmark no longer measures it, stop replaying this fix and note why. Skip a point that cannot be measured and note why. Do not switch PHP versions between points. + +## 7. Take the best numbers as the target + +For each case and metric, the best number is the lowest one measured at the fix or at any replay point, ignoring differences within the tolerance. Note it and the commit that measured it. + +When the start commit is slower than the best, find the commit that made it slower. Binary-search the first-parent commits between the last point at the best numbers and the next point with `measure.sh`, as `git bisect` would, skipping a commit that cannot be measured as `git bisect skip` does, and read the commit you find. If it slowed the code down on purpose, for correctness or a bug fix, note that and keep its behavior. + +## 8. Make today's code beat the best numbers + +1. Improve the code paths the fixes touched, as they are today, including what the slowing commits changed, starting with the paths that take the largest share of the profile. Run the project's tests after every change. Do not special-case the benchmark's input. +2. Commit each attempt and measure it with `measure.sh`, next to the start commit. Judge it by cycles, the metric closest to time: fewer instructions with more cache misses can be slower. Local counts repeat closely, so a drop smaller than the tolerance can still be real; the wall clock decides. Memory must not grow in a way the workload notices. +3. Before timing, estimate the most a change can save on the workload: its cycle drop times the path's share of the profile. When that is smaller than the noise of the timing, stack it with other changes on the workload's paths and judge them together. +4. Time the workload in wall-clock time before and after, with the same dependencies on both sides, such as in the worktree of the start commit that `measure.sh` made, before and after applying the change. Run at least 20 pairs, alternating before and after, pinned to one CPU with `taskset -c`. Report the mean ± standard deviation of both. Keep a change only when the 95 % confidence interval of the difference of the means excludes zero. A change that only moves Tally's numbers did not have real-world impact. Leave rejected changes out of the project's branch and list them in the report with their numbers. + +## 9. Commit and hand over + +The expected numbers in the project must come from its CI job, because numbers measured on this machine do not fit CI's environment. Use the numbers of this machine only for the comparisons above and the report. + +1. Commit the Tally setup and the bench files first, on top of the start commit and before any improvement, without `#[Expected]` and `#[ExpectedFor]` attributes. Without CI, run `tally run --output=tmp/tally-results` and `tally accept tmp/tally-results` in the working copy to put this machine's numbers in, and say so in the commit message. +2. Commit each kept improvement separately, with its benchmark numbers and wall-clock result in the commit message, so it can be reviewed and reverted on its own. +3. Ask the user before pushing. Once they push, CI prints a patch with its numbers for the new cases. Download its results with `gh run download -p 'tally-*' -D tmp/tally` and run `tally accept tmp/tally`. Put the numbers of today's code into the setup commit and those of each improvement into its commit, from runs on those commits; an improvement's run that reports its case as faster is the proof in CI's environment. +4. Report each fix: its commit and what it sped up, the case, and a table of instructions, cycles, and memory at "before", at the fix, at each replay point, at the start commit, and after your change. Then say where the best numbers came from, what slowed the code down since and why, and the wall-clock result on the real workload as mean ± standard deviation, before and after. List the fixes and changes you dropped and why, and what you set up in the project. +5. Remove the worktrees: `git worktree remove --force` each one under `$TALLY_REWIND_DIR/worktrees/`, then `git worktree prune`. diff --git a/skills/tally-rewind/scripts/measure.sh b/skills/tally-rewind/scripts/measure.sh new file mode 100755 index 0000000..f8b356a --- /dev/null +++ b/skills/tally-rewind/scripts/measure.sh @@ -0,0 +1,282 @@ +#!/usr/bin/env bash +# Measures Tally benchmark files at commits of the project's history, each in a git worktree of its +# own, with the Tally version of the main checkout, so that numbers measured at different commits +# compare. +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: measure.sh [...] ... -- ... + measure.sh [...] ... + +Copies the benchmark files or directories, with the data files in them, as they are in the working +copy when it starts, into a git worktree at each commit, installs the project's dependencies there, +and runs tally run on them. Run it anywhere inside the project's main checkout. + +The commits are resolved and the files copied once, when it starts, so later edits and commits in +the working copy do not change what a running batch measures. Tally itself is installed apart from +the project, in exactly the version the working copy has installed, in vendor/ or in a tools +directory such as tools/tally/ or tools/.tally/. It runs with the working copy's tally.php, or with +the defaults when there is none. + +Prints one tab-separated line per measured case: + commit description date case instructions cycles memory-bytes +and appends them to $TALLY_REWIND_DIR/measurements.tsv. The full Tally report is in +$TALLY_REWIND_DIR/results/.log, its results in $TALLY_REWIND_DIR/results//. + +Options starting with -- are passed to tally run, such as --filter=RouteMatchingBench. + +Environment: + TALLY_REWIND_DIR Where the worktrees, results, and measurements.tsv are kept. + Default: ${TMPDIR:-/tmp}/tally-rewind/ + TALLY_REWIND_PREPARE A shell command run in each new worktree before its dependencies are + installed, such as 'cp .env.example .env'. + COMPOSER_ROOT_VERSION Set it when the project needs it. Otherwise, for a composer.json that uses + self.version, it is taken from the branch alias or the nearest tag. + +Dependencies are installed as the commit locks them. When that fails, it retries with platform +requirements ignored, then without the dev dependencies, and says so in the install log. + +Exit codes: + 0 every commit measured, also when cases are slower or faster than their expected numbers + 1 a commit was not measurable: its dependencies did not install, a benchmark is invalid there, + or a case threw or timed out; the output says which and where the log is + 2 invalid usage +EOF +} + +tally_options=() +positional=() +separated=false +commits=() +for argument in "$@"; do + case "$argument" in + -h | --help) usage; exit 0 ;; + --) separated=true; commits=(${positional[@]+"${positional[@]}"}); positional=() ;; + --*) tally_options+=("$argument") ;; + *) positional+=("$argument") ;; + esac +done +if $separated; then + paths=(${positional[@]+"${positional[@]}"}) +elif ((${#positional[@]} > 0)); then + commits=("${positional[0]}") + paths=("${positional[@]:1}") +else + paths=() +fi +if ((${#commits[@]} == 0 || ${#paths[@]} == 0)); then + usage >&2 + exit 2 +fi + +top=$(git rev-parse --show-toplevel) +top=$(cd "$top" && pwd -P) + +shas=() +for commit in "${commits[@]}"; do + if ! sha=$(git -C "$top" rev-parse --verify --quiet "$commit^{commit}"); then + echo "Unknown commit: $commit" >&2 + exit 2 + fi + shas+=("$sha") +done + +relative=() +for path in "${paths[@]}"; do + if [[ -d $path ]]; then + absolute=$(cd "$path" && pwd -P) + elif [[ -f $path ]]; then + absolute=$(cd "$(dirname "$path")" && pwd -P)/$(basename "$path") + else + echo "No such benchmark file or directory: $path" >&2 + exit 2 + fi + case "$absolute" in + "$top"/*) relative+=("${absolute#"$top"/}") ;; + *) echo "The benchmark path $path is outside the project in $top." >&2; exit 2 ;; + esac +done + +# a dev version is pinned to its commit, so every commit runs the same build +mapfile -t installed_files < <(find "$top" -maxdepth 5 \( -name .git -o -name node_modules \) -prune -o -path '*/vendor/composer/installed.json' -print) +if ! tally=$(php -r ' + foreach (array_slice($argv, 1) as $file) { + $installed = json_decode((string) file_get_contents($file), true); + foreach ($installed["packages"] ?? [] as $package) { + if ($package["name"] !== "phpstan/tally") { + continue; + } + $version = $package["version"]; + $reference = $package["source"]["reference"] ?? $package["dist"]["reference"] ?? ""; + $dev = str_starts_with($version, "dev-") || str_ends_with($version, "-dev"); + echo $dev && $reference !== "" ? $package["version"] . "#" . $reference : $version; + exit(0); + } + } + exit(1); +' "${installed_files[@]}"); then + echo "Tally is not installed in $top, neither in vendor/ nor in a tools directory. Set it up there first." >&2 + exit 2 +fi + +base=${TALLY_REWIND_DIR:-${TMPDIR:-/tmp}/tally-rewind/$(basename "$top")} +mkdir -p "$base/worktrees" "$base/results" "$base/tally" +base=$(cd "$base" && pwd -P) + +# Tally apart from the project, so that no commit's dependencies have to fit it +tally_project=$base/tally/$(printf '%s' "$tally" | sha1sum | cut -c1-12) +if [[ ! -x $tally_project/vendor/bin/tally ]]; then + mkdir -p "$tally_project" + php -r 'echo json_encode(["require" => ["phpstan/tally" => $argv[1]], "minimum-stability" => "dev", "prefer-stable" => true], JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES), "\n";' "$tally" >"$tally_project/composer.json" + if ! composer update --working-dir="$tally_project" --no-interaction --no-progress >"$tally_project/install.log" 2>&1; then + echo "Cannot install phpstan/tally $tally, see $tally_project/install.log" >&2 + exit 2 + fi +fi +tally_vendor=$(cd "$tally_project/vendor" && pwd -P) + +snapshot=$(mktemp -d "$base/snapshot.XXXXXX") +trap 'rm -rf "$snapshot"' EXIT +for path in "${relative[@]}"; do + mkdir -p "$snapshot/$(dirname "$path")" + cp -R "$top/$path" "$snapshot/$path" +done +has_config=false +if [[ -f $top/tally.php ]]; then + has_config=true + cp "$top/tally.php" "$snapshot/.tally-rewind-config.php" +fi + +install() { + if [[ -f composer.lock ]]; then + composer install --no-interaction --no-progress + else + composer update --no-interaction --no-progress + fi +} + +prepare() { + local worktree=$1 sha=$2 + cd "$worktree" + if [[ -n ${TALLY_REWIND_PREPARE:-} ]]; then + bash -c "$TALLY_REWIND_PREPARE" + fi + if [[ -z ${COMPOSER_ROOT_VERSION:-} ]] && grep -q '"self.version"' composer.json; then + COMPOSER_ROOT_VERSION=$(php -r ' + $alias = json_decode((string) file_get_contents("composer.json"), true)["extra"]["branch-alias"] ?? []; + echo is_array($alias) && $alias !== [] ? reset($alias) : ""; + ') + if [[ -z $COMPOSER_ROOT_VERSION ]]; then + COMPOSER_ROOT_VERSION=$(git describe --tags --abbrev=0 "$sha" 2>/dev/null | sed 's/^v//') + fi + export COMPOSER_ROOT_VERSION + echo "COMPOSER_ROOT_VERSION=$COMPOSER_ROOT_VERSION" + fi + # old commits lock packages that have advisories since + export COMPOSER_NO_SECURITY_BLOCKING=1 COMPOSER_NO_AUDIT=1 + install && return + echo "Retrying with platform requirements ignored." + COMPOSER_IGNORE_PLATFORM_REQS=1 install && return + echo "Retrying without the dev dependencies." + php -r ' + $json = json_decode((string) file_get_contents("composer.json"), true); + unset($json["require-dev"]); + file_put_contents("composer.json", json_encode($json, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)); + ' + rm -f composer.lock + COMPOSER_IGNORE_PLATFORM_REQS=1 composer update --no-dev --no-interaction --no-progress +} + +measurements=$base/measurements.tsv +if [[ ! -f $measurements ]]; then + printf 'commit\tdescription\tdate\tcase\tinstructions\tcycles\tmemory-bytes\n' >"$measurements" +fi + +# 0 measured, 1 not measurable +measure() { + local sha=$1 + local worktree=$base/worktrees/$sha + if [[ ! -d $worktree ]]; then + git -C "$top" worktree add --detach --quiet "$worktree" "$sha" + fi + + # inside vendor/, so it goes when the worktree or its dependencies go + if [[ ! -f $worktree/vendor/.tally-rewind-installed ]]; then + local install_log=$base/results/$sha.install.log + if ! (prepare "$worktree" "$sha") >"$install_log" 2>&1; then + echo "$sha: not measurable: its dependencies do not install, see $install_log" >&2 + return 1 + fi + touch "$worktree/vendor/.tally-rewind-installed" + fi + + for path in "${relative[@]}"; do + rm -rf "${worktree:?}/$path" + mkdir -p "$worktree/$(dirname "$path")" + cp -R "$snapshot/$path" "$worktree/$path" + done + if $has_config; then + cp "$snapshot/.tally-rewind-config.php" "$worktree/.tally-rewind-config.php" + fi + # the project's autoloader prepends itself, so Tally's own takes the lead again: a commit may + # have another version of Tally installed + cat >"$worktree/.tally-rewind.php" <unregister(); + \$tally->register(true); +} + +return $($has_config && echo "require __DIR__ . '/.tally-rewind-config.php'" || echo 'Tally\Configuration::create()'); +PHP + + local results=$base/results/$sha + local report=$results.log + local status=0 + rm -rf "$results" + ( + cd "$worktree" + NO_COLOR=1 "$tally_vendor/bin/tally" run --config="$worktree/.tally-rewind.php" --output="$results" ${tally_options[@]+"${tally_options[@]}"} "${relative[@]}" + ) >"$report" 2>&1 || status=$? + if ((status == 2)) || [[ ! -d $results ]]; then + echo "$sha: not measurable: tally run exited with $status, see $report" >&2 + tail -n 20 "$report" >&2 + return 1 + fi + + local description date + description=$(git -C "$top" describe --tags --always "$sha") + date=$(git -C "$top" show -s --format=%cs "$sha") + local measured=0 + php -r ' + $failed = false; + $files = glob($argv[1] . "/*.json"); + sort($files); + foreach ($files as $file) { + $result = json_decode((string) file_get_contents($file), true); + if (isset($result["metrics"])) { + $metrics = $result["metrics"]; + echo implode("\t", [$argv[2], $argv[3], $argv[4], $result["id"], $metrics["instructions"], $metrics["cycles"], $metrics["memory"]]), "\n"; + } else { + $failed = true; + fwrite(STDERR, $result["id"] . ": not measured: " . strtok($result["failure"], "\n") . "\n"); + } + } + exit($failed ? 1 : 0); + ' "$results" "$sha" "$description" "$date" | tee -a "$measurements" || measured=$? + if ((measured != 0)); then + echo "$sha: not measurable, see $report" >&2 + return 1 + fi +} + +exit_code=0 +for sha in "${shas[@]}"; do + measure "$sha" || exit_code=1 +done +exit "$exit_code" diff --git a/src/Cli/ApplicationFactory.php b/src/Cli/ApplicationFactory.php index 55c6470..96e8722 100644 --- a/src/Cli/ApplicationFactory.php +++ b/src/Cli/ApplicationFactory.php @@ -70,9 +70,10 @@ final class ApplicationFactory * @param string $phpBinary runs the workers * @param list $iniFiles the ini files the PHP binary loaded, in the order it loaded them * @param string $tallyBinary the bin/tally the workers run + * @param string $program how the user ran Tally, argv[0], which the commands in the report run * @param string $codspeed the builds of CodSpeed's Valgrind and tally_hooks for this platform */ - public function create($stdout, $stderr, string $workingDirectory, array $environment, bool $colors, bool $errorColors, string $phpBinary, array $iniFiles, string $tallyBinary, string $codspeed): Application + public function create($stdout, $stderr, string $workingDirectory, array $environment, bool $colors, bool $errorColors, string $phpBinary, array $iniFiles, string $tallyBinary, string $program, string $codspeed): Application { $relativePath = new RelativePath(); $closestName = new ClosestName(); @@ -99,7 +100,7 @@ public function create($stdout, $stderr, string $workingDirectory, array $enviro new ConsoleReport(new TextTable(), $rows), new MarkdownReport($rows), new GitHubAnnotations($rows, $relativePath), - new ReportCommands(), + new ReportCommands($relativePath->of($program, $workingDirectory)), $expectationFiles, $expectationChanges, new ExpectationsPatch(new UnifiedDiff(), $relativePath), diff --git a/src/Discovery/DirectorySuiteLoader.php b/src/Discovery/DirectorySuiteLoader.php index 41b02e1..1ddb1c1 100644 --- a/src/Discovery/DirectorySuiteLoader.php +++ b/src/Discovery/DirectorySuiteLoader.php @@ -14,10 +14,11 @@ use Tally\Expectation\ExpectedAttributeRewriter; use Tally\Report\RelativePath; use Throwable; -use function array_fill_keys; +use function array_combine; use function array_filter; use function array_map; use function array_merge; +use function basename; use function file_get_contents; use function get_declared_classes; use function is_file; @@ -31,7 +32,8 @@ /** * Finds benchmarks like PHPUnit finds tests: the public, non-static methods whose names start - * with bench, in the instantiable classes declared in *Bench.php files. + * with bench, in the instantiable class named like its *Bench.php file. Other classes of the file, + * such as helpers, are not benchmarks. */ final class DirectorySuiteLoader { @@ -61,13 +63,13 @@ public function load(array $paths, string $workingDirectory, Configuration $conf $expanded = []; // the attributes of each file that declares a benchmark method, read once for all its cases $attributes = []; - $isLoaded = array_fill_keys($files, true); + $benchClasses = array_combine($files, array_map(static fn (string $file): string => basename($file, '.php'), $files)); $classNames = get_declared_classes(); // the order PHP declared them in depends on what was autoloaded before, so the first invalid benchmark would too sort($classNames); foreach ($classNames as $className) { $class = new ReflectionClass($className); - if (!isset($isLoaded[(string) $class->getFileName()]) || !$class->isInstantiable()) { + if (($benchClasses[(string) $class->getFileName()] ?? null) !== $class->getShortName() || !$class->isInstantiable()) { continue; } $this->constructor->check($class); diff --git a/src/Report/ReportCommands.php b/src/Report/ReportCommands.php index fbdcffe..bba6edd 100644 --- a/src/Report/ReportCommands.php +++ b/src/Report/ReportCommands.php @@ -4,6 +4,7 @@ use Tally\Comparison\CaseComparison; use function escapeshellarg; +use function preg_match; use function sprintf; /** @@ -13,17 +14,26 @@ final class ReportCommands { - public const BINARY = 'vendor/bin/tally'; - /** Where the accept commands download or write the results */ public const RESULTS_DIRECTORY = 'tmp/tally'; /** The artifacts of a GitHub Actions run that hold the results */ public const ARTIFACT_PATTERN = 'tally-*'; + /** The program as the commands write it, quoted only when the shell would split it */ + private string $program; + + /** + * @param string $program how the user ran Tally, such as vendor/bin/tally or tools/tally/vendor/bin/tally + */ + public function __construct(string $program) + { + $this->program = preg_match('~^[\w./-]+$~', $program) === 1 ? $program : escapeshellarg($program); + } + public function reproduce(?CaseComparison $worst): string { - return $worst === null ? '' : sprintf('%s run --filter=%s', self::BINARY, escapeshellarg($worst->id->toString())); + return $worst === null ? '' : sprintf('%s run --filter=%s', $this->program, escapeshellarg($worst->id->toString())); } /** @@ -41,14 +51,14 @@ public function accept(array $environment, ?string $resultsDirectory): string $repository, self::ARTIFACT_PATTERN, self::RESULTS_DIRECTORY, - self::BINARY, + $this->program, ); } if ($resultsDirectory !== null) { - return sprintf('%s accept %s', self::BINARY, escapeshellarg($resultsDirectory)); + return sprintf('%s accept %s', $this->program, escapeshellarg($resultsDirectory)); } - return sprintf("%s run --output=%s\n%1\$s accept %2\$s", self::BINARY, self::RESULTS_DIRECTORY); + return sprintf("%s run --output=%s\n%1\$s accept %2\$s", $this->program, self::RESULTS_DIRECTORY); } } diff --git a/tests/Fixtures/Loading/Helpers/MoneyBench.php b/tests/Fixtures/Loading/Helpers/MoneyBench.php new file mode 100644 index 0000000..c2949a3 --- /dev/null +++ b/tests/Fixtures/Loading/Helpers/MoneyBench.php @@ -0,0 +1,43 @@ +add(new Money(2)); + } + +} + +/** A helper whose constructor Tally could not call */ +final class Money +{ + + public function __construct(private int $cents) + { + } + + public function add(self $other): self + { + return new self($this->cents + $other->cents); + } + +} + +/** Named like a benchmark class, but not like the file */ +final class OtherBench +{ + + public function __construct(private int $size) + { + } + + public function benchSize(): int + { + return $this->size; + } + +} diff --git a/tests/InProcessTally.php b/tests/InProcessTally.php index 456d01b..b99b080 100644 --- a/tests/InProcessTally.php +++ b/tests/InProcessTally.php @@ -29,7 +29,7 @@ final class InProcessTally */ public static function run(string $workingDirectory, array $environment, array $arguments): array { - return self::runWithCodSpeed(ValgrindFixture::codspeed(), $workingDirectory, $environment, $arguments); + return self::runAs('vendor/bin/tally', ValgrindFixture::codspeed(), $workingDirectory, $environment, $arguments); } /** @@ -39,6 +39,18 @@ public static function run(string $workingDirectory, array $environment, array $ * @return array{int, string, string} the exit code, the output, and the error output */ public static function runWithCodSpeed(string $codspeed, string $workingDirectory, array $environment, array $arguments): array + { + return self::runAs('vendor/bin/tally', $codspeed, $workingDirectory, $environment, $arguments); + } + + /** + * @param string $program how the user ran Tally, argv[0] + * @param string $codspeed the build of CodSpeed's Valgrind that tally run counts with + * @param array $environment + * @param list $arguments without the program + * @return array{int, string, string} the exit code, the output, and the error output + */ + public static function runAs(string $program, string $codspeed, string $workingDirectory, array $environment, array $arguments): array { $output = fopen('php://memory', 'w+'); $errorOutput = fopen('php://memory', 'w+'); @@ -57,8 +69,8 @@ public static function runWithCodSpeed(string $codspeed, string $workingDirector } $exitCode = (new ApplicationFactory()) - ->create($output, $errorOutput, $workingDirectory, $environment, false, false, PHP_BINARY, $iniFiles, self::BINARY, $codspeed) - ->run(['tally', ...$arguments]); + ->create($output, $errorOutput, $workingDirectory, $environment, false, false, PHP_BINARY, $iniFiles, self::BINARY, $program, $codspeed) + ->run([$program, ...$arguments]); rewind($output); rewind($errorOutput); diff --git a/tests/Integration/Cli/CompareAndAcceptCommandTest.php b/tests/Integration/Cli/CompareAndAcceptCommandTest.php index ebc95f2..d51ee58 100644 --- a/tests/Integration/Cli/CompareAndAcceptCommandTest.php +++ b/tests/Integration/Cli/CompareAndAcceptCommandTest.php @@ -6,6 +6,7 @@ use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use Tally\Tests\InProcessTally; +use Tally\Tests\ValgrindFixture; use function chmod; use function file_get_contents; use function file_put_contents; @@ -50,6 +51,14 @@ public function testComparesTheResultsOfAllShards(): void $this->assertStringContainsString("Accept the new numbers\n vendor/bin/tally accept '" . self::RESULTS . "'\n", $output); } + public function testSuggestsCommandsThatRunTallyTheWayTheUserDid(): void + { + [, $output] = InProcessTally::runAs(__DIR__ . '/tools/tally/vendor/bin/tally', ValgrindFixture::codspeed(), __DIR__, [], ['compare', self::RESULTS, self::COMPARE]); + + $this->assertStringContainsString("Reproduce locally\n tools/tally/vendor/bin/tally run --filter='" . self::BENCH . "::benchSlower'\n", $output); + $this->assertStringContainsString("Accept the new numbers\n tools/tally/vendor/bin/tally accept '" . self::RESULTS . "'\n", $output); + } + public function testWritesTheAnnotationsAndJobSummaryOfAShardInGitHubActions(): void { $summary = $this->directory . '/summary.md'; diff --git a/tests/Integration/Discovery/DirectorySuiteLoaderTest.php b/tests/Integration/Discovery/DirectorySuiteLoaderTest.php index 20bd205..d76155a 100644 --- a/tests/Integration/Discovery/DirectorySuiteLoaderTest.php +++ b/tests/Integration/Discovery/DirectorySuiteLoaderTest.php @@ -132,6 +132,19 @@ public function testNoBenchmarks(): void $this->assertSame([], self::ids([__DIR__ . '/../../Fixtures/Loading/NoBenchmarks'])); } + /** + * The other classes of a file, such as helpers, are not benchmarks and may take any constructor arguments. + * + * @throws InvalidBenchmarkException + */ + public function testOnlyTheClassNamedLikeItsFile(): void + { + $this->assertSame( + ['Tally\\Tests\\Fixtures\\Loading\\Helpers\\MoneyBench::benchAdd'], + self::ids([__DIR__ . '/../../Fixtures/Loading/Helpers']), + ); + } + /** * @param list $paths * @return list diff --git a/tests/Unit/Report/ReportCommandsTest.php b/tests/Unit/Report/ReportCommandsTest.php index 385466d..889e533 100644 --- a/tests/Unit/Report/ReportCommandsTest.php +++ b/tests/Unit/Report/ReportCommandsTest.php @@ -13,8 +13,26 @@ public function testReproduceRunsTheWorstCase(): void { $worst = Comparisons::compare("App\\ABench::benchSlower@it's", 1, ['instructions' => 1], ['instructions' => 2], null); - $this->assertSame("vendor/bin/tally run --filter='App\\ABench::benchSlower@it'\\''s'", (new ReportCommands())->reproduce($worst)); - $this->assertSame('', (new ReportCommands())->reproduce(null)); + $this->assertSame("vendor/bin/tally run --filter='App\\ABench::benchSlower@it'\\''s'", (new ReportCommands('vendor/bin/tally'))->reproduce($worst)); + $this->assertSame('', (new ReportCommands('vendor/bin/tally'))->reproduce(null)); + } + + /** + * @return iterable + */ + public static function dataProgram(): iterable + { + yield 'a Composer project of its own' => ['tools/tally/vendor/bin/tally', "tools/tally/vendor/bin/tally run --filter='App\\ABench::benchSlower'\ntools/tally/vendor/bin/tally run --output=tmp/tally\ntools/tally/vendor/bin/tally accept tmp/tally"]; + yield 'a path the shell splits' => ['/home/me/my project/tally', "'/home/me/my project/tally' run --filter='App\\ABench::benchSlower'\n'/home/me/my project/tally' run --output=tmp/tally\n'/home/me/my project/tally' accept tmp/tally"]; + } + + #[DataProvider('dataProgram')] + public function testCommandsRunTheProgramAsGiven(string $program, string $commands): void + { + $worst = Comparisons::compare('App\\ABench::benchSlower', 1, ['instructions' => 1], ['instructions' => 2], null); + $reportCommands = new ReportCommands($program); + + $this->assertSame($commands, $reportCommands->reproduce($worst) . "\n" . $reportCommands->accept([], null)); } /** @@ -38,7 +56,7 @@ public static function dataAccept(): iterable #[DataProvider('dataAccept')] public function testAccept(array $environment, ?string $resultsDirectory, string $command): void { - $this->assertSame($command, (new ReportCommands())->accept($environment, $resultsDirectory)); + $this->assertSame($command, (new ReportCommands('vendor/bin/tally'))->accept($environment, $resultsDirectory)); } }