Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .bashunitrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# E2E tests in e2e/, run from the repository root: e2e/bashunit test e2e
BASHUNIT_BOOTSTRAP=e2e/bootstrap.sh
BASHUNIT_STRICT_MODE=true
2 changes: 2 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
*.php text eol=lf

.agents export-ignore
.bashunitrc export-ignore
.github export-ignore
bench export-ignore
e2e export-ignore
examples export-ignore
tests export-ignore
tmp export-ignore
Expand Down
81 changes: 80 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,85 @@ jobs:
path: tmp/tally-examples
if-no-files-found: ignore

e2e:
name: "E2E tests (${{ matrix.name }})"
# The cases run without debuggers and profilers, however PHP loads them: each row loads them
# the way a PHP setup does, and the tests load them in other ways on top of it
needs: ["codspeed", "tally-hooks"]
# runs on release/1.x too, with the committed builds, when the builds are skipped
if: ${{ !cancelled() && !failure() }}
runs-on: "ubuntu-24.04"
timeout-minutes: 30

strategy:
fail-fast: false
matrix:
include:
# from setup-php's scan directory, in the coverage mode
- name: "Xdebug"
coverage: "xdebug"
extensions: "posix"
ini-values: ""
stripped: "xdebug"
# collecting coverage from the start
- name: "PCOV"
coverage: "pcov"
extensions: "posix"
ini-values: "pcov.enabled=1"
stripped: "pcov"
# both at once, which the coverage option does not load, Xdebug in its default develop mode
- name: "Xdebug and PCOV"
coverage: ""
extensions: "posix, xdebug, pcov"
ini-values: "pcov.enabled=1"
stripped: "xdebug pcov"

steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
with:
egress-policy: audit

- name: "Checkout"
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: "Install PHP"
uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # v2
with:
coverage: ${{ matrix.coverage }}
php-version: "8.5"
extensions: ${{ matrix.extensions }}
ini-values: ${{ matrix.ini-values }}

- name: "Install dependencies"
run: "composer install --no-interaction --no-progress"

- name: "Download CodSpeed's Valgrind"
if: ${{ !startsWith(github.ref_name, 'release/') }}
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: codspeed-ubuntu-24.04
path: codspeed

- name: "Download tally_hooks"
if: ${{ !startsWith(github.ref_name, 'release/') }}
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: tally-hooks-ubuntu-24.04
path: codspeed

- name: "Install bashunit"
uses: "TypedDevs/bashunit@8d4f357f52df7596f92ece2d6c049d91d45517a2" # 0.52.0
with:
directory: "e2e"

- name: "E2E tests"
env:
E2E_EXTENSIONS: ${{ matrix.stripped }}
run: "e2e/bashunit test e2e"

release-tree:
name: "Release tree (${{ matrix.runner }}, PHP ${{ matrix.php-version }} ${{ matrix.phpts }})"
# Checks the tree release/1.x gets from this commit the way a project installs it, on each platform
Expand Down Expand Up @@ -513,7 +592,7 @@ jobs:

release-branch:
name: "Push release/1.x"
needs: ["release-tree", "tests", "static-analysis", "benchmarks"]
needs: ["release-tree", "tests", "static-analysis", "benchmarks", "e2e"]
if: ${{ github.event_name == 'push' && github.ref_name == '1.x' }}
runs-on: "ubuntu-latest"
timeout-minutes: 10
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
/build-cs
/composer.lock
/vendor
/e2e/bashunit
/.bashunit
/.amp/portals/*
/codspeed
/codspeed/*/.key
9 changes: 5 additions & 4 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,16 @@ Tally benchmarks PHP code by counting instructions, cycles, and memory under Cod
- `src/Report` – `Report` groups comparisons into `Section`s by their worst status. Renderers are pure and return strings: `ConsoleReport` (summary and full table, with or without colors), `MarkdownReport` (GitHub job summary), `GitHubAnnotations` (workflow commands). `Rows` and `TextTable` lay out the tables. Cases measured without an expectation become `NewExpectations`: `ExpectationsPatch` renders the planned file changes as one patch with `UnifiedDiff` (the hunks GNU `diff -u` prints, applicable with `git apply`), so a contributor adds a benchmark without numbers and copies the expectations from the CI output. `RelativePath` makes paths relative to the working directory. The golden files in `tests/Unit/Report/golden` are the expected output; regenerate them only when the output changes on purpose, and read the diff.
- `src/Result` – `MeasurementJson`, the format of a measured case in results files; `ResultsDirectory` reads and writes a directory of them; `CallgrindProfile` reads the instructions and estimated cycles of each named part of a profile CodSpeed wrote.
- `src/Process` – a generic process pool. `ProcessPool` runs `Job`s, each a list of `ProcessCommand` steps that run while each succeeds, in parallel, and kills a step that exceeds its time limit with all its descendants through `ProcessKiller`. `ProcessTable` is the pure part: parsing `/proc/<pid>/stat` and finding descendants. Descendants matter because killing `codspeed` leaves Valgrind running.
- `src/Execution` – measuring a case in worker processes. `CaseJobs` builds a case's job (a native run, then a run under CodSpeed's Valgrind with the simulated caches of `codspeed run --mode=simulation`, sharing a scratch directory) and reads what the job left. `WorkerCommandLine` builds the commands, `CaseOutcome` turns the runs and the profile parts into a `Measurement` (both pure). `CodSpeedInstallation` finds the builds for the platform in `codspeed/linux-<machine>`: Valgrind, whose tools the workers find through `VALGRIND_LIB`, and `tally_hooks-<PHP minor>[-zts].so`, which the counted worker loads with `-d extension=` unless PHP loads `tally_hooks` already, `DirectoryRemover` cleans up the work directory.
- `src/Cli` – `Application`, the commands (`list`, `run`, `compare`, `accept`, and the hidden `worker` that `run` starts), `ArgumentParser`, `Input`, `Output`. `ReportOutput` writes the report, with the patch for cases without expectations, and, in GitHub Actions, the annotations (relative to `GITHUB_WORKSPACE`) and job summary. `run` checks that every selected case's expectations can be rewritten before it counts anything; `accept` writes each file once. `ApplicationFactory` is the composition root; `bin/tally` passes it the streams, the environment, and the binaries the workers run. Every error goes through `Output::error()`: the first line says what is wrong (bold red on a terminal), the lines after it what to do, such as `Run tally list --help for its options.`; an unknown command or option suggests the closest name through `Text\ClosestName`.
- `src/Execution` – measuring a case in worker processes. `CaseJobs` builds a case's job (a native run, then a run under CodSpeed's Valgrind with the simulated caches of `codspeed run --mode=simulation`, sharing a scratch directory) and reads what the job left. `WorkerCommandLine` builds the commands, `CaseOutcome` turns the runs and the profile parts into a `Measurement` (both pure). `StrippedIni` composes the php.ini of the workers from PHP's ini files without the lines that load the extensions the cases run without, such as Xdebug and PCOV, because PHP often loads them enabled, which multiplies the counts, and a setting cannot unload an extension (pure); `StrippedIniFile` writes it into the work directory when there is something to strip, and the workers then start with `-c` and a `PHP_INI_SCAN_DIR` that does not exist; the worker fails a case when PHP loads one of them anyway. `CodSpeedInstallation` finds the builds for the platform in `codspeed/linux-<machine>`: Valgrind, whose tools the workers find through `VALGRIND_LIB`, and `tally_hooks-<PHP minor>[-zts].so`, which the counted worker loads with `-d extension=` unless PHP loads `tally_hooks` already, `DirectoryRemover` cleans up the work directory.
- `src/Cli` – `Application`, the commands (`list`, `run`, `compare`, `accept`, and the hidden `worker` that `run` starts), `ArgumentParser`, `Input`, `Output`. `ReportOutput` writes the report, with the patch for cases without expectations, and, in GitHub Actions, the annotations (relative to `GITHUB_WORKSPACE`) and job summary. `run` checks that every selected case's expectations can be rewritten before it counts anything; `accept` writes each file once. `ApplicationFactory` is the composition root; `bin/tally` passes it the streams, the environment, the binaries the workers run, and the ini files PHP loaded. Every error goes through `Output::error()`: the first line says what is wrong (bold red on a terminal), the lines after it what to do, such as `Run tally list --help for its options.`; an unknown command or option suggests the closest name through `Text\ClosestName`.
- `src/Text` – `ClosestName`, the "did you mean" suggestion for a mistyped name of a command, option, variable, function, or key.
- `src/` root – value objects shared by everything: `Metric`, `Metrics`, `CaseId`, `BenchmarkCase`, `DataSet`, `Measurement`, `ScratchDirectory`, `Configuration` (what `tally.php` returns, and where every default is chosen, such as the tolerance, whether `tally accept` writes `dataSet:` as a named argument, and the environment variables the workers keep, because the size of the environment moves the counts), and the `Meter` interface.
- `src/` root – value objects shared by everything: `Metric`, `Metrics`, `CaseId`, `BenchmarkCase`, `DataSet`, `Measurement`, `ScratchDirectory`, `Configuration` (what `tally.php` returns, and where every default is chosen, such as the tolerance, whether `tally accept` writes `dataSet:` as a named argument, the environment variables the workers keep, because the size of the environment moves the counts, and the extensions they run without), and the `Meter` interface.
- `bin/tally` – finds the autoloader and runs `ApplicationFactory`.
- `ext/tally_hooks` – the PHP extension the counted worker calls: functions in `Tally\Hooks` that make Callgrind's client requests through Valgrind's `valgrind.h` and `callgrind.h`. Built with `-DTALLY_HOOKS_RECORDING`, it records the calls instead, for the tests. Keep it to these few functions; the logic belongs in PHP.
- `bench/` – Tally's benchmarks of itself, listed in `tally.php`. They cover the code whose cost grows with the suite, a benchmark file, or a profile, and that runs once per case or once per worker: locating and rewriting expectations, planning and diffing new expectations, expanding cases, sharding, reading profiles, and rendering reports. `SuiteSource` generates the code and cases of large benchmark classes as their input. Build inputs in data providers and before methods, which are not counted, and keep the counted call to the one method being measured. `#[Warmup(1)]` keeps autoloading out of the count.
- `examples/` – short, artificial benchmarks for README.md, one feature each. `examples/Scripts` shows a data provider with its own `ExpectationSource`, which keeps the numbers in a comment at the top of each script. Keep them readable over realistic; real measurements of Tally belong in `bench/`. CI runs them under CodSpeed next to `bench/`, so they stay valid.
- `build/` – scripts for the build, such as the coverage check. `build-codspeed.sh` builds CodSpeed's Valgrind (Callgrind only, static launcher) at a pinned tag into `codspeed/linux-<machine>`, which `1.x` ignores, and `build-tally-hooks.sh` builds `ext/tally_hooks` with `phpize` next to it; `make codspeed` runs both for this machine and PHP, CI builds `tally_hooks` for PHP 8.2 to 8.6, NTS and ZTS, in the official `php` Docker images. `build-release.sh` turns `1.x` into the `release/1.x` tree by committing the builds of both platforms; CI pushes it on every push to `1.x`, and releases are tagged there with `tag-release.sh <version>`, which checks that the generated commit sits on the `1.x` commit to release. `check-release.sh` installs a release tree into a new project with Composer and counts a benchmark, which CI runs on every pull request on x86_64 and aarch64, with PHP 8.2 and 8.5, and 8.5 ZTS. `build/PHPStan` holds Tally's own PHPStan rules and `BenchmarkUsageProvider`, which tells the dead code detector about the methods Tally calls through reflection. `build/stubs` holds the stubs for what PHPStan cannot see, such as the functions of `tally_hooks`.
- `e2e/` – end-to-end tests with [bashunit](https://bashunit.com), `<name>/<name>_test.sh`; `e2e/bootstrap.sh`, set in `.bashunitrc`, gives each test a directory of its own, which is also its `TMPDIR`. `stripped-extensions` counts a benchmark with CodSpeed's Valgrind while PHP loads the extensions in `E2E_EXTENSIONS` in several ways (its scan directory, another one, the php.ini in `PHPRC`, `-c`): the case must not see them and must count what it counts on a PHP without them. CI installs bashunit into `e2e/` and runs it once per PHP setup (Xdebug, PCOV, both); locally, with the extensions loaded and bashunit installed the same way: `E2E_EXTENSIONS='xdebug pcov' e2e/bashunit test e2e`. Like `tests/Fixtures`, their PHP files are only linted.
- `tests/Unit` – pure logic, no file system, processes, or network. `tests/Integration` – adapters and the CLI. `tests/Fixtures` – benchmark classes and configuration files the tests load; excluded from PHPStan and the coding standard because some are invalid on purpose.

## Design rules
Expand Down Expand Up @@ -61,7 +62,7 @@ make infection # mutation testing of the whole codebase, fails below 100 % MSI
make bench # bench/ under CodSpeed's Valgrind, builds it and tally_hooks with make codspeed first
```

`make cs-install` clones build-cs. PHPStan runs with no ignores and no baseline: fix the types. The dead code detector reports unused code; delete it, or mark it `@api` when it is public API. A stale result cache of the dead code detector makes PHPStan fail with "DCD cache file not found"; `make infection` clears the result cache first, run `vendor/bin/phpstan clear-result-cache` before other runs when it happens. CI runs the lint and the tests on PHP 8.2 to 8.6, the rest on 8.5 and 8.6; Infection runs on the whole codebase on pushes and on the changed lines in pull requests. The Benchmarks job runs `bench/` under CodSpeed on a pinned runner, because the counts depend on the PHP build. Take expected numbers from that job, not from a local run: a benchmark without them gets a patch in the job's output, and changed numbers get `tally accept` with the `tally-results` artifact.
`make cs-install` clones build-cs. PHPStan runs with no ignores and no baseline: fix the types. The dead code detector reports unused code; delete it, or mark it `@api` when it is public API. A stale result cache of the dead code detector makes PHPStan fail with "DCD cache file not found"; `make infection` clears the result cache first, run `vendor/bin/phpstan clear-result-cache` before other runs when it happens. CI runs the lint and the tests on PHP 8.2 to 8.6, the rest on 8.5 and 8.6; Infection runs on the whole codebase on pushes and on the changed lines in pull requests. The E2E job runs `e2e/` with Xdebug and PCOV loaded. The Benchmarks job runs `bench/` under CodSpeed on a pinned runner, because the counts depend on the PHP build. Take expected numbers from that job, not from a local run: a benchmark without them gets a patch in the job's output, and changed numbers get `tally accept` with the `tally-results` artifact.

## Conventions

Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ $(CODSPEED)/tally_hooks-%.so: build/build-tally-hooks.sh $(wildcard ext/tally_ho
lint:
php vendor/bin/parallel-lint --colors bin/tally \
--exclude tests/Fixtures/Loading/Syntax \
bench build examples src tests tally.php
bench build e2e examples src tests tally.php

.PHONY: cs-install
cs-install:
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -432,13 +432,14 @@ Every value Tally uses when a benchmark or the command line does not give one is
| `withGrowthTolerance(TolerancePolicy)` | The tolerance of benchmarks with a complexity class and without a policy on their method or class. `RelativeTolerance(0.1)` by default. |
| `withNamedArguments(bool)` | Whether `tally accept` and the patch of new expectations write `#[ExpectedFor(dataSet: 'size-10', …)]` instead of `#[ExpectedFor('size-10', …)]`. Off by default. |
| `withEnvironmentVariables(list<string>)` | The environment variables the cases run with; `tally run` leaves out the others. The environment is on the stack of the counted process, so a variable that differs from run to run, such as the branch name in CI, moves its memory layout, and with it the counts of benchmarks whose allocations depend on it. `PATH`, `HOME`, `TMPDIR`, `LANG`, `LC_ALL`, `PHPRC`, and `PHP_INI_SCAN_DIR` by default; list them too when you add one. |
| `withStrippedExtensions(list<string>)` | The extensions the cases run without, even when PHP's ini files load them. PHP often loads them enabled, which multiplies the counts: Xdebug in its default develop mode and PCOV with `pcov.enabled=1` count a loop of method calls 6.5 and 3.4 times over. A setting cannot unload an extension, and the cases do not get the `-d` options of `tally run`. When an ini file loads one, `tally run` starts the cases with a php.ini made of PHP's ini files with those lines commented out. A case fails when PHP loads one some other way, such as built in. `xdebug`, `pcov`, `uopz`, `blackfire`, `tideways`, `tideways_xhprof`, `xhprof`, `spx`, `excimer`, `memprof`, and `opentelemetry` by default; leave out one that your benchmarks need. |

## In CI

The counts depend on everything that runs the benchmark:

- the PHP version, including the patch release,
- how PHP was built, which extensions are loaded, and the PHP settings, such as whether opcache and JIT are on,
- how PHP was built, which extensions are loaded, and the PHP settings, such as whether opcache and JIT are on; the cases run without debuggers and profilers, see [`withStrippedExtensions()`](#configuration),
- the system libraries, such as glibc, which come with the runner's operating system image,
- the environment variables the cases keep, see [`withEnvironmentVariables()`](#configuration).

Expand Down
13 changes: 12 additions & 1 deletion bin/tally
Original file line number Diff line number Diff line change
Expand Up @@ -46,5 +46,16 @@ use Tally\Cli\ApplicationFactory;
// built by build/build-codspeed.sh, the releases include it
$codspeed = sprintf('%s/codspeed/%s-%s', dirname(__DIR__), strtolower(PHP_OS_FAMILY), php_uname('m'));

exit((new ApplicationFactory())->create(STDOUT, STDERR, $workingDirectory, $environment, $colors, $errorColors, PHP_BINARY, __FILE__, $codspeed)->run($arguments));
// the workers start with these, or with a php.ini composed of them without the stripped extensions
$iniFiles = [];
foreach ([(string) php_ini_loaded_file(), ...explode(',', (string) php_ini_scanned_files())] as $iniFile) {
$iniFile = trim($iniFile);
if ($iniFile === '') {
continue;
}

$iniFiles[] = $iniFile;
}

exit((new ApplicationFactory())->create(STDOUT, STDERR, $workingDirectory, $environment, $colors, $errorColors, PHP_BINARY, $iniFiles, __FILE__, $codspeed)->run($arguments));
})();
15 changes: 15 additions & 0 deletions e2e/bootstrap.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
# Loaded by bashunit before the E2E tests, see BASHUNIT_BOOTSTRAP in .bashunitrc.
# Run the tests from the repository root: e2e/bashunit test e2e

# A test runs in the directory of its test file and gets E2E_TMPDIR, a directory of its own,
# which is also its TMPDIR: tally run keeps its work directory in sys_get_temp_dir().
function e2e::set_up() {
E2E_TMPDIR=$(mktemp -d)
export TMPDIR="$E2E_TMPDIR"
cd "$(dirname "${BASH_SOURCE[1]}")" || return
}

function e2e::tear_down() {
rm -rf "${E2E_TMPDIR:?}"
}
Loading
Loading