From dc55bbfc199677c209db6aa55c6eaa84fee2fc28 Mon Sep 17 00:00:00 2001 From: "ondrej@mirtes.cz" Date: Sat, 14 Nov 2026 21:47:14 +0000 Subject: [PATCH 1/2] Start the workers without debuggers and profilers that PHP loads PHP often loads Xdebug, PCOV, and other profilers enabled, which multiplies the counts: Xdebug in its default develop mode and PCOV with pcov.enabled=1 count a loop of method calls 6.5 and 3.4 times over. Switching them off does not reach the workers, which get neither the -d options of tally run nor XDEBUG_MODE, and a setting cannot unload an extension. When PHP's ini files load one of Configuration::strippedExtensions(), tally run composes a php.ini of them with those lines commented out, like the restart of phpstan-src, and both runs of a case start with -c and a PHP_INI_SCAN_DIR that does not exist, because proc_open leaves out a variable whose value is empty. Without such a line, the command line stays as it was, so the expected numbers of CI do not move. A worker that still loads one fails its case instead of counting with it. Amp-Thread-ID: https://ampcode.com/threads/T-01a1dbdd-1694-7100-875a-74f105f1221f --- CLAUDE.md | 6 +- README.md | 3 +- bin/tally | 13 +++- src/Cli/ApplicationFactory.php | 6 +- src/Cli/RunCommand.php | 28 +++++--- src/Cli/WorkerCommand.php | 23 ++++++- src/Configuration.php | 31 ++++++++- src/Execution/StrippedIni.php | 48 +++++++++++++ src/Execution/StrippedIniFile.php | 49 ++++++++++++++ src/Execution/WorkerCommandLine.php | 10 +++ src/Execution/WorkerSettings.php | 3 + .../StrippedExtensions/StrippedBench.php | 25 +++++++ tests/Fixtures/StrippedExtensions/none.php | 9 +++ tests/Fixtures/StrippedExtensions/tally.php | 9 +++ tests/InProcessTally.php | 17 ++++- tests/Integration/Cli/RunCommandTest.php | 24 +++++++ tests/Integration/Cli/WorkerCommandTest.php | 32 ++++++++- tests/Integration/Execution/CaseJobsTest.php | 2 +- .../Execution/StrippedIniFileTest.php | 67 +++++++++++++++++++ tests/Unit/ConfigurationTest.php | 2 + tests/Unit/Execution/StrippedIniTest.php | 59 ++++++++++++++++ .../Unit/Execution/WorkerCommandLineTest.php | 23 +++++-- 22 files changed, 462 insertions(+), 27 deletions(-) create mode 100644 src/Execution/StrippedIni.php create mode 100644 src/Execution/StrippedIniFile.php create mode 100644 tests/Fixtures/StrippedExtensions/StrippedBench.php create mode 100644 tests/Fixtures/StrippedExtensions/none.php create mode 100644 tests/Fixtures/StrippedExtensions/tally.php create mode 100644 tests/Integration/Execution/StrippedIniFileTest.php create mode 100644 tests/Unit/Execution/StrippedIniTest.php diff --git a/CLAUDE.md b/CLAUDE.md index 05a77bd..64689a6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,10 +16,10 @@ Tally benchmarks PHP code by counting instructions, cycles, and memory under Cod - `src/Report` – `Report` groups comparisons into `Section`s by their worst status. Renderers are pure and return strings: `ConsoleReport` (summary and full table, with or without colors), `MarkdownReport` (GitHub job summary), `GitHubAnnotations` (workflow commands). `Rows` and `TextTable` lay out the tables. Cases measured without an expectation become `NewExpectations`: `ExpectationsPatch` renders the planned file changes as one patch with `UnifiedDiff` (the hunks GNU `diff -u` prints, applicable with `git apply`), so a contributor adds a benchmark without numbers and copies the expectations from the CI output. `RelativePath` makes paths relative to the working directory. The golden files in `tests/Unit/Report/golden` are the expected output; regenerate them only when the output changes on purpose, and read the diff. - `src/Result` – `MeasurementJson`, the format of a measured case in results files; `ResultsDirectory` reads and writes a directory of them; `CallgrindProfile` reads the instructions and estimated cycles of each named part of a profile CodSpeed wrote. - `src/Process` – a generic process pool. `ProcessPool` runs `Job`s, each a list of `ProcessCommand` steps that run while each succeeds, in parallel, and kills a step that exceeds its time limit with all its descendants through `ProcessKiller`. `ProcessTable` is the pure part: parsing `/proc//stat` and finding descendants. Descendants matter because killing `codspeed` leaves Valgrind running. -- `src/Execution` – measuring a case in worker processes. `CaseJobs` builds a case's job (a native run, then a run under CodSpeed's Valgrind with the simulated caches of `codspeed run --mode=simulation`, sharing a scratch directory) and reads what the job left. `WorkerCommandLine` builds the commands, `CaseOutcome` turns the runs and the profile parts into a `Measurement` (both pure). `CodSpeedInstallation` finds the builds for the platform in `codspeed/linux-`: Valgrind, whose tools the workers find through `VALGRIND_LIB`, and `tally_hooks-[-zts].so`, which the counted worker loads with `-d extension=` unless PHP loads `tally_hooks` already, `DirectoryRemover` cleans up the work directory. -- `src/Cli` – `Application`, the commands (`list`, `run`, `compare`, `accept`, and the hidden `worker` that `run` starts), `ArgumentParser`, `Input`, `Output`. `ReportOutput` writes the report, with the patch for cases without expectations, and, in GitHub Actions, the annotations (relative to `GITHUB_WORKSPACE`) and job summary. `run` checks that every selected case's expectations can be rewritten before it counts anything; `accept` writes each file once. `ApplicationFactory` is the composition root; `bin/tally` passes it the streams, the environment, and the binaries the workers run. Every error goes through `Output::error()`: the first line says what is wrong (bold red on a terminal), the lines after it what to do, such as `Run tally list --help for its options.`; an unknown command or option suggests the closest name through `Text\ClosestName`. +- `src/Execution` – measuring a case in worker processes. `CaseJobs` builds a case's job (a native run, then a run under CodSpeed's Valgrind with the simulated caches of `codspeed run --mode=simulation`, sharing a scratch directory) and reads what the job left. `WorkerCommandLine` builds the commands, `CaseOutcome` turns the runs and the profile parts into a `Measurement` (both pure). `StrippedIni` composes the php.ini of the workers from PHP's ini files without the lines that load the extensions the cases run without, such as Xdebug and PCOV, because PHP often loads them enabled, which multiplies the counts, and a setting cannot unload an extension (pure); `StrippedIniFile` writes it into the work directory when there is something to strip, and the workers then start with `-c` and a `PHP_INI_SCAN_DIR` that does not exist; the worker fails a case when PHP loads one of them anyway. `CodSpeedInstallation` finds the builds for the platform in `codspeed/linux-`: Valgrind, whose tools the workers find through `VALGRIND_LIB`, and `tally_hooks-[-zts].so`, which the counted worker loads with `-d extension=` unless PHP loads `tally_hooks` already, `DirectoryRemover` cleans up the work directory. +- `src/Cli` – `Application`, the commands (`list`, `run`, `compare`, `accept`, and the hidden `worker` that `run` starts), `ArgumentParser`, `Input`, `Output`. `ReportOutput` writes the report, with the patch for cases without expectations, and, in GitHub Actions, the annotations (relative to `GITHUB_WORKSPACE`) and job summary. `run` checks that every selected case's expectations can be rewritten before it counts anything; `accept` writes each file once. `ApplicationFactory` is the composition root; `bin/tally` passes it the streams, the environment, the binaries the workers run, and the ini files PHP loaded. Every error goes through `Output::error()`: the first line says what is wrong (bold red on a terminal), the lines after it what to do, such as `Run tally list --help for its options.`; an unknown command or option suggests the closest name through `Text\ClosestName`. - `src/Text` – `ClosestName`, the "did you mean" suggestion for a mistyped name of a command, option, variable, function, or key. -- `src/` root – value objects shared by everything: `Metric`, `Metrics`, `CaseId`, `BenchmarkCase`, `DataSet`, `Measurement`, `ScratchDirectory`, `Configuration` (what `tally.php` returns, and where every default is chosen, such as the tolerance, whether `tally accept` writes `dataSet:` as a named argument, and the environment variables the workers keep, because the size of the environment moves the counts), and the `Meter` interface. +- `src/` root – value objects shared by everything: `Metric`, `Metrics`, `CaseId`, `BenchmarkCase`, `DataSet`, `Measurement`, `ScratchDirectory`, `Configuration` (what `tally.php` returns, and where every default is chosen, such as the tolerance, whether `tally accept` writes `dataSet:` as a named argument, the environment variables the workers keep, because the size of the environment moves the counts, and the extensions they run without), and the `Meter` interface. - `bin/tally` – finds the autoloader and runs `ApplicationFactory`. - `ext/tally_hooks` – the PHP extension the counted worker calls: functions in `Tally\Hooks` that make Callgrind's client requests through Valgrind's `valgrind.h` and `callgrind.h`. Built with `-DTALLY_HOOKS_RECORDING`, it records the calls instead, for the tests. Keep it to these few functions; the logic belongs in PHP. - `bench/` – Tally's benchmarks of itself, listed in `tally.php`. They cover the code whose cost grows with the suite, a benchmark file, or a profile, and that runs once per case or once per worker: locating and rewriting expectations, planning and diffing new expectations, expanding cases, sharding, reading profiles, and rendering reports. `SuiteSource` generates the code and cases of large benchmark classes as their input. Build inputs in data providers and before methods, which are not counted, and keep the counted call to the one method being measured. `#[Warmup(1)]` keeps autoloading out of the count. diff --git a/README.md b/README.md index 342b2c3..29befb4 100644 --- a/README.md +++ b/README.md @@ -432,13 +432,14 @@ Every value Tally uses when a benchmark or the command line does not give one is | `withGrowthTolerance(TolerancePolicy)` | The tolerance of benchmarks with a complexity class and without a policy on their method or class. `RelativeTolerance(0.1)` by default. | | `withNamedArguments(bool)` | Whether `tally accept` and the patch of new expectations write `#[ExpectedFor(dataSet: 'size-10', …)]` instead of `#[ExpectedFor('size-10', …)]`. Off by default. | | `withEnvironmentVariables(list)` | The environment variables the cases run with; `tally run` leaves out the others. The environment is on the stack of the counted process, so a variable that differs from run to run, such as the branch name in CI, moves its memory layout, and with it the counts of benchmarks whose allocations depend on it. `PATH`, `HOME`, `TMPDIR`, `LANG`, `LC_ALL`, `PHPRC`, and `PHP_INI_SCAN_DIR` by default; list them too when you add one. | +| `withStrippedExtensions(list)` | The extensions the cases run without, even when PHP's ini files load them. PHP often loads them enabled, which multiplies the counts: Xdebug in its default develop mode and PCOV with `pcov.enabled=1` count a loop of method calls 6.5 and 3.4 times over. A setting cannot unload an extension, and the cases do not get the `-d` options of `tally run`. When an ini file loads one, `tally run` starts the cases with a php.ini made of PHP's ini files with those lines commented out. A case fails when PHP loads one some other way, such as built in. `xdebug`, `pcov`, `uopz`, `blackfire`, `tideways`, `tideways_xhprof`, `xhprof`, `spx`, `excimer`, `memprof`, and `opentelemetry` by default; leave out one that your benchmarks need. | ## In CI The counts depend on everything that runs the benchmark: - the PHP version, including the patch release, -- how PHP was built, which extensions are loaded, and the PHP settings, such as whether opcache and JIT are on, +- how PHP was built, which extensions are loaded, and the PHP settings, such as whether opcache and JIT are on; the cases run without debuggers and profilers, see [`withStrippedExtensions()`](#configuration), - the system libraries, such as glibc, which come with the runner's operating system image, - the environment variables the cases keep, see [`withEnvironmentVariables()`](#configuration). diff --git a/bin/tally b/bin/tally index e92ccef..45b909b 100755 --- a/bin/tally +++ b/bin/tally @@ -46,5 +46,16 @@ use Tally\Cli\ApplicationFactory; // built by build/build-codspeed.sh, the releases include it $codspeed = sprintf('%s/codspeed/%s-%s', dirname(__DIR__), strtolower(PHP_OS_FAMILY), php_uname('m')); - exit((new ApplicationFactory())->create(STDOUT, STDERR, $workingDirectory, $environment, $colors, $errorColors, PHP_BINARY, __FILE__, $codspeed)->run($arguments)); + // the workers start with these, or with a php.ini composed of them without the stripped extensions + $iniFiles = []; + foreach ([(string) php_ini_loaded_file(), ...explode(',', (string) php_ini_scanned_files())] as $iniFile) { + $iniFile = trim($iniFile); + if ($iniFile === '') { + continue; + } + + $iniFiles[] = $iniFile; + } + + exit((new ApplicationFactory())->create(STDOUT, STDERR, $workingDirectory, $environment, $colors, $errorColors, PHP_BINARY, $iniFiles, __FILE__, $codspeed)->run($arguments)); })(); diff --git a/src/Cli/ApplicationFactory.php b/src/Cli/ApplicationFactory.php index c83d06f..25e7670 100644 --- a/src/Cli/ApplicationFactory.php +++ b/src/Cli/ApplicationFactory.php @@ -14,6 +14,8 @@ use Tally\Execution\CaseOutcome; use Tally\Execution\CodSpeedInstallation; use Tally\Execution\DirectoryRemover; +use Tally\Execution\StrippedIni; +use Tally\Execution\StrippedIniFile; use Tally\Execution\WorkerCommandLine; use Tally\Expectation\ExpectationChanges; use Tally\Expectation\ExpectationFiles; @@ -65,10 +67,11 @@ final class ApplicationFactory * @param bool $colors whether to color the standard output * @param bool $errorColors whether to color the error output * @param string $phpBinary runs the workers + * @param list $iniFiles the ini files the PHP binary loaded, in the order it loaded them * @param string $tallyBinary the bin/tally the workers run * @param string $codspeed the builds of CodSpeed's Valgrind and tally_hooks for this platform */ - public function create($stdout, $stderr, string $workingDirectory, array $environment, bool $colors, bool $errorColors, string $phpBinary, string $tallyBinary, string $codspeed): Application + public function create($stdout, $stderr, string $workingDirectory, array $environment, bool $colors, bool $errorColors, string $phpBinary, array $iniFiles, string $tallyBinary, string $codspeed): Application { $relativePath = new RelativePath(); $closestName = new ClosestName(); @@ -109,6 +112,7 @@ public function create($stdout, $stderr, string $workingDirectory, array $enviro $selectedCases, new CodSpeedInstallation($codspeed), new CaseJobs(new WorkerCommandLine($phpBinary, $tallyBinary), new CaseOutcome(), new CallgrindProfile(), $json), + new StrippedIniFile(new StrippedIni(), $iniFiles), new ProcessPool(new ProcessKiller(new ProcessTable())), $comparator, new ProgressFormat(), diff --git a/src/Cli/RunCommand.php b/src/Cli/RunCommand.php index b004451..b44996a 100644 --- a/src/Cli/RunCommand.php +++ b/src/Cli/RunCommand.php @@ -10,6 +10,7 @@ use Tally\Execution\CaseJobs; use Tally\Execution\CodSpeedInstallation; use Tally\Execution\DirectoryRemover; +use Tally\Execution\StrippedIniFile; use Tally\Execution\WorkerSettings; use Tally\Expectation\SourceLocation; use Tally\Measurement; @@ -50,6 +51,7 @@ public function __construct( private SelectedCases $selectedCases, private CodSpeedInstallation $installation, private CaseJobs $caseJobs, + private StrippedIniFile $strippedIni, private ProcessPool $pool, private Comparator $comparator, private ProgressFormat $progress, @@ -116,15 +118,6 @@ public function execute(Input $input, Output $output): int $located[] = [$case, $case->expectations->location()]; } $workDirectory = sys_get_temp_dir() . '/tally-' . getmypid(); - $settings = new WorkerSettings( - $configuration->iniEntries(), - $this->suiteOptions($input), - $valgrind, - $extension, - $workDirectory, - $configuration->nativeTimeLimitSeconds(), - $configuration->countedTimeLimitSeconds(), - ); $output->write(sprintf( "Tally counts %d %s under CodSpeed's CPU simulation, %d at once.\n\n", @@ -133,7 +126,24 @@ public function execute(Input $input, Output $output): int $jobs, )); try { + $phpIni = $this->strippedIni->write($configuration->strippedExtensions(), $workDirectory); + $settings = new WorkerSettings( + $configuration->iniEntries(), + $phpIni, + $this->suiteOptions($input), + $valgrind, + $extension, + $workDirectory, + $configuration->nativeTimeLimitSeconds(), + $configuration->countedTimeLimitSeconds(), + ); $environment = [...array_intersect_key($this->environment, array_flip($configuration->environmentVariables())), 'VALGRIND_LIB' => $valgrindTools]; + if ($phpIni !== null) { + // The php.ini has the contents of the files PHP would scan. In a directory that does not + // exist, PHP scans none; proc_open leaves out a variable whose value is empty, and PHP + // would scan its own directory. + $environment['PHP_INI_SCAN_DIR'] = $workDirectory . '/no-ini-files'; + } $measurements = $this->measure($located, $settings, $environment, $jobs, $input->option('output'), $output); } catch (RuntimeException $e) { $output->error($e->getMessage()); diff --git a/src/Cli/WorkerCommand.php b/src/Cli/WorkerCommand.php index fc43792..4e19125 100644 --- a/src/Cli/WorkerCommand.php +++ b/src/Cli/WorkerCommand.php @@ -13,11 +13,14 @@ use Tally\Runner\IdleCounter; use Tally\ScratchDirectory; use Throwable; +use function array_uintersect; use function file_put_contents; use function get_loaded_extensions; +use function implode; use function is_dir; use function mkdir; use function sprintf; +use function strcasecmp; /** * Runs one case in this process and writes its measurement. tally run starts a worker natively, @@ -74,9 +77,10 @@ public function execute(Input $input, Output $output): int $id = $this->required($input, 'case'); $scratch = $this->required($input, 'scratch'); $resultFile = $this->required($input, 'result'); - $case = $this->find($this->selectedCases->load($input, $this->selectedCases->configuration($input), $input->arguments()), $id); + $configuration = $this->selectedCases->configuration($input); + $case = $this->find($this->selectedCases->load($input, $configuration, $input->arguments()), $id); - $measurement = $this->run($case, $scratch, $input->flag('counted')); + $measurement = $this->run($case, $scratch, $input->flag('counted'), $configuration->strippedExtensions()); if (@file_put_contents($resultFile, $this->json->encode($measurement)) === false) { $output->error(sprintf('Cannot write %s.', $resultFile)); @@ -86,8 +90,21 @@ public function execute(Input $input, Output $output): int return $measurement->failure === null ? Application::EXIT_SUCCESS : Application::EXIT_FAILURE; } - private function run(BenchmarkCase $case, string $scratch, bool $counted): Measurement + /** + * @param list $strippedExtensions + */ + private function run(BenchmarkCase $case, string $scratch, bool $counted, array $strippedExtensions): Measurement { + $loaded = array_uintersect($strippedExtensions, get_loaded_extensions(), strcasecmp(...)); + if ($loaded !== []) { + return Measurement::failed($case->id, sprintf( + "PHP loads extensions that the cases run without: %s.\n" + . 'tally run comments out the lines of the ini files that load them, so this PHP loads them some other way, such as built in. ' + . 'Leave them out of Configuration::withStrippedExtensions() to count with them.', + implode(', ', $loaded), + )); + } + try { if (!is_dir($scratch)) { mkdir($scratch, 0777, true); diff --git a/src/Configuration.php b/src/Configuration.php index 1337329..e14fdec 100644 --- a/src/Configuration.php +++ b/src/Configuration.php @@ -30,6 +30,7 @@ final class Configuration * @param list $paths * @param list $iniEntries * @param list $environmentVariables + * @param list $strippedExtensions */ private function __construct( private array $paths, @@ -39,7 +40,8 @@ private function __construct( private TolerancePolicy $tolerance, private TolerancePolicy $growthTolerance, private bool $namedArguments, - private array $environmentVariables + private array $environmentVariables, + private array $strippedExtensions ) { } @@ -62,6 +64,9 @@ public static function create(): self // what PHP, a shell, and the locale need; the rest, such as the branch name in CI, would // move the memory layout of the counted process, and with it the counts of some benchmarks ['PATH', 'HOME', 'TMPDIR', 'LANG', 'LC_ALL', 'PHPRC', 'PHP_INI_SCAN_DIR'], + // debuggers, coverage, profilers, and tracers, which hook into running PHP code and + // multiply the counts when they are enabled + ['xdebug', 'pcov', 'uopz', 'blackfire', 'tideways', 'tideways_xhprof', 'xhprof', 'spx', 'excimer', 'memprof', 'opentelemetry'], ); } @@ -153,6 +158,22 @@ public function withEnvironmentVariables(array $names): self return $configuration; } + /** + * The extensions the processes that run the cases start without, even when the ini files of PHP + * load them: tally run comments out the lines that load them. PHP often loads them enabled, + * such as Xdebug in its default develop mode, which multiplies the counts, and the settings + * that switch them off for tally run, such as -d or XDEBUG_MODE, do not reach the cases. + * + * @param list $names + */ + public function withStrippedExtensions(array $names): self + { + $configuration = clone $this; + $configuration->strippedExtensions = $names; + + return $configuration; + } + /** * @return list */ @@ -202,4 +223,12 @@ public function environmentVariables(): array return $this->environmentVariables; } + /** + * @return list + */ + public function strippedExtensions(): array + { + return $this->strippedExtensions; + } + } diff --git a/src/Execution/StrippedIni.php b/src/Execution/StrippedIni.php new file mode 100644 index 0000000..83f0dc4 --- /dev/null +++ b/src/Execution/StrippedIni.php @@ -0,0 +1,48 @@ + $files the contents of the ini files PHP loaded, in the order it loaded them + * @param list $extensions the names of the extensions to strip + * @return string|null null when no file loads any of the extensions + */ + public function compose(array $files, array $extensions): ?string + { + $composed = ''; + $stripped = 0; + foreach ($files as $file) { + // The CLI never applies what follows a [PATH=] or [HOST=] section, and joined with the + // next file, the section would swallow that file too. + $file = (string) preg_replace('~^[ \t]*\[(?:PATH|HOST)[ \t]*=.*~ims', '', $file); + foreach ($extensions as $extension) { + // the file name, not a directory, starts with the extension's name, which ends there: + // tideways-php-8.5.so is Tideways, tideways_xhprof.so is not + $file = (string) preg_replace( + '~^[ \t]*(?:zend_)?extension[ \t]*=[ \t]*["\']?(?:[^"\';\n]*/)?' . preg_quote($extension, '~') . '(?![\w/])~mi', + ';$0', + $file, + -1, + $count, + ); + $stripped += $count; + } + $composed .= $file . "\n"; + } + + return $stripped === 0 ? null : $composed; + } + +} diff --git a/src/Execution/StrippedIniFile.php b/src/Execution/StrippedIniFile.php new file mode 100644 index 0000000..db0de17 --- /dev/null +++ b/src/Execution/StrippedIniFile.php @@ -0,0 +1,49 @@ + $iniFiles the ini files PHP loaded, in the order it loaded them + */ + public function __construct(private StrippedIni $ini, private array $iniFiles) + { + } + + /** + * @param list $extensions the names of the extensions to strip + * @return string|null the php.ini written into the directory, null when no ini file loads any of + * the extensions, and the workers start with PHP's own + * @throws RuntimeException when the php.ini cannot be written + */ + public function write(array $extensions, string $directory): ?string + { + $ini = $this->ini->compose(array_map(static fn (string $file): string => (string) file_get_contents($file), $this->iniFiles), $extensions); + if ($ini === null) { + return null; + } + + $file = $directory . '/php.ini'; + // writing the file fails when the directory cannot be created + @mkdir($directory, 0777, true); + if (@file_put_contents($file, $ini) === false) { + throw new RuntimeException(sprintf('Cannot write %s.', $file)); + } + + return $file; + } + +} diff --git a/src/Execution/WorkerCommandLine.php b/src/Execution/WorkerCommandLine.php index b55e6e3..d6b705f 100644 --- a/src/Execution/WorkerCommandLine.php +++ b/src/Execution/WorkerCommandLine.php @@ -55,6 +55,7 @@ public function native(WorkerSettings $settings, BenchmarkCase $case, string $sc { return [ $this->phpBinary, + ...$this->phpIni($settings), ...$this->ini($settings->iniEntries), ...$this->worker($settings, $case, $scratch, $result, []), ]; @@ -73,6 +74,7 @@ public function counted(WorkerSettings $settings, BenchmarkCase $case, string $s ...self::VALGRIND_OPTIONS, '--callgrind-out-file=' . $profileFolder . '/%p.out', $this->phpBinary, + ...$this->phpIni($settings), ...$this->ini([ ...self::COUNTED_INI_ENTRIES, ...($settings->extension === null ? [] : ['extension=' . $settings->extension]), @@ -82,6 +84,14 @@ public function counted(WorkerSettings $settings, BenchmarkCase $case, string $s ]; } + /** + * @return list + */ + private function phpIni(WorkerSettings $settings): array + { + return $settings->phpIni === null ? [] : ['-c', $settings->phpIni]; + } + /** * @param list $entries * @return list diff --git a/src/Execution/WorkerSettings.php b/src/Execution/WorkerSettings.php index ffff73f..32108bc 100644 --- a/src/Execution/WorkerSettings.php +++ b/src/Execution/WorkerSettings.php @@ -10,6 +10,8 @@ final class WorkerSettings /** * @param list $iniEntries PHP settings of the workers, like memory_limit=-1 + * @param string|null $phpIni the php.ini the workers start with instead of PHP's own, null for + * PHP's own * @param list $suiteOptions the options with which the workers load the benchmark file of their case * @param string $valgrind the launcher of CodSpeed's Valgrind * @param string|null $extension the tally_hooks extension the counted worker loads, null when PHP @@ -18,6 +20,7 @@ final class WorkerSettings */ public function __construct( public readonly array $iniEntries, + public readonly ?string $phpIni, public readonly array $suiteOptions, public readonly string $valgrind, public readonly ?string $extension, diff --git a/tests/Fixtures/StrippedExtensions/StrippedBench.php b/tests/Fixtures/StrippedExtensions/StrippedBench.php new file mode 100644 index 0000000..468ddd4 --- /dev/null +++ b/tests/Fixtures/StrippedExtensions/StrippedBench.php @@ -0,0 +1,25 @@ +withPaths([__DIR__]) + ->withEnvironmentVariables(['PATH', 'PHP_INI_SCAN_DIR']) + ->withStrippedExtensions([]); diff --git a/tests/Fixtures/StrippedExtensions/tally.php b/tests/Fixtures/StrippedExtensions/tally.php new file mode 100644 index 0000000..76a3999 --- /dev/null +++ b/tests/Fixtures/StrippedExtensions/tally.php @@ -0,0 +1,9 @@ +withPaths([__DIR__]) + ->withEnvironmentVariables(['PATH', 'PHP_INI_SCAN_DIR']) + ->withStrippedExtensions(['not_an_extension', 'TALLY_HOOKS']); diff --git a/tests/InProcessTally.php b/tests/InProcessTally.php index b6f7d22..456d01b 100644 --- a/tests/InProcessTally.php +++ b/tests/InProcessTally.php @@ -4,9 +4,13 @@ use PHPUnit\Framework\Assert; use Tally\Cli\ApplicationFactory; +use function explode; use function fopen; +use function php_ini_loaded_file; +use function php_ini_scanned_files; use function rewind; use function stream_get_contents; +use function trim; use const PHP_BINARY; /** @@ -41,8 +45,19 @@ public static function runWithCodSpeed(string $codspeed, string $workingDirector Assert::assertNotFalse($output); Assert::assertNotFalse($errorOutput); + // like bin/tally, the ini files of this process + $iniFiles = []; + foreach ([(string) php_ini_loaded_file(), ...explode(',', (string) php_ini_scanned_files())] as $iniFile) { + $iniFile = trim($iniFile); + if ($iniFile === '') { + continue; + } + + $iniFiles[] = $iniFile; + } + $exitCode = (new ApplicationFactory()) - ->create($output, $errorOutput, $workingDirectory, $environment, false, false, PHP_BINARY, self::BINARY, $codspeed) + ->create($output, $errorOutput, $workingDirectory, $environment, false, false, PHP_BINARY, $iniFiles, self::BINARY, $codspeed) ->run(['tally', ...$arguments]); rewind($output); diff --git a/tests/Integration/Cli/RunCommandTest.php b/tests/Integration/Cli/RunCommandTest.php index f327d95..7cecc84 100644 --- a/tests/Integration/Cli/RunCommandTest.php +++ b/tests/Integration/Cli/RunCommandTest.php @@ -21,12 +21,14 @@ use function getmypid; use function json_encode; use function ksort; +use function php_ini_loaded_file; use function proc_close; use function proc_open; use function realpath; use function sprintf; use function str_repeat; use function sys_get_temp_dir; +use function var_export; use const JSON_THROW_ON_ERROR; final class RunCommandTest extends TestCase @@ -309,6 +311,28 @@ public function testTheCasesGetOnlyTheEnvironmentVariablesOfTheConfiguration(arr $this->assertStringContainsString("\n RuntimeException: " . $seen . "\n", $output); } + /** + * @return iterable + */ + public static function dataStrippedExtensions(): iterable + { + yield 'an extension an ini file loads' => [ + 'tally.php', + sprintf('tally_hooks is not loaded, the php.ini is %s, PHP scans no files.', var_export(sys_get_temp_dir() . '/tally-' . getmypid() . '/php.ini', true)), + ]; + yield 'none' => ['none.php', sprintf('tally_hooks is loaded, the php.ini is %s, PHP scans files.', var_export(php_ini_loaded_file(), true))]; + } + + #[DataProvider('dataStrippedExtensions')] + public function testTheCasesRunWithoutTheStrippedExtensions(string $config, string $seen): void + { + $fixture = __DIR__ . '/../../Fixtures/StrippedExtensions'; + [$exitCode, $output] = InProcessTally::run(__DIR__, $this->environment(), ['run', '--config=' . $fixture . '/' . $config, $fixture]); + + $this->assertSame(1, $exitCode); + $this->assertStringContainsString("\n RuntimeException: " . $seen . "\n", $output); + } + /** * @return array */ diff --git a/tests/Integration/Cli/WorkerCommandTest.php b/tests/Integration/Cli/WorkerCommandTest.php index 4538880..916af4c 100644 --- a/tests/Integration/Cli/WorkerCommandTest.php +++ b/tests/Integration/Cli/WorkerCommandTest.php @@ -31,6 +31,9 @@ final class WorkerCommandTest extends TestCase private const WORKER = __DIR__ . '/../../Fixtures/Worker'; private const BENCH = 'Tally\\Tests\\Fixtures\\Worker\\WorkerBench'; + /** Strips no extensions, this process loads PCOV for coverage */ + private const CONFIG = '--config=' . __DIR__ . '/../../Fixtures/StrippedExtensions/none.php'; + private string $directory; #[Override] @@ -100,6 +103,7 @@ public function testMeasuresTheMemoryOfTheBenchmarkAlone(string $method, int $mi PHP_BINARY, InProcessTally::BINARY, 'worker', + self::CONFIG, '--case=' . self::BENCH . '::' . $method, '--scratch=' . $this->directory, '--result=' . $this->directory . '/result.json', @@ -154,9 +158,32 @@ public function testRefusesToCountOutsideTheSimulation(): void $this->assertStringStartsWith("The benchmark does not run under CodSpeed's CPU simulation.\n", (string) $this->writtenResult()->failure); } + /** + * @throws InvalidResultException + */ + public function testRefusesToRunWithAStrippedExtension(): void + { + [$exitCode] = InProcessTally::run(__DIR__, [], [ + 'worker', + '--config=' . __DIR__ . '/../../Fixtures/StrippedExtensions/tally.php', + '--case=' . self::BENCH . '::benchAllocate', + '--scratch=' . $this->directory, + '--result=' . $this->directory . '/result.json', + self::WORKER, + ]); + + $this->assertSame(1, $exitCode); + $this->assertSame( + "PHP loads extensions that the cases run without: TALLY_HOOKS.\n" + . 'tally run comments out the lines of the ini files that load them, so this PHP loads them some other way, such as built in. ' + . 'Leave them out of Configuration::withStrippedExtensions() to count with them.', + $this->writtenResult()->failure, + ); + } + public function testReportsAResultThatCannotBeWritten(): void { - [$exitCode, , $errorOutput] = InProcessTally::run(__DIR__, [], ['worker', '--case=' . self::BENCH . '::benchAllocate', '--scratch=' . $this->directory, '--result=/nonexistent/result.json', self::WORKER]); + [$exitCode, , $errorOutput] = InProcessTally::run(__DIR__, [], ['worker', self::CONFIG, '--case=' . self::BENCH . '::benchAllocate', '--scratch=' . $this->directory, '--result=/nonexistent/result.json', self::WORKER]); $this->assertSame(2, $exitCode); $this->assertSame("Cannot write /nonexistent/result.json.\n", $errorOutput); @@ -205,7 +232,7 @@ public function testRequiredOption(string $option): void $arguments = ['case' => '--case=' . self::BENCH . '::benchAllocate', 'scratch' => '--scratch=' . $this->directory, 'result' => '--result=' . $this->directory . '/result.json']; unset($arguments[$option]); - [$exitCode, , $errorOutput] = InProcessTally::run(__DIR__, [], ['worker', ...array_values($arguments), self::WORKER]); + [$exitCode, , $errorOutput] = InProcessTally::run(__DIR__, [], ['worker', self::CONFIG, ...array_values($arguments), self::WORKER]); $this->assertSame(2, $exitCode); $this->assertStringStartsWith(sprintf("Option --%s is required.\nRun tally worker --help for its options.\n", $option), $errorOutput); @@ -219,6 +246,7 @@ private function worker(string $id, array $options, string $scratch): array { return InProcessTally::run(dirname(__DIR__, 3), [], [ 'worker', + self::CONFIG, '--case=' . $id, '--scratch=' . $scratch, '--result=' . $this->directory . '/result.json', diff --git a/tests/Integration/Execution/CaseJobsTest.php b/tests/Integration/Execution/CaseJobsTest.php index 444e7a2..6d1bdab 100644 --- a/tests/Integration/Execution/CaseJobsTest.php +++ b/tests/Integration/Execution/CaseJobsTest.php @@ -131,7 +131,7 @@ private static function jobs(): CaseJobs private function settings(string $workDirectory): WorkerSettings { - return new WorkerSettings([], [], '/opt/codspeed/valgrind', '/opt/codspeed/tally_hooks-8.5.so', $workDirectory, 60, 1800); + return new WorkerSettings([], null, [], '/opt/codspeed/valgrind', '/opt/codspeed/tally_hooks-8.5.so', $workDirectory, 60, 1800); } private static function memory(int $bytes): Measurement diff --git a/tests/Integration/Execution/StrippedIniFileTest.php b/tests/Integration/Execution/StrippedIniFileTest.php new file mode 100644 index 0000000..7fc7cae --- /dev/null +++ b/tests/Integration/Execution/StrippedIniFileTest.php @@ -0,0 +1,67 @@ +directory = sys_get_temp_dir() . '/tally-ini-' . getmypid(); + mkdir($this->directory . '/conf.d', 0777, true); + file_put_contents($this->directory . '/php.ini', "memory_limit=1G\n"); + file_put_contents($this->directory . '/conf.d/20-xdebug.ini', "zend_extension=xdebug.so\n"); + } + + #[Override] + protected function tearDown(): void + { + (new DirectoryRemover())->remove($this->directory); + } + + public function testWritesTheFilesWithoutTheStrippedExtensions(): void + { + $work = $this->directory . '/work/tally'; + + $file = $this->iniFile()->write(['xdebug'], $work); + + $this->assertSame($work . '/php.ini', $file); + $this->assertStringEqualsFile($work . '/php.ini', "memory_limit=1G\n\n;zend_extension=xdebug.so\n\n"); + } + + public function testWritesNothingWhenNoFileLoadsAStrippedExtension(): void + { + $work = $this->directory . '/work'; + + $this->assertNull($this->iniFile()->write(['pcov'], $work)); + $this->assertDirectoryDoesNotExist($work); + } + + public function testReportsAFileThatCannotBeWritten(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Cannot write /nonexistent/tally/php.ini.'); + + $this->iniFile()->write(['xdebug'], '/nonexistent/tally'); + } + + private function iniFile(): StrippedIniFile + { + return new StrippedIniFile(new StrippedIni(), [$this->directory . '/php.ini', $this->directory . '/conf.d/20-xdebug.ini']); + } + +} diff --git a/tests/Unit/ConfigurationTest.php b/tests/Unit/ConfigurationTest.php index 996dc43..8a7832f 100644 --- a/tests/Unit/ConfigurationTest.php +++ b/tests/Unit/ConfigurationTest.php @@ -21,6 +21,7 @@ public function testDefaults(): void $this->assertSame('±10 % of each metric', $configuration->growthTolerance()->describe()); $this->assertFalse($configuration->namedArguments()); $this->assertSame(['PATH', 'HOME', 'TMPDIR', 'LANG', 'LC_ALL', 'PHPRC', 'PHP_INI_SCAN_DIR'], $configuration->environmentVariables()); + $this->assertSame(['xdebug', 'pcov', 'uopz', 'blackfire', 'tideways', 'tideways_xhprof', 'xhprof', 'spx', 'excimer', 'memprof', 'opentelemetry'], $configuration->strippedExtensions()); } public function testWithersKeepTheOriginal(): void @@ -36,6 +37,7 @@ public function testWithersKeepTheOriginal(): void $this->assertSame('±40 % of each metric', $original->withGrowthTolerance(new RelativeTolerance(0.4))->growthTolerance()->describe()); $this->assertTrue($original->withNamedArguments(true)->namedArguments()); $this->assertSame(['PATH', 'CI'], $original->withEnvironmentVariables(['PATH', 'CI'])->environmentVariables()); + $this->assertSame(['xdebug'], $original->withStrippedExtensions(['xdebug'])->strippedExtensions()); $this->assertEquals(Configuration::create(), $original); } diff --git a/tests/Unit/Execution/StrippedIniTest.php b/tests/Unit/Execution/StrippedIniTest.php new file mode 100644 index 0000000..19c0561 --- /dev/null +++ b/tests/Unit/Execution/StrippedIniTest.php @@ -0,0 +1,59 @@ +, list, string|null}> + */ + public static function dataCompose(): iterable + { + yield 'the line that loads a stripped extension is commented out' => [ + ["extension=intl\nzend_extension=xdebug.so\nmemory_limit=1G\n"], + ['xdebug'], + "extension=intl\n;zend_extension=xdebug.so\nmemory_limit=1G\n\n", + ]; + yield 'a path, quotes, spaces, and other letter case' => [ + [" Zend_Extension = \"/usr/lib/php/20240924/Xdebug.so\"\nextension = '/opt/pcov.so' ; coverage\n"], + ['xdebug', 'pcov'], + "; Zend_Extension = \"/usr/lib/php/20240924/Xdebug.so\"\n;extension = '/opt/pcov.so' ; coverage\n\n", + ]; + yield 'a file name that goes on with something other than a letter, digit, or underscore' => [ + ["extension=/usr/lib/tideways/tideways-php-8.5.so\n"], + ['tideways'], + ";extension=/usr/lib/tideways/tideways-php-8.5.so\n\n", + ]; + yield 'blank lines before the line' => [ + ["\n\nextension=pcov"], + ['pcov'], + "\n\n;extension=pcov\n", + ]; + yield 'every file, each up to its first section for a path or a host, which the CLI never applies' => [ + ["precision=10\n[PATH=/srv/www]\nextension=xdebug\n", "extension=pcov\n [host=example.com]\nprecision=12\n", "precision=14\n"], + ['xdebug', 'pcov'], + "precision=10\n\n;extension=pcov\n\nprecision=14\n\n", + ]; + yield 'no file loads a stripped extension' => [["extension=intl\n", "memory_limit=1G\n"], ['xdebug'], null]; + yield 'no stripped extensions' => [["extension=xdebug\n"], [], null]; + yield 'an extension whose name starts with that of a stripped one' => [["extension=tideways_xhprof.so\n"], ['tideways'], null]; + yield 'a directory named after a stripped extension' => [["extension=/opt/pcov/intl.so\n"], ['pcov'], null]; + yield 'a line that is commented out already, and a setting of the extension' => [[";extension=xdebug\nxdebug.mode=debug\n"], ['xdebug'], null]; + } + + /** + * @param list $files + * @param list $extensions + */ + #[DataProvider('dataCompose')] + public function testCompose(array $files, array $extensions, ?string $expected): void + { + $this->assertSame($expected, (new StrippedIni())->compose($files, $extensions)); + } + +} diff --git a/tests/Unit/Execution/WorkerCommandLineTest.php b/tests/Unit/Execution/WorkerCommandLineTest.php index b2a5c66..2c4cd55 100644 --- a/tests/Unit/Execution/WorkerCommandLineTest.php +++ b/tests/Unit/Execution/WorkerCommandLineTest.php @@ -30,7 +30,7 @@ public function testNative(): void '--config=tally.php', '--', '/project/benchmarks/FooBench.php', - ], self::commandLine()->native(self::settings('/opt/codspeed/tally_hooks-8.5.so'), self::case(), '/work/scratch', '/work/native.json')); + ], self::commandLine()->native(self::settings('/opt/codspeed/tally_hooks-8.5.so', null), self::case(), '/work/scratch', '/work/native.json')); } public function testCountedRunsTheWorkerUnderCallgrindWithTheCachesOfCodSpeed(): void @@ -72,7 +72,7 @@ public function testCountedRunsTheWorkerUnderCallgrindWithTheCachesOfCodSpeed(): '--config=tally.php', '--', '/project/benchmarks/FooBench.php', - ], self::commandLine()->counted(self::settings('/opt/codspeed/tally_hooks-8.5.so'), self::case(), '/work/scratch', '/work/counted.json', '/work/profile')); + ], self::commandLine()->counted(self::settings('/opt/codspeed/tally_hooks-8.5.so', null), self::case(), '/work/scratch', '/work/counted.json', '/work/profile')); } private static function commandLine(): WorkerCommandLine @@ -84,14 +84,29 @@ public function testCountedWithoutLoadingTheExtensionPhpLoadsAlready(): void { $this->assertSame( ['-d', 'zend.max_allowed_stack_size=-1', '-d', 'memory_limit=-1', '-d', 'opcache.enable_cli=1', 'vendor/bin/tally'], - array_slice(self::commandLine()->counted(self::settings(null), self::case(), '/work/scratch', '/work/counted.json', '/work/profile'), 19, 7), + array_slice(self::commandLine()->counted(self::settings(null, null), self::case(), '/work/scratch', '/work/counted.json', '/work/profile'), 19, 7), ); } - private static function settings(?string $extension): WorkerSettings + public function testBothRunsStartWithThePhpIniWithoutTheStrippedExtensions(): void + { + $settings = self::settings(null, '/work/php.ini'); + + $this->assertSame( + ['/usr/bin/php', '-c', '/work/php.ini', '-d', 'memory_limit=-1'], + array_slice(self::commandLine()->native($settings, self::case(), '/work/scratch', '/work/native.json'), 0, 5), + ); + $this->assertSame( + ['/usr/bin/php', '-c', '/work/php.ini', '-d', 'zend.max_allowed_stack_size=-1'], + array_slice(self::commandLine()->counted($settings, self::case(), '/work/scratch', '/work/counted.json', '/work/profile'), 18, 5), + ); + } + + private static function settings(?string $extension, ?string $phpIni): WorkerSettings { return new WorkerSettings( ['memory_limit=-1', 'opcache.enable_cli=1'], + $phpIni, ['--config=tally.php'], '/opt/codspeed/valgrind', $extension, From b9d9a46450536dda4477a047fd2e095de7143add Mon Sep 17 00:00:00 2001 From: "ondrej@mirtes.cz" Date: Sat, 14 Nov 2026 21:47:23 +0000 Subject: [PATCH 2/2] Test the stripped extensions end to end with bashunit The unit and integration tests run with the fake Valgrind, and only the Code Coverage and Mutation Testing jobs load an extension to strip, PCOV, which no test asserts on. The E2E job loads Xdebug, PCOV, or both the way setup-php does, and the tests load them in other ways on top of it: another scan directory, the php.ini in PHPRC, and -c. With CodSpeed's Valgrind, the case must not see them and must count within 0.1 % of a PHP without them, which also catches a php.ini of the workers that loses another setting. Two controls keep the checks from passing for the wrong reason: the case sees the extensions the configuration keeps, and a worker that loads one with -d refuses to count. Like the e2e tests of phpstan-src, they are e2e//_test.sh with a bootstrap in .bashunitrc, and CI installs bashunit with its pinned action. Amp-Thread-ID: https://ampcode.com/threads/T-01a1dbdd-1694-7100-875a-74f105f1221f --- .bashunitrc | 3 + .gitattributes | 2 + .github/workflows/ci.yml | 81 +++++++++++- .gitignore | 2 + CLAUDE.md | 3 +- Makefile | 2 +- e2e/bootstrap.sh | 15 +++ e2e/stripped-extensions/bench/ProbeBench.php | 32 +++++ e2e/stripped-extensions/kept.php | 7 ++ .../stripped-extensions_test.sh | 118 ++++++++++++++++++ e2e/stripped-extensions/tally.php | 6 + 11 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 .bashunitrc create mode 100755 e2e/bootstrap.sh create mode 100644 e2e/stripped-extensions/bench/ProbeBench.php create mode 100644 e2e/stripped-extensions/kept.php create mode 100755 e2e/stripped-extensions/stripped-extensions_test.sh create mode 100644 e2e/stripped-extensions/tally.php diff --git a/.bashunitrc b/.bashunitrc new file mode 100644 index 0000000..c49b28d --- /dev/null +++ b/.bashunitrc @@ -0,0 +1,3 @@ +# E2E tests in e2e/, run from the repository root: e2e/bashunit test e2e +BASHUNIT_BOOTSTRAP=e2e/bootstrap.sh +BASHUNIT_STRICT_MODE=true diff --git a/.gitattributes b/.gitattributes index 394aede..19cb0ae 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,8 +1,10 @@ *.php text eol=lf .agents export-ignore +.bashunitrc export-ignore .github export-ignore bench export-ignore +e2e export-ignore examples export-ignore tests export-ignore tmp export-ignore diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c7259a..81c3d4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -430,6 +430,85 @@ jobs: path: tmp/tally-examples if-no-files-found: ignore + e2e: + name: "E2E tests (${{ matrix.name }})" + # The cases run without debuggers and profilers, however PHP loads them: each row loads them + # the way a PHP setup does, and the tests load them in other ways on top of it + needs: ["codspeed", "tally-hooks"] + # runs on release/1.x too, with the committed builds, when the builds are skipped + if: ${{ !cancelled() && !failure() }} + runs-on: "ubuntu-24.04" + timeout-minutes: 30 + + strategy: + fail-fast: false + matrix: + include: + # from setup-php's scan directory, in the coverage mode + - name: "Xdebug" + coverage: "xdebug" + extensions: "posix" + ini-values: "" + stripped: "xdebug" + # collecting coverage from the start + - name: "PCOV" + coverage: "pcov" + extensions: "posix" + ini-values: "pcov.enabled=1" + stripped: "pcov" + # both at once, which the coverage option does not load, Xdebug in its default develop mode + - name: "Xdebug and PCOV" + coverage: "" + extensions: "posix, xdebug, pcov" + ini-values: "pcov.enabled=1" + stripped: "xdebug pcov" + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2 + with: + egress-policy: audit + + - name: "Checkout" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: "Install PHP" + uses: "shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962" # v2 + with: + coverage: ${{ matrix.coverage }} + php-version: "8.5" + extensions: ${{ matrix.extensions }} + ini-values: ${{ matrix.ini-values }} + + - name: "Install dependencies" + run: "composer install --no-interaction --no-progress" + + - name: "Download CodSpeed's Valgrind" + if: ${{ !startsWith(github.ref_name, 'release/') }} + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: codspeed-ubuntu-24.04 + path: codspeed + + - name: "Download tally_hooks" + if: ${{ !startsWith(github.ref_name, 'release/') }} + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: tally-hooks-ubuntu-24.04 + path: codspeed + + - name: "Install bashunit" + uses: "TypedDevs/bashunit@8d4f357f52df7596f92ece2d6c049d91d45517a2" # 0.52.0 + with: + directory: "e2e" + + - name: "E2E tests" + env: + E2E_EXTENSIONS: ${{ matrix.stripped }} + run: "e2e/bashunit test e2e" + release-tree: name: "Release tree (${{ matrix.runner }}, PHP ${{ matrix.php-version }} ${{ matrix.phpts }})" # Checks the tree release/1.x gets from this commit the way a project installs it, on each platform @@ -513,7 +592,7 @@ jobs: release-branch: name: "Push release/1.x" - needs: ["release-tree", "tests", "static-analysis", "benchmarks"] + needs: ["release-tree", "tests", "static-analysis", "benchmarks", "e2e"] if: ${{ github.event_name == 'push' && github.ref_name == '1.x' }} runs-on: "ubuntu-latest" timeout-minutes: 10 diff --git a/.gitignore b/.gitignore index 68fc8a2..423bae5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ /build-cs /composer.lock /vendor +/e2e/bashunit +/.bashunit /.amp/portals/* /codspeed /codspeed/*/.key diff --git a/CLAUDE.md b/CLAUDE.md index 64689a6..3b70306 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,6 +25,7 @@ Tally benchmarks PHP code by counting instructions, cycles, and memory under Cod - `bench/` – Tally's benchmarks of itself, listed in `tally.php`. They cover the code whose cost grows with the suite, a benchmark file, or a profile, and that runs once per case or once per worker: locating and rewriting expectations, planning and diffing new expectations, expanding cases, sharding, reading profiles, and rendering reports. `SuiteSource` generates the code and cases of large benchmark classes as their input. Build inputs in data providers and before methods, which are not counted, and keep the counted call to the one method being measured. `#[Warmup(1)]` keeps autoloading out of the count. - `examples/` – short, artificial benchmarks for README.md, one feature each. `examples/Scripts` shows a data provider with its own `ExpectationSource`, which keeps the numbers in a comment at the top of each script. Keep them readable over realistic; real measurements of Tally belong in `bench/`. CI runs them under CodSpeed next to `bench/`, so they stay valid. - `build/` – scripts for the build, such as the coverage check. `build-codspeed.sh` builds CodSpeed's Valgrind (Callgrind only, static launcher) at a pinned tag into `codspeed/linux-`, which `1.x` ignores, and `build-tally-hooks.sh` builds `ext/tally_hooks` with `phpize` next to it; `make codspeed` runs both for this machine and PHP, CI builds `tally_hooks` for PHP 8.2 to 8.6, NTS and ZTS, in the official `php` Docker images. `build-release.sh` turns `1.x` into the `release/1.x` tree by committing the builds of both platforms; CI pushes it on every push to `1.x`, and releases are tagged there with `tag-release.sh `, which checks that the generated commit sits on the `1.x` commit to release. `check-release.sh` installs a release tree into a new project with Composer and counts a benchmark, which CI runs on every pull request on x86_64 and aarch64, with PHP 8.2 and 8.5, and 8.5 ZTS. `build/PHPStan` holds Tally's own PHPStan rules and `BenchmarkUsageProvider`, which tells the dead code detector about the methods Tally calls through reflection. `build/stubs` holds the stubs for what PHPStan cannot see, such as the functions of `tally_hooks`. +- `e2e/` – end-to-end tests with [bashunit](https://bashunit.com), `/_test.sh`; `e2e/bootstrap.sh`, set in `.bashunitrc`, gives each test a directory of its own, which is also its `TMPDIR`. `stripped-extensions` counts a benchmark with CodSpeed's Valgrind while PHP loads the extensions in `E2E_EXTENSIONS` in several ways (its scan directory, another one, the php.ini in `PHPRC`, `-c`): the case must not see them and must count what it counts on a PHP without them. CI installs bashunit into `e2e/` and runs it once per PHP setup (Xdebug, PCOV, both); locally, with the extensions loaded and bashunit installed the same way: `E2E_EXTENSIONS='xdebug pcov' e2e/bashunit test e2e`. Like `tests/Fixtures`, their PHP files are only linted. - `tests/Unit` – pure logic, no file system, processes, or network. `tests/Integration` – adapters and the CLI. `tests/Fixtures` – benchmark classes and configuration files the tests load; excluded from PHPStan and the coding standard because some are invalid on purpose. ## Design rules @@ -61,7 +62,7 @@ make infection # mutation testing of the whole codebase, fails below 100 % MSI make bench # bench/ under CodSpeed's Valgrind, builds it and tally_hooks with make codspeed first ``` -`make cs-install` clones build-cs. PHPStan runs with no ignores and no baseline: fix the types. The dead code detector reports unused code; delete it, or mark it `@api` when it is public API. A stale result cache of the dead code detector makes PHPStan fail with "DCD cache file not found"; `make infection` clears the result cache first, run `vendor/bin/phpstan clear-result-cache` before other runs when it happens. CI runs the lint and the tests on PHP 8.2 to 8.6, the rest on 8.5 and 8.6; Infection runs on the whole codebase on pushes and on the changed lines in pull requests. The Benchmarks job runs `bench/` under CodSpeed on a pinned runner, because the counts depend on the PHP build. Take expected numbers from that job, not from a local run: a benchmark without them gets a patch in the job's output, and changed numbers get `tally accept` with the `tally-results` artifact. +`make cs-install` clones build-cs. PHPStan runs with no ignores and no baseline: fix the types. The dead code detector reports unused code; delete it, or mark it `@api` when it is public API. A stale result cache of the dead code detector makes PHPStan fail with "DCD cache file not found"; `make infection` clears the result cache first, run `vendor/bin/phpstan clear-result-cache` before other runs when it happens. CI runs the lint and the tests on PHP 8.2 to 8.6, the rest on 8.5 and 8.6; Infection runs on the whole codebase on pushes and on the changed lines in pull requests. The E2E job runs `e2e/` with Xdebug and PCOV loaded. The Benchmarks job runs `bench/` under CodSpeed on a pinned runner, because the counts depend on the PHP build. Take expected numbers from that job, not from a local run: a benchmark without them gets a patch in the job's output, and changed numbers get `tally accept` with the `tally-results` artifact. ## Conventions diff --git a/Makefile b/Makefile index 999713e..6e69d04 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ $(CODSPEED)/tally_hooks-%.so: build/build-tally-hooks.sh $(wildcard ext/tally_ho lint: php vendor/bin/parallel-lint --colors bin/tally \ --exclude tests/Fixtures/Loading/Syntax \ - bench build examples src tests tally.php + bench build e2e examples src tests tally.php .PHONY: cs-install cs-install: diff --git a/e2e/bootstrap.sh b/e2e/bootstrap.sh new file mode 100755 index 0000000..94610a8 --- /dev/null +++ b/e2e/bootstrap.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Loaded by bashunit before the E2E tests, see BASHUNIT_BOOTSTRAP in .bashunitrc. +# Run the tests from the repository root: e2e/bashunit test e2e + +# A test runs in the directory of its test file and gets E2E_TMPDIR, a directory of its own, +# which is also its TMPDIR: tally run keeps its work directory in sys_get_temp_dir(). +function e2e::set_up() { + E2E_TMPDIR=$(mktemp -d) + export TMPDIR="$E2E_TMPDIR" + cd "$(dirname "${BASH_SOURCE[1]}")" || return +} + +function e2e::tear_down() { + rm -rf "${E2E_TMPDIR:?}" +} diff --git a/e2e/stripped-extensions/bench/ProbeBench.php b/e2e/stripped-extensions/bench/ProbeBench.php new file mode 100644 index 0000000..4f2a52c --- /dev/null +++ b/e2e/stripped-extensions/bench/ProbeBench.php @@ -0,0 +1,32 @@ +step($i); + } + } + + private function step(int $i): int + { + return $i % 7; + } + +} diff --git a/e2e/stripped-extensions/kept.php b/e2e/stripped-extensions/kept.php new file mode 100644 index 0000000..fa3e629 --- /dev/null +++ b/e2e/stripped-extensions/kept.php @@ -0,0 +1,7 @@ +withPaths([__DIR__ . '/bench']) + ->withStrippedExtensions([]); diff --git a/e2e/stripped-extensions/stripped-extensions_test.sh b/e2e/stripped-extensions/stripped-extensions_test.sh new file mode 100755 index 0000000..2754109 --- /dev/null +++ b/e2e/stripped-extensions/stripped-extensions_test.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# The cases run without the extensions in E2E_EXTENSIONS, such as "xdebug pcov", however PHP's ini +# files load them. Each test loads them another way, and checks that the case does not see them +# and counts what it counts on a PHP without them. CI runs the tests once for each PHP setup. + +function set_up() { + e2e::set_up + split_ini +} + +function tear_down() { + e2e::tear_down +} + +function test_extensions_from_the_scan_directory_of_php() { + # as the PHP setup installs them, such as conf.d/20-xdebug.ini + assert_counts_without_the_extensions php +} + +function test_extensions_from_an_ini_file_in_another_scan_directory() { + mkdir "$E2E_TMPDIR/extra" + cp "$E2E_TMPDIR/lines.ini" "$E2E_TMPDIR/extra/99-profilers.ini" + assert_counts_without_the_extensions env PHPRC="$E2E_TMPDIR/without" PHP_INI_SCAN_DIR="$E2E_TMPDIR/without/conf.d:$E2E_TMPDIR/extra" php +} + +function test_extensions_from_the_php_ini_in_phprc() { + assert_counts_without_the_extensions env PHPRC="$E2E_TMPDIR/with" PHP_INI_SCAN_DIR="$E2E_TMPDIR/without/conf.d" php +} + +function test_extensions_from_the_php_ini_given_with_c() { + # tally run gives the workers neither its -c nor its -d options, so without stripping they would + # not get the extensions either: this one checks only that they do not reach the case + assert_counts_without_the_extensions env PHP_INI_SCAN_DIR="$E2E_TMPDIR/without/conf.d" php -c "$E2E_TMPDIR/with/php.ini" +} + +function test_the_case_sees_the_extensions_the_configuration_keeps() { + # the probe of the other tests would pass for the wrong reason if it did not see them + tally_run php -- --config=kept.php && code=0 || code=$? + + assert_same "1" "$code" + assert_file_contains "$E2E_TMPDIR/output.txt" "RuntimeException: The case runs with ${E2E_EXTENSIONS// /, }." +} + +function test_refuses_to_count_with_extensions_that_no_ini_file_loads() { + # loaded with -d, which tally run cannot comment out + php -r 'echo "withPaths([", var_export($argv[1], true), "])->withIniEntries(", var_export(array_map("trim", file($argv[2], FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES)), true), ");\n";' \ + "$PWD/bench" "$E2E_TMPDIR/lines.ini" > "$E2E_TMPDIR/tally.php" + + tally_run "${WITHOUT[@]}" -- --config="$E2E_TMPDIR/tally.php" && code=0 || code=$? + + assert_same "1" "$code" + assert_file_contains "$E2E_TMPDIR/output.txt" "PHP loads extensions that the cases run without: ${E2E_EXTENSIONS// /, }." +} + +# Splits PHP's ini files into $E2E_TMPDIR/without, the php.ini and the scanned files in conf.d +# without the lines that load the extensions, and $E2E_TMPDIR/lines.ini, those lines. +# $E2E_TMPDIR/with/php.ini is the php.ini with them, and WITHOUT the PHP command without them. +function split_ini() { + local loaded file + mkdir -p "$E2E_TMPDIR/without/conf.d" "$E2E_TMPDIR/with" + touch "$E2E_TMPDIR/without/php.ini" "$E2E_TMPDIR/lines.ini" + loaded=$(php -r 'echo (string) php_ini_loaded_file();') + if [ -n "$loaded" ]; then + split_file "$loaded" "$E2E_TMPDIR/without/php.ini" + fi + while IFS= read -r file; do + split_file "$file" "$E2E_TMPDIR/without/conf.d/$(basename "$file")" + done < <(php -r 'foreach (explode(",", (string) php_ini_scanned_files()) as $file) { if (trim($file) !== "") { echo trim($file), "\n"; } }') + cat "$E2E_TMPDIR/without/php.ini" "$E2E_TMPDIR/lines.ini" > "$E2E_TMPDIR/with/php.ini" + WITHOUT=(env PHPRC="$E2E_TMPDIR/without" PHP_INI_SCAN_DIR="$E2E_TMPDIR/without/conf.d" php) +} + +function split_file() { + local pattern="^[[:space:]]*(zend_)?extension[[:space:]]*=.*(${E2E_EXTENSIONS// /|})" + grep -Eiv "$pattern" "$1" > "$2" || true + grep -Ei "$pattern" "$1" >> "$E2E_TMPDIR/lines.ini" || true +} + +# Checks the case with the PHP command given, such as env PHPRC=dir php: tally run counts it +# without the extensions, which the command loads, and like a PHP without them. +function assert_counts_without_the_extensions() { + local code counted without + assert_same "$E2E_EXTENSIONS" "$(loaded_extensions "$@")" + assert_same "" "$(loaded_extensions "${WITHOUT[@]}")" + + tally_run "$@" -- && code=0 || code=$? + assert_same "0" "$code" || cat "$E2E_TMPDIR/output.txt" + counted=$(instructions) + tally_run "${WITHOUT[@]}" -- && code=0 || code=$? + assert_same "0" "$code" || cat "$E2E_TMPDIR/output.txt" + without=$(instructions) + + # The php.ini of the workers keeps every other setting, a lost one would move the counts. The + # command lines and the environments of the workers differ in length, which can move a few. + assert_within_delta "$without" "$counted" "$((without / 1000))" +} + +function loaded_extensions() { + "$@" -r 'echo implode(" ", array_intersect(explode(" ", $argv[1]), get_loaded_extensions()));' "$E2E_EXTENSIONS" +} + +# Runs tally run on the probe with the PHP command before --, and the options after it. The output +# goes to $E2E_TMPDIR/output.txt, the results to $E2E_TMPDIR/results. +function tally_run() { + local php=() + while [ "$1" != "--" ]; do + php+=("$1") + shift + done + shift + rm -rf "$E2E_TMPDIR/results" + # outside GitHub Actions, where tally run would write annotations and the job summary + env -u GITHUB_ACTIONS "${php[@]}" ../../bin/tally run --output="$E2E_TMPDIR/results" "$@" > "$E2E_TMPDIR/output.txt" 2>&1 +} + +function instructions() { + php -r 'echo json_decode(file_get_contents($argv[1]), true)["metrics"]["instructions"];' "$E2E_TMPDIR"/results/*.json +} diff --git a/e2e/stripped-extensions/tally.php b/e2e/stripped-extensions/tally.php new file mode 100644 index 0000000..9df798c --- /dev/null +++ b/e2e/stripped-extensions/tally.php @@ -0,0 +1,6 @@ +withPaths([__DIR__ . '/bench']);