From 71e775a831ceb5b4927f9155d9650817775b8daa Mon Sep 17 00:00:00 2001 From: "ondrej@mirtes.cz" Date: Sat, 14 Nov 2026 11:53:40 +0000 Subject: [PATCH] Run the cases with only the environment variables the configuration keeps The environment is on the stack of the counted process, so variables that differ from run to run, such as the branch name in CI, moved its memory layout and the counts of benchmarks whose allocations depend on it: 3,000 more bytes of environment moved phpstan-src's FileCacheStorageBench::benchSave@entries-10000 by 1 % of instructions and 8 % of cycles. withEnvironmentVariables() lists the variables the workers keep, PATH, HOME, TMPDIR, LANG, LC_ALL, PHPRC, and PHP_INI_SCAN_DIR by default. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d73a-bab0-7223-9245-50c2c00f22c7 --- CLAUDE.md | 2 +- README.md | 4 +- src/Cli/RunCommand.php | 6 +- src/Configuration.php | 31 +++++++++- .../Fixtures/Environment/EnvironmentBench.php | 18 ++++++ tests/Fixtures/Environment/tally.php | 7 +++ tests/Fixtures/Run/tally.php | 7 +++ tests/Integration/Cli/RunCommandTest.php | 58 +++++++++++++++---- tests/Unit/ConfigurationTest.php | 2 + 9 files changed, 118 insertions(+), 17 deletions(-) create mode 100644 tests/Fixtures/Environment/EnvironmentBench.php create mode 100644 tests/Fixtures/Environment/tally.php create mode 100644 tests/Fixtures/Run/tally.php diff --git a/CLAUDE.md b/CLAUDE.md index fee5ad8..c873aa4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ Tally benchmarks PHP code by counting instructions, cycles, and memory under Cod - `src/Execution` – measuring a case in worker processes. `CaseJobs` builds a case's job (a native run, then a run under CodSpeed's Valgrind with the simulated caches of `codspeed run --mode=simulation`, sharing a scratch directory) and reads what the job left. `WorkerCommandLine` builds the commands, `CaseOutcome` turns the runs and the profile parts into a `Measurement` (both pure). `CodSpeedInstallation` finds the builds for the platform in `codspeed/linux-`: Valgrind, whose tools the workers find through `VALGRIND_LIB`, and `tally_hooks-[-zts].so`, which the counted worker loads with `-d extension=` unless PHP loads `tally_hooks` already, `DirectoryRemover` cleans up the work directory. - `src/Cli` – `Application`, the commands (`list`, `run`, `compare`, `accept`, and the hidden `worker` that `run` starts), `ArgumentParser`, `Input`, `Output`. `ReportOutput` writes the report, with the patch for cases without expectations, and, in GitHub Actions, the annotations (relative to `GITHUB_WORKSPACE`) and job summary. `run` checks that every selected case's expectations can be rewritten before it counts anything; `accept` writes each file once. `ApplicationFactory` is the composition root; `bin/tally` passes it the streams, the environment, and the binaries the workers run. Every error goes through `Output::error()`: the first line says what is wrong (bold red on a terminal), the lines after it what to do, such as `Run tally list --help for its options.`; an unknown command or option suggests the closest name through `Text\ClosestName`. - `src/Text` – `ClosestName`, the "did you mean" suggestion for a mistyped name of a command, option, variable, function, or key. -- `src/` root – value objects shared by everything: `Metric`, `Metrics`, `CaseId`, `BenchmarkCase`, `DataSet`, `Measurement`, `ScratchDirectory`, `Configuration` (what `tally.php` returns, and where every default is chosen, such as the tolerance and whether `tally accept` writes `dataSet:` as a named argument), and the `Meter` interface. +- `src/` root – value objects shared by everything: `Metric`, `Metrics`, `CaseId`, `BenchmarkCase`, `DataSet`, `Measurement`, `ScratchDirectory`, `Configuration` (what `tally.php` returns, and where every default is chosen, such as the tolerance, whether `tally accept` writes `dataSet:` as a named argument, and the environment variables the workers keep, because the size of the environment moves the counts), and the `Meter` interface. - `bin/tally` – finds the autoloader and runs `ApplicationFactory`. - `ext/tally_hooks` – the PHP extension the counted worker calls: functions in `Tally\Hooks` that make Callgrind's client requests through Valgrind's `valgrind.h` and `callgrind.h`. Built with `-DTALLY_HOOKS_RECORDING`, it records the calls instead, for the tests. Keep it to these few functions; the logic belongs in PHP. - `bench/` – Tally's benchmarks of itself, listed in `tally.php`. They cover the code whose cost grows with the suite, a benchmark file, or a profile, and that runs once per case or once per worker: locating and rewriting expectations, planning and diffing new expectations, expanding cases, sharding, reading profiles, and rendering reports. `SuiteSource` generates the code and cases of large benchmark classes as their input. Build inputs in data providers and before methods, which are not counted, and keep the counted call to the one method being measured. `#[Warmup(1)]` keeps autoloading out of the count. diff --git a/README.md b/README.md index 600a26d..b013084 100644 --- a/README.md +++ b/README.md @@ -431,6 +431,7 @@ Every value Tally uses when a benchmark or the command line does not give one is | `withTolerance(TolerancePolicy)` | The tolerance of benchmarks without a policy on their method or class: any of the tolerance attributes, or your own `TolerancePolicy`. [`MetricTolerance`](#attributes) with ±2 % instructions, ±5 % cycles but at least 2,000, and ±5 % memory by default. | | `withGrowthTolerance(TolerancePolicy)` | The tolerance of benchmarks with a complexity class and without a policy on their method or class. `RelativeTolerance(0.1)` by default. | | `withNamedArguments(bool)` | Whether `tally accept` and the patch of new expectations write `#[ExpectedFor(dataSet: 'size-10', …)]` instead of `#[ExpectedFor('size-10', …)]`. Off by default. | +| `withEnvironmentVariables(list)` | The environment variables the cases run with; `tally run` leaves out the others. The environment is on the stack of the counted process, so a variable that differs from run to run, such as the branch name in CI, moves its memory layout, and with it the counts of benchmarks whose allocations depend on it. `PATH`, `HOME`, `TMPDIR`, `LANG`, `LC_ALL`, `PHPRC`, and `PHP_INI_SCAN_DIR` by default; list them too when you add one. | ## In CI @@ -438,7 +439,8 @@ The counts depend on everything that runs the benchmark: - the PHP version, including the patch release, - how PHP was built, which extensions are loaded, and the PHP settings, such as whether opcache and JIT are on, -- the system libraries, such as glibc, which come with the runner's operating system image. +- the system libraries, such as glibc, which come with the runner's operating system image, +- the environment variables the cases keep, see [`withEnvironmentVariables()`](#configuration). Expected numbers measured in one environment only fit that environment. So: diff --git a/src/Cli/RunCommand.php b/src/Cli/RunCommand.php index a7eeda7..e242613 100644 --- a/src/Cli/RunCommand.php +++ b/src/Cli/RunCommand.php @@ -18,6 +18,8 @@ use Tally\Report\Report; use Tally\Result\ResultsDirectory; use Tally\Runner\CodSpeedException; +use function array_flip; +use function array_intersect_key; use function array_map; use function count; use function ctype_digit; @@ -42,7 +44,7 @@ final class RunCommand implements Command { /** - * @param array $environment of tally, the workers get it too + * @param array $environment of tally, the workers get the variables the configuration keeps */ public function __construct( private SelectedCases $selectedCases, @@ -131,7 +133,7 @@ public function execute(Input $input, Output $output): int $jobs, )); try { - $environment = [...$this->environment, 'VALGRIND_LIB' => $valgrindTools]; + $environment = [...array_intersect_key($this->environment, array_flip($configuration->environmentVariables())), 'VALGRIND_LIB' => $valgrindTools]; $measurements = $this->measure($located, $settings, $environment, $jobs, $input->option('output'), $output); } catch (RuntimeException $e) { $output->error($e->getMessage()); diff --git a/src/Configuration.php b/src/Configuration.php index c489876..1337329 100644 --- a/src/Configuration.php +++ b/src/Configuration.php @@ -29,6 +29,7 @@ final class Configuration /** * @param list $paths * @param list $iniEntries + * @param list $environmentVariables */ private function __construct( private array $paths, @@ -37,7 +38,8 @@ private function __construct( private int $countedTimeLimitSeconds, private TolerancePolicy $tolerance, private TolerancePolicy $growthTolerance, - private bool $namedArguments + private bool $namedArguments, + private array $environmentVariables ) { } @@ -57,6 +59,9 @@ public static function create(): self // grows makes the counts of PHPStan's benchmarks up to 9 % more than the formula. new RelativeTolerance(0.1), false, + // what PHP, a shell, and the locale need; the rest, such as the branch name in CI, would + // move the memory layout of the counted process, and with it the counts of some benchmarks + ['PATH', 'HOME', 'TMPDIR', 'LANG', 'LC_ALL', 'PHPRC', 'PHP_INI_SCAN_DIR'], ); } @@ -132,6 +137,22 @@ public function withNamedArguments(bool $namedArguments): self return $configuration; } + /** + * The environment variables the processes that run the cases keep from that of tally run. The + * others are left out: the environment is on the stack of the counted process, so a variable + * that differs from run to run, such as the branch name in CI, moves its memory layout, and + * that moves the instructions and cycles of a benchmark whose allocations depend on it. + * + * @param list $names + */ + public function withEnvironmentVariables(array $names): self + { + $configuration = clone $this; + $configuration->environmentVariables = $names; + + return $configuration; + } + /** * @return list */ @@ -173,4 +194,12 @@ public function namedArguments(): bool return $this->namedArguments; } + /** + * @return list + */ + public function environmentVariables(): array + { + return $this->environmentVariables; + } + } diff --git a/tests/Fixtures/Environment/EnvironmentBench.php b/tests/Fixtures/Environment/EnvironmentBench.php new file mode 100644 index 0000000..03b1d81 --- /dev/null +++ b/tests/Fixtures/Environment/EnvironmentBench.php @@ -0,0 +1,18 @@ +withPaths([__DIR__]) + ->withEnvironmentVariables(['PATH', 'PHP_INI_SCAN_DIR', 'TALLY_EXAMPLE']); diff --git a/tests/Fixtures/Run/tally.php b/tests/Fixtures/Run/tally.php new file mode 100644 index 0000000..798f2ce --- /dev/null +++ b/tests/Fixtures/Run/tally.php @@ -0,0 +1,7 @@ +withEnvironmentVariables(['PATH', 'PHP_INI_SCAN_DIR', 'TALLY_FAKE_COUNTS']); diff --git a/tests/Integration/Cli/RunCommandTest.php b/tests/Integration/Cli/RunCommandTest.php index 7a9e922..55f204e 100644 --- a/tests/Integration/Cli/RunCommandTest.php +++ b/tests/Integration/Cli/RunCommandTest.php @@ -12,6 +12,7 @@ use Tally\Result\ResultsDirectory; use Tally\Tests\InProcessTally; use function array_map; +use function array_slice; use function fclose; use function file_get_contents; use function fwrite; @@ -46,7 +47,7 @@ protected function setUp(): void */ public function testCountsComparesAndKeepsTheResults(): void { - [$exitCode, $output, $errorOutput] = InProcessTally::run(__DIR__, $this->environment(), ['run', '--jobs=2', '--output=' . $this->output, self::RUN]); + [$exitCode, $output, $errorOutput] = self::runWithFixtureConfiguration($this->environment(), ['run', '--jobs=2', '--output=' . $this->output, self::RUN]); $this->assertSame(1, $exitCode, $errorOutput); $this->assertStringStartsWith("Tally counts 4 cases under CodSpeed's CPU simulation, 2 at once.\n\n", $output); @@ -83,7 +84,7 @@ public function testComparesTheGrowthWithTheBaseThatFinishesLater(): void $growth . '@large' => [2500, 3000, 100, 150], $growth . '@small' => [1100, 1500, 100, 150], ], JSON_THROW_ON_ERROR); - [$exitCode, $output, $errorOutput] = InProcessTally::run(__DIR__, $environment, ['run', '--jobs=1', '--filter=@large', '--output=' . $this->output . '-growth', __DIR__ . '/../../Fixtures/Growth']); + [$exitCode, $output, $errorOutput] = self::runWithFixtureConfiguration($environment, ['run', '--jobs=1', '--filter=@large', '--output=' . $this->output . '-growth', __DIR__ . '/../../Fixtures/Growth']); $this->assertSame(1, $exitCode, $errorOutput); $this->assertSame( @@ -108,7 +109,7 @@ public function testComparesTheGrowthWithTheBaseThatFinishesLater(): void public function testPassesWhenEveryCaseMatches(): void { - [$exitCode, $output, $errorOutput] = InProcessTally::run(__DIR__, $this->environment(), ['run', '--filter=benchOk', self::RUN]); + [$exitCode, $output, $errorOutput] = self::runWithFixtureConfiguration($this->environment(), ['run', '--filter=benchOk', self::RUN]); $this->assertSame(0, $exitCode, $errorOutput); $this->assertStringContainsString('Tally counts 1 case under', $output); @@ -117,7 +118,7 @@ public function testPassesWhenEveryCaseMatches(): void public function testAShardWithoutCases(): void { - [$exitCode, $output, $errorOutput] = InProcessTally::run(__DIR__, $this->environment(), ['run', '--shard=5/5', self::RUN]); + [$exitCode, $output, $errorOutput] = self::runWithFixtureConfiguration($this->environment(), ['run', '--shard=5/5', self::RUN]); $this->assertSame(0, $exitCode, $errorOutput); $this->assertSame("This shard has no cases to count.\n", $output); @@ -125,7 +126,7 @@ public function testAShardWithoutCases(): void public function testFailsBeforeCountingWhenAnExpectationCannotBeFound(): void { - [$exitCode, $output, $errorOutput] = InProcessTally::run(__DIR__, $this->environment(), ['run', __DIR__ . '/../../Fixtures/NonLiteral']); + [$exitCode, $output, $errorOutput] = self::runWithFixtureConfiguration($this->environment(), ['run', __DIR__ . '/../../Fixtures/NonLiteral']); $this->assertSame(2, $exitCode); $this->assertSame('', $output); @@ -136,7 +137,7 @@ public function testAPartMissingFromTheProfileIsNotMeasured(): void { $environment = ['TALLY_FAKE_COUNTS' => '{}'] + $this->environment(); - [$exitCode, $output] = InProcessTally::run(__DIR__, $environment, ['run', '--filter=benchOk', self::RUN]); + [$exitCode, $output] = self::runWithFixtureConfiguration($environment, ['run', '--filter=benchOk', self::RUN]); $this->assertSame(1, $exitCode); $this->assertStringContainsString("Callgrind's profile has no part named " . self::BENCH . '::benchOk.', $output); @@ -161,7 +162,7 @@ public function testWritesTheJobSummaryAndAnnotationsInGitHubActions(): void ] + $this->environment(); $repository = (string) realpath(__DIR__ . '/../../..'); - [$exitCode, $output] = InProcessTally::run($repository, $environment, ['run', '--filter=/benchSlower|benchNew/', self::RUN]); + [$exitCode, $output] = InProcessTally::run($repository, $environment, ['run', '--config=' . self::RUN . '/tally.php', '--filter=/benchSlower|benchNew/', self::RUN]); $this->assertSame(1, $exitCode); $this->assertStringContainsString('::error file=tests/Fixtures/Run/RunBench.php,line=16,title=', $output); @@ -188,7 +189,7 @@ public function testAShardWritesNoAnnotationsAndNoJobSummary(): void $summary = $this->output . '-shard-summary.md'; $environment = ['GITHUB_ACTIONS' => 'true', 'GITHUB_STEP_SUMMARY' => $summary] + $this->environment(); - [$exitCode, $output] = InProcessTally::run((string) realpath(__DIR__ . '/../../..'), $environment, ['run', '--filter=benchSlower', '--shard=1/1', self::RUN]); + [$exitCode, $output] = InProcessTally::run((string) realpath(__DIR__ . '/../../..'), $environment, ['run', '--config=' . self::RUN . '/tally.php', '--filter=benchSlower', '--shard=1/1', self::RUN]); $this->assertSame(1, $exitCode); $this->assertStringContainsString('Benchmarks failed: 1 slower', $output); @@ -200,7 +201,7 @@ public function testAnnotationsPointToFilesInTheWorkspaceFromASubdirectory(): vo { $environment = ['GITHUB_ACTIONS' => 'true', 'GITHUB_WORKSPACE' => (string) realpath(__DIR__ . '/../../..')] + $this->environment(); - [, $output] = InProcessTally::run(__DIR__, $environment, ['run', '--filter=benchSlower', self::RUN]); + [, $output] = self::runWithFixtureConfiguration($environment, ['run', '--filter=benchSlower', self::RUN]); $this->assertStringContainsString('::error file=tests/Fixtures/Run/RunBench.php,line=16,title=', $output); } @@ -209,7 +210,7 @@ public function testReportsAJobSummaryThatCannotBeWritten(): void { $environment = ['GITHUB_ACTIONS' => 'true', 'GITHUB_STEP_SUMMARY' => '/nonexistent/summary.md'] + $this->environment(); - [, , $errorOutput] = InProcessTally::run(__DIR__, $environment, ['run', '--filter=benchOk', self::RUN]); + [, , $errorOutput] = self::runWithFixtureConfiguration($environment, ['run', '--filter=benchOk', self::RUN]); $this->assertSame("Cannot write the job summary to /nonexistent/summary.md.\n", $errorOutput); } @@ -229,7 +230,7 @@ public function testRefusesToStartWithoutABuildOfValgrind(): void public function testReportsAResultsDirectoryThatCannotBeCreated(): void { - [$exitCode, , $errorOutput] = InProcessTally::run(__DIR__, $this->environment(), ['run', '--filter=benchOk', '--output=/nonexistent/results', self::RUN]); + [$exitCode, , $errorOutput] = self::runWithFixtureConfiguration($this->environment(), ['run', '--filter=benchOk', '--output=/nonexistent/results', self::RUN]); $this->assertSame(2, $exitCode); $this->assertSame("Cannot create /nonexistent/results.\n", $errorOutput); @@ -248,12 +249,45 @@ public static function dataInvalidJobs(): iterable #[DataProvider('dataInvalidJobs')] public function testInvalidJobs(string $jobs): void { - [$exitCode, , $errorOutput] = InProcessTally::run(__DIR__, $this->environment(), ['run', '--jobs=' . $jobs, self::RUN]); + [$exitCode, , $errorOutput] = self::runWithFixtureConfiguration($this->environment(), ['run', '--jobs=' . $jobs, self::RUN]); $this->assertSame(2, $exitCode); $this->assertStringStartsWith(sprintf("Option --jobs needs a positive number, %s given.\n", $jobs), $errorOutput); } + /** + * Runs tally with the configuration of the fixtures, which keeps TALLY_FAKE_COUNTS for the fake Valgrind. + * + * @param array $environment + * @param non-empty-list $arguments the command first + * @return array{int, string, string} + */ + private static function runWithFixtureConfiguration(array $environment, array $arguments): array + { + return InProcessTally::run(__DIR__, $environment, [$arguments[0], '--config=' . self::RUN . '/tally.php', ...array_slice($arguments, 1)]); + } + + /** + * @return iterable, string}> + */ + public static function dataEnvironment(): iterable + { + yield 'the default leaves out the others' => [[], 'TALLY_EXAMPLE is false, PATH is set.']; + yield 'a configured variable is kept' => [['--config=' . __DIR__ . '/../../Fixtures/Environment/tally.php'], "TALLY_EXAMPLE is 'kept', PATH is set."]; + } + + /** + * @param list $config + */ + #[DataProvider('dataEnvironment')] + public function testTheCasesGetOnlyTheEnvironmentVariablesOfTheConfiguration(array $config, string $seen): void + { + [$exitCode, $output] = InProcessTally::run(__DIR__, ['TALLY_EXAMPLE' => 'kept'] + $this->environment(), ['run', ...$config, __DIR__ . '/../../Fixtures/Environment']); + + $this->assertSame(1, $exitCode); + $this->assertStringContainsString("\n RuntimeException: " . $seen . "\n", $output); + } + /** * @return array */ diff --git a/tests/Unit/ConfigurationTest.php b/tests/Unit/ConfigurationTest.php index 8f7c78d..996dc43 100644 --- a/tests/Unit/ConfigurationTest.php +++ b/tests/Unit/ConfigurationTest.php @@ -20,6 +20,7 @@ public function testDefaults(): void $this->assertSame('±2 % instructions, ±5 % cycles but at least ±2,000, ±5 % memory', $configuration->tolerance()->describe()); $this->assertSame('±10 % of each metric', $configuration->growthTolerance()->describe()); $this->assertFalse($configuration->namedArguments()); + $this->assertSame(['PATH', 'HOME', 'TMPDIR', 'LANG', 'LC_ALL', 'PHPRC', 'PHP_INI_SCAN_DIR'], $configuration->environmentVariables()); } public function testWithersKeepTheOriginal(): void @@ -34,6 +35,7 @@ public function testWithersKeepTheOriginal(): void $this->assertSame('±30 % of each metric', $original->withTolerance(new RelativeTolerance(0.3))->tolerance()->describe()); $this->assertSame('±40 % of each metric', $original->withGrowthTolerance(new RelativeTolerance(0.4))->growthTolerance()->describe()); $this->assertTrue($original->withNamedArguments(true)->namedArguments()); + $this->assertSame(['PATH', 'CI'], $original->withEnvironmentVariables(['PATH', 'CI'])->environmentVariables()); $this->assertEquals(Configuration::create(), $original); }