From 812545b137c1c82b6ab821468bef01e1edbe4e3b Mon Sep 17 00:00:00 2001
From: Bill Schnurr
Date: Tue, 27 Oct 2026 16:15:32 -0700
Subject: [PATCH 1/6] Isolate benchmark configs and enforce source-file parity
Use explicit checker configurations and a shared implementation-first source manifest, reject results without verified native file counts, and report inference and memory-comparison limitations.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 35cc6021-56a0-45b2-9b66-3817d947cd99
---
.../workflows/typecheck_benchmark_weekly.yml | 3 +
build/benchmark/README.md | 116 +++++-
build/benchmark/compare_benchmarks.py | 4 +-
build/benchmark/render_benchmark_html.py | 39 +-
build/benchmark/test_compare_benchmarks.py | 33 +-
build/benchmark/test_render_benchmark_html.py | 39 ++
build/benchmark/test_typecheck_benchmark.py | 348 +++++++++++++++++-
build/benchmark/typecheck_benchmark.py | 285 +++++++++++---
8 files changed, 796 insertions(+), 71 deletions(-)
diff --git a/.github/workflows/typecheck_benchmark_weekly.yml b/.github/workflows/typecheck_benchmark_weekly.yml
index 163f1408737e..3fe5f2845665 100644
--- a/.github/workflows/typecheck_benchmark_weekly.yml
+++ b/.github/workflows/typecheck_benchmark_weekly.yml
@@ -92,6 +92,9 @@ jobs:
checkers+=(pyright-threads)
extra_args+=(--skip-pyright-build)
fi
+ if [[ '${{ matrix.checker }}' == 'ty' ]]; then
+ checkers+=(ty-infer)
+ fi
python build/benchmark/typecheck_benchmark.py \
-c "${checkers[@]}" -r 3 -w 1 -t 1800 \
--memory-limit-mb 8192 --os-name linux-x64 \
diff --git a/build/benchmark/README.md b/build/benchmark/README.md
index 0ccec430ef5b..585240361834 100644
--- a/build/benchmark/README.md
+++ b/build/benchmark/README.md
@@ -17,8 +17,9 @@ additional dependencies. Repository commits are pinned so pull-request compariso
source revisions.
Entries can define `exclude_directories` to prune named directories while resolving their check
-paths. The benchmark expands those entries to `.py` and `.pyi` files before invoking any checker, so
-all checkers analyze the same scope. NumPy excludes its test directories; pandas includes its test
+paths. The benchmark expands every entry to a sorted, deduplicated `.py`/`.pyi` manifest before
+invoking any checker. When both are selected for the same module, the `.py` implementation takes
+precedence over its `.pyi`; standalone stubs remain included. NumPy excludes its test directories; pandas includes its test
and `_testing` directories. Generated Pyright configurations explicitly retain
its standard `**/node_modules`, `**/__pycache__`, and `**/.*` exclusions.
@@ -92,8 +93,111 @@ it afterward. If Pyright is selected, the local Pyright production bundle is sti
`--skip-pyright-build` is supplied.
Checker configuration files use unique temporary names in the target directory and are removed after
-each invocation. Existing `pyrightconfig.json`, mypy, ty, or Pyrefly configuration files are not
-overwritten. The temporary configuration selects the full local directory.
+each invocation. Every checker receives an explicit configuration, including Zuban, whose mode is
+also explicitly `default`. Existing repository configuration files are not overwritten or extended.
+The temporary configuration selects the full local directory.
+
+## Fairness and interpretation
+
+The benchmark is an end-to-end CLI throughput measurement, not an equal-work or quality-adjusted
+ranking. A slower result does not by itself prove that a checker does more analysis. Different
+inference algorithms, import graphs, bundled stubs, diagnostics, threading, startup, and diagnostic
+formatting all contribute to wall time.
+
+All checkers target the running interpreter's Python major/minor version and receive its interpreter
+path. ty and Pyrefly ignore `.gitignore` and `.ignore`; Zuban disables gitignore exclusions.
+ty also merges user-level configuration even when `--config-file` is supplied, so both ty profiles
+run with fresh empty `APPDATA`/`XDG_CONFIG_HOME` config directories, removed after each invocation.
+ty's built-in directory exclusions are reset before applying Pyright's standard exclusions, and
+Pyrefly's exclusion heuristics are disabled in favor of those same exclusions. This avoids silently
+skipping selected `dist`, `venv`, or ignored source files, or checking hidden directories only in ty.
+Imported dependencies may still be analyzed differently; matching entry-point scope does not imply
+matching transitive work.
+
+Every checker receives the same explicit source-file manifest, not independently expanded directory
+roots. A missing configured path or empty selection is an error; neither falls back to checking the
+whole repository. mypy and Zuban use explicit TOML file lists to avoid command-line size limits and
+Windows INI backslash escaping. ty's source selectors use gitignore-pattern separators even on
+Windows, where native backslashes can silently match no files.
+
+Before any warmups or measured runs, each checker performs one **untimed file-count audit** using
+the benchmark configuration and the selected files. Pyright reports checked files in `--stats`,
+threaded Pyright reports files to analyze with `--verbose`, Pyrefly reports project modules with
+`--summary=full`, ty logs checked file paths with `-vv`, and mypy/Zuban report checked source-file
+counts in their summaries. Missing counts, fatal errors, and counts different from the manifest
+reject that checker's measurements. Verbose/debug audit output is not enabled for measured runs.
+Counts in ordinary checker output are also checked during warmups and measured runs.
+The audit is recorded separately as `uncounted_validation_runs_per_checker = 1`, including when
+`--warmup 0` is selected; this is therefore not a cold-filesystem measurement.
+
+Result JSON retains `source_files`, `source_file_count`, and `source_manifest_sha256` per package;
+each successful checker records its native `files_selected`, `file_count_verified`, and separate
+`file_count_audit_time_s`. HTML cells show the verified count. The report rejects different
+manifests for the same package even if counts match, and the regression comparator also checks the
+manifest hash and audit-run setting. These are source entry-point counts, **not total imported or
+parsed files**; forcing equal dependency counts across different checkers would change their work.
+Counts also do not prove equal diagnostic coverage: repository and user configs are tested
+separately with deliberately suppressed diagnostics, and source-level ignore comments retain each
+tool's semantics.
+
+The `ty` profile retains ty's default return-type inference. The separately selectable `ty-infer`
+profile uses the same binary and configuration except for
+`analysis.return-type-inference = "always"`. Pyrefly explicitly uses its `default` preset,
+`check-unannotated-defs = true`, and `infer-return-types = "checked"`. Pyright uses `basic` mode with
+unannotated-function analysis enabled by default; mypy uses `check_untyped_defs = True`.
+Similar option names or presets do not imply equivalent inference. In particular, ty documents
+limitations for generator and asynchronous return inference even with `"always"`.
+
+```console
+python build/benchmark/typecheck_benchmark.py -c ty ty-infer -r 3 -w 1
+```
+
+For defensible cross-checker comparisons, use the same machine, pinned source commits, dependency
+versions, interpreter, target platform, source scope, and resource/thread budgets. The weekly jobs
+currently use separate hosted runners and install dependencies without a shared version lock, so
+their results are independent trends rather than a controlled speed ranking. Repeated fresh
+processes after a warmup measure a warm-filesystem CLI workload, not editor incrementality or a
+cold-disk workload. Alternate run order for paired experiments and report the individual samples
+and dispersion, not only a speedup.
+
+Do not divide total runtime by enabled rule count or emitted diagnostic count. Rules differ in
+granularity and share parsing, import resolution, inference, constraint solving, and caches.
+Disabling a diagnostic often suppresses reporting without removing its prerequisite analysis.
+For a feature-cost investigation, use focused workloads (for example overload resolution,
+protocol matching, generics, narrowing, unannotated returns, and typed versus untyped imports),
+verify expected diagnostics and inferred types outside the timed runs, and compare paired
+feature-on/feature-off measurements within each checker where supported. Report these as marginal
+costs on that workload, not additive or portable "time per rule". Report type-system conformance
+and curated false-positive/false-negative cases separately from throughput; the
+[Python typing conformance suite](https://github.com/python/typing/tree/main/conformance) is a useful
+shared specification, but not a performance weighting scheme.
+
+Linux peak RSS currently polls only `/proc//status`; it does not sum child-process memory.
+In particular, Pyright `--threads` RSS must not be interpreted as total worker memory or a memory
+improvement over single-process Pyright. Windows RSS is unavailable. Neither limitation prevents
+wall-time measurements, but neither supports cross-process memory rankings.
+
+Configuration changes are recorded as `checker_config_version` in result JSON. Comparisons reject
+different versions, including pre-versioned results, so regenerate baselines rather than comparing
+new file-selection or interpreter settings against old measurements.
+Pyrefly's exit-code-1 "no files matched" error is a failed measurement, not an ordinary type error.
+
+The opt-in integration tests execute the real binaries against conflicting root/nested configs,
+ignored files, directory-exclusion probes, and paired implementations/stubs. They verify every
+checker's native count against the same manifest, including clean files and standalone stubs,
+and check that `ty-infer` catches an inferred-return
+assignment that the default profile does not. Install all default checkers and build Pyright first,
+then set `TYPECHECK_BENCHMARK_INTEGRATION=1` and run:
+
+```console
+python -m unittest discover -s build/benchmark -p test_typecheck_benchmark.py
+```
+
+Configuration references:
+[ty configuration](https://docs.astral.sh/ty/configuration/),
+[ty inference and source options](https://docs.astral.sh/ty/reference/configuration/),
+[Pyrefly configuration selection and presets](https://pyrefly.org/en/docs/configuration/),
+and [Zuban modes and configuration](https://docs.zubanls.com/en/latest/usage.html).
## Developer workflow
@@ -171,7 +275,7 @@ that first introduces them cannot trigger itself.
| Flag | Description |
| --- | --- |
-| `-c, --checkers NAME [NAME ...]` | Checkers to run; choices are `pyright`, `pyright-threads`, `pyright-pip`, `pyrefly`, `ty`, `mypy`, and `zuban` |
+| `-c, --checkers NAME [NAME ...]` | Checkers to run; choices are `pyright`, `pyright-threads`, `pyright-pip`, `pyrefly`, `ty`, `ty-infer`, `mypy`, and `zuban` |
| `-r, --runs N` | Measured runs per checker (default: 5) |
| `-w, --warmup N` | Warmup runs discarded before measurement (default: 1) |
| `-t, --timeout SECONDS` | Timeout for each checker invocation (default: 300) |
@@ -242,6 +346,8 @@ peak-memory charts. Reports and artifacts are published before a failed comparis
unsuccessful.
The weekly workflow runs Pyright, Pyrefly, ty, mypy, and Zuban in independent hosted-runner jobs.
+The ty job runs `ty` and `ty-infer` sequentially against the same prepared corpus, with separate
+labels and results; it does not replace ty's default behavior with a stricter configuration.
The Pyright job measures single-threaded mode and `--threads` mode sequentially on the same runner and
prepared package corpus. Each mode performs three measured runs after one warmup. The report includes
a per-package Pyright threading chart with wall time, peak RSS, and speedup. The aggregate job stores
diff --git a/build/benchmark/compare_benchmarks.py b/build/benchmark/compare_benchmarks.py
index bbc265e9a5a2..ab50c3f4d473 100644
--- a/build/benchmark/compare_benchmarks.py
+++ b/build/benchmark/compare_benchmarks.py
@@ -157,7 +157,9 @@ def _analyze(
"node_options",
"runs_per_package",
"warmup_runs",
+ "uncounted_validation_runs_per_checker",
"dependency_isolation",
+ "checker_config_version",
):
if baseline.get(field) != candidate.get(field):
failures.append(
@@ -257,7 +259,7 @@ def _analyze(
)
continue
scope_changed = False
- for field in ("check_paths", "exclude_directories"):
+ for field in ("check_paths", "exclude_directories", "source_manifest_sha256"):
if old_package.get(field, []) != new_package.get(field, []):
failures.append(
f"{package}/{checker}: package {field} changed from "
diff --git a/build/benchmark/render_benchmark_html.py b/build/benchmark/render_benchmark_html.py
index e06740b47a7d..4f33e60dd538 100644
--- a/build/benchmark/render_benchmark_html.py
+++ b/build/benchmark/render_benchmark_html.py
@@ -9,10 +9,12 @@
from typing import Any
-CHECKER_ORDER = ["pyright", "pyright-threads", "pyrefly", "ty", "mypy", "zuban"]
+CHECKER_ORDER = ["pyright", "pyright-threads", "pyrefly", "ty", "ty-infer", "mypy", "zuban"]
def _checker_label(checker: str) -> str:
+ if checker == "ty-infer":
+ return "ty (return inference)"
return "pyright (threads)" if checker == "pyright-threads" else checker
@@ -56,6 +58,9 @@ def _phase_time(phase_times: dict[str, Any], name: str) -> str:
def render_html(results: dict[str, dict[str, Any]]) -> str:
if not results:
raise ValueError("At least one checker result is required")
+ config_version = next(iter(results.values())).get("checker_config_version")
+ if any(data.get("checker_config_version") != config_version for data in results.values()):
+ raise ValueError("Checker results use different benchmark configuration versions")
checkers = _ordered_checkers(results)
packages = sorted(
{
@@ -68,6 +73,23 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
metrics = {
checker: _package_metrics(results[checker], checker) for checker in checkers
}
+ source_scopes: dict[str, tuple[int, str]] = {}
+ for checker in checkers:
+ for package in results[checker].get("results", []):
+ manifest_hash = package.get("source_manifest_sha256")
+ if manifest_hash is None:
+ continue
+ name = package["package_name"]
+ scope = (package["source_file_count"], manifest_hash)
+ if name in source_scopes and source_scopes[name] != scope:
+ raise ValueError(f"{name}: checker results use different source manifests")
+ source_scopes[name] = scope
+ metric = metrics[checker].get(name, {})
+ if metric.get("ok") and (
+ not metric.get("file_count_verified")
+ or metric.get("files_selected") != scope[0]
+ ):
+ raise ValueError(f"{name}/{checker}: source-file count is not verified")
totals = {
checker: float(
results[checker].get("aggregate", {})
@@ -110,10 +132,18 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
continue
cells.append(
f'| {float(metric["execution_time_s"]):.2f}s'
- f'{float(metric.get("peak_memory_mb", 0.0)):.0f} MB | '
+ f'{float(metric.get("peak_memory_mb", 0.0)):.0f} MB'
+ + (
+ f'{metric["files_selected"]} source files verified'
+ if metric.get("file_count_verified") else ""
+ ) + ''
)
+ scope_label = (
+ f'{source_scopes[package][0]} source files selected'
+ if package in source_scopes else ""
+ )
package_rows.append(
- f'| {html.escape(package)} | {"".join(cells)}
'
+ f'| {html.escape(package)}{scope_label} | {"".join(cells)}
'
)
pyright_stats_rows: list[str] = []
@@ -234,6 +264,9 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
Run summary
Generated {html.escape(timestamp)} · Python {html.escape(str(environment.get('python_version', 'unknown')))} · {html.escape(str(environment.get('platform_details', environment.get('platform', 'unknown'))))}
| Checker | Measured | Failed | Total time | Median time | Median RSS |
{''.join(summary_rows)}
Package comparison
Different checker jobs run on separate hosted runners, so timings are independent trends rather than a fastest-checker ranking. The two Pyright modes run sequentially in one job. Cells show wall time and peak RSS.
+Checker defaults, inference, and diagnostic coverage differ; these timings do not measure equivalent analysis or time per rule. The two ty modes run sequentially in one job: ty retains its default return inference, while ty (return inference) enables inference for unannotated returns. Neither mode guarantees Pyright-equivalent inference.
+Linux RSS measures the parent checker process, not its child processes; multi-process memory totals are not comparable.
+Verified source-file counts come from an untimed checker audit against a shared implementation-first manifest. Imported dependencies are not included. Historical results without verified counts have not passed this audit.
| Package | {checker_headers}
{''.join(package_rows)}
{threading_section}
diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py
index a1086b4e7971..65ad663649c5 100644
--- a/build/benchmark/test_compare_benchmarks.py
+++ b/build/benchmark/test_compare_benchmarks.py
@@ -238,6 +238,18 @@ def test_accepts_timeout_limit_change(self) -> None:
self.assertEqual(failures, [])
+ def test_rejects_changed_checker_configuration(self) -> None:
+ candidate = _result(10.0, 100.0)
+ candidate["checker_config_version"] = 2
+ with redirect_stdout(io.StringIO()):
+ failures = compare_benchmarks.compare(
+ _result(10.0, 100.0), candidate, 10.0
+ )
+ self.assertEqual(
+ failures,
+ ["environment mismatch for checker_config_version: None != 2"],
+ )
+
def test_reports_success_without_failed_baseline(self) -> None:
baseline = _result(0.0, 0.0, ok=False)
candidate = _result(12.0, 345.0)
@@ -326,7 +338,7 @@ def test_rejects_node_options_mismatch(self) -> None:
],
)
- def test_allows_uncounted_validation_run_mismatch(self) -> None:
+ def test_rejects_uncounted_validation_run_mismatch(self) -> None:
candidate = _result(10.0, 100.0)
candidate["uncounted_validation_runs_per_checker"] = 1
@@ -335,7 +347,22 @@ def test_allows_uncounted_validation_run_mismatch(self) -> None:
_result(10.0, 100.0), candidate, 10.0
)
- self.assertEqual(failures, [])
+ self.assertEqual(
+ failures,
+ ["environment mismatch for uncounted_validation_runs_per_checker: 0 != 1"],
+ )
+
+ def test_rejects_different_source_manifests(self) -> None:
+ baseline = _result(10.0, 100.0)
+ candidate = _result(10.0, 100.0)
+ baseline["results"][0]["source_manifest_sha256"] = "old"
+ candidate["results"][0]["source_manifest_sha256"] = "new"
+ with redirect_stdout(io.StringIO()):
+ failures = compare_benchmarks.compare(baseline, candidate, 10.0)
+ self.assertEqual(
+ failures,
+ ["example/pyright: package source_manifest_sha256 changed from old to new"],
+ )
def test_rejects_package_commit_mismatch(self) -> None:
candidate = _result(10.0, 100.0)
@@ -579,7 +606,7 @@ def test_workflows_use_current_pnpm_setup(self) -> None:
)
self.assertEqual(
weekly_benchmark_steps[9]["run"],
- "checkers=('${{ matrix.checker }}')\nextra_args=()\nif [[ '${{ matrix.checker }}' == 'pyright' ]]; then\n checkers+=(pyright-threads)\n extra_args+=(--skip-pyright-build)\nfi\npython build/benchmark/typecheck_benchmark.py \\\n -c \"${checkers[@]}\" -r 3 -w 1 -t 1800 \\\n --memory-limit-mb 8192 --os-name linux-x64 \\\n --output build/benchmark/results \\\n \"${extra_args[@]}\"\n",
+ "checkers=('${{ matrix.checker }}')\nextra_args=()\nif [[ '${{ matrix.checker }}' == 'pyright' ]]; then\n checkers+=(pyright-threads)\n extra_args+=(--skip-pyright-build)\nfi\nif [[ '${{ matrix.checker }}' == 'ty' ]]; then\n checkers+=(ty-infer)\nfi\npython build/benchmark/typecheck_benchmark.py \\\n -c \"${checkers[@]}\" -r 3 -w 1 -t 1800 \\\n --memory-limit-mb 8192 --os-name linux-x64 \\\n --output build/benchmark/results \\\n \"${extra_args[@]}\"\n",
)
weekly_report_steps = weekly_workflow_data["jobs"]["report"]["steps"]
self.assertEqual(
diff --git a/build/benchmark/test_render_benchmark_html.py b/build/benchmark/test_render_benchmark_html.py
index 4bd37c38627a..8f33c70d7215 100644
--- a/build/benchmark/test_render_benchmark_html.py
+++ b/build/benchmark/test_render_benchmark_html.py
@@ -69,6 +69,45 @@ def handle_data(self, data: str) -> None:
class RenderBenchmarkHtmlTest(unittest.TestCase):
+ def test_requires_matching_manifests_and_verified_native_counts(self) -> None:
+ pyright = _result("pyright", 2.0, 200.0)
+ ty = _result("ty", 1.0, 100.0)
+ for checker, result in (("pyright", pyright), ("ty", ty)):
+ result["checker_config_version"] = 4
+ result["results"][0].update(
+ source_file_count=5, source_manifest_sha256="same"
+ )
+ result["results"][0]["metrics"][checker].update(
+ files_selected=5, file_count_verified=True
+ )
+ report = render_benchmark_html.render_html({"pyright": pyright, "ty": ty})
+ self.assertIn("5 source files selected", report)
+ self.assertEqual(report.count("5 source files verified"), 2)
+
+ ty["checker_config_version"] = 3
+ with self.assertRaisesRegex(ValueError, "different benchmark configuration versions"):
+ render_benchmark_html.render_html({"pyright": pyright, "ty": ty})
+ ty["checker_config_version"] = 4
+ ty["results"][0]["source_manifest_sha256"] = "different"
+ with self.assertRaisesRegex(ValueError, "different source manifests"):
+ render_benchmark_html.render_html({"pyright": pyright, "ty": ty})
+ ty["results"][0]["source_manifest_sha256"] = "same"
+ ty["results"][0]["metrics"]["ty"]["files_selected"] = 4
+ with self.assertRaisesRegex(ValueError, "source-file count is not verified"):
+ render_benchmark_html.render_html({"pyright": pyright, "ty": ty})
+
+ def test_renders_ty_inference_as_a_separate_profile(self) -> None:
+ report = render_benchmark_html.render_html(
+ {
+ "ty-infer": _result("ty-infer", 2.0, 200.0),
+ "ty": _result("ty", 1.0, 100.0),
+ }
+ )
+ parser = _SemanticParser()
+ parser.feed(report)
+ self.assertLess(parser.cells.index("ty"), parser.cells.index("ty (return inference)"))
+ self.assertIn("do not measure equivalent analysis or time per rule", report)
+
def test_loads_one_result_per_checker(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
diff --git a/build/benchmark/test_typecheck_benchmark.py b/build/benchmark/test_typecheck_benchmark.py
index 96be4b97e6a2..85b934dc1342 100644
--- a/build/benchmark/test_typecheck_benchmark.py
+++ b/build/benchmark/test_typecheck_benchmark.py
@@ -1,7 +1,9 @@
import io
import json
import os
+import subprocess
import tempfile
+import tomllib
import unittest
from contextlib import redirect_stdout
from pathlib import Path
@@ -30,6 +32,45 @@ def test_resolve_check_paths_excludes_named_directories(self) -> None:
{"source/module.py", "source/types.pyi"},
)
+ def test_manifest_prefers_implementations_deduplicates_and_prunes_defaults(self) -> None:
+ source = self.root / "source"
+ source.mkdir()
+ for name in ("module.py", "module.pyi", "stub.pyi", "clean.py", ".hidden.py"):
+ (source / name).write_text("")
+ for name in ("node_modules", "__pycache__", ".hidden", "tests"):
+ (source / name).mkdir()
+ (source / name / "ignored.py").write_text("")
+ paths, missing = benchmark._resolve_check_paths(
+ self.root, ["source", "source/module.py", "missing"], ["tests"]
+ )
+ self.assertEqual(missing, ["missing"])
+ self.assertEqual(
+ [path.relative_to(source).name for path in paths],
+ ["clean.py", "module.py", "stub.pyi"],
+ )
+ stub_only, _ = benchmark._resolve_check_paths(
+ self.root, ["source/module.pyi"], []
+ )
+ self.assertEqual(stub_only, [source / "module.pyi"])
+
+ def test_source_file_counts_do_not_include_imported_dependencies(self) -> None:
+ cases = [
+ ("pyright", "Total files parsed and bound: 500\nTotal files checked: 5\n", 5),
+ ("pyright-threads", "Found 5 files to analyze\nFound 5 source files\n", 5),
+ ("pyrefly", " INFO 5 modules (500 dependent modules); 1000 lines\n", 5),
+ ("mypy", "Found 1 error in 1 file (checked 5 source files)\n", 5),
+ ("zuban", "Success: no issues found in 1 source file\n", 1),
+ ("ty", "DEBUG Checking file 'first.py'\nDEBUG Checking file 'first.py'\n"
+ "DEBUG Checking file 'second.pyi'\nDEBUG Checking all files took 0.1s\n", 2),
+ ("ty", "DEBUG Checking all files took 0.0s\n", 0),
+ ("ty", "All checks passed!\n", None),
+ ("pyrefly", " INFO 0 errors\n", None),
+ ("mypy", "errors prevented further checking\n", None),
+ ]
+ for checker, output, expected in cases:
+ with self.subTest(checker=checker, output=output):
+ self.assertEqual(benchmark._parse_source_file_count(checker, output), expected)
+
def setUp(self) -> None:
self.temp_dir = tempfile.TemporaryDirectory(
prefix=".typecheck-benchmark-test-",
@@ -173,6 +214,8 @@ def test_pyright_config_and_command(self) -> None:
config = json.loads(configs[0].read_text(encoding="utf-8"))
self.assertEqual(config["include"], ["src"])
self.assertEqual(config["exclude"], benchmark.PYRIGHT_DEFAULT_EXCLUDES)
+ self.assertEqual(config["pythonVersion"], benchmark.TARGET_PYTHON_VERSION)
+ self.assertEqual(command[-3:-1], ["--pythonpath", benchmark.sys.executable])
self.assertIs(config["useLibraryCodeForTypes"], True)
configs[0].unlink()
@@ -193,6 +236,8 @@ def test_threaded_pyright_uses_runner_cpu_count_without_stats(self) -> None:
"pyright.js",
"--project",
str(configs[0]),
+ "--pythonpath",
+ benchmark.sys.executable,
"--threads",
],
)
@@ -336,16 +381,68 @@ def test_packages_use_independent_dependency_environments(self) -> None:
)
self.assertEqual(checker_targets, install_targets)
- def test_zuban_uses_positional_paths_without_config(self) -> None:
- source_dir = self.root / "src"
- source_dir.mkdir()
+ def test_zuban_uses_explicit_config_and_mode(self) -> None:
+ source_dir = self.root / "src" / "package"
+ source_dir.mkdir(parents=True)
with patch.object(benchmark, "_checker_command", return_value=["zuban"]):
command, configs = benchmark._build_checker_command(
"zuban", self.root, [source_dir]
)
- self.assertEqual(command, ["zuban", "check", "src"])
- self.assertEqual(configs, [])
+ self.assertEqual(
+ command,
+ [
+ "zuban", "check", "--mode", "default",
+ "--config-file", str(configs[0]),
+ "--python-executable", benchmark.sys.executable,
+ ],
+ )
+ config = tomllib.loads(configs[0].read_text())["tool"]["mypy"]
+ self.assertEqual(config["files"], [str(source_dir.relative_to(self.root))])
+ self.assertIs(config["exclude_gitignore"], False)
+ configs[0].unlink()
+
+ def test_ty_profiles_share_scope_but_not_return_inference(self) -> None:
+ with patch.object(benchmark, "_executable", return_value="ty") as executable:
+ self.assertEqual(benchmark._checker_command("ty-infer"), ["ty"])
+ executable.assert_called_once_with("ty")
+ for checker in ("ty", "ty-infer"):
+ with self.subTest(checker=checker), patch.object(
+ benchmark, "_checker_command", return_value=["ty"]
+ ):
+ command, configs = benchmark._build_checker_command(
+ checker, self.root, [self.root / "source"]
+ )
+ config = tomllib.loads(configs[0].read_text())
+ self.assertEqual(
+ command, ["ty", "check", "--config-file", str(configs[0])]
+ )
+ self.assertEqual(
+ config["src"],
+ {
+ "include": ["source"],
+ "respect-ignore-files": False,
+ "exclude": ["!**", *benchmark.PYRIGHT_DEFAULT_EXCLUDES],
+ },
+ )
+ self.assertEqual(
+ config["environment"]["python"], benchmark.sys.executable
+ )
+ self.assertEqual(
+ config.get("analysis", {}).get("return-type-inference"),
+ "always" if checker == "ty-infer" else None,
+ )
+ configs[0].unlink()
+
+ def test_pyrefly_config_disables_ignore_files_and_pins_inference(self) -> None:
+ config_path = benchmark._write_pyrefly_config(self.root, ["source"])
+ config = tomllib.loads(config_path.read_text())
+ self.assertEqual(config["project-includes"], ["source"])
+ self.assertIs(config["use-ignore-files"], False)
+ self.assertIs(config["check-unannotated-defs"], True)
+ self.assertEqual(config["infer-return-types"], "checked")
+ self.assertEqual(config["python-interpreter-path"], benchmark.sys.executable)
+ config_path.unlink()
def test_mypy_config_preserves_testing_paths(self) -> None:
testing_dir = self.root / "testing"
@@ -360,8 +457,13 @@ def test_mypy_config_preserves_testing_paths(self) -> None:
["mypy", "--no-incremental", "--config-file", str(configs[0])],
)
self.assertEqual(
- configs[0].read_text(encoding="utf-8"),
- "[mypy]\nfiles = testing\ncheck_untyped_defs = True\n",
+ tomllib.loads(configs[0].read_text(encoding="utf-8"))["tool"]["mypy"],
+ {
+ "files": ["testing"],
+ "check_untyped_defs": True,
+ "python_version": benchmark.TARGET_PYTHON_VERSION,
+ "python_executable": benchmark.sys.executable,
+ },
)
configs[0].unlink()
@@ -454,6 +556,26 @@ def test_uncaptured_run_rejects_fatal_exit_code(self) -> None:
self.assertFalse(result["ok"])
self.assertEqual(result["error_message"], "Fatal error (exit code 2)")
+ def test_pyrefly_no_matching_files_is_not_a_successful_measurement(self) -> None:
+ process_result: benchmark.ProcessResult = {
+ "stdout": "",
+ "stderr": "No Python files matched pattern `source`\n",
+ "returncode": 1,
+ "timed_out": False,
+ "execution_time_s": 0.1,
+ "peak_memory_mb": 10.0,
+ "oom_killed": False,
+ }
+ with (
+ patch.object(benchmark, "_checker_command", return_value=["pyrefly"]),
+ patch.object(
+ benchmark, "run_process_with_timeout", return_value=process_result
+ ),
+ ):
+ result = benchmark.run_checker("pyrefly", self.root, None, 30, 4096)
+ self.assertFalse(result["ok"])
+ self.assertIn("No Python files matched", result["error_message"])
+
def test_results_include_upstream_provenance_and_memory_limit(self) -> None:
output_file = benchmark._save_results(
[],
@@ -472,11 +594,12 @@ def test_results_include_upstream_provenance_and_memory_limit(self) -> None:
self.assertEqual(output["upstream_source"], benchmark.UPSTREAM_SOURCE)
self.assertEqual(output["memory_limit_mb"], 2048)
+ self.assertEqual(output["checker_config_version"], benchmark.CHECKER_CONFIG_VERSION)
self.assertEqual(
output["node_options"], benchmark.os.environ.get("NODE_OPTIONS", "")
)
self.assertEqual(output["warmup_runs"], 0)
- self.assertEqual(output["uncounted_validation_runs_per_checker"], 0)
+ self.assertEqual(output["uncounted_validation_runs_per_checker"], 1)
self.assertEqual(output["python_version"], benchmark.platform.python_version())
self.assertEqual(output["runner_image"], "local")
self.assertEqual(output["architecture"], benchmark.platform.machine())
@@ -533,8 +656,9 @@ def test_local_mode_builds_pyright_without_clone_or_install(self) -> None:
clone.assert_not_called()
install.assert_not_called()
- def test_zero_warmups_runs_only_measured_checks(self) -> None:
+ def test_zero_warmups_runs_an_audit_before_measured_checks(self) -> None:
calls: list[bool] = []
+ (self.root / "clean.py").write_text("")
def run_checker(
checker: str,
@@ -544,13 +668,15 @@ def run_checker(
memory_limit_mb: int,
capture_output: bool = True,
environment: dict[str, str] | None = None,
+ audit_files: bool = False,
) -> benchmark.TimingMetrics:
- calls.append(capture_output)
+ calls.append(audit_files)
return {
"ok": True,
"execution_time_s": 1.0,
"peak_memory_mb": 2.0,
"oom_killed": False,
+ "files_selected": 1,
}
stdout = io.StringIO()
@@ -571,9 +697,10 @@ def run_checker(
memory_limit_mb=4096,
)
- self.assertEqual(calls, [True, True])
+ self.assertEqual(calls, [True, False, False])
self.assertEqual(
stdout.getvalue(),
+ " pyright source-file audit: 1/1 verified\n"
" Running pyright (2 measured)...\n"
" Run 1/2... 1.000s, 2.0 MB\n"
" Run 2/2... 1.000s, 2.0 MB\n"
@@ -582,6 +709,7 @@ def run_checker(
def test_all_warmups_and_runs_capture_output(self) -> None:
captures: list[bool] = []
+ (self.root / "clean.py").write_text("")
def run_checker(
checker: str,
@@ -591,6 +719,7 @@ def run_checker(
memory_limit_mb: int,
capture_output: bool = True,
environment: dict[str, str] | None = None,
+ audit_files: bool = False,
) -> benchmark.TimingMetrics:
captures.append(capture_output)
return {
@@ -598,6 +727,7 @@ def run_checker(
"execution_time_s": 1.0,
"peak_memory_mb": 2.0,
"oom_killed": False,
+ "files_selected": 1,
}
with (
@@ -617,7 +747,201 @@ def run_checker(
memory_limit_mb=4096,
)
- self.assertEqual(captures, [True, True, True])
+ self.assertEqual(captures, [True, True, True, True])
+
+ def test_file_count_mismatch_prevents_timed_runs(self) -> None:
+ for name in ("clean.py", "second.py"):
+ (self.root / name).write_text("")
+ audit: benchmark.TimingMetrics = {
+ "ok": True, "execution_time_s": 0.1, "peak_memory_mb": 10.0,
+ "oom_killed": False, "files_selected": 1,
+ }
+ with (
+ patch.object(benchmark, "_checker_command", return_value=["checker"]),
+ patch.object(benchmark, "run_checker", return_value=audit) as run,
+ redirect_stdout(io.StringIO()),
+ ):
+ result = benchmark._benchmark_directory(
+ name="local", github_url=None, package_path=self.root,
+ resolved_paths=None, type_checkers=["ty"], timeout=30,
+ runs=3, warmup=1, memory_limit_mb=4096,
+ )
+ run.assert_called_once()
+ self.assertIs(run.call_args.kwargs["audit_files"], True)
+ self.assertEqual(result["source_file_count"], 2)
+ self.assertEqual(result["source_files"], ["clean.py", "second.py"])
+ self.assertEqual(len(result["source_manifest_sha256"]), 64)
+ self.assertFalse(result["metrics"]["ty"]["ok"])
+ self.assertFalse(result["metrics"]["ty"]["file_count_verified"])
+ self.assertIn("expected 2, ty reported 1", result["metrics"]["ty"]["error_message"])
+
+ def test_empty_source_selection_is_not_a_full_repository_fallback(self) -> None:
+ with patch.object(benchmark, "run_checker") as run:
+ result = benchmark._benchmark_directory(
+ name="local", github_url=None, package_path=self.root,
+ resolved_paths=[], type_checkers=["ty"], timeout=30,
+ runs=1, warmup=0, memory_limit_mb=4096,
+ )
+ run.assert_not_called()
+ self.assertEqual(result["error"], "No Python source files selected")
+
+ def test_count_changes_after_the_audit_reject_measured_results(self) -> None:
+ (self.root / "clean.py").write_text("")
+ audit: benchmark.TimingMetrics = {
+ "ok": True, "execution_time_s": 0.1, "peak_memory_mb": 10.0,
+ "oom_killed": False, "files_selected": 1,
+ }
+ measured: benchmark.TimingMetrics = {**audit, "files_selected": 0}
+ with (
+ patch.object(benchmark, "_checker_command", return_value=["checker"]),
+ patch.object(benchmark, "run_checker", side_effect=[audit, measured]),
+ redirect_stdout(io.StringIO()),
+ ):
+ result = benchmark._benchmark_directory(
+ name="local", github_url=None, package_path=self.root,
+ resolved_paths=None, type_checkers=["pyright"], timeout=30,
+ runs=1, warmup=0, memory_limit_mb=4096,
+ )
+ self.assertFalse(result["metrics"]["pyright"]["ok"])
+ self.assertIn("Source-file count changed", result["metrics"]["pyright"]["error_message"])
+
+
+@unittest.skipUnless(
+ os.environ.get("TYPECHECK_BENCHMARK_INTEGRATION") == "1",
+ "Set TYPECHECK_BENCHMARK_INTEGRATION=1 with all checkers installed",
+)
+class CheckerConfigIntegrationTest(unittest.TestCase):
+ def setUp(self) -> None:
+ self.temp_dir = tempfile.TemporaryDirectory(dir=benchmark.SCRIPT_DIR)
+ self.addCleanup(self.temp_dir.cleanup)
+ self.root = Path(self.temp_dir.name)
+
+ def _run(
+ self, checker: str, check_paths: list[Path] | None = None
+ ) -> subprocess.CompletedProcess[str]:
+ command, configs = benchmark._build_checker_command(
+ checker, self.root, check_paths or [self.root / "source"]
+ )
+ self.assertIsNotNone(command, f"{checker} must be installed for integration tests")
+ try:
+ return subprocess.run(
+ command, cwd=self.root, capture_output=True, text=True, timeout=60
+ )
+ finally:
+ for config in configs:
+ config.unlink()
+
+ def test_native_configs_and_ignore_files_do_not_change_benchmark_scope(self) -> None:
+ source = self.root / "source"
+ source.mkdir()
+ (source / "selected.py").write_text('value: int = "wrong"\n')
+ (source / "ignored.py").write_text('value: int = "wrong"\n')
+ (self.root / "outside.py").write_text('value: int = "wrong"\n')
+ for directory in ("dist", "venv", "tests", "node_modules", ".hidden"):
+ path = source / directory
+ path.mkdir()
+ (path / f"{directory.strip('.')}_probe.py").write_text(
+ 'value: int = "wrong"\n'
+ )
+ for name, content in (
+ ("paired.py", 'value: int = "wrong"\n'),
+ ("paired.pyi", "value: int\n"),
+ ("standalone.pyi", "value: int\n"),
+ ("clean.py", "value: int = 1\n"),
+ ):
+ (source / name).write_text(content)
+ (self.root / ".ignore").write_text("source/ignored.py\n")
+ (self.root / ".gitignore").write_text("source/ignored.py\n")
+ native_configs = {
+ "pyrightconfig.json": '{"typeCheckingMode":"off","exclude":["**"]}',
+ "ty.toml": '[rules]\nall = "ignore"\n',
+ "pyrefly.toml": 'project-includes = ["outside.py"]\n'
+ "[errors]\nbad-assignment = false\n",
+ "mypy.ini": "[mypy]\nignore_errors = True\n",
+ "pyproject.toml": '[project]\nname = "hostile"\nversion = "0"\n'
+ 'requires-python = ">=3.8"\n'
+ '[tool.ty.rules]\nall = "ignore"\n'
+ "[tool.pyrefly.errors]\nbad-assignment = false\n"
+ "[tool.mypy]\nignore_errors = true\n"
+ "[tool.zuban]\nignore_errors = true\n",
+ }
+ for directory in (self.root, source):
+ for name, content in native_configs.items():
+ (directory / name).write_text(content)
+
+ manifest, missing = benchmark._resolve_check_paths(self.root, ["source"], [])
+ self.assertEqual(missing, [])
+ self.assertEqual(len(manifest), 8)
+ self.assertNotIn(source / "paired.pyi", manifest)
+ for checker in [*benchmark.DEFAULT_TYPE_CHECKERS, "ty-infer", "pyright-threads"]:
+ with self.subTest(checker=checker):
+ audit = benchmark.run_checker(
+ checker, self.root, manifest, 60, 0, audit_files=True
+ )
+ self.assertTrue(audit["ok"], audit)
+ self.assertEqual(audit["files_selected"], len(manifest), audit)
+ result = self._run(checker, manifest)
+ output = result.stdout + result.stderr
+ self.assertEqual(result.returncode, 1, output)
+ for name in ("selected", "ignored", "dist_probe", "venv_probe", "tests_probe"):
+ self.assertIn(f"{name}.py", output, output)
+ for name in ("outside", "node_modules_probe", "hidden_probe"):
+ self.assertNotIn(f"{name}.py", output, output)
+ for directory in (self.root, source):
+ for name, content in native_configs.items():
+ self.assertEqual((directory / name).read_text(), content)
+
+ def test_ty_inference_profile_changes_inferred_return_checking(self) -> None:
+ source = self.root / "source"
+ source.mkdir()
+ (source / "inferred.py").write_text(
+ 'def inferred():\n return "wrong"\n\nvalue: int = inferred()\n'
+ )
+ default = self._run("ty")
+ inferred = self._run("ty-infer")
+ self.assertEqual(default.returncode, 0, default.stdout + default.stderr)
+ self.assertEqual(inferred.returncode, 1, inferred.stdout + inferred.stderr)
+ self.assertIn("invalid-assignment", inferred.stdout + inferred.stderr)
+
+ def test_zuban_preserves_nested_source_paths(self) -> None:
+ source = self.root / "source" / "nested"
+ source.mkdir(parents=True)
+ (source / "selected.py").write_text('value: int = "wrong"\n')
+ result = self._run("zuban", [source])
+ self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
+ self.assertIn("selected.py", result.stdout + result.stderr)
+
+ def test_ty_does_not_merge_user_level_rule_suppression(self) -> None:
+ source = self.root / "source"
+ source.mkdir()
+ selected = source / "selected.py"
+ selected.write_text('value: int = "wrong"\n')
+ user_config = self.root / "user-config"
+ (user_config / "ty").mkdir(parents=True)
+ (user_config / "ty" / "ty.toml").write_text('[rules]\nall = "ignore"\n')
+ outputs: list[benchmark.ProcessResult] = []
+ original_run = benchmark.run_process_with_timeout
+
+ def capture(*args, **kwargs):
+ output = original_run(*args, **kwargs)
+ outputs.append(output)
+ return output
+
+ with (
+ patch.dict(os.environ, {
+ "APPDATA": str(user_config), "XDG_CONFIG_HOME": str(user_config)
+ }),
+ patch.object(benchmark, "run_process_with_timeout", side_effect=capture),
+ ):
+ for checker in ("ty", "ty-infer"):
+ with self.subTest(checker=checker):
+ result = benchmark.run_checker(
+ checker, self.root, [selected], 60, 0, audit_files=True
+ )
+ self.assertTrue(result["ok"], result)
+ self.assertEqual(result["files_selected"], 1)
+ self.assertEqual(outputs[-1]["returncode"], 1, outputs[-1])
+ self.assertIn("invalid-assignment", outputs[-1]["stdout"])
if __name__ == "__main__":
diff --git a/build/benchmark/typecheck_benchmark.py b/build/benchmark/typecheck_benchmark.py
index 7d5d5c4e00a4..1f794b203bd0 100644
--- a/build/benchmark/typecheck_benchmark.py
+++ b/build/benchmark/typecheck_benchmark.py
@@ -19,6 +19,7 @@
from __future__ import annotations
import argparse
+import hashlib
import json
import os
import platform
@@ -31,6 +32,7 @@
import tempfile
import threading
import time
+from contextlib import nullcontext
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Sequence, TypedDict
@@ -43,7 +45,8 @@
PYRIGHT_ENTRY_POINT_ENV = "PYRIGHT_BENCHMARK_ENTRY_POINT"
DEFAULT_TYPE_CHECKERS = ["pyright", "pyrefly", "ty", "mypy", "zuban"]
-AVAILABLE_TYPE_CHECKERS = [*DEFAULT_TYPE_CHECKERS, "pyright-threads", "pyright-pip"]
+AVAILABLE_TYPE_CHECKERS = [*DEFAULT_TYPE_CHECKERS, "pyright-threads", "pyright-pip", "ty-infer"]
+TARGET_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
DEFAULT_TIMEOUT = 300
DEFAULT_MEMORY_LIMIT_MB = 4096
CLONE_TIMEOUT = 300
@@ -60,6 +63,7 @@
}
PYRIGHT_DEFAULT_EXCLUDES = ["**/node_modules", "**/__pycache__", "**/.*"]
DEPENDENCY_ISOLATION = "pip-target-per-package"
+CHECKER_CONFIG_VERSION = 4
class BenchmarkError(Exception):
@@ -91,6 +95,9 @@ class TimingMetrics(RequiredTimingMetrics, total=False):
peak_memories_mb: list[float]
execution_time_stats: RunStats
peak_memory_stats: RunStats
+ files_selected: int
+ file_count_verified: bool
+ file_count_audit_time_s: float
class PackageResult(TypedDict, total=False):
@@ -102,6 +109,9 @@ class PackageResult(TypedDict, total=False):
local_path: str
error: str | None
metrics: dict[str, TimingMetrics]
+ source_files: list[str]
+ source_file_count: int
+ source_manifest_sha256: str
class AggregateStats(TypedDict):
@@ -157,6 +167,8 @@ def _pyright_command() -> list[str] | None:
def _checker_command(checker: str) -> list[str] | None:
+ if checker == "ty-infer":
+ checker = "ty"
if checker in ("pyright", "pyright-threads"):
return _pyright_command()
if checker == "pyright-pip":
@@ -604,28 +616,43 @@ def _resolve_check_paths(
for path in configured_paths
if not path.exists()
]
- existing_paths = [path for path in configured_paths if path.exists()]
- if not exclude_directories:
- return existing_paths, missing
+ excluded_names = {"node_modules", "__pycache__", *exclude_directories}
+ resolved_paths: set[Path] = set()
- excluded_names = set(exclude_directories)
- resolved_paths: list[Path] = []
- for path in existing_paths:
+ def is_excluded(path: Path) -> bool:
+ return any(
+ part.startswith(".") or part in excluded_names
+ for part in path.relative_to(package_path).parts
+ )
+
+ def walk_error(error: OSError) -> None:
+ raise error
+
+ for path in configured_paths:
+ if not path.exists() or is_excluded(path):
+ continue
if path.is_file():
- resolved_paths.append(path)
+ if path.suffix in (".py", ".pyi"):
+ resolved_paths.add(path)
continue
- for root, directory_names, file_names in os.walk(path):
+ for root, directory_names, file_names in os.walk(path, onerror=walk_error):
+ root_path = Path(root)
directory_names[:] = [
- name for name in directory_names if name not in excluded_names
+ name for name in directory_names if not is_excluded(root_path / name)
]
- directory_names.sort()
- root_path = Path(root)
- resolved_paths.extend(
+ resolved_paths.update(
root_path / name
- for name in sorted(file_names)
+ for name in file_names
if Path(name).suffix in (".py", ".pyi")
+ and not is_excluded(root_path / name)
)
- return resolved_paths, missing
+ return sorted(
+ (
+ path for path in resolved_paths
+ if path.suffix != ".pyi" or path.with_suffix(".py") not in resolved_paths
+ ),
+ key=lambda path: path.relative_to(package_path).as_posix(),
+ ), missing
def _new_config_path(package_path: Path, checker: str, suffix: str) -> Path:
@@ -645,6 +672,7 @@ def _write_pyright_config(package_path: Path, check_paths: list[str] | None) ->
"exclude": PYRIGHT_DEFAULT_EXCLUDES,
"typeCheckingMode": "basic",
"useLibraryCodeForTypes": True,
+ "pythonVersion": TARGET_PYTHON_VERSION,
}
config_path.write_text(json.dumps(config), encoding="utf-8")
return config_path
@@ -653,31 +681,55 @@ def _write_pyright_config(package_path: Path, check_paths: list[str] | None) ->
def _write_mypy_config(
package_path: Path, check_paths: list[str] | None, checker: str
) -> Path:
- config_path = _new_config_path(package_path, checker, ".ini")
- lines = ["[mypy]"]
- if check_paths:
- lines.append(f"files = {', '.join(check_paths)}")
- lines.extend(["check_untyped_defs = True", ""])
+ config_path = _new_config_path(package_path, checker, ".toml")
+ lines = [
+ "[tool.mypy]",
+ f"files = {json.dumps(check_paths or ['.'])}",
+ "check_untyped_defs = true",
+ f"python_version = {json.dumps(TARGET_PYTHON_VERSION)}",
+ f"python_executable = {json.dumps(sys.executable)}",
+ ]
+ if checker == "zuban":
+ lines.append("exclude_gitignore = false")
+ lines.append("")
config_path.write_text("\n".join(lines), encoding="utf-8")
return config_path
-def _write_ty_config(package_path: Path, check_paths: list[str] | None) -> Path:
+def _write_ty_config(
+ package_path: Path, check_paths: list[str] | None, *, infer_returns: bool = False
+) -> Path:
config_path = _new_config_path(package_path, "ty", ".toml")
- lines: list[str] = []
- if check_paths:
- paths = ", ".join(json.dumps(path) for path in check_paths)
- lines.extend(["[src]", f"include = [{paths}]"])
+ lines = [
+ "[environment]",
+ f"python = {json.dumps(sys.executable)}",
+ f"python-version = {json.dumps(TARGET_PYTHON_VERSION)}",
+ "",
+ "[src]",
+ "respect-ignore-files = false",
+ # These are gitignore-style patterns, not native Windows paths.
+ f"include = {json.dumps([Path(path).as_posix() for path in check_paths or ['.']])}",
+ f"exclude = {json.dumps(['!**', *PYRIGHT_DEFAULT_EXCLUDES])}",
+ ]
+ if infer_returns:
+ lines.extend(["", "[analysis]", 'return-type-inference = "always"'])
config_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
return config_path
def _write_pyrefly_config(package_path: Path, check_paths: list[str] | None) -> Path:
config_path = _new_config_path(package_path, "pyrefly", ".toml")
- lines: list[str] = []
- if check_paths:
- paths = ", ".join(json.dumps(path) for path in check_paths)
- lines.append(f"project_includes = [{paths}]")
+ lines = [
+ f"project-includes = {json.dumps(check_paths or ['.'])}",
+ "use-ignore-files = false",
+ "disable-project-excludes-heuristics = true",
+ f"project-excludes = {json.dumps(PYRIGHT_DEFAULT_EXCLUDES)}",
+ f"python-interpreter-path = {json.dumps(sys.executable)}",
+ f"python-version = {json.dumps(TARGET_PYTHON_VERSION)}",
+ 'preset = "default"',
+ "check-unannotated-defs = true",
+ 'infer-return-types = "checked"',
+ ]
config_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
return config_path
@@ -723,6 +775,25 @@ def _parse_pyright_stats(
)
+def _parse_source_file_count(checker: str, output: str) -> int | None:
+ if checker in ("ty", "ty-infer"):
+ if not re.search(r"\bDEBUG Checking all files took ", output):
+ return None
+ return len(set(re.findall(
+ r"\bDEBUG Checking file '(.+)'$", output, re.MULTILINE
+ )))
+ if checker in ("pyright", "pyright-pip"):
+ pattern = r"^Total files checked:\s+(\d+)$"
+ elif checker == "pyright-threads":
+ pattern = r"^Found (\d+) files? to analyze$"
+ elif checker == "pyrefly":
+ pattern = r"\bINFO (\d+) modules? \(\d+ dependent modules?\)"
+ else:
+ pattern = r"(?:checked |Success: no issues found in )(\d+) source files?\b"
+ counts = {int(count) for count in re.findall(pattern, output, re.MULTILINE)}
+ return counts.pop() if len(counts) == 1 else None
+
+
def _build_checker_command(
checker: str,
package_path: Path,
@@ -740,6 +811,8 @@ def _build_checker_command(
*base_command,
"--project",
str(config_path),
+ "--pythonpath",
+ sys.executable,
]
if checker == "pyright-threads":
command.append("--threads")
@@ -754,8 +827,10 @@ def _build_checker_command(
"--config",
str(config_path),
], [config_path]
- if checker == "ty":
- config_path = _write_ty_config(package_path, relative_paths)
+ if checker in ("ty", "ty-infer"):
+ config_path = _write_ty_config(
+ package_path, relative_paths, infer_returns=checker == "ty-infer"
+ )
return [
*base_command,
"check",
@@ -770,11 +845,20 @@ def _build_checker_command(
"--config-file",
str(config_path),
]
- if not relative_paths:
- command.append(".")
return command, [config_path]
if checker == "zuban":
- return [*base_command, "check", *(relative_paths or ["."])], []
+ config_path = _write_mypy_config(package_path, relative_paths, "zuban")
+ command = [
+ *base_command,
+ "check",
+ "--mode",
+ "default",
+ "--config-file",
+ str(config_path),
+ "--python-executable",
+ sys.executable,
+ ]
+ return command, [config_path]
return None, []
@@ -786,6 +870,7 @@ def run_checker(
memory_limit_mb: int,
capture_output: bool = True,
environment: dict[str, str] | None = None,
+ audit_files: bool = False,
) -> TimingMetrics:
"""Run one checker once using a minimal checker-specific configuration."""
command, config_paths = _build_checker_command(
@@ -805,16 +890,39 @@ def run_checker(
"error_message": error_message,
}
+ if audit_files:
+ if checker == "pyright-threads":
+ command.append("--verbose")
+ elif checker == "pyrefly":
+ command.append("--summary=full")
+ elif checker in ("ty", "ty-infer"):
+ command.append("-vv")
+
try:
try:
- result = run_process_with_timeout(
- command,
- cwd=package_path,
- timeout=timeout,
- memory_limit_mb=memory_limit_mb,
- capture_output=capture_output,
- environment=environment,
- )
+ # ty merges user configuration even with an explicit project config.
+ with (
+ tempfile.TemporaryDirectory(
+ prefix=".typecheck-benchmark-user-", dir=package_path
+ )
+ if checker in ("ty", "ty-infer")
+ else nullcontext(None)
+ ) as config_home:
+ checker_environment = environment
+ if config_home is not None:
+ checker_environment = (
+ environment if environment is not None else os.environ
+ ).copy()
+ checker_environment["APPDATA"] = config_home
+ checker_environment["XDG_CONFIG_HOME"] = config_home
+ result = run_process_with_timeout(
+ command,
+ cwd=package_path,
+ timeout=timeout,
+ memory_limit_mb=memory_limit_mb,
+ capture_output=capture_output,
+ environment=checker_environment,
+ )
except OSError as exc:
return {
"ok": False,
@@ -849,13 +957,25 @@ def run_checker(
}
combined_output = f"{result['stderr']}\n{result['stdout']}"
+ if checker == "pyrefly" and re.search(
+ r"^No Python files matched pattern", combined_output, re.MULTILINE
+ ):
+ return {
+ "ok": False,
+ "execution_time_s": result["execution_time_s"],
+ "peak_memory_mb": result["peak_memory_mb"],
+ "oom_killed": False,
+ "error_message": "No Python files matched the configured include patterns",
+ }
if capture_output and "errors prevented further checking" in combined_output:
return {
"ok": False,
"execution_time_s": result["execution_time_s"],
"peak_memory_mb": result["peak_memory_mb"],
"oom_killed": False,
- "error_message": "Fatal: errors prevented further checking",
+ "error_message": (
+ "Fatal: errors prevented further checking; " + _failure_message(result)
+ ),
}
# All supported checkers use exit code 1 for ordinary type errors.
@@ -882,6 +1002,12 @@ def run_checker(
metrics["files_checked"] = files_checked
if phase_times:
metrics["phase_times_s"] = phase_times
+ files_selected = _parse_source_file_count(checker, combined_output)
+ if files_selected is not None:
+ metrics["files_selected"] = files_selected
+ elif audit_files:
+ metrics["ok"] = False
+ metrics["error_message"] = f"{checker} did not report a verifiable source-file count"
return metrics
@@ -967,6 +1093,26 @@ def _benchmark_directory(
local_path: str | None = None,
environment: dict[str, str] | None = None,
) -> PackageResult:
+ source_files, missing = _resolve_check_paths(
+ package_path,
+ (_relative_check_paths(package_path, resolved_paths) or [])
+ if resolved_paths is not None else ["."],
+ [],
+ )
+ if missing or not source_files:
+ return {
+ "package_name": name,
+ "github_url": github_url,
+ "error": (
+ f"Missing configured source paths: {', '.join(missing)}"
+ if missing else "No Python source files selected"
+ ),
+ "metrics": {},
+ }
+ manifest = [path.relative_to(package_path).as_posix() for path in source_files]
+ manifest_hash = hashlib.sha256(
+ json.dumps(manifest, ensure_ascii=True).encode("utf-8")
+ ).hexdigest()
metrics: dict[str, TimingMetrics] = {}
for checker in type_checkers:
if not _checker_command(checker):
@@ -980,6 +1126,27 @@ def _benchmark_directory(
}
continue
+ audit = run_checker(
+ checker, package_path, source_files, timeout, memory_limit_mb,
+ environment=environment, audit_files=True,
+ )
+ selected_count = audit.get("files_selected")
+ if not audit["ok"] or selected_count is None or selected_count != len(source_files):
+ audit["ok"] = False
+ audit["file_count_verified"] = False
+ if not audit.get("error_message"):
+ audit["error_message"] = (
+ f"Source-file count mismatch: expected {len(source_files)}, "
+ f"{checker} reported {selected_count}"
+ )
+ audit["file_count_audit_time_s"] = audit["execution_time_s"]
+ audit["execution_time_s"] = 0.0
+ audit["peak_memory_mb"] = 0.0
+ metrics[checker] = audit
+ print(f" {checker} source-file audit failed: {audit['error_message']}")
+ continue
+ print(f" {checker} source-file audit: {selected_count}/{len(source_files)} verified")
+
if warmup == 0:
run_description = f"{runs} measured"
else:
@@ -1002,11 +1169,18 @@ def _benchmark_directory(
metric = run_checker(
checker,
package_path,
- resolved_paths,
+ source_files,
timeout,
memory_limit_mb,
environment=environment,
)
+ run_count = metric.get("files_selected")
+ if metric["ok"] and run_count is not None and run_count != len(source_files):
+ metric["ok"] = False
+ metric["error_message"] = (
+ f"Source-file count changed: expected {len(source_files)}, "
+ f"{checker} reported {run_count}"
+ )
if not metric.get("ok"):
print(f"failed: {metric.get('error_message', 'Unknown error')}")
failure = metric
@@ -1050,6 +1224,9 @@ def _benchmark_directory(
"peak_memories_mb": peak_memories,
"execution_time_stats": compute_run_stats(execution_times),
"peak_memory_stats": compute_run_stats(peak_memories),
+ "files_selected": selected_count,
+ "file_count_verified": True,
+ "file_count_audit_time_s": audit["execution_time_s"],
}
if parsed_file_counts:
result_metric["files_parsed"] = parsed_file_counts[0]
@@ -1077,6 +1254,9 @@ def _benchmark_directory(
"github_url": github_url,
"error": None,
"metrics": metrics,
+ "source_files": manifest,
+ "source_file_count": len(manifest),
+ "source_manifest_sha256": manifest_hash,
}
if local_path:
result["local_path"] = local_path
@@ -1128,7 +1308,12 @@ def _benchmark_package(
package_path, raw_check_paths, exclude_directories
)
if missing:
- print(f" Warning: missing configured check paths: {', '.join(missing)}")
+ return {
+ **package_metadata,
+ "commit": commit or "unknown",
+ "error": f"Missing configured check paths: {', '.join(missing)}",
+ "metrics": {},
+ }
if resolved_paths:
check_text = ", ".join(raw_check_paths)
if exclude_directories:
@@ -1137,7 +1322,12 @@ def _benchmark_package(
) + ")"
print(f" Checking: {check_text}")
else:
- print(" Warning: configured check paths do not exist; checking full repo")
+ return {
+ **package_metadata,
+ "commit": commit or "unknown",
+ "error": "No Python source files selected",
+ "metrics": {},
+ }
result = _benchmark_directory(
name=name,
@@ -1196,12 +1386,13 @@ def _save_results(
"memory_limit_mb": memory_limit_mb,
"node_options": os.environ.get("NODE_OPTIONS", ""),
"dependency_isolation": DEPENDENCY_ISOLATION,
+ "checker_config_version": CHECKER_CONFIG_VERSION,
"type_checkers": type_checkers,
"type_checker_versions": versions,
"package_count": len(results),
"runs_per_package": runs,
"warmup_runs": warmup,
- "uncounted_validation_runs_per_checker": 0,
+ "uncounted_validation_runs_per_checker": 1,
"timeout_s": timeout,
"aggregate": aggregate,
"results": results,
From c08cd1b2605a5a7775e7ef2f09696df34a557535 Mon Sep 17 00:00:00 2001
From: Bill Schnurr
Date: Tue, 27 Oct 2026 16:35:11 -0700
Subject: [PATCH 2/6] Enable all Pyrefly diagnostics in benchmark profile
Use Pyrefly's all preset, prove an optional diagnostic is enabled with a real-binary regression test, and invalidate older configuration baselines. Document that this is not equivalent to Pyright basic analysis.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 35cc6021-56a0-45b2-9b66-3817d947cd99
---
build/benchmark/README.md | 8 +++++---
build/benchmark/render_benchmark_html.py | 2 +-
build/benchmark/test_typecheck_benchmark.py | 9 +++++++++
build/benchmark/typecheck_benchmark.py | 4 ++--
4 files changed, 17 insertions(+), 6 deletions(-)
diff --git a/build/benchmark/README.md b/build/benchmark/README.md
index 585240361834..039266d2c102 100644
--- a/build/benchmark/README.md
+++ b/build/benchmark/README.md
@@ -142,11 +142,13 @@ tool's semantics.
The `ty` profile retains ty's default return-type inference. The separately selectable `ty-infer`
profile uses the same binary and configuration except for
-`analysis.return-type-inference = "always"`. Pyrefly explicitly uses its `default` preset,
+`analysis.return-type-inference = "always"`. Pyrefly explicitly uses its `all` preset,
`check-unannotated-defs = true`, and `infer-return-types = "checked"`. Pyright uses `basic` mode with
unannotated-function analysis enabled by default; mypy uses `check_untyped_defs = True`.
-Similar option names or presets do not imply equivalent inference. In particular, ty documents
-limitations for generator and asynchronous return inference even with `"always"`.
+Pyrefly's `all` preset enables every error kind and strict callable subtyping; it is not equivalent
+to Pyright's `basic` mode. Similar option names or presets do not imply equivalent inference.
+Changing diagnostic severity does not necessarily change how much analysis a checker performs.
+In particular, ty documents limitations for generator and asynchronous return inference even with `"always"`.
```console
python build/benchmark/typecheck_benchmark.py -c ty ty-infer -r 3 -w 1
diff --git a/build/benchmark/render_benchmark_html.py b/build/benchmark/render_benchmark_html.py
index 4f33e60dd538..39379068edba 100644
--- a/build/benchmark/render_benchmark_html.py
+++ b/build/benchmark/render_benchmark_html.py
@@ -264,7 +264,7 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
Run summary
Generated {html.escape(timestamp)} · Python {html.escape(str(environment.get('python_version', 'unknown')))} · {html.escape(str(environment.get('platform_details', environment.get('platform', 'unknown'))))}
| Checker | Measured | Failed | Total time | Median time | Median RSS |
{''.join(summary_rows)}
Package comparison
Different checker jobs run on separate hosted runners, so timings are independent trends rather than a fastest-checker ranking. The two Pyright modes run sequentially in one job. Cells show wall time and peak RSS.
-Checker defaults, inference, and diagnostic coverage differ; these timings do not measure equivalent analysis or time per rule. The two ty modes run sequentially in one job: ty retains its default return inference, while ty (return inference) enables inference for unannotated returns. Neither mode guarantees Pyright-equivalent inference.
+Checker defaults, inference, and diagnostic coverage differ; these timings do not measure equivalent analysis or time per rule. Pyrefly uses its all preset; Pyright uses basic mode. These presets are not equivalent. The two ty modes run sequentially in one job: ty retains its default return inference, while ty (return inference) enables inference for unannotated returns. Neither mode guarantees Pyright-equivalent inference.
Linux RSS measures the parent checker process, not its child processes; multi-process memory totals are not comparable.
Verified source-file counts come from an untimed checker audit against a shared implementation-first manifest. Imported dependencies are not included. Historical results without verified counts have not passed this audit.
| Package | {checker_headers}
{''.join(package_rows)}
diff --git a/build/benchmark/test_typecheck_benchmark.py b/build/benchmark/test_typecheck_benchmark.py
index 85b934dc1342..6fa4d63d4439 100644
--- a/build/benchmark/test_typecheck_benchmark.py
+++ b/build/benchmark/test_typecheck_benchmark.py
@@ -891,6 +891,15 @@ def test_native_configs_and_ignore_files_do_not_change_benchmark_scope(self) ->
for name, content in native_configs.items():
self.assertEqual((directory / name).read_text(), content)
+ def test_pyrefly_enables_optional_diagnostics(self) -> None:
+ source = self.root / "source"
+ source.mkdir()
+ (source / "unused_ignore.py").write_text("value: int = 1 # type: ignore\n")
+ result = self._run("pyrefly")
+ output = result.stdout + result.stderr
+ self.assertEqual(result.returncode, 1, output)
+ self.assertIn("[unused-type-ignore]", output)
+
def test_ty_inference_profile_changes_inferred_return_checking(self) -> None:
source = self.root / "source"
source.mkdir()
diff --git a/build/benchmark/typecheck_benchmark.py b/build/benchmark/typecheck_benchmark.py
index 1f794b203bd0..2202b3f2c295 100644
--- a/build/benchmark/typecheck_benchmark.py
+++ b/build/benchmark/typecheck_benchmark.py
@@ -63,7 +63,7 @@
}
PYRIGHT_DEFAULT_EXCLUDES = ["**/node_modules", "**/__pycache__", "**/.*"]
DEPENDENCY_ISOLATION = "pip-target-per-package"
-CHECKER_CONFIG_VERSION = 4
+CHECKER_CONFIG_VERSION = 5
class BenchmarkError(Exception):
@@ -726,7 +726,7 @@ def _write_pyrefly_config(package_path: Path, check_paths: list[str] | None) ->
f"project-excludes = {json.dumps(PYRIGHT_DEFAULT_EXCLUDES)}",
f"python-interpreter-path = {json.dumps(sys.executable)}",
f"python-version = {json.dumps(TARGET_PYTHON_VERSION)}",
- 'preset = "default"',
+ 'preset = "all"',
"check-unannotated-defs = true",
'infer-return-types = "checked"',
]
From 6defaec0b6007b80c11a22d90049e45d842d2d97 Mon Sep 17 00:00:00 2001
From: Bill Schnurr
Date: Tue, 27 Oct 2026 16:46:19 -0700
Subject: [PATCH 3/6] Report untimed behavioral capability probes alongside
timings
Verify exact inferred types and intended negative diagnostics on paired examples using the same isolated checker configurations. Preserve native counts and diagnostic evidence, distinguish behavior mismatches from evaluation failures, and reject incompatible suite identities or outcomes during comparisons and shard merging. Do not normalize runtime by probe or rule counts.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 35cc6021-56a0-45b2-9b66-3817d947cd99
---
build/benchmark/README.md | 32 ++-
build/benchmark/benchmark_capabilities.py | 213 ++++++++++++++++++
build/benchmark/compare_benchmarks.py | 16 ++
build/benchmark/merge_benchmark_results.py | 11 +
build/benchmark/render_benchmark_html.py | 76 ++++++-
.../benchmark/test_benchmark_capabilities.py | 126 +++++++++++
build/benchmark/test_compare_benchmarks.py | 17 ++
.../benchmark/test_merge_benchmark_results.py | 29 +++
build/benchmark/test_render_benchmark_html.py | 46 ++++
build/benchmark/test_typecheck_benchmark.py | 1 +
build/benchmark/typecheck_benchmark.py | 131 +++++++----
11 files changed, 651 insertions(+), 47 deletions(-)
create mode 100644 build/benchmark/benchmark_capabilities.py
create mode 100644 build/benchmark/test_benchmark_capabilities.py
diff --git a/build/benchmark/README.md b/build/benchmark/README.md
index 039266d2c102..e1ae54e98ef8 100644
--- a/build/benchmark/README.md
+++ b/build/benchmark/README.md
@@ -8,7 +8,8 @@ The upstream copyright and MIT license are reproduced in
[`UPSTREAM_LICENSE.txt`](UPSTREAM_LICENSE.txt).
The local integration preserves repository-specific behavior: it builds and invokes the current
checkout's Pyright production bundle rather than resolving Pyright from `PATH`. The benchmark
-measures performance only, not diagnostic accuracy or type precision.
+measures corpus performance, with a separate small behavioral audit rather than a comprehensive
+diagnostic-accuracy or type-precision comparison.
The checked corpus is a curated set of source repositories derived from
[`lolpack/type_coverage_py`](https://github.com/lolpack/type_coverage_py/blob/main/included_packages.txt).
@@ -154,6 +155,35 @@ In particular, ty documents limitations for generator and asynchronous return in
python build/benchmark/typecheck_benchmark.py -c ty ty-infer -r 3 -w 1
```
+After corpus timings finish, the harness runs an **untimed behavioral capability suite** with two
+batch invocations per installed checker, using the same configurations and interpreter.
+Eight paired probes cover an exact-type assertion
+control, unannotated return inference, unannotated function body checking, generics, overloads,
+protocols, narrowing, and propagation from a local annotated import. The import probe does not
+model all third-party package or stub-resolution behavior.
+
+The positive examples must be accepted, including exact `typing.assert_type` checks; unexpected
+`Any` or `Unknown` therefore cannot satisfy an `int` assertion. Negative examples must report a
+diagnostic at exactly the intended line, with no other diagnostic lines. The assertion control
+must pass before any case can match. Both batches require native file counts covering all eight
+probe files; crashes, timeouts, missing counts, and unrecognized output are **not evaluated**,
+not zero capability or a successful result.
+
+JSON stores the versioned suite identity, per-case diagnostic lines, and raw process evidence in
+`capability_suite`. HTML adds a **Behavior matches** column and a per-case matrix. Mismatches can
+mean a missed error, lost precision, or additional strictness diagnostics: for example, Pyrefly's
+`all` preset also requires return annotations. Do not equate every mismatch with weaker inference.
+Comparisons reject changed suite identities or behavioral outcomes rather than accepting a
+speedup that silently changed these observed behaviors.
+Package-shard merges require identical suite identities and outcomes but permit different raw
+process evidence (such as temporary paths); the merged result retains the first shard's evidence.
+
+These are a few examples, not a weighted quality score, a measurement of internal work, or proof
+that the timed corpus receives equivalent analysis. Runtime is not divided by matches, rules, or
+diagnostic counts. Even passing every probe does not justify a quality-adjusted corpus speed ratio;
+that requires a separately controlled workload with verified shared behavioral requirements.
+Historical results without this audit have no verified behavior-match count.
+
For defensible cross-checker comparisons, use the same machine, pinned source commits, dependency
versions, interpreter, target platform, source scope, and resource/thread budgets. The weekly jobs
currently use separate hosted runners and install dependencies without a shared version lock, so
diff --git a/build/benchmark/benchmark_capabilities.py b/build/benchmark/benchmark_capabilities.py
new file mode 100644
index 000000000000..6543d7ff1602
--- /dev/null
+++ b/build/benchmark/benchmark_capabilities.py
@@ -0,0 +1,213 @@
+"""Small behavioral probes, evaluated separately from corpus timings."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import tempfile
+from pathlib import Path
+from typing import Callable, TypedDict
+
+
+CASES = {
+ "assert_type": (
+ "Exact-type assertion control",
+ 'from typing import assert_type\n'
+ 'assert_type(int("1"), int)\n'
+ 'assert_type(int("1"), str) # expect-error\n',
+ ),
+ "unannotated_return": (
+ "Unannotated return inference",
+ 'from typing import assert_type\n'
+ 'def inferred():\n'
+ ' return int("1")\n'
+ 'assert_type(inferred(), int)\n'
+ 'bad: str = inferred() # expect-error\n',
+ ),
+ "unannotated_body": (
+ "Unannotated function body checking",
+ 'def checked():\n'
+ ' good: int = 1\n'
+ ' bad: str = 1 # expect-error\n',
+ ),
+ "generics": (
+ "Generic type propagation",
+ 'from typing import TypeVar, assert_type\n'
+ 'T = TypeVar("T")\n'
+ 'def first(values: list[T]) -> T:\n'
+ ' return values[0]\n'
+ 'values: list[int] = [1]\n'
+ 'assert_type(first(values), int)\n'
+ 'bad: str = first(values) # expect-error\n',
+ ),
+ "overloads": (
+ "Overload selection",
+ 'from typing import assert_type, overload\n'
+ '@overload\n'
+ 'def choose(value: int) -> int: ...\n'
+ '@overload\n'
+ 'def choose(value: str) -> str: ...\n'
+ 'def choose(value: int | str) -> int | str:\n'
+ ' return value\n'
+ 'assert_type(choose(int("1")), int)\n'
+ 'bad: str = choose(int("1")) # expect-error\n',
+ ),
+ "protocols": (
+ "Structural protocol checking",
+ 'from typing import Protocol, assert_type\n'
+ 'class Reader(Protocol):\n'
+ ' def read(self) -> int: ...\n'
+ 'class Good:\n'
+ ' def read(self) -> int:\n'
+ ' return 1\n'
+ 'class Bad:\n'
+ ' def read(self) -> str:\n'
+ ' return "wrong"\n'
+ 'good: Reader = Good()\n'
+ 'assert_type(good.read(), int)\n'
+ 'bad: Reader = Bad() # expect-error\n',
+ ),
+ "narrowing": (
+ "Union narrowing",
+ 'from typing import assert_type\n'
+ 'def narrow(value: int | str) -> None:\n'
+ ' if isinstance(value, int):\n'
+ ' assert_type(value, int)\n'
+ ' bad: str = value # expect-error\n'
+ ' else:\n'
+ ' assert_type(value, str)\n',
+ ),
+ "imported_types": (
+ "Imported type propagation",
+ 'from typing import assert_type\n'
+ 'from capability_support import typed_value\n'
+ 'assert_type(typed_value(), int)\n'
+ 'bad: str = typed_value() # expect-error\n',
+ ),
+}
+SUPPORT_SOURCE = "def typed_value() -> int:\n return 1\n"
+SUITE_VERSION = 1
+SUITE_SHA256 = hashlib.sha256(
+ json.dumps([SUITE_VERSION, CASES, SUPPORT_SOURCE], sort_keys=True).encode("utf-8")
+).hexdigest()
+
+
+class CapabilityOutput(TypedDict):
+ stdout: str
+ stderr: str
+ returncode: int
+ timed_out: bool
+ oom_killed: bool
+ files_selected: int | None
+
+
+class CaseResult(TypedDict):
+ passed: bool
+ positive_diagnostic_lines: list[int]
+ negative_diagnostic_lines: list[int]
+ expected_error_line: int
+
+
+class CheckerResult(TypedDict, total=False):
+ ok: bool
+ error: str
+ cases: dict[str, CaseResult]
+ evidence: dict[str, CapabilityOutput]
+
+
+class CapabilitySuite(TypedDict):
+ version: int
+ sha256: str
+ cases: dict[str, str]
+ results: dict[str, CheckerResult]
+
+
+def _diagnostic_lines(output: str) -> dict[str, set[int]]:
+ locations: dict[str, set[int]] = {}
+ for match in re.finditer(r"([\w.-]+\.pyi?):(\d+)(?::\d+)?", output):
+ locations.setdefault(Path(match[1]).stem, set()).add(int(match[2]))
+ return locations
+
+
+def run_capability_suite(
+ checkers: list[str],
+ run: Callable[[str, Path, list[Path]], CapabilityOutput],
+) -> CapabilitySuite:
+ suite: CapabilitySuite = {
+ "version": SUITE_VERSION,
+ "sha256": SUITE_SHA256,
+ "cases": {name: label for name, (label, _) in CASES.items()},
+ "results": {},
+ }
+ with tempfile.TemporaryDirectory(prefix="typecheck-capabilities-") as directory:
+ root = Path(directory)
+ paths: dict[str, list[Path]] = {}
+ expected: dict[str, int] = {}
+ for phase in ("positive", "negative"):
+ project = root / phase
+ project.mkdir()
+ (project / "capability_support.py").write_text(SUPPORT_SOURCE, encoding="utf-8")
+ paths[phase] = []
+ for name, (_, source) in CASES.items():
+ lines = source.splitlines(keepends=True)
+ expected[name] = next(
+ index for index, line in enumerate(lines, 1) if "# expect-error" in line
+ )
+ if phase == "positive":
+ lines = [
+ "\n" if "# expect-error" in line else line for line in lines
+ ]
+ path = project / f"{name}.py"
+ path.write_text("".join(lines), encoding="utf-8")
+ paths[phase].append(path)
+
+ for checker in checkers:
+ evidence: dict[str, CapabilityOutput] = {}
+ locations: dict[str, dict[str, set[int]]] = {}
+ error = None
+ for phase in ("positive", "negative"):
+ try:
+ output = run(checker, root / phase, paths[phase])
+ except OSError as exc:
+ error = f"{phase}: unable to evaluate capabilities: {exc}"
+ break
+ evidence[phase] = output
+ text = output["stdout"] + "\n" + output["stderr"]
+ if output["timed_out"] or output["oom_killed"]:
+ error = f"{phase}: capability checker timed out or exceeded memory limit"
+ elif output["returncode"] not in (0, 1):
+ error = f"{phase}: fatal checker exit {output['returncode']}"
+ elif output["files_selected"] != len(CASES):
+ error = f"{phase}: capability source-file count is not verified"
+ else:
+ locations[phase] = _diagnostic_lines(text)
+ if set(locations[phase]) - CASES.keys():
+ error = f"{phase}: unexpected diagnostics outside capability cases"
+ elif bool(locations[phase]) != (output["returncode"] == 1):
+ error = f"{phase}: unrecognized or inconsistent diagnostic output"
+ if error:
+ break
+ if error:
+ suite["results"][checker] = {
+ "ok": False, "error": error, "evidence": evidence,
+ }
+ continue
+ cases: dict[str, CaseResult] = {}
+ control_passed = (
+ not locations["positive"].get("assert_type")
+ and locations["negative"].get("assert_type") == {expected["assert_type"]}
+ )
+ for name in CASES:
+ positive = locations["positive"].get(name, set())
+ negative = locations["negative"].get(name, set())
+ cases[name] = {
+ "passed": control_passed and not positive and negative == {expected[name]},
+ "positive_diagnostic_lines": sorted(positive),
+ "negative_diagnostic_lines": sorted(negative),
+ "expected_error_line": expected[name],
+ }
+ suite["results"][checker] = {
+ "ok": True, "cases": cases, "evidence": evidence,
+ }
+ return suite
diff --git a/build/benchmark/compare_benchmarks.py b/build/benchmark/compare_benchmarks.py
index ab50c3f4d473..baf2adc3cda0 100644
--- a/build/benchmark/compare_benchmarks.py
+++ b/build/benchmark/compare_benchmarks.py
@@ -170,6 +170,22 @@ def _analyze(
candidate_metrics = _metrics_by_package(candidate)
baseline_packages = _packages_by_name(baseline)
candidate_packages = _packages_by_name(candidate)
+ old_suite = baseline.get("capability_suite")
+ new_suite = candidate.get("capability_suite")
+ if old_suite is not None or new_suite is not None:
+ if old_suite is None or new_suite is None or any(
+ old_suite.get(field) != new_suite.get(field)
+ for field in ("version", "sha256", "cases")
+ ):
+ failures.append("capability suite changed or was not evaluated")
+ else:
+ for checker, old_result in old_suite["results"].items():
+ new_result = new_suite["results"].get(checker, {})
+ if (
+ old_result.get("ok") != new_result.get("ok")
+ or old_result.get("cases") != new_result.get("cases")
+ ):
+ failures.append(f"{checker}: behavioral capability outcomes changed")
for key, old in sorted(baseline_metrics.items()):
package, checker = key
diff --git a/build/benchmark/merge_benchmark_results.py b/build/benchmark/merge_benchmark_results.py
index bffe4282e1f1..baeb014e1f41 100644
--- a/build/benchmark/merge_benchmark_results.py
+++ b/build/benchmark/merge_benchmark_results.py
@@ -40,6 +40,17 @@ def merge_results(paths: list[Path]) -> dict[str, Any]:
metadata = {
key: value for key, value in data.items() if key not in VARIABLE_FIELDS
}
+ if "capability_suite" in metadata:
+ suite = metadata["capability_suite"]
+ metadata["capability_suite"] = {
+ **suite,
+ "results": {
+ checker: {
+ key: value for key, value in result.items() if key != "evidence"
+ }
+ for checker, result in suite["results"].items()
+ },
+ }
if expected_metadata is None:
expected_metadata = metadata
merged = copy.deepcopy(data)
diff --git a/build/benchmark/render_benchmark_html.py b/build/benchmark/render_benchmark_html.py
index 39379068edba..ee08129261a4 100644
--- a/build/benchmark/render_benchmark_html.py
+++ b/build/benchmark/render_benchmark_html.py
@@ -73,6 +73,26 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
metrics = {
checker: _package_metrics(results[checker], checker) for checker in checkers
}
+ suites = {
+ checker: data["capability_suite"]
+ for checker, data in results.items() if "capability_suite" in data
+ }
+ identities = {
+ json.dumps(
+ [suite["version"], suite["sha256"], suite["cases"]], sort_keys=True
+ )
+ for suite in suites.values()
+ }
+ if len(identities) > 1:
+ raise ValueError("Checker results use different capability suites")
+ capability_cases = next(iter(suites.values()))["cases"] if suites else {}
+ capabilities = {
+ checker: suite["results"].get(checker, {})
+ for checker, suite in suites.items()
+ }
+ for checker, result in capabilities.items():
+ if result.get("ok") and set(result["cases"]) != set(capability_cases):
+ raise ValueError(f"{checker}: incomplete capability results")
source_scopes: dict[str, tuple[int, str]] = {}
for checker in checkers:
for package in results[checker].get("results", []):
@@ -110,6 +130,15 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
total = totals[checker]
share = total / max_total * 100 if max_total else 0.0
version = results[checker].get("type_checker_versions", {}).get(checker, "unknown")
+ capability_cell = ""
+ if suites:
+ capability = capabilities.get(checker, {})
+ text = (
+ f'{sum(case["passed"] for case in capability["cases"].values())}'
+ f' / {len(capability_cases)}'
+ if capability.get("ok") else "Not evaluated"
+ )
+ capability_cell = f"{text} | "
summary_rows.append(
""
f"| {html.escape(_checker_label(checker))}"
@@ -119,9 +148,53 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
f"{total:.1f}s"
f" | {float(aggregate.get('p50_execution_time_s', 0.0)):.1f}s | "
f"{float(aggregate.get('p50_peak_memory_mb', 0.0)):.0f} MB | "
+ + capability_cell +
"
"
)
+ capability_section = ""
+ if suites:
+ rows = []
+ for name, label in capability_cases.items():
+ cells = []
+ for checker in checkers:
+ capability = capabilities.get(checker, {})
+ if not capability.get("ok"):
+ error = html.escape(capability.get("error", "No capability audit recorded"))
+ cells.append(f'Not evaluated{error} | ')
+ continue
+ case = capability["cases"][name]
+ if case["passed"]:
+ cells.append('Match | ')
+ else:
+ details = (
+ f'Positive diagnostic lines: {case["positive_diagnostic_lines"]}; '
+ f'negative diagnostic lines: {case["negative_diagnostic_lines"]}; '
+ f'expected negative line: {case["expected_error_line"]}.'
+ )
+ cells.append(
+ f'Mismatch{html.escape(details)} | '
+ )
+ rows.append(
+ f'| {html.escape(label)} | {"".join(cells)}
'
+ )
+ headers = "".join(
+ f'{html.escape(_checker_label(checker))} | ' for checker in checkers
+ )
+ capability_section = (
+ 'Behavioral capability probes
'
+ 'Untimed paired probes run after corpus timings with the same '
+ 'checker configurations. A match requires accepting the positive example, preserving '
+ 'asserted types, and reporting exactly the intended negative diagnostic line. '
+ 'The assertion control must also pass. Extra strictness diagnostics can cause a mismatch; '
+ 'a mismatch does not necessarily mean less inference. Raw JSON retains diagnostic evidence.
'
+ 'These few examples are not a comprehensive conformance or quality score, '
+ 'do not certify equivalent analysis on the timed projects, and are not a denominator '
+ 'for normalizing runtime. Historical results without probes are not evaluated.
'
+ '| Behavior | '
+ f'{headers}
{"".join(rows)}
'
+ )
+
package_rows: list[str] = []
for package in packages:
cells = []
@@ -262,13 +335,14 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
Run summary
Generated {html.escape(timestamp)} · Python {html.escape(str(environment.get('python_version', 'unknown')))} · {html.escape(str(environment.get('platform_details', environment.get('platform', 'unknown'))))}
-| Checker | Measured | Failed | Total time | Median time | Median RSS |
{''.join(summary_rows)}
+| Checker | Measured | Failed | Total time | Median time | Median RSS | {'Behavior matches | ' if suites else ''}
{''.join(summary_rows)}
Package comparison
Different checker jobs run on separate hosted runners, so timings are independent trends rather than a fastest-checker ranking. The two Pyright modes run sequentially in one job. Cells show wall time and peak RSS.
Checker defaults, inference, and diagnostic coverage differ; these timings do not measure equivalent analysis or time per rule. Pyrefly uses its all preset; Pyright uses basic mode. These presets are not equivalent. The two ty modes run sequentially in one job: ty retains its default return inference, while ty (return inference) enables inference for unannotated returns. Neither mode guarantees Pyright-equivalent inference.
Linux RSS measures the parent checker process, not its child processes; multi-process memory totals are not comparable.
Verified source-file counts come from an untimed checker audit against a shared implementation-first manifest. Imported dependencies are not included. Historical results without verified counts have not passed this audit.
| Package | {checker_headers}
{''.join(package_rows)}
{threading_section}
+{capability_section}
{pyright_stats_section}
diff --git a/build/benchmark/test_benchmark_capabilities.py b/build/benchmark/test_benchmark_capabilities.py
new file mode 100644
index 000000000000..678c7679b0fe
--- /dev/null
+++ b/build/benchmark/test_benchmark_capabilities.py
@@ -0,0 +1,126 @@
+import os
+import unittest
+from pathlib import Path
+
+import benchmark_capabilities as capabilities
+import typecheck_benchmark as benchmark
+
+
+def _output(paths: list[Path], phase: str) -> capabilities.CapabilityOutput:
+ diagnostics = []
+ for path in paths:
+ for line, text in enumerate(path.read_text().splitlines(), 1):
+ if phase == "negative" and "# expect-error" in text:
+ diagnostics.append(f"{path}:{line}:1: error: expected")
+ return {
+ "stdout": "\n".join(diagnostics),
+ "stderr": "",
+ "returncode": 1 if diagnostics else 0,
+ "timed_out": False,
+ "oom_killed": False,
+ "files_selected": len(paths),
+ }
+
+
+class BenchmarkCapabilitiesTest(unittest.TestCase):
+ def test_requires_positive_precision_and_negative_diagnostics(self) -> None:
+ def run(checker, root, paths):
+ output = _output(paths, root.name)
+ if root.name == "positive":
+ output["stdout"] = "generics.py:6:1: error: Any is not int"
+ output["returncode"] = 1
+ else:
+ output["stdout"] = "\n".join(
+ line for line in output["stdout"].splitlines()
+ if "unannotated_return.py:" not in line
+ )
+ output["stdout"] += "\nprotocols.py:4:1: error: unexpected"
+ return output
+
+ suite = capabilities.run_capability_suite(["checker"], run)
+ result = suite["results"]["checker"]
+ self.assertTrue(result["ok"])
+ self.assertFalse(result["cases"]["generics"]["passed"])
+ self.assertFalse(result["cases"]["unannotated_return"]["passed"])
+ self.assertFalse(result["cases"]["protocols"]["passed"])
+ self.assertTrue(result["cases"]["overloads"]["passed"])
+ self.assertIn("Any is not int", result["evidence"]["positive"]["stdout"])
+
+ def test_missing_assert_type_control_cannot_pass_other_cases(self) -> None:
+ def run(checker, root, paths):
+ output = _output(paths, root.name)
+ output["stdout"] = "\n".join(
+ line for line in output["stdout"].splitlines()
+ if "assert_type.py:" not in line
+ )
+ return output
+
+ suite = capabilities.run_capability_suite(["checker"], run)
+ cases = suite["results"]["checker"]["cases"]
+ self.assertTrue(all(not case["passed"] for case in cases.values()))
+
+ def test_incomplete_or_unrecognized_runs_are_not_coverage(self) -> None:
+ for fault in ("count", "timeout", "fatal", "format", "outside"):
+ with self.subTest(fault=fault):
+ def run(checker, root, paths):
+ output = _output(paths, root.name)
+ if fault == "count":
+ output["files_selected"] = len(paths) - 1
+ elif fault == "timeout":
+ output["timed_out"] = True
+ elif fault == "fatal":
+ output["returncode"] = 2
+ elif fault == "format":
+ output["returncode"] = 1
+ output["stdout"] = "Unrecognized diagnostics"
+ else:
+ output["returncode"] = 1
+ output["stdout"] = "capability_support.py:1: error: unexpected"
+ return output
+
+ result = capabilities.run_capability_suite(["checker"], run)["results"]["checker"]
+ self.assertFalse(result["ok"])
+ self.assertIn("error", result)
+ self.assertNotIn("cases", result)
+
+ def test_checker_start_failure_does_not_hide_other_checkers(self) -> None:
+ def run(checker, root, paths):
+ if checker == "missing":
+ raise FileNotFoundError("Not installed")
+ return _output(paths, root.name)
+
+ results = capabilities.run_capability_suite(["missing", "installed"], run)["results"]
+ self.assertFalse(results["missing"]["ok"])
+ self.assertIn("Not installed", results["missing"]["error"])
+ self.assertTrue(all(case["passed"] for case in results["installed"]["cases"].values()))
+
+ @unittest.skipUnless(
+ os.environ.get("TYPECHECK_BENCHMARK_INTEGRATION") == "1",
+ "Set TYPECHECK_BENCHMARK_INTEGRATION=1 with all checkers installed",
+ )
+ def test_real_profiles_verify_precision_and_inference_difference(self) -> None:
+ def run(checker, root, paths):
+ result = benchmark._run_checker_process(
+ checker, root, paths, 60, 0, audit_files=True,
+ )
+ return {
+ **result,
+ "files_selected": benchmark._parse_source_file_count(
+ checker, result["stdout"] + "\n" + result["stderr"]
+ ),
+ }
+
+ checkers = [*benchmark.DEFAULT_TYPE_CHECKERS, "ty-infer", "pyright-threads"]
+ suite = capabilities.run_capability_suite(checkers, run)
+ for checker, result in suite["results"].items():
+ with self.subTest(checker=checker):
+ self.assertTrue(result["ok"], result)
+ self.assertTrue(result["cases"]["assert_type"]["passed"], result)
+ self.assertTrue(result["cases"]["generics"]["passed"], result)
+ self.assertTrue(result["cases"]["imported_types"]["passed"], result)
+ self.assertFalse(suite["results"]["ty"]["cases"]["unannotated_return"]["passed"])
+ self.assertTrue(suite["results"]["ty-infer"]["cases"]["unannotated_return"]["passed"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py
index 65ad663649c5..c1ebea145e37 100644
--- a/build/benchmark/test_compare_benchmarks.py
+++ b/build/benchmark/test_compare_benchmarks.py
@@ -77,6 +77,23 @@ def _result(time: float, memory: float, ok: bool = True) -> dict:
class CompareBenchmarksTest(unittest.TestCase):
+ def test_rejects_faster_results_with_changed_capability_evidence(self) -> None:
+ baseline = _result(10.0, 100.0)
+ candidate = _result(5.0, 100.0)
+ for result in (baseline, candidate):
+ result["capability_suite"] = {
+ "version": 1, "sha256": "same", "cases": {"returns": "Return inference"},
+ "results": {
+ "pyright": {"ok": True, "cases": {"returns": {"passed": True}}},
+ },
+ }
+ candidate["capability_suite"]["results"]["pyright"]["cases"]["returns"]["passed"] = False
+ failures, _ = compare_benchmarks._analyze(baseline, candidate, 10.0)
+ self.assertTrue(any("capability outcomes changed" in failure for failure in failures))
+ candidate["capability_suite"]["sha256"] = "different"
+ failures, _ = compare_benchmarks._analyze(baseline, candidate, 10.0)
+ self.assertIn("capability suite changed or was not evaluated", failures)
+
def test_accepts_changes_within_threshold(self) -> None:
with redirect_stdout(io.StringIO()):
failures = compare_benchmarks.compare(
diff --git a/build/benchmark/test_merge_benchmark_results.py b/build/benchmark/test_merge_benchmark_results.py
index 132a0a199b75..915dca2fa06e 100644
--- a/build/benchmark/test_merge_benchmark_results.py
+++ b/build/benchmark/test_merge_benchmark_results.py
@@ -38,6 +38,35 @@ def _result(package: str, time: float) -> dict:
class MergeBenchmarkResultsTest(unittest.TestCase):
+ def test_merges_variable_capability_evidence_but_rejects_changed_behavior(self) -> None:
+ with tempfile.TemporaryDirectory() as temp_dir:
+ root = Path(temp_dir)
+ paths = []
+ data = []
+ for index, package in enumerate(("click", "numpy")):
+ result = _result(package, float(index + 1))
+ result["capability_suite"] = {
+ "version": 1, "sha256": "same", "cases": {"returns": "Return inference"},
+ "results": {
+ "pyright": {
+ "ok": True, "cases": {"returns": {"passed": True}},
+ "evidence": {"stdout": f"Different temp path {index}", "time": index},
+ },
+ },
+ }
+ path = root / f"{index}.json"
+ path.write_text(json.dumps(result), encoding="utf-8")
+ paths.append(path)
+ data.append(result)
+ merged = merge_benchmark_results.merge_results(paths)
+ self.assertEqual(merged["package_count"], 2)
+ self.assertEqual(merged["capability_suite"], data[0]["capability_suite"])
+
+ data[1]["capability_suite"]["results"]["pyright"]["cases"]["returns"]["passed"] = False
+ paths[1].write_text(json.dumps(data[1]), encoding="utf-8")
+ with self.assertRaisesRegex(ValueError, "different benchmark metadata"):
+ merge_benchmark_results.merge_results(paths)
+
def test_merges_packages_and_recomputes_aggregate(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
diff --git a/build/benchmark/test_render_benchmark_html.py b/build/benchmark/test_render_benchmark_html.py
index 8f33c70d7215..d973cbdcfb64 100644
--- a/build/benchmark/test_render_benchmark_html.py
+++ b/build/benchmark/test_render_benchmark_html.py
@@ -69,6 +69,52 @@ def handle_data(self, data: str) -> None:
class RenderBenchmarkHtmlTest(unittest.TestCase):
+ def test_renders_behavior_matches_without_quality_adjusting_times(self) -> None:
+ pyright = _result("pyright", 2.0, 200.0)
+ ty = _result("ty", 1.0, 100.0)
+ for checker, result in (("pyright", pyright), ("ty", ty)):
+ result["capability_suite"] = {
+ "version": 1,
+ "sha256": "same",
+ "cases": {"returns": "Return inference"},
+ "results": {
+ checker: {
+ "ok": True,
+ "cases": {
+ "returns": {
+ "passed": checker == "pyright",
+ "positive_diagnostic_lines": [] if checker == "pyright" else [3],
+ "negative_diagnostic_lines": [4] if checker == "pyright" else [],
+ "expected_error_line": 4,
+ }
+ },
+ }
+ },
+ }
+ report = render_benchmark_html.render_html({"pyright": pyright, "ty": ty})
+ parser = _SemanticParser()
+ parser.feed(report)
+ self.assertIn("Behavior matches", parser.cells)
+ self.assertIn("1 / 1", parser.cells)
+ self.assertIn("0 / 1", parser.cells)
+ self.assertIn("2.00s", parser.cells)
+ self.assertIn("1.00s", parser.cells)
+ self.assertIn("Behavioral capability probes", parser.headings)
+ self.assertIn("Positive diagnostic lines: [3]", report)
+ self.assertIn("do not certify equivalent analysis", report)
+
+ ty["capability_suite"]["sha256"] = "different"
+ with self.assertRaisesRegex(ValueError, "different capability suites"):
+ render_benchmark_html.render_html({"pyright": pyright, "ty": ty})
+ ty["capability_suite"]["sha256"] = "same"
+ ty["capability_suite"]["results"]["ty"] = {
+ "ok": False, "error": "",
+ }
+ report = render_benchmark_html.render_html({"pyright": pyright, "ty": ty})
+ self.assertIn("Not evaluated", report)
+ self.assertIn("<failed audit>", report)
+ self.assertNotIn("", report)
+
def test_requires_matching_manifests_and_verified_native_counts(self) -> None:
pyright = _result("pyright", 2.0, 200.0)
ty = _result("ty", 1.0, 100.0)
diff --git a/build/benchmark/test_typecheck_benchmark.py b/build/benchmark/test_typecheck_benchmark.py
index 6fa4d63d4439..cf21a9ab3a21 100644
--- a/build/benchmark/test_typecheck_benchmark.py
+++ b/build/benchmark/test_typecheck_benchmark.py
@@ -630,6 +630,7 @@ def test_local_mode_builds_pyright_without_clone_or_install(self) -> None:
return_value={"pyright": "1.2.3"},
),
patch.object(benchmark, "_save_results", return_value=result_path),
+ patch.object(benchmark, "run_capability_suite", return_value={"results": {}}),
patch.object(benchmark, "_print_summary"),
patch.object(benchmark, "clone_package") as clone,
patch.object(benchmark, "install_deps") as install,
diff --git a/build/benchmark/typecheck_benchmark.py b/build/benchmark/typecheck_benchmark.py
index 2202b3f2c295..19ac8b348509 100644
--- a/build/benchmark/typecheck_benchmark.py
+++ b/build/benchmark/typecheck_benchmark.py
@@ -10,8 +10,9 @@
"""Compare type-checker wall time and peak memory on configured Python projects.
-This benchmark measures speed and peak RSS only. It does not compare diagnostics
-or type-checking precision. Configured projects are cloned into a temporary
+Corpus timings measure speed and peak RSS, with a separate small behavioral
+capability audit that is not a comprehensive precision comparison.
+Configured projects are cloned into a temporary
directory and installed into package-specific dependency directories before
they are checked; --local benchmarks an existing directory without clone or installation.
"""
@@ -37,6 +38,8 @@
from pathlib import Path
from typing import Any, Sequence, TypedDict
+from benchmark_capabilities import CapabilityOutput, CapabilitySuite, run_capability_suite
+
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[1]
PYRIGHT_PACKAGE_DIR = REPO_ROOT / "packages" / "pyright"
@@ -63,7 +66,7 @@
}
PYRIGHT_DEFAULT_EXCLUDES = ["**/node_modules", "**/__pycache__", "**/.*"]
DEPENDENCY_ISOLATION = "pip-target-per-package"
-CHECKER_CONFIG_VERSION = 5
+CHECKER_CONFIG_VERSION = 6
class BenchmarkError(Exception):
@@ -862,7 +865,7 @@ def _build_checker_command(
return None, []
-def run_checker(
+def _run_checker_process(
checker: str,
package_path: Path,
check_paths: list[Path] | None,
@@ -871,8 +874,8 @@ def run_checker(
capture_output: bool = True,
environment: dict[str, str] | None = None,
audit_files: bool = False,
-) -> TimingMetrics:
- """Run one checker once using a minimal checker-specific configuration."""
+) -> ProcessResult:
+ """Execute a checker with the shared benchmark configuration and isolation."""
command, config_paths = _build_checker_command(
checker, package_path, check_paths
)
@@ -882,13 +885,7 @@ def run_checker(
if checker in AVAILABLE_TYPE_CHECKERS
else f"Unknown checker: {checker}"
)
- return {
- "ok": False,
- "execution_time_s": 0.0,
- "peak_memory_mb": 0.0,
- "oom_killed": False,
- "error_message": error_message,
- }
+ raise FileNotFoundError(error_message)
if audit_files:
if checker == "pyright-threads":
@@ -899,42 +896,59 @@ def run_checker(
command.append("-vv")
try:
- try:
- # ty merges user configuration even with an explicit project config.
- with (
- tempfile.TemporaryDirectory(
- prefix=".typecheck-benchmark-user-", dir=package_path
- )
- if checker in ("ty", "ty-infer")
- else nullcontext(None)
- ) as config_home:
- checker_environment = environment
- if config_home is not None:
- checker_environment = (
- environment if environment is not None else os.environ
- ).copy()
- checker_environment["APPDATA"] = config_home
- checker_environment["XDG_CONFIG_HOME"] = config_home
- result = run_process_with_timeout(
- command,
- cwd=package_path,
- timeout=timeout,
- memory_limit_mb=memory_limit_mb,
- capture_output=capture_output,
- environment=checker_environment,
- )
- except OSError as exc:
- return {
- "ok": False,
- "execution_time_s": 0.0,
- "peak_memory_mb": 0.0,
- "oom_killed": False,
- "error_message": f"Unable to start {checker}: {exc}",
- }
+ # ty merges user configuration even with an explicit project config.
+ with (
+ tempfile.TemporaryDirectory(
+ prefix=".typecheck-benchmark-user-", dir=package_path
+ )
+ if checker in ("ty", "ty-infer")
+ else nullcontext(None)
+ ) as config_home:
+ checker_environment = environment
+ if config_home is not None:
+ checker_environment = (
+ environment if environment is not None else os.environ
+ ).copy()
+ checker_environment["APPDATA"] = config_home
+ checker_environment["XDG_CONFIG_HOME"] = config_home
+ return run_process_with_timeout(
+ command,
+ cwd=package_path,
+ timeout=timeout,
+ memory_limit_mb=memory_limit_mb,
+ capture_output=capture_output,
+ environment=checker_environment,
+ )
finally:
for config_path in config_paths:
config_path.unlink(missing_ok=True)
+
+def run_checker(
+ checker: str,
+ package_path: Path,
+ check_paths: list[Path] | None,
+ timeout: int,
+ memory_limit_mb: int,
+ capture_output: bool = True,
+ environment: dict[str, str] | None = None,
+ audit_files: bool = False,
+) -> TimingMetrics:
+ """Run one checker once using a minimal checker-specific configuration."""
+ try:
+ result = _run_checker_process(
+ checker, package_path, check_paths, timeout, memory_limit_mb,
+ capture_output=capture_output, environment=environment, audit_files=audit_files,
+ )
+ except OSError as exc:
+ return {
+ "ok": False,
+ "execution_time_s": 0.0,
+ "peak_memory_mb": 0.0,
+ "oom_killed": False,
+ "error_message": f"Unable to start {checker}: {exc}",
+ }
+
if result["oom_killed"]:
memory = result["peak_memory_mb"]
memory_text = f" at {memory:.1f} MB" if memory > 0 else ""
@@ -1358,6 +1372,7 @@ def _save_results(
memory_limit_mb: int,
os_name: str | None,
local_dir: Path | None,
+ capability_suite: CapabilitySuite | None = None,
) -> Path:
timestamp = datetime.now(timezone.utc)
date = timestamp.strftime("%Y-%m-%d")
@@ -1401,6 +1416,8 @@ def _save_results(
output["os"] = os_name
if local_dir:
output["local_directory"] = str(local_dir)
+ if capability_suite is not None:
+ output["capability_suite"] = capability_suite
serialized = json.dumps(output, indent=2) + "\n"
dated_file.write_text(serialized, encoding="utf-8")
@@ -1531,6 +1548,29 @@ def run_benchmark(
)
)
+ print("\nUntimed behavioral capability audit:")
+
+ def run_capability(
+ checker: str, root: Path, paths: list[Path]
+ ) -> CapabilityOutput:
+ result = _run_checker_process(
+ checker, root, paths, min(timeout, 60), memory_limit_mb, audit_files=True,
+ )
+ return {
+ **result,
+ "files_selected": _parse_source_file_count(
+ checker, result["stdout"] + "\n" + result["stderr"]
+ ),
+ }
+
+ capability_suite = run_capability_suite(type_checkers, run_capability)
+ for checker, result in capability_suite["results"].items():
+ if result["ok"]:
+ passed = sum(case["passed"] for case in result["cases"].values())
+ print(f" {checker}: {passed}/{len(capability_suite['cases'])} cases passed")
+ else:
+ print(f" {checker}: not evaluated: {result['error']}")
+
aggregate = compute_aggregate_stats(results, type_checkers)
dated_file = _save_results(
results,
@@ -1544,6 +1584,7 @@ def run_benchmark(
memory_limit_mb,
os_name,
resolved_local_dir,
+ capability_suite,
)
print("\nBenchmark complete.")
From 198bfc5be2adcd3e7a4be287a4254b7817c1fa32 Mon Sep 17 00:00:00 2001
From: Bill Schnurr
Date: Wed, 28 Oct 2026 14:44:00 -0700
Subject: [PATCH 4/6] Audit corpus configs and preserve native checker file
discovery
Recursively inventory native checker configs and TOML/INI settings, including Pyrefly sub-config overrides, without publishing values. Surface parse and traversal failures and include discovered settings in JSON, console output, and weekly HTML.
Pass configured roots and benchmark exclusions to each checker instead of selecting implementations or stubs. Record native counts, mark differing scopes non-comparable, and exclude those timings from report totals. Keep explicit configuration isolation and validate it against hostile root/nested settings using real native discovery.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 35cc6021-56a0-45b2-9b66-3817d947cd99
---
build/benchmark/README.md | 50 ++--
build/benchmark/compare_benchmarks.py | 3 +
build/benchmark/render_benchmark_html.py | 95 ++++++-
build/benchmark/test_compare_benchmarks.py | 9 +
build/benchmark/test_render_benchmark_html.py | 37 +++
build/benchmark/test_typecheck_benchmark.py | 150 +++++++++--
build/benchmark/typecheck_benchmark.py | 247 +++++++++++++-----
7 files changed, 472 insertions(+), 119 deletions(-)
diff --git a/build/benchmark/README.md b/build/benchmark/README.md
index e1ae54e98ef8..752255ed34bc 100644
--- a/build/benchmark/README.md
+++ b/build/benchmark/README.md
@@ -17,11 +17,11 @@ Each entry in `install_envs.json` defines its repository, check paths, installat
additional dependencies. Repository commits are pinned so pull-request comparisons analyze the same
source revisions.
-Entries can define `exclude_directories` to prune named directories while resolving their check
-paths. The benchmark expands every entry to a sorted, deduplicated `.py`/`.pyi` manifest before
-invoking any checker. When both are selected for the same module, the `.py` implementation takes
-precedence over its `.pyi`; standalone stubs remain included. NumPy excludes its test directories; pandas includes its test
-and `_testing` directories. Generated Pyright configurations explicitly retain
+Entries can define `exclude_directories`, translated into exclusions in each benchmark config.
+The benchmark passes configured roots to each checker and preserves native file discovery; it
+does not select implementations over stubs or stubs over implementations. Native import resolution
+normally prefers stubs where available. NumPy excludes its test directories; pandas includes its test
+and `_testing` directories. Generated benchmark configurations explicitly retain
its standard `**/node_modules`, `**/__pycache__`, and `**/.*` exclusions.
## Prerequisites
@@ -115,29 +115,41 @@ skipping selected `dist`, `venv`, or ignored source files, or checking hidden di
Imported dependencies may still be analyzed differently; matching entry-point scope does not imply
matching transitive work.
-Every checker receives the same explicit source-file manifest, not independently expanded directory
-roots. A missing configured path or empty selection is an error; neither falls back to checking the
-whole repository. mypy and Zuban use explicit TOML file lists to avoid command-line size limits and
+Each checker discovers files natively within the configured roots. A missing configured path or
+empty root selection is an error; neither falls back to checking the whole repository.
+mypy and Zuban receive roots through TOML file lists to avoid command-line size limits and
Windows INI backslash escaping. ty's source selectors use gitignore-pattern separators even on
Windows, where native backslashes can silently match no files.
+Before checking, an **untimed repository configuration search** recursively inventories native
+`pyrightconfig.json`, `ty.toml`, `pyrefly.toml`, `zuban.toml`, `mypy.ini`, and `.mypy.ini` candidates.
+It also searches TOML files for checker-specific `[tool.*]` settings and `project.requires-python`,
+and INI settings in `setup.cfg` and mypy configs. JSON records `repository_configs` per package;
+console output and the weekly HTML show paths, sections/setting names, and that explicit benchmark
+configs are passed instead. Configuration values are not published, and detected does not mean
+applied. Invalid TOML/INI files produce visible audit parse errors rather than disappearing from
+the inventory. The search skips VCS metadata, `node_modules`, `__pycache__`, and harness-generated
+configs. It does not modify or delete repository configs. The real-binary suppression tests below,
+not discovery alone, provide evidence that conflicting corpus configs cannot override the harness.
+
Before any warmups or measured runs, each checker performs one **untimed file-count audit** using
-the benchmark configuration and the selected files. Pyright reports checked files in `--stats`,
+the benchmark configuration and native discovery. Pyright reports checked files in `--stats`,
threaded Pyright reports files to analyze with `--verbose`, Pyrefly reports project modules with
`--summary=full`, ty logs checked file paths with `-vv`, and mypy/Zuban report checked source-file
-counts in their summaries. Missing counts, fatal errors, and counts different from the manifest
-reject that checker's measurements. Verbose/debug audit output is not enabled for measured runs.
+counts in their summaries. Missing/zero counts and fatal errors reject that checker's measurements.
+Verbose/debug audit output is not enabled for measured runs.
Counts in ordinary checker output are also checked during warmups and measured runs.
The audit is recorded separately as `uncounted_validation_runs_per_checker = 1`, including when
`--warmup 0` is selected; this is therefore not a cold-filesystem measurement.
-Result JSON retains `source_files`, `source_file_count`, and `source_manifest_sha256` per package;
-each successful checker records its native `files_selected`, `file_count_verified`, and separate
-`file_count_audit_time_s`. HTML cells show the verified count. The report rejects different
-manifests for the same package even if counts match, and the regression comparator also checks the
-manifest hash and audit-run setting. These are source entry-point counts, **not total imported or
-parsed files**; forcing equal dependency counts across different checkers would change their work.
-Counts also do not prove equal diagnostic coverage: repository and user configs are tested
+Each successful checker records its native `files_selected`, `file_count_verified`, and separate
+`file_count_audit_time_s`. Different audited counts make that package **Not comparable**, including
+across separate weekly checker jobs; its timings are excluded from report totals rather than
+forcing the tools to discover the same files. Raw JSON retains the measurements and differences.
+The regression comparator rejects changed native counts, roots, exclusions, or audit-run settings.
+These are source entry-point counts, **not total imported or parsed files**. Equal counts do not
+prove identical file identities, equal dependency work, or equal diagnostic coverage.
+Repository and user configs are tested
separately with deliberately suppressed diagnostics, and source-level ignore comments retain each
tool's semantics.
@@ -216,7 +228,7 @@ Pyrefly's exit-code-1 "no files matched" error is a failed measurement, not an o
The opt-in integration tests execute the real binaries against conflicting root/nested configs,
ignored files, directory-exclusion probes, and paired implementations/stubs. They verify every
-checker's native count against the same manifest, including clean files and standalone stubs,
+checker's native count before and after the hostile configs, including clean files and standalone stubs,
and check that `ty-infer` catches an inferred-return
assignment that the default profile does not. Install all default checkers and build Pyright first,
then set `TYPECHECK_BENCHMARK_INTEGRATION=1` and run:
diff --git a/build/benchmark/compare_benchmarks.py b/build/benchmark/compare_benchmarks.py
index baf2adc3cda0..4134c7e35a04 100644
--- a/build/benchmark/compare_benchmarks.py
+++ b/build/benchmark/compare_benchmarks.py
@@ -275,6 +275,9 @@ def _analyze(
)
continue
scope_changed = False
+ if old.get("files_selected") != new.get("files_selected"):
+ failures.append(f"{package}/{checker}: native source-file count changed")
+ scope_changed = True
for field in ("check_paths", "exclude_directories", "source_manifest_sha256"):
if old_package.get(field, []) != new_package.get(field, []):
failures.append(
diff --git a/build/benchmark/render_benchmark_html.py b/build/benchmark/render_benchmark_html.py
index ee08129261a4..e35c4d77c790 100644
--- a/build/benchmark/render_benchmark_html.py
+++ b/build/benchmark/render_benchmark_html.py
@@ -5,6 +5,7 @@
import argparse
import html
import json
+import statistics
from pathlib import Path
from typing import Any
@@ -44,7 +45,7 @@ def _ordered_checkers(results: dict[str, dict[str, Any]]) -> list[str]:
def _package_metrics(data: dict[str, Any], checker: str) -> dict[str, dict[str, Any]]:
return {
- package["package_name"]: package.get("metrics", {}).get(checker, {})
+ package["package_name"]: package.get("metrics", {}).get(checker, {}).copy()
for package in data.get("results", [])
if package.get("package_name")
}
@@ -110,11 +111,41 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
or metric.get("files_selected") != scope[0]
):
raise ValueError(f"{name}/{checker}: source-file count is not verified")
+ mismatched_packages = set()
+ for package in packages:
+ native_counts = {
+ metric["files_selected"]
+ for checker in checkers
+ if (metric := metrics[checker].get(package, {})).get("file_count_verified")
+ }
+ if len(native_counts) > 1:
+ mismatched_packages.add(package)
+ for checker in checkers:
+ metric = metrics[checker].get(package, {})
+ if metric.get("file_count_verified"):
+ metric["ok"] = False
+ metric["source_count_mismatch"] = True
+ aggregates = {
+ checker: results[checker].get("aggregate", {}).get(checker, {})
+ for checker in checkers
+ }
+ if mismatched_packages:
+ for checker in checkers:
+ accepted = [metric for metric in metrics[checker].values() if metric.get("ok")]
+ aggregates[checker] = {
+ "packages_tested": len(accepted),
+ "packages_failed": len(metrics[checker]) - len(accepted),
+ "total_execution_time_s": sum(metric["execution_time_s"] for metric in accepted),
+ "p50_execution_time_s": statistics.median(
+ [metric["execution_time_s"] for metric in accepted]
+ ) if accepted else 0,
+ "p50_peak_memory_mb": statistics.median(
+ [metric.get("peak_memory_mb", 0) for metric in accepted]
+ ) if accepted else 0,
+ }
totals = {
checker: float(
- results[checker].get("aggregate", {})
- .get(checker, {})
- .get("total_execution_time_s", 0.0)
+ aggregates[checker].get("total_execution_time_s", 0.0)
)
for checker in checkers
}
@@ -124,7 +155,7 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
summary_rows: list[str] = []
for checker in checkers:
- aggregate = results[checker].get("aggregate", {}).get(checker, {})
+ aggregate = aggregates[checker]
tested = int(aggregate.get("packages_tested", 0))
failed = int(aggregate.get("packages_failed", 0))
total = totals[checker]
@@ -201,7 +232,12 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
for checker in checkers:
metric = metrics[checker].get(package, {})
if not metric.get("ok"):
- cells.append('Not measured | ')
+ cells.append(
+ 'Not comparable'
+ f'{metric["files_selected"]} native source files | '
+ if metric.get("source_count_mismatch")
+ else 'Not measured | '
+ )
continue
cells.append(
f'{float(metric["execution_time_s"]):.2f}s'
@@ -219,6 +255,50 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
f' | | {html.escape(package)}{scope_label} | {"".join(cells)}
'
)
+ config_rows = []
+ for package in packages:
+ records = {
+ json.dumps(record, sort_keys=True)
+ for data in results.values()
+ for item in data.get("results", [])
+ if item.get("package_name") == package
+ for record in item.get("repository_configs", [])
+ }
+ audited = any(
+ "repository_configs" in item
+ for data in results.values() for item in data.get("results", [])
+ if item.get("package_name") == package
+ )
+ if not records and audited:
+ config_rows.append(
+ f'| {html.escape(package)} | '
+ 'No checker configs or TOML settings detected |
'
+ )
+ for serialized in sorted(records):
+ record = json.loads(serialized)
+ detail = ", ".join(
+ record.get("settings") or record["sections"] or record["checkers"]
+ ) or "Unknown"
+ handling = (
+ f'Scan parse error: {record["parse_error"]}'
+ if "parse_error" in record else record["disposition"]
+ )
+ config_rows.append(
+ f'| {html.escape(package)} | {html.escape(record["path"])} | '
+ f'{html.escape(detail)} | {html.escape(handling)} |
'
+ )
+ config_section = (
+ 'Repository configuration audit
'
+ 'Untimed recursive search of each checkout for native checker configs, '
+ 'TOML tool sections, and mypy INI settings. Detected does not mean applied: the harness '
+ 'passes explicit benchmark configurations instead. Values are not published. Parse errors '
+ 'are shown rather than treated as absence. The search skips VCS metadata, node_modules, '
+ 'and __pycache__; source-level ignore comments retain their native semantics.
'
+ '| Package | File | '
+ 'Checker / sections | Handling |
'
+ f'{"".join(config_rows)}
'
+ ) if config_rows else ""
+
pyright_stats_rows: list[str] = []
for package, metric in sorted(metrics.get("pyright", {}).items()):
phase_times = metric.get("phase_times_s")
@@ -339,10 +419,11 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
Package comparison
Different checker jobs run on separate hosted runners, so timings are independent trends rather than a fastest-checker ranking. The two Pyright modes run sequentially in one job. Cells show wall time and peak RSS.
Checker defaults, inference, and diagnostic coverage differ; these timings do not measure equivalent analysis or time per rule. Pyrefly uses its all preset; Pyright uses basic mode. These presets are not equivalent. The two ty modes run sequentially in one job: ty retains its default return inference, while ty (return inference) enables inference for unannotated returns. Neither mode guarantees Pyright-equivalent inference.
Linux RSS measures the parent checker process, not its child processes; multi-process memory totals are not comparable.
-Verified source-file counts come from an untimed checker audit against a shared implementation-first manifest. Imported dependencies are not included. Historical results without verified counts have not passed this audit.
+Each checker discovers files natively within the configured roots; the harness does not choose .py over .pyi or vice versa. Untimed audits record native source-file counts, not imported dependencies. Different counts make a package not comparable and exclude its timings from report totals. Equal counts do not prove identical files or analysis. Historical results without audits are not verified.
| Package | {checker_headers}
{''.join(package_rows)}
{threading_section}
{capability_section}
+{config_section}
{pyright_stats_section}
diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py
index c1ebea145e37..486052017693 100644
--- a/build/benchmark/test_compare_benchmarks.py
+++ b/build/benchmark/test_compare_benchmarks.py
@@ -77,6 +77,15 @@ def _result(time: float, memory: float, ok: bool = True) -> dict:
class CompareBenchmarksTest(unittest.TestCase):
+ def test_rejects_a_speedup_with_a_smaller_native_file_count(self) -> None:
+ baseline = _result(10.0, 100.0)
+ candidate = _result(5.0, 100.0)
+ baseline["results"][0]["metrics"]["pyright"]["files_selected"] = 2
+ candidate["results"][0]["metrics"]["pyright"]["files_selected"] = 1
+ failures, rows = compare_benchmarks._analyze(baseline, candidate, 10.0)
+ self.assertIn("example/pyright: native source-file count changed", failures)
+ self.assertEqual(rows, [])
+
def test_rejects_faster_results_with_changed_capability_evidence(self) -> None:
baseline = _result(10.0, 100.0)
candidate = _result(5.0, 100.0)
diff --git a/build/benchmark/test_render_benchmark_html.py b/build/benchmark/test_render_benchmark_html.py
index d973cbdcfb64..549a83a3eee1 100644
--- a/build/benchmark/test_render_benchmark_html.py
+++ b/build/benchmark/test_render_benchmark_html.py
@@ -69,6 +69,43 @@ def handle_data(self, data: str) -> None:
class RenderBenchmarkHtmlTest(unittest.TestCase):
+ def test_native_count_mismatches_are_not_comparable_or_in_summary_totals(self) -> None:
+ pyright = _result("pyright", 2.0, 200.0)
+ mypy = _result("mypy", 1.0, 100.0)
+ for checker, result, count in (("pyright", pyright, 2), ("mypy", mypy, 1)):
+ result["results"][0]["metrics"][checker].update(
+ files_selected=count, file_count_verified=True
+ )
+ report = render_benchmark_html.render_html({"pyright": pyright, "mypy": mypy})
+ self.assertEqual(report.count("Not comparable"), 2)
+ self.assertIn("2 native source files", report)
+ self.assertIn("1 native source files", report)
+ self.assertNotIn("2.00s", report)
+ self.assertNotIn("1.00s", report)
+ self.assertNotIn("2.0s", report)
+ self.assertTrue(pyright["results"][0]["metrics"]["pyright"]["ok"])
+
+ def test_repository_configuration_audit_is_visible_and_escaped(self) -> None:
+ result = _result("ty", 1.0, 100.0)
+ result["results"][0]["repository_configs"] = [
+ {
+ "path": "nested/pyproject.toml", "checkers": ["ty"],
+ "sections": ["tool.ty"], "settings": ["tool.ty.rules.all"],
+ "disposition": "not applied: explicit benchmark config",
+ },
+ {
+ "path": "broken.toml", "checkers": [], "sections": [], "settings": [],
+ "parse_error": "",
+ },
+ ]
+ report = render_benchmark_html.render_html({"ty": result})
+ self.assertIn("Repository configuration audit", report)
+ self.assertIn("nested/pyproject.toml", report)
+ self.assertIn("tool.ty.rules.all", report)
+ self.assertIn("not applied: explicit benchmark config", report)
+ self.assertIn("Scan parse error: <invalid TOML>", report)
+ self.assertNotIn("", report)
+
def test_renders_behavior_matches_without_quality_adjusting_times(self) -> None:
pyright = _result("pyright", 2.0, 200.0)
ty = _result("ty", 1.0, 100.0)
diff --git a/build/benchmark/test_typecheck_benchmark.py b/build/benchmark/test_typecheck_benchmark.py
index cf21a9ab3a21..cb44dcbb9dd8 100644
--- a/build/benchmark/test_typecheck_benchmark.py
+++ b/build/benchmark/test_typecheck_benchmark.py
@@ -13,7 +13,88 @@
class TypecheckBenchmarkTest(unittest.TestCase):
- def test_resolve_check_paths_excludes_named_directories(self) -> None:
+ def test_repository_config_audit_finds_native_files_and_toml_settings_without_values(self) -> None:
+ nested = self.root / "source"
+ nested.mkdir()
+ (self.root / "pyproject.toml").write_text(
+ '[project]\nrequires-python = ">=3.10"\n'
+ '[tool.ty.rules]\nall = "ignore"\n'
+ '[tool.pyrefly.errors]\nbad-assignment = false\n'
+ '[[tool.pyrefly.sub-config]]\nmatches = "**/*.py"\n'
+ '[tool.pyrefly.sub-config.errors]\nbad-return = false\n'
+ )
+ (self.root / "pyrightconfig.json").write_text('{"typeCheckingMode":"off"}')
+ (nested / "ty.toml").write_text('[src]\nexclude = ["sentinel-value"]\n')
+ (nested / "pyrefly.toml").write_text('project-includes = ["sentinel-value"]\n')
+ (nested / "setup.cfg").write_text("[mypy]\nignore_errors = True\n")
+ (nested / "custom.toml").write_text('[tool.mypy]\nignore_errors = true\n')
+ (self.root / ".git").mkdir()
+ (self.root / ".git" / "ty.toml").write_text('[rules]\nall = "ignore"\n')
+ (self.root / ".typecheck-benchmark-ty-generated.toml").write_text("[rules]\n")
+ records = benchmark._audit_repository_configs(self.root)
+ by_path = {record["path"]: record for record in records}
+ self.assertEqual(set(by_path), {
+ "pyproject.toml", "pyrightconfig.json", "source/ty.toml",
+ "source/pyrefly.toml", "source/setup.cfg", "source/custom.toml",
+ })
+ self.assertIn("tool.ty.rules.all", by_path["pyproject.toml"]["settings"])
+ self.assertIn("tool.pyrefly.errors.bad-assignment", by_path["pyproject.toml"]["settings"])
+ self.assertIn("tool.pyrefly.sub-config[0].errors.bad-return", by_path["pyproject.toml"]["settings"])
+ self.assertIn("mypy.ignore_errors", by_path["source/setup.cfg"]["settings"])
+ self.assertEqual(by_path["source/custom.toml"]["checkers"], ["mypy", "zuban"])
+ self.assertIn("not applied", by_path["source/ty.toml"]["disposition"])
+ self.assertNotIn("sentinel-value", json.dumps(records))
+
+ def test_repository_config_audit_surfaces_parse_errors_without_config_contents(self) -> None:
+ (self.root / "pyproject.toml").write_text('invalid = "sentinel-value')
+ (self.root / "mypy.ini").write_text("[mypy]\nsentinel-value\n")
+ records = benchmark._audit_repository_configs(self.root)
+ self.assertEqual(len(records), 2)
+ self.assertTrue(all("parse_error" in record for record in records))
+ self.assertNotIn("sentinel-value", json.dumps(records))
+
+ def test_incomplete_repository_search_is_not_a_successful_empty_audit(self) -> None:
+ with (
+ patch.object(benchmark, "_audit_repository_configs", side_effect=PermissionError("denied")),
+ patch.object(benchmark, "run_checker") as run,
+ redirect_stdout(io.StringIO()),
+ ):
+ result = benchmark._benchmark_directory(
+ name="local", github_url=None, package_path=self.root, resolved_paths=None,
+ type_checkers=["ty"], timeout=30, runs=1, warmup=0, memory_limit_mb=0,
+ )
+ self.assertIn("Repository configuration audit failed", result["error"])
+ self.assertNotIn("repository_configs", result)
+ run.assert_not_called()
+
+ def test_different_native_counts_are_not_accepted_as_comparable_timings(self) -> None:
+ (self.root / "module.py").write_text("")
+ (self.root / "module.pyi").write_text("")
+ roots = []
+
+ def run(checker, package_path, check_paths, *args, **kwargs):
+ roots.append(check_paths)
+ return {
+ "ok": True, "execution_time_s": 0.1, "peak_memory_mb": 10.0,
+ "oom_killed": False, "files_selected": 2 if checker == "pyright" else 1,
+ }
+
+ with (
+ patch.object(benchmark, "_checker_command", return_value=["checker"]),
+ patch.object(benchmark, "run_checker", side_effect=run),
+ redirect_stdout(io.StringIO()),
+ ):
+ result = benchmark._benchmark_directory(
+ name="local", github_url=None, package_path=self.root, resolved_paths=None,
+ type_checkers=["pyright", "mypy"], timeout=30, runs=1, warmup=0,
+ memory_limit_mb=0,
+ )
+ self.assertTrue(all(paths == [self.root] for paths in roots))
+ self.assertTrue(all(not metric["ok"] for metric in result["metrics"].values()))
+ self.assertTrue(all(metric["source_count_mismatch"] for metric in result["metrics"].values()))
+ self.assertEqual(benchmark.compute_aggregate_stats([result], ["pyright"])["pyright"]["packages_tested"], 0)
+
+ def test_resolve_check_paths_preserves_directories_for_native_discovery(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
package_path = Path(temp_dir)
(package_path / "source" / "tests").mkdir(parents=True)
@@ -23,16 +104,15 @@ def test_resolve_check_paths_excludes_named_directories(self) -> None:
(package_path / "source" / "tests" / "test_module.py").write_text("")
paths, missing = benchmark._resolve_check_paths(
- package_path, ["source"], ["tests"]
+ package_path, ["source"]
)
self.assertEqual(missing, [])
self.assertEqual(
- {path.relative_to(package_path).as_posix() for path in paths},
- {"source/module.py", "source/types.pyi"},
+ paths, [package_path / "source"],
)
- def test_manifest_prefers_implementations_deduplicates_and_prunes_defaults(self) -> None:
+ def test_resolve_check_paths_does_not_choose_implementations_or_stubs(self) -> None:
source = self.root / "source"
source.mkdir()
for name in ("module.py", "module.pyi", "stub.pyi", "clean.py", ".hidden.py"):
@@ -41,15 +121,14 @@ def test_manifest_prefers_implementations_deduplicates_and_prunes_defaults(self)
(source / name).mkdir()
(source / name / "ignored.py").write_text("")
paths, missing = benchmark._resolve_check_paths(
- self.root, ["source", "source/module.py", "missing"], ["tests"]
+ self.root, ["source", "source/module.py", "missing"]
)
self.assertEqual(missing, ["missing"])
self.assertEqual(
- [path.relative_to(source).name for path in paths],
- ["clean.py", "module.py", "stub.pyi"],
+ paths, [source, source / "module.py"],
)
stub_only, _ = benchmark._resolve_check_paths(
- self.root, ["source/module.pyi"], []
+ self.root, ["source/module.pyi"]
)
self.assertEqual(stub_only, [source / "module.pyi"])
@@ -463,6 +542,7 @@ def test_mypy_config_preserves_testing_paths(self) -> None:
"check_untyped_defs": True,
"python_version": benchmark.TARGET_PYTHON_VERSION,
"python_executable": benchmark.sys.executable,
+ "exclude": [r"(^|[/\\])(?:node_modules|__pycache__|\.[^/\\]+)([/\\]|$)"],
},
)
configs[0].unlink()
@@ -670,6 +750,7 @@ def run_checker(
capture_output: bool = True,
environment: dict[str, str] | None = None,
audit_files: bool = False,
+ exclude_directories: list[str] | None = None,
) -> benchmark.TimingMetrics:
calls.append(audit_files)
return {
@@ -701,7 +782,8 @@ def run_checker(
self.assertEqual(calls, [True, False, False])
self.assertEqual(
stdout.getvalue(),
- " pyright source-file audit: 1/1 verified\n"
+ " Repository config audit: 0 candidate files\n"
+ " pyright source-file audit: 1 native files verified\n"
" Running pyright (2 measured)...\n"
" Run 1/2... 1.000s, 2.0 MB\n"
" Run 2/2... 1.000s, 2.0 MB\n"
@@ -721,6 +803,7 @@ def run_checker(
capture_output: bool = True,
environment: dict[str, str] | None = None,
audit_files: bool = False,
+ exclude_directories: list[str] | None = None,
) -> benchmark.TimingMetrics:
captures.append(capture_output)
return {
@@ -750,12 +833,12 @@ def run_checker(
self.assertEqual(captures, [True, True, True, True])
- def test_file_count_mismatch_prevents_timed_runs(self) -> None:
+ def test_zero_native_file_count_prevents_timed_runs(self) -> None:
for name in ("clean.py", "second.py"):
(self.root / name).write_text("")
audit: benchmark.TimingMetrics = {
"ok": True, "execution_time_s": 0.1, "peak_memory_mb": 10.0,
- "oom_killed": False, "files_selected": 1,
+ "oom_killed": False, "files_selected": 0,
}
with (
patch.object(benchmark, "_checker_command", return_value=["checker"]),
@@ -769,12 +852,10 @@ def test_file_count_mismatch_prevents_timed_runs(self) -> None:
)
run.assert_called_once()
self.assertIs(run.call_args.kwargs["audit_files"], True)
- self.assertEqual(result["source_file_count"], 2)
- self.assertEqual(result["source_files"], ["clean.py", "second.py"])
- self.assertEqual(len(result["source_manifest_sha256"]), 64)
+ self.assertEqual(result["check_paths"], ["."])
self.assertFalse(result["metrics"]["ty"]["ok"])
self.assertFalse(result["metrics"]["ty"]["file_count_verified"])
- self.assertIn("expected 2, ty reported 1", result["metrics"]["ty"]["error_message"])
+ self.assertIn("ty reported 0", result["metrics"]["ty"]["error_message"])
def test_empty_source_selection_is_not_a_full_repository_fallback(self) -> None:
with patch.object(benchmark, "run_checker") as run:
@@ -818,10 +899,11 @@ def setUp(self) -> None:
self.root = Path(self.temp_dir.name)
def _run(
- self, checker: str, check_paths: list[Path] | None = None
+ self, checker: str, check_paths: list[Path] | None = None,
+ exclude_directories: list[str] | None = None,
) -> subprocess.CompletedProcess[str]:
command, configs = benchmark._build_checker_command(
- checker, self.root, check_paths or [self.root / "source"]
+ checker, self.root, check_paths or [self.root / "source"], exclude_directories
)
self.assertIsNotNone(command, f"{checker} must be installed for integration tests")
try:
@@ -846,13 +928,21 @@ def test_native_configs_and_ignore_files_do_not_change_benchmark_scope(self) ->
)
for name, content in (
("paired.py", 'value: int = "wrong"\n'),
- ("paired.pyi", "value: int\n"),
+ ("paired.pyi", "value: MissingType\n"),
("standalone.pyi", "value: int\n"),
("clean.py", "value: int = 1\n"),
):
(source / name).write_text(content)
(self.root / ".ignore").write_text("source/ignored.py\n")
(self.root / ".gitignore").write_text("source/ignored.py\n")
+ checkers = [*benchmark.DEFAULT_TYPE_CHECKERS, "ty-infer", "pyright-threads"]
+ baselines = {}
+ for checker in checkers:
+ baseline = benchmark.run_checker(
+ checker, self.root, [source], 60, 0, audit_files=True
+ )
+ self.assertTrue(baseline["ok"], baseline)
+ baselines[checker] = baseline["files_selected"]
native_configs = {
"pyrightconfig.json": '{"typeCheckingMode":"off","exclude":["**"]}',
"ty.toml": '[rules]\nall = "ignore"\n',
@@ -870,24 +960,32 @@ def test_native_configs_and_ignore_files_do_not_change_benchmark_scope(self) ->
for name, content in native_configs.items():
(directory / name).write_text(content)
- manifest, missing = benchmark._resolve_check_paths(self.root, ["source"], [])
+ roots, missing = benchmark._resolve_check_paths(self.root, ["source"])
self.assertEqual(missing, [])
- self.assertEqual(len(manifest), 8)
- self.assertNotIn(source / "paired.pyi", manifest)
- for checker in [*benchmark.DEFAULT_TYPE_CHECKERS, "ty-infer", "pyright-threads"]:
+ self.assertEqual(roots, [source])
+ for checker in checkers:
with self.subTest(checker=checker):
audit = benchmark.run_checker(
- checker, self.root, manifest, 60, 0, audit_files=True
+ checker, self.root, roots, 60, 0, audit_files=True
)
self.assertTrue(audit["ok"], audit)
- self.assertEqual(audit["files_selected"], len(manifest), audit)
- result = self._run(checker, manifest)
+ self.assertEqual(audit["files_selected"], baselines[checker], audit)
+ result = self._run(checker, roots)
output = result.stdout + result.stderr
self.assertEqual(result.returncode, 1, output)
for name in ("selected", "ignored", "dist_probe", "venv_probe", "tests_probe"):
self.assertIn(f"{name}.py", output, output)
+ self.assertIn("paired.pyi", output, output)
for name in ("outside", "node_modules_probe", "hidden_probe"):
self.assertNotIn(f"{name}.py", output, output)
+ excluded = benchmark.run_checker(
+ checker, self.root, roots, 60, 0, audit_files=True,
+ exclude_directories=["tests"],
+ )
+ self.assertTrue(excluded["ok"], excluded)
+ self.assertEqual(excluded["files_selected"], baselines[checker] - 1)
+ result = self._run(checker, roots, ["tests"])
+ self.assertNotIn("tests_probe.py", result.stdout + result.stderr)
for directory in (self.root, source):
for name, content in native_configs.items():
self.assertEqual((directory / name).read_text(), content)
diff --git a/build/benchmark/typecheck_benchmark.py b/build/benchmark/typecheck_benchmark.py
index 19ac8b348509..105dde8b526d 100644
--- a/build/benchmark/typecheck_benchmark.py
+++ b/build/benchmark/typecheck_benchmark.py
@@ -20,7 +20,7 @@
from __future__ import annotations
import argparse
-import hashlib
+import configparser
import json
import os
import platform
@@ -33,6 +33,7 @@
import tempfile
import threading
import time
+import tomllib
from contextlib import nullcontext
from datetime import datetime, timezone
from pathlib import Path
@@ -66,7 +67,7 @@
}
PYRIGHT_DEFAULT_EXCLUDES = ["**/node_modules", "**/__pycache__", "**/.*"]
DEPENDENCY_ISOLATION = "pip-target-per-package"
-CHECKER_CONFIG_VERSION = 6
+CHECKER_CONFIG_VERSION = 7
class BenchmarkError(Exception):
@@ -101,6 +102,7 @@ class TimingMetrics(RequiredTimingMetrics, total=False):
files_selected: int
file_count_verified: bool
file_count_audit_time_s: float
+ source_count_mismatch: bool
class PackageResult(TypedDict, total=False):
@@ -115,6 +117,7 @@ class PackageResult(TypedDict, total=False):
source_files: list[str]
source_file_count: int
source_manifest_sha256: str
+ repository_configs: list[dict[str, Any]]
class AggregateStats(TypedDict):
@@ -611,7 +614,6 @@ def _relative_check_paths(
def _resolve_check_paths(
package_path: Path,
check_paths: list[str],
- exclude_directories: list[str],
) -> tuple[list[Path], list[str]]:
configured_paths = [package_path / path for path in check_paths]
missing = [
@@ -619,43 +621,110 @@ def _resolve_check_paths(
for path in configured_paths
if not path.exists()
]
- excluded_names = {"node_modules", "__pycache__", *exclude_directories}
- resolved_paths: set[Path] = set()
+ return list(dict.fromkeys(path for path in configured_paths if path.exists())), missing
- def is_excluded(path: Path) -> bool:
- return any(
- part.startswith(".") or part in excluded_names
- for part in path.relative_to(package_path).parts
- )
+
+def _audit_repository_configs(package_path: Path) -> list[dict[str, Any]]:
+ native_names = {
+ "pyrightconfig.json": ["pyright"],
+ "ty.toml": ["ty"],
+ "pyrefly.toml": ["pyrefly"],
+ "mypy.ini": ["mypy", "zuban"],
+ ".mypy.ini": ["mypy", "zuban"],
+ "zuban.toml": ["zuban"],
+ }
+ tool_names = {
+ "pyright": ["pyright"],
+ "basedpyright": ["basedpyright"],
+ "ty": ["ty"],
+ "pyrefly": ["pyrefly"],
+ "mypy": ["mypy", "zuban"],
+ "zuban": ["zuban"],
+ }
+ records: list[dict[str, Any]] = []
def walk_error(error: OSError) -> None:
raise error
- for path in configured_paths:
- if not path.exists() or is_excluded(path):
- continue
- if path.is_file():
- if path.suffix in (".py", ".pyi"):
- resolved_paths.add(path)
- continue
- for root, directory_names, file_names in os.walk(path, onerror=walk_error):
- root_path = Path(root)
- directory_names[:] = [
- name for name in directory_names if not is_excluded(root_path / name)
- ]
- resolved_paths.update(
- root_path / name
- for name in file_names
- if Path(name).suffix in (".py", ".pyi")
- and not is_excluded(root_path / name)
- )
- return sorted(
- (
- path for path in resolved_paths
- if path.suffix != ".pyi" or path.with_suffix(".py") not in resolved_paths
- ),
- key=lambda path: path.relative_to(package_path).as_posix(),
- ), missing
+ for root, directories, files in os.walk(package_path, onerror=walk_error):
+ directories[:] = sorted(
+ name for name in directories
+ if name not in {".git", ".hg", ".svn", "node_modules", "__pycache__"}
+ )
+ for name in sorted(files):
+ if name.startswith(".typecheck-benchmark-"):
+ continue
+ path = Path(root) / name
+ checkers = set(native_names.get(name, []))
+ sections: list[str] = []
+ settings: list[str] = []
+ error = None
+
+ def setting_names(data: dict[str, Any], prefix: str) -> None:
+ for key, value in data.items():
+ setting = f"{prefix}.{key}"
+ settings.append(setting)
+ if isinstance(value, dict):
+ setting_names(value, setting)
+ elif isinstance(value, list):
+ for index, item in enumerate(value):
+ if isinstance(item, dict):
+ setting_names(item, f"{setting}[{index}]")
+
+ if path.suffix == ".toml":
+ try:
+ data = tomllib.loads(path.read_text(encoding="utf-8-sig"))
+ except (tomllib.TOMLDecodeError, UnicodeError) as exc:
+ error = f"{type(exc).__name__}: unable to parse TOML"
+ else:
+ if name in native_names:
+ setting_names(data, path.stem)
+ tools = data.get("tool", {})
+ if isinstance(tools, dict):
+ for tool, affected in tool_names.items():
+ if tool in tools:
+ sections.append(f"tool.{tool}")
+ checkers.update(affected)
+ if isinstance(tools[tool], dict):
+ setting_names(tools[tool], f"tool.{tool}")
+ if isinstance(data.get("project"), dict) and "requires-python" in data["project"]:
+ sections.append("project.requires-python")
+ checkers.update(("ty", "pyrefly"))
+ settings.append("project.requires-python")
+ elif name == "setup.cfg" or name in ("mypy.ini", ".mypy.ini"):
+ parser = configparser.ConfigParser(interpolation=None)
+ try:
+ parser.read_string(path.read_text(encoding="utf-8-sig"))
+ except (configparser.Error, UnicodeError) as exc:
+ error = f"{type(exc).__name__}: unable to parse INI"
+ else:
+ sections = [
+ section for section in parser.sections()
+ if section == "mypy" or section.startswith("mypy-")
+ ]
+ if sections:
+ checkers.update(("mypy", "zuban"))
+ settings.extend(
+ f"{section}.{key}" for section in sections for key in parser[section]
+ )
+ if checkers or sections or error:
+ record: dict[str, Any] = {
+ "path": path.relative_to(package_path).as_posix(),
+ "checkers": sorted(checkers),
+ "sections": sections,
+ "settings": sorted(settings),
+ "disposition": "not applied: explicit benchmark config",
+ }
+ if error:
+ record["parse_error"] = error
+ records.append(record)
+ return sorted(records, key=lambda record: record["path"])
+
+
+def _glob_excludes(exclude_directories: list[str] | None) -> list[str]:
+ return [*PYRIGHT_DEFAULT_EXCLUDES, *(
+ f"**/{name}" for name in exclude_directories or []
+ )]
def _new_config_path(package_path: Path, checker: str, suffix: str) -> Path:
@@ -668,11 +737,14 @@ def _new_config_path(package_path: Path, checker: str, suffix: str) -> Path:
return Path(path)
-def _write_pyright_config(package_path: Path, check_paths: list[str] | None) -> Path:
+def _write_pyright_config(
+ package_path: Path, check_paths: list[str] | None,
+ exclude_directories: list[str] | None = None,
+) -> Path:
config_path = _new_config_path(package_path, "pyright", ".json")
config = {
"include": check_paths or ["."],
- "exclude": PYRIGHT_DEFAULT_EXCLUDES,
+ "exclude": _glob_excludes(exclude_directories),
"typeCheckingMode": "basic",
"useLibraryCodeForTypes": True,
"pythonVersion": TARGET_PYTHON_VERSION,
@@ -682,7 +754,8 @@ def _write_pyright_config(package_path: Path, check_paths: list[str] | None) ->
def _write_mypy_config(
- package_path: Path, check_paths: list[str] | None, checker: str
+ package_path: Path, check_paths: list[str] | None, checker: str,
+ exclude_directories: list[str] | None = None,
) -> Path:
config_path = _new_config_path(package_path, checker, ".toml")
lines = [
@@ -691,6 +764,13 @@ def _write_mypy_config(
"check_untyped_defs = true",
f"python_version = {json.dumps(TARGET_PYTHON_VERSION)}",
f"python_executable = {json.dumps(sys.executable)}",
+ "exclude = " + json.dumps([
+ r"(^|[/\\])(?:node_modules|__pycache__|\.[^/\\]+)([/\\]|$)",
+ *(
+ rf"(^|[/\\]){re.escape(name)}([/\\]|$)"
+ for name in exclude_directories or []
+ ),
+ ]),
]
if checker == "zuban":
lines.append("exclude_gitignore = false")
@@ -700,7 +780,8 @@ def _write_mypy_config(
def _write_ty_config(
- package_path: Path, check_paths: list[str] | None, *, infer_returns: bool = False
+ package_path: Path, check_paths: list[str] | None, *, infer_returns: bool = False,
+ exclude_directories: list[str] | None = None,
) -> Path:
config_path = _new_config_path(package_path, "ty", ".toml")
lines = [
@@ -712,7 +793,7 @@ def _write_ty_config(
"respect-ignore-files = false",
# These are gitignore-style patterns, not native Windows paths.
f"include = {json.dumps([Path(path).as_posix() for path in check_paths or ['.']])}",
- f"exclude = {json.dumps(['!**', *PYRIGHT_DEFAULT_EXCLUDES])}",
+ f"exclude = {json.dumps(['!**', *_glob_excludes(exclude_directories)])}",
]
if infer_returns:
lines.extend(["", "[analysis]", 'return-type-inference = "always"'])
@@ -720,13 +801,16 @@ def _write_ty_config(
return config_path
-def _write_pyrefly_config(package_path: Path, check_paths: list[str] | None) -> Path:
+def _write_pyrefly_config(
+ package_path: Path, check_paths: list[str] | None,
+ exclude_directories: list[str] | None = None,
+) -> Path:
config_path = _new_config_path(package_path, "pyrefly", ".toml")
lines = [
f"project-includes = {json.dumps(check_paths or ['.'])}",
"use-ignore-files = false",
"disable-project-excludes-heuristics = true",
- f"project-excludes = {json.dumps(PYRIGHT_DEFAULT_EXCLUDES)}",
+ f"project-excludes = {json.dumps(_glob_excludes(exclude_directories))}",
f"python-interpreter-path = {json.dumps(sys.executable)}",
f"python-version = {json.dumps(TARGET_PYTHON_VERSION)}",
'preset = "all"',
@@ -801,6 +885,7 @@ def _build_checker_command(
checker: str,
package_path: Path,
check_paths: list[Path] | None,
+ exclude_directories: list[str] | None = None,
) -> tuple[list[str] | None, list[Path]]:
"""Build one checker command and return temporary configs to remove."""
base_command = _checker_command(checker)
@@ -809,7 +894,7 @@ def _build_checker_command(
relative_paths = _relative_check_paths(package_path, check_paths)
if checker in ("pyright", "pyright-threads", "pyright-pip"):
- config_path = _write_pyright_config(package_path, relative_paths)
+ config_path = _write_pyright_config(package_path, relative_paths, exclude_directories)
command = [
*base_command,
"--project",
@@ -823,7 +908,7 @@ def _build_checker_command(
command.append("--stats")
return command, [config_path]
if checker == "pyrefly":
- config_path = _write_pyrefly_config(package_path, relative_paths)
+ config_path = _write_pyrefly_config(package_path, relative_paths, exclude_directories)
return [
*base_command,
"check",
@@ -832,7 +917,8 @@ def _build_checker_command(
], [config_path]
if checker in ("ty", "ty-infer"):
config_path = _write_ty_config(
- package_path, relative_paths, infer_returns=checker == "ty-infer"
+ package_path, relative_paths, infer_returns=checker == "ty-infer",
+ exclude_directories=exclude_directories,
)
return [
*base_command,
@@ -841,7 +927,7 @@ def _build_checker_command(
str(config_path),
], [config_path]
if checker == "mypy":
- config_path = _write_mypy_config(package_path, relative_paths, "mypy")
+ config_path = _write_mypy_config(package_path, relative_paths, "mypy", exclude_directories)
command = [
*base_command,
"--no-incremental",
@@ -850,7 +936,7 @@ def _build_checker_command(
]
return command, [config_path]
if checker == "zuban":
- config_path = _write_mypy_config(package_path, relative_paths, "zuban")
+ config_path = _write_mypy_config(package_path, relative_paths, "zuban", exclude_directories)
command = [
*base_command,
"check",
@@ -874,10 +960,11 @@ def _run_checker_process(
capture_output: bool = True,
environment: dict[str, str] | None = None,
audit_files: bool = False,
+ exclude_directories: list[str] | None = None,
) -> ProcessResult:
"""Execute a checker with the shared benchmark configuration and isolation."""
command, config_paths = _build_checker_command(
- checker, package_path, check_paths
+ checker, package_path, check_paths, exclude_directories
)
if not command:
error_message = (
@@ -933,12 +1020,14 @@ def run_checker(
capture_output: bool = True,
environment: dict[str, str] | None = None,
audit_files: bool = False,
+ exclude_directories: list[str] | None = None,
) -> TimingMetrics:
"""Run one checker once using a minimal checker-specific configuration."""
try:
result = _run_checker_process(
checker, package_path, check_paths, timeout, memory_limit_mb,
capture_output=capture_output, environment=environment, audit_files=audit_files,
+ exclude_directories=exclude_directories,
)
except OSError as exc:
return {
@@ -1106,14 +1195,14 @@ def _benchmark_directory(
memory_limit_mb: int,
local_path: str | None = None,
environment: dict[str, str] | None = None,
+ exclude_directories: list[str] | None = None,
) -> PackageResult:
- source_files, missing = _resolve_check_paths(
+ check_roots, missing = _resolve_check_paths(
package_path,
(_relative_check_paths(package_path, resolved_paths) or [])
if resolved_paths is not None else ["."],
- [],
)
- if missing or not source_files:
+ if missing or not check_roots:
return {
"package_name": name,
"github_url": github_url,
@@ -1123,10 +1212,20 @@ def _benchmark_directory(
),
"metrics": {},
}
- manifest = [path.relative_to(package_path).as_posix() for path in source_files]
- manifest_hash = hashlib.sha256(
- json.dumps(manifest, ensure_ascii=True).encode("utf-8")
- ).hexdigest()
+ try:
+ repository_configs = _audit_repository_configs(package_path)
+ except OSError as exc:
+ message = f"Repository configuration audit failed: {exc}"
+ print(f" {message}")
+ return {
+ "package_name": name, "github_url": github_url,
+ "error": message, "metrics": {},
+ }
+ print(f" Repository config audit: {len(repository_configs)} candidate files")
+ for config in repository_configs:
+ detail = ", ".join(config["sections"] or config["checkers"]) or "unparsed TOML"
+ status = "parse error" if "parse_error" in config else config["disposition"]
+ print(f" {config['path']}: {detail} ({status})")
metrics: dict[str, TimingMetrics] = {}
for checker in type_checkers:
if not _checker_command(checker):
@@ -1141,17 +1240,17 @@ def _benchmark_directory(
continue
audit = run_checker(
- checker, package_path, source_files, timeout, memory_limit_mb,
+ checker, package_path, check_roots, timeout, memory_limit_mb,
environment=environment, audit_files=True,
+ exclude_directories=exclude_directories,
)
selected_count = audit.get("files_selected")
- if not audit["ok"] or selected_count is None or selected_count != len(source_files):
+ if not audit["ok"] or selected_count is None or selected_count == 0:
audit["ok"] = False
audit["file_count_verified"] = False
if not audit.get("error_message"):
audit["error_message"] = (
- f"Source-file count mismatch: expected {len(source_files)}, "
- f"{checker} reported {selected_count}"
+ f"No verifiable nonzero native source-file count: {checker} reported {selected_count}"
)
audit["file_count_audit_time_s"] = audit["execution_time_s"]
audit["execution_time_s"] = 0.0
@@ -1159,7 +1258,7 @@ def _benchmark_directory(
metrics[checker] = audit
print(f" {checker} source-file audit failed: {audit['error_message']}")
continue
- print(f" {checker} source-file audit: {selected_count}/{len(source_files)} verified")
+ print(f" {checker} source-file audit: {selected_count} native files verified")
if warmup == 0:
run_description = f"{runs} measured"
@@ -1183,16 +1282,17 @@ def _benchmark_directory(
metric = run_checker(
checker,
package_path,
- source_files,
+ check_roots,
timeout,
memory_limit_mb,
environment=environment,
+ exclude_directories=exclude_directories,
)
run_count = metric.get("files_selected")
- if metric["ok"] and run_count is not None and run_count != len(source_files):
+ if metric["ok"] and run_count is not None and run_count != selected_count:
metric["ok"] = False
metric["error_message"] = (
- f"Source-file count changed: expected {len(source_files)}, "
+ f"Source-file count changed: expected {selected_count}, "
f"{checker} reported {run_count}"
)
if not metric.get("ok"):
@@ -1263,14 +1363,26 @@ def _benchmark_directory(
f"(stddev: {result_metric['execution_time_stats']['stddev']:.3f}s)"
)
+ native_counts = {
+ checker: metric["files_selected"]
+ for checker, metric in metrics.items() if metric.get("file_count_verified")
+ }
+ if len(set(native_counts.values())) > 1:
+ message = f"Native source-file counts differ: {native_counts}; timings are not comparable"
+ print(f" {message}")
+ for checker in native_counts:
+ metrics[checker]["ok"] = False
+ metrics[checker]["source_count_mismatch"] = True
+ metrics[checker]["error_message"] = message
+
result: PackageResult = {
"package_name": name,
"github_url": github_url,
"error": None,
"metrics": metrics,
- "source_files": manifest,
- "source_file_count": len(manifest),
- "source_manifest_sha256": manifest_hash,
+ "check_paths": _relative_check_paths(package_path, check_roots) or [],
+ "exclude_directories": exclude_directories or [],
+ "repository_configs": repository_configs,
}
if local_path:
result["local_path"] = local_path
@@ -1319,7 +1431,7 @@ def _benchmark_package(
resolved_paths: list[Path] | None = None
if raw_check_paths:
resolved_paths, missing = _resolve_check_paths(
- package_path, raw_check_paths, exclude_directories
+ package_path, raw_check_paths
)
if missing:
return {
@@ -1354,6 +1466,7 @@ def _benchmark_package(
warmup=warmup,
memory_limit_mb=memory_limit_mb,
environment=_dependency_environment(dependency_path, package),
+ exclude_directories=exclude_directories,
)
result["commit"] = commit or "unknown"
result.update(package_metadata)
From f907f97e26af7fb01c21cdb6c804cbcfdb3731bb Mon Sep 17 00:00:00 2001
From: Bill Schnurr
Date: Mon, 2 Nov 2026 16:18:27 -0700
Subject: [PATCH 5/6] Reject incompatible benchmark evidence and verify
analysis differences
Require environment, corpus, dependency and native-count evidence; protect history compatibility and handle missing metrics. Separate individual totals from shared successful coverage without implying equivalent analysis. Retain configuration inventories on preparation failures.
Validate diagnostic categories and assertion controls, add installed typed/untyped dependency probes, and pin a common Python checker environment for weekly jobs. Record resolved package versions and import-path override fingerprints. Preserve native discovery and explicitly withhold apples-to-apples speed claims.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 35cc6021-56a0-45b2-9b66-3817d947cd99
---
.../workflows/typecheck_benchmark_weekly.yml | 6 +-
build/benchmark/README.md | 42 ++++--
build/benchmark/benchmark_capabilities.py | 129 +++++++++++++++---
build/benchmark/benchmark_evidence.py | 72 ++++++++++
build/benchmark/checker_requirements.txt | 4 +
build/benchmark/compare_benchmarks.py | 19 ++-
build/benchmark/merge_benchmark_results.py | 2 +
build/benchmark/render_benchmark_html.py | 69 +++++++---
build/benchmark/render_pyright_history.py | 57 +++++++-
.../benchmark/test_benchmark_capabilities.py | 58 +++++---
build/benchmark/test_benchmark_evidence.py | 63 +++++++++
build/benchmark/test_compare_benchmarks.py | 17 ++-
build/benchmark/test_render_benchmark_html.py | 55 +++++++-
.../benchmark/test_render_pyright_history.py | 28 ++++
build/benchmark/test_typecheck_benchmark.py | 5 +
build/benchmark/typecheck_benchmark.py | 42 +++++-
16 files changed, 583 insertions(+), 85 deletions(-)
create mode 100644 build/benchmark/benchmark_evidence.py
create mode 100644 build/benchmark/checker_requirements.txt
create mode 100644 build/benchmark/test_benchmark_evidence.py
diff --git a/.github/workflows/typecheck_benchmark_weekly.yml b/.github/workflows/typecheck_benchmark_weekly.yml
index 3fe5f2845665..ff39d56971c1 100644
--- a/.github/workflows/typecheck_benchmark_weekly.yml
+++ b/.github/workflows/typecheck_benchmark_weekly.yml
@@ -37,6 +37,7 @@ jobs:
cache: 'pip'
cache-dependency-path: |
build/benchmark/install_envs.json
+ build/benchmark/checker_requirements.txt
.github/workflows/typecheck_benchmark_weekly.yml
- name: Verify build prerequisites
@@ -45,9 +46,8 @@ jobs:
g++ --version
make --version
- - name: Install ${{ matrix.checker }}
- if: ${{ matrix.checker != 'pyright' }}
- run: python -m pip install --upgrade '${{ matrix.checker }}'
+ - name: Install shared checker environment
+ run: python -m pip install -r build/benchmark/checker_requirements.txt
- uses: pnpm/action-setup@f520eceda224fe1a4aed5a2a27a194379a409996 # v6
if: ${{ matrix.checker == 'pyright' }}
diff --git a/build/benchmark/README.md b/build/benchmark/README.md
index 752255ed34bc..48bb4755db9c 100644
--- a/build/benchmark/README.md
+++ b/build/benchmark/README.md
@@ -121,7 +121,8 @@ mypy and Zuban receive roots through TOML file lists to avoid command-line size
Windows INI backslash escaping. ty's source selectors use gitignore-pattern separators even on
Windows, where native backslashes can silently match no files.
-Before checking, an **untimed repository configuration search** recursively inventories native
+Immediately after cloning, before dependency installation, an **untimed repository configuration search**
+recursively inventories native
`pyrightconfig.json`, `ty.toml`, `pyrefly.toml`, `zuban.toml`, `mypy.ini`, and `.mypy.ini` candidates.
It also searches TOML files for checker-specific `[tool.*]` settings and `project.requires-python`,
and INI settings in `setup.cfg` and mypy configs. JSON records `repository_configs` per package;
@@ -129,7 +130,8 @@ console output and the weekly HTML show paths, sections/setting names, and that
configs are passed instead. Configuration values are not published, and detected does not mean
applied. Invalid TOML/INI files produce visible audit parse errors rather than disappearing from
the inventory. The search skips VCS metadata, `node_modules`, `__pycache__`, and harness-generated
-configs. It does not modify or delete repository configs. The real-binary suppression tests below,
+configs. Preparation failures retain the inventory. It does not modify or delete repository configs.
+The real-binary suppression tests below,
not discovery alone, provide evidence that conflicting corpus configs cannot override the harness.
Before any warmups or measured runs, each checker performs one **untimed file-count audit** using
@@ -147,6 +149,14 @@ Each successful checker records its native `files_selected`, `file_count_verifie
across separate weekly checker jobs; its timings are excluded from report totals rather than
forcing the tools to discover the same files. Raw JSON retains the measurements and differences.
The regression comparator rejects changed native counts, roots, exclusions, or audit-run settings.
+Configuration version 8 additionally requires successful results to carry positive audited counts,
+source revisions, preparation evidence, and resolved dependency versions. The weekly renderer
+rejects differing package sets/revisions, roots, exclusions, target dependencies, Python/platform/
+runner profiles, and ambient Python distribution versions. It also checks hashes of inherited
+`PYTHONPATH`/`MYPYPATH` overrides without publishing their values.
+Regenerate older baselines and release history rather than mixing pre-contract measurements.
+History now retains native counts and rejects incompatible configuration, audit, import environment,
+and successful native-scope evidence.
These are source entry-point counts, **not total imported or parsed files**. Equal counts do not
prove identical file identities, equal dependency work, or equal diagnostic coverage.
Repository and user configs are tested
@@ -169,19 +179,22 @@ python build/benchmark/typecheck_benchmark.py -c ty ty-infer -r 3 -w 1
After corpus timings finish, the harness runs an **untimed behavioral capability suite** with two
batch invocations per installed checker, using the same configurations and interpreter.
-Eight paired probes cover an exact-type assertion
+Ten paired probes cover an exact-type assertion
control, unannotated return inference, unannotated function body checking, generics, overloads,
-protocols, narrowing, and propagation from a local annotated import. The import probe does not
-model all third-party package or stub-resolution behavior.
+protocols, narrowing, propagation from a local annotated import, and installed packages with and
+without a `py.typed` marker. The installed-package fixtures use the same dependency-path mechanism
+as the corpus. These probes do not model all third-party package or stub-resolution behavior.
The positive examples must be accepted, including exact `typing.assert_type` checks; unexpected
`Any` or `Unknown` therefore cannot satisfy an `int` assertion. Negative examples must report a
-diagnostic at exactly the intended line, with no other diagnostic lines. The assertion control
-must pass before any case can match. Both batches require native file counts covering all eight
+diagnostic in the intended category at exactly the intended line, with no other primary diagnostics.
+Checker-specific adapters read primary diagnostics and rule IDs; notes and source excerpts are
+not diagnostics, and unrelated same-line lint errors cannot satisfy the assertion control.
+The assertion control must pass before any case can match. Both batches require native counts covering all ten
probe files; crashes, timeouts, missing counts, and unrecognized output are **not evaluated**,
not zero capability or a successful result.
-JSON stores the versioned suite identity, per-case diagnostic lines, and raw process evidence in
+JSON stores the versioned suite identity, per-case diagnostic categories/rule IDs/lines, and raw process evidence in
`capability_suite`. HTML adds a **Behavior matches** column and a per-case matrix. Mismatches can
mean a missed error, lost precision, or additional strictness diagnostics: for example, Pyrefly's
`all` preset also requires return annotations. Do not equate every mismatch with weaker inference.
@@ -198,8 +211,14 @@ Historical results without this audit have no verified behavior-match count.
For defensible cross-checker comparisons, use the same machine, pinned source commits, dependency
versions, interpreter, target platform, source scope, and resource/thread budgets. The weekly jobs
-currently use separate hosted runners and install dependencies without a shared version lock, so
-their results are independent trends rather than a controlled speed ranking. Repeated fresh
+use separate hosted runners. Every weekly job installs the same pinned checker requirements from
+`checker_requirements.txt`; snapshots of ambient and per-project resolved distribution versions
+detect differing environments, including unlocked transitive dependencies. This is not a shared
+corpus dependency lock, identical physical hardware, or proof of equivalent inference.
+Results remain independent trends rather than a controlled speed ranking. The summary distinguishes
+individual successful-package totals from totals over the shared successful package intersection.
+Never rank individual totals with differing coverage; even shared counts are not equal analysis.
+Repeated fresh
processes after a warmup measure a warm-filesystem CLI workload, not editor incrementality or a
cold-disk workload. Alternate run order for paired experiments and report the individual samples
and dispersion, not only a speedup.
@@ -216,7 +235,8 @@ and curated false-positive/false-negative cases separately from throughput; the
[Python typing conformance suite](https://github.com/python/typing/tree/main/conformance) is a useful
shared specification, but not a performance weighting scheme.
-Linux peak RSS currently polls only `/proc//status`; it does not sum child-process memory.
+Linux peak RSS and memory-limit enforcement currently use only `/proc//status`;
+they do not sum child-process memory.
In particular, Pyright `--threads` RSS must not be interpreted as total worker memory or a memory
improvement over single-process Pyright. Windows RSS is unavailable. Neither limitation prevents
wall-time measurements, but neither supports cross-process memory rankings.
diff --git a/build/benchmark/benchmark_capabilities.py b/build/benchmark/benchmark_capabilities.py
index 6543d7ff1602..66ef6691b245 100644
--- a/build/benchmark/benchmark_capabilities.py
+++ b/build/benchmark/benchmark_capabilities.py
@@ -87,7 +87,18 @@
),
}
SUPPORT_SOURCE = "def typed_value() -> int:\n return 1\n"
-SUITE_VERSION = 1
+for _name, _label in (
+ ("installed_typed", "Installed typed-package precision"),
+ ("installed_untyped", "Installed untyped-package precision"),
+):
+ CASES[_name] = (
+ _label,
+ "from typing import assert_type\n"
+ f"from capability_{_name} import typed_value\n"
+ "assert_type(typed_value(), int)\n"
+ "bad: str = typed_value() # expect-error\n",
+ )
+SUITE_VERSION = 2
SUITE_SHA256 = hashlib.sha256(
json.dumps([SUITE_VERSION, CASES, SUPPORT_SOURCE], sort_keys=True).encode("utf-8")
).hexdigest()
@@ -102,11 +113,20 @@ class CapabilityOutput(TypedDict):
files_selected: int | None
+class Diagnostic(TypedDict):
+ file: str
+ line: int
+ rule: str
+ category: str
+
+
class CaseResult(TypedDict):
passed: bool
positive_diagnostic_lines: list[int]
negative_diagnostic_lines: list[int]
expected_error_line: int
+ positive_diagnostics: list[Diagnostic]
+ negative_diagnostics: list[Diagnostic]
class CheckerResult(TypedDict, total=False):
@@ -123,16 +143,61 @@ class CapabilitySuite(TypedDict):
results: dict[str, CheckerResult]
-def _diagnostic_lines(output: str) -> dict[str, set[int]]:
- locations: dict[str, set[int]] = {}
- for match in re.finditer(r"([\w.-]+\.pyi?):(\d+)(?::\d+)?", output):
- locations.setdefault(Path(match[1]).stem, set()).add(int(match[2]))
- return locations
+def parse_diagnostics(checker: str, output: str) -> list[Diagnostic]:
+ """Read primary diagnostics and rule IDs, never source excerpts or notes."""
+ diagnostics: list[Diagnostic] = []
+ if checker in ("pyright", "pyright-threads", "pyright-pip"):
+ pattern = r"(?m)^\s+([^\n]+\.pyi?):(\d+):\d+ - (?:error|warning|information): "
+ matches = list(re.finditer(pattern, output))
+ for index, match in enumerate(matches):
+ block = output[match.end():matches[index + 1].start() if index + 1 < len(matches) else len(output)]
+ rule = re.search(r"\((report\w+)\)", block)
+ if rule is None:
+ raise ValueError("Pyright diagnostic has no recognized rule ID")
+ diagnostics.append(_diagnostic(match[1], int(match[2]), rule[1]))
+ elif checker in ("ty", "ty-infer", "pyrefly"):
+ pattern = (
+ r"(?m)^(?:error|warning|info)\[([\w-]+)\]:[^\n]*"
+ if checker in ("ty", "ty-infer")
+ else r"(?m)^(?:ERROR|WARN|INFO) [^\n]* \[([\w-]+)\]"
+ )
+ matches = list(re.finditer(pattern, output))
+ for index, match in enumerate(matches):
+ block = output[match.end():matches[index + 1].start() if index + 1 < len(matches) else len(output)]
+ location = re.search(r"(?m)^\s*-->\s+([^\n]+\.pyi?):(\d+):\d+", block)
+ if location is None:
+ raise ValueError(f"{checker} diagnostic has no primary source location")
+ diagnostics.append(_diagnostic(location[1], int(location[2]), match[1]))
+ elif checker in ("mypy", "zuban"):
+ for match in re.finditer(r"(?m)^([^\n]+\.pyi?):(\d+)(?::\d+)?: (?:error|warning): (.+)$", output):
+ rule = re.search(r"\[([\w-]+)\]\s*$", match[3])
+ if rule is None:
+ raise ValueError(f"{checker} diagnostic has no recognized rule ID")
+ code = rule[1]
+ if checker == "zuban" and code == "misc" and match[3].startswith("Expression is of type "):
+ code = "assert-type"
+ diagnostics.append(_diagnostic(match[1], int(match[2]), code))
+ else:
+ raise ValueError(f"No diagnostic adapter for {checker}")
+ return diagnostics
+
+
+def _diagnostic(path: str, line: int, rule: str) -> Diagnostic:
+ categories = {
+ "reportAssertTypeFailure": "assert-type", "type-assertion-failure": "assert-type",
+ "assert-type": "assert-type", "reportAssignmentType": "assignment",
+ "invalid-assignment": "assignment", "bad-assignment": "assignment",
+ "assignment": "assignment", "arg-type": "argument",
+ }
+ return {
+ "file": Path(path.strip()).stem, "line": line, "rule": rule,
+ "category": categories.get(rule, f"other:{rule}"),
+ }
def run_capability_suite(
checkers: list[str],
- run: Callable[[str, Path, list[Path]], CapabilityOutput],
+ run: Callable[[str, Path, list[Path], Path], CapabilityOutput],
) -> CapabilitySuite:
suite: CapabilitySuite = {
"version": SUITE_VERSION,
@@ -142,6 +207,13 @@ def run_capability_suite(
}
with tempfile.TemporaryDirectory(prefix="typecheck-capabilities-") as directory:
root = Path(directory)
+ dependencies = root / "dependencies"
+ for name in ("installed_typed", "installed_untyped"):
+ package = dependencies / f"capability_{name}"
+ package.mkdir(parents=True)
+ (package / "__init__.py").write_text(SUPPORT_SOURCE, encoding="utf-8")
+ if name == "installed_typed":
+ (package / "py.typed").write_text("", encoding="utf-8")
paths: dict[str, list[Path]] = {}
expected: dict[str, int] = {}
for phase in ("positive", "negative"):
@@ -164,16 +236,15 @@ def run_capability_suite(
for checker in checkers:
evidence: dict[str, CapabilityOutput] = {}
- locations: dict[str, dict[str, set[int]]] = {}
+ locations: dict[str, list[Diagnostic]] = {}
error = None
for phase in ("positive", "negative"):
try:
- output = run(checker, root / phase, paths[phase])
+ output = run(checker, root / phase, paths[phase], dependencies)
except OSError as exc:
error = f"{phase}: unable to evaluate capabilities: {exc}"
break
evidence[phase] = output
- text = output["stdout"] + "\n" + output["stderr"]
if output["timed_out"] or output["oom_killed"]:
error = f"{phase}: capability checker timed out or exceeded memory limit"
elif output["returncode"] not in (0, 1):
@@ -181,8 +252,12 @@ def run_capability_suite(
elif output["files_selected"] != len(CASES):
error = f"{phase}: capability source-file count is not verified"
else:
- locations[phase] = _diagnostic_lines(text)
- if set(locations[phase]) - CASES.keys():
+ try:
+ locations[phase] = parse_diagnostics(checker, output["stdout"])
+ except ValueError as exc:
+ error = f"{phase}: {exc}"
+ break
+ if {item["file"] for item in locations[phase]} - CASES.keys():
error = f"{phase}: unexpected diagnostics outside capability cases"
elif bool(locations[phase]) != (output["returncode"] == 1):
error = f"{phase}: unrecognized or inconsistent diagnostic output"
@@ -194,18 +269,32 @@ def run_capability_suite(
}
continue
cases: dict[str, CaseResult] = {}
- control_passed = (
- not locations["positive"].get("assert_type")
- and locations["negative"].get("assert_type") == {expected["assert_type"]}
+ def case_diagnostics(phase: str, name: str) -> list[Diagnostic]:
+ return [item for item in locations[phase] if item["file"] == name]
+
+ def expected_diagnostics(name: str, items: list[Diagnostic]) -> bool:
+ category = "assert-type" if name == "assert_type" else "assignment"
+ return (
+ len(items) == 1 and items[0]["line"] == expected[name]
+ and (
+ items[0]["category"] == category
+ or (name == "generics" and items[0]["category"] == "argument")
+ )
+ )
+
+ control_passed = not case_diagnostics("positive", "assert_type") and expected_diagnostics(
+ "assert_type", case_diagnostics("negative", "assert_type")
)
for name in CASES:
- positive = locations["positive"].get(name, set())
- negative = locations["negative"].get(name, set())
+ positive = case_diagnostics("positive", name)
+ negative = case_diagnostics("negative", name)
cases[name] = {
- "passed": control_passed and not positive and negative == {expected[name]},
- "positive_diagnostic_lines": sorted(positive),
- "negative_diagnostic_lines": sorted(negative),
+ "passed": control_passed and not positive and expected_diagnostics(name, negative),
+ "positive_diagnostic_lines": sorted({item["line"] for item in positive}),
+ "negative_diagnostic_lines": sorted({item["line"] for item in negative}),
"expected_error_line": expected[name],
+ "positive_diagnostics": positive,
+ "negative_diagnostics": negative,
}
suite["results"][checker] = {
"ok": True, "cases": cases, "evidence": evidence,
diff --git a/build/benchmark/benchmark_evidence.py b/build/benchmark/benchmark_evidence.py
new file mode 100644
index 000000000000..05eea9c45d49
--- /dev/null
+++ b/build/benchmark/benchmark_evidence.py
@@ -0,0 +1,72 @@
+"""Compatibility checks shared by benchmark reports and regression history."""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+
+EVIDENCE_VERSION = 8
+ENVIRONMENT_FIELDS = (
+ "platform", "architecture", "runner_class", "runner_image", "cpu_count",
+ "python_version", "memory_measurement", "memory_limit_mb", "dependency_isolation",
+ "checker_config_version", "uncounted_validation_runs_per_checker",
+ "runs_per_package", "warmup_runs", "timeout_s",
+ "import_path_overrides",
+)
+SCOPE_FIELDS = ("commit", "check_paths", "exclude_directories", "resolved_dependencies")
+
+
+def validate_evidence(data: dict[str, Any]) -> None:
+ if data.get("checker_config_version", 0) < EVIDENCE_VERSION:
+ return
+ for field in (*ENVIRONMENT_FIELDS, "installed_packages"):
+ if field not in data:
+ raise ValueError(f"Missing benchmark evidence field: {field}")
+ names: set[str] = set()
+ for package in data.get("results", []):
+ name = package["package_name"]
+ if name in names:
+ raise ValueError(f"Duplicate package evidence: {name}")
+ names.add(name)
+ if not package.get("error"):
+ for field in (*SCOPE_FIELDS, "repository_configs"):
+ if field not in package:
+ raise ValueError(f"{name}: missing package evidence: {field}")
+ for checker, metric in package.get("metrics", {}).items():
+ if not metric.get("ok"):
+ continue
+ count = metric.get("files_selected")
+ if (
+ metric.get("file_count_verified") is not True
+ or not isinstance(count, int) or isinstance(count, bool) or count <= 0
+ ):
+ raise ValueError(f"{name}/{checker}: source-file count is not verified")
+ for field in ("execution_time_s", "peak_memory_mb"):
+ value = metric.get(field)
+ if (
+ not isinstance(value, (int, float)) or isinstance(value, bool)
+ or not math.isfinite(value) or value < 0
+ ):
+ raise ValueError(f"{name}/{checker}: invalid {field}")
+
+
+def validate_compatibility(left: dict[str, Any], right: dict[str, Any]) -> None:
+ validate_evidence(left)
+ validate_evidence(right)
+ for field in ENVIRONMENT_FIELDS:
+ if left.get(field) != right.get(field):
+ raise ValueError(f"Incompatible benchmark environment: {field}")
+ if left.get("installed_packages") != right.get("installed_packages"):
+ raise ValueError("Incompatible benchmark import environments: installed_packages")
+ left_packages = {item["package_name"]: item for item in left.get("results", [])}
+ right_packages = {item["package_name"]: item for item in right.get("results", [])}
+ if left_packages.keys() != right_packages.keys():
+ raise ValueError("Incompatible benchmark package sets")
+ for name in left_packages:
+ old, new = left_packages[name], right_packages[name]
+ # A failed preparation may lack dependency evidence. It is not shared successful work.
+ fields = SCOPE_FIELDS if not old.get("error") and not new.get("error") else SCOPE_FIELDS[:3]
+ for field in fields:
+ if old.get(field) != new.get(field):
+ raise ValueError(f"{name}: incompatible package {field}")
diff --git a/build/benchmark/checker_requirements.txt b/build/benchmark/checker_requirements.txt
new file mode 100644
index 000000000000..6c8e5c3db0a0
--- /dev/null
+++ b/build/benchmark/checker_requirements.txt
@@ -0,0 +1,4 @@
+pyrefly==1.4.0
+ty==0.0.90
+mypy==2.4.0
+zuban==0.10.1
diff --git a/build/benchmark/compare_benchmarks.py b/build/benchmark/compare_benchmarks.py
index 4134c7e35a04..456f5db086e0 100644
--- a/build/benchmark/compare_benchmarks.py
+++ b/build/benchmark/compare_benchmarks.py
@@ -9,6 +9,8 @@
from pathlib import Path
from typing import Any, Literal, TypedDict
+from benchmark_evidence import validate_evidence
+
DEFAULT_THRESHOLD_PERCENT = 20.0
DEFAULT_TIME_NOISE_FLOOR_SECONDS = 1.0
DEFAULT_MEMORY_NOISE_FLOOR_MB = 100.0
@@ -146,6 +148,11 @@ def _analyze(
rows: list[ComparisonRow] = []
if failures:
return failures, rows
+ try:
+ validate_evidence(baseline)
+ validate_evidence(candidate)
+ except ValueError as exc:
+ return [str(exc)], rows
for field in (
"platform",
"architecture",
@@ -160,6 +167,8 @@ def _analyze(
"uncounted_validation_runs_per_checker",
"dependency_isolation",
"checker_config_version",
+ "installed_packages",
+ "import_path_overrides",
):
if baseline.get(field) != candidate.get(field):
failures.append(
@@ -274,11 +283,19 @@ def _analyze(
}
)
continue
+ if not new:
+ failures.append(f"{package}/{checker}: candidate result failed or is missing")
+ rows.append({
+ "package": package, "checker": checker, "execution_time_s": None,
+ "time_delta": None, "peak_memory_mb": None, "memory_delta": None,
+ "status": "Missing",
+ })
+ continue
scope_changed = False
if old.get("files_selected") != new.get("files_selected"):
failures.append(f"{package}/{checker}: native source-file count changed")
scope_changed = True
- for field in ("check_paths", "exclude_directories", "source_manifest_sha256"):
+ for field in ("check_paths", "exclude_directories", "source_manifest_sha256", "resolved_dependencies"):
if old_package.get(field, []) != new_package.get(field, []):
failures.append(
f"{package}/{checker}: package {field} changed from "
diff --git a/build/benchmark/merge_benchmark_results.py b/build/benchmark/merge_benchmark_results.py
index baeb014e1f41..ba5242a8cedf 100644
--- a/build/benchmark/merge_benchmark_results.py
+++ b/build/benchmark/merge_benchmark_results.py
@@ -9,6 +9,7 @@
from typing import Any
import typecheck_benchmark
+from benchmark_evidence import validate_evidence
VARIABLE_FIELDS = {
@@ -36,6 +37,7 @@ def merge_results(paths: list[Path]) -> dict[str, Any]:
data = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(data, dict):
raise ValueError(f"{path} must contain a JSON object")
+ validate_evidence(data)
metadata = {
key: value for key, value in data.items() if key not in VARIABLE_FIELDS
diff --git a/build/benchmark/render_benchmark_html.py b/build/benchmark/render_benchmark_html.py
index e35c4d77c790..8d9b32e14491 100644
--- a/build/benchmark/render_benchmark_html.py
+++ b/build/benchmark/render_benchmark_html.py
@@ -9,6 +9,8 @@
from pathlib import Path
from typing import Any
+from benchmark_evidence import validate_compatibility, validate_evidence
+
CHECKER_ORDER = ["pyright", "pyright-threads", "pyrefly", "ty", "ty-infer", "mypy", "zuban"]
@@ -62,6 +64,18 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
config_version = next(iter(results.values())).get("checker_config_version")
if any(data.get("checker_config_version") != config_version for data in results.values()):
raise ValueError("Checker results use different benchmark configuration versions")
+ reference = next(iter(results.values()))
+ for data in results.values():
+ validate_evidence(data)
+ validate_compatibility(reference, data)
+ if "pyright" in results and "pyright-threads" in results:
+ single, threaded = results["pyright"], results["pyright-threads"]
+ if (
+ single.get("type_checker_versions", {}).get("pyright")
+ != threaded.get("type_checker_versions", {}).get("pyright-threads")
+ or single.get("node_options") != threaded.get("node_options")
+ ):
+ raise ValueError("Pyright threading comparison requires matching versions and Node options")
checkers = _ordered_checkers(results)
packages = sorted(
{
@@ -129,27 +143,29 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
checker: results[checker].get("aggregate", {}).get(checker, {})
for checker in checkers
}
- if mismatched_packages:
- for checker in checkers:
- accepted = [metric for metric in metrics[checker].values() if metric.get("ok")]
- aggregates[checker] = {
- "packages_tested": len(accepted),
- "packages_failed": len(metrics[checker]) - len(accepted),
- "total_execution_time_s": sum(metric["execution_time_s"] for metric in accepted),
- "p50_execution_time_s": statistics.median(
- [metric["execution_time_s"] for metric in accepted]
- ) if accepted else 0,
- "p50_peak_memory_mb": statistics.median(
- [metric.get("peak_memory_mb", 0) for metric in accepted]
- ) if accepted else 0,
- }
+ for checker in checkers:
+ accepted = [metric for metric in metrics[checker].values() if metric.get("ok")]
+ aggregates[checker] = {
+ "packages_tested": len(accepted),
+ "packages_failed": len(packages) - len(accepted),
+ "total_execution_time_s": sum(metric["execution_time_s"] for metric in accepted),
+ "p50_execution_time_s": statistics.median(
+ [metric["execution_time_s"] for metric in accepted]
+ ) if accepted else 0,
+ "p50_peak_memory_mb": statistics.median(
+ [metric.get("peak_memory_mb", 0) for metric in accepted]
+ ) if accepted else 0,
+ }
+ shared_packages = [
+ package for package in packages
+ if all(metrics[checker].get(package, {}).get("ok") for checker in checkers)
+ ]
totals = {
checker: float(
aggregates[checker].get("total_execution_time_s", 0.0)
)
for checker in checkers
}
- max_total = max(totals.values(), default=0.0)
timestamp = max(str(data.get("timestamp", "")) for data in results.values())
environment = next(iter(results.values()))
@@ -159,7 +175,8 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
tested = int(aggregate.get("packages_tested", 0))
failed = int(aggregate.get("packages_failed", 0))
total = totals[checker]
- share = total / max_total * 100 if max_total else 0.0
+ shared_total = sum(metrics[checker][package]["execution_time_s"] for package in shared_packages)
+ shared_text = f"{shared_total:.1f}s" if shared_packages else "N/A"
version = results[checker].get("type_checker_versions", {}).get(checker, "unknown")
capability_cell = ""
if suites:
@@ -175,8 +192,8 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
f"{html.escape(_checker_label(checker))}"
f"{html.escape(str(version))} | "
f"{tested} | {failed} | "
- f""
- f"{total:.1f}s | "
+ f"{total:.1f}s | "
+ f"{len(shared_packages)} | {shared_text} | "
f"{float(aggregate.get('p50_execution_time_s', 0.0)):.1f}s | "
f"{float(aggregate.get('p50_peak_memory_mb', 0.0)):.0f} MB | "
+ capability_cell +
@@ -203,6 +220,11 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
f'negative diagnostic lines: {case["negative_diagnostic_lines"]}; '
f'expected negative line: {case["expected_error_line"]}.'
)
+ if "negative_diagnostics" in case:
+ details += (
+ f' Positive rules: {[item["rule"] for item in case["positive_diagnostics"]]};'
+ f' negative rules: {[item["rule"] for item in case["negative_diagnostics"]]}.'
+ )
cells.append(
f'Mismatch{html.escape(details)} | '
)
@@ -216,9 +238,11 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
'Behavioral capability probes
'
'Untimed paired probes run after corpus timings with the same '
'checker configurations. A match requires accepting the positive example, preserving '
- 'asserted types, and reporting exactly the intended negative diagnostic line. '
+ 'asserted types, and reporting exactly the intended negative diagnostic category and line. '
'The assertion control must also pass. Extra strictness diagnostics can cause a mismatch; '
- 'a mismatch does not necessarily mean less inference. Raw JSON retains diagnostic evidence.
'
+ 'a mismatch does not necessarily mean less inference. Installed-package probes separately '
+ 'test imports with and without py.typed; matching project-file counts cannot establish '
+ 'equal dependency analysis. Raw JSON retains rule IDs and diagnostic evidence.
'
'These few examples are not a comprehensive conformance or quality score, '
'do not certify equivalent analysis on the timed projects, and are not a denominator '
'for normalizing runtime. Historical results without probes are not evaluated.
'
@@ -415,10 +439,11 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:
Run summary
Generated {html.escape(timestamp)} · Python {html.escape(str(environment.get('python_version', 'unknown')))} · {html.escape(str(environment.get('platform_details', environment.get('platform', 'unknown'))))}
-| Checker | Measured | Failed | Total time | Median time | Median RSS | {'Behavior matches | ' if suites else ''}
{''.join(summary_rows)}
+Individual totals cover each checker's successful packages and must not be ranked against different coverage. Shared totals cover only packages successful for every displayed profile with matching audited counts. Shared source scope is not equivalent analysis; these remain independent CLI measurements, not an apples-to-apples speed ranking.
+| Checker | Measured | Failed | Individual total | Shared packages | Shared total | Median time | Median RSS | {'Behavior matches | ' if suites else ''}
{''.join(summary_rows)}
Package comparison
Different checker jobs run on separate hosted runners, so timings are independent trends rather than a fastest-checker ranking. The two Pyright modes run sequentially in one job. Cells show wall time and peak RSS.
Checker defaults, inference, and diagnostic coverage differ; these timings do not measure equivalent analysis or time per rule. Pyrefly uses its all preset; Pyright uses basic mode. These presets are not equivalent. The two ty modes run sequentially in one job: ty retains its default return inference, while ty (return inference) enables inference for unannotated returns. Neither mode guarantees Pyright-equivalent inference.
-Linux RSS measures the parent checker process, not its child processes; multi-process memory totals are not comparable.
+Linux RSS and the memory cap cover only the parent checker process, not its child processes; process-tree memory budgets and multi-process memory totals are not comparable.
Each checker discovers files natively within the configured roots; the harness does not choose .py over .pyi or vice versa. Untimed audits record native source-file counts, not imported dependencies. Different counts make a package not comparable and exclude its timings from report totals. Equal counts do not prove identical files or analysis. Historical results without audits are not verified.
| Package | {checker_headers}
{''.join(package_rows)}
{threading_section}
diff --git a/build/benchmark/render_pyright_history.py b/build/benchmark/render_pyright_history.py
index 72f4c5d8e631..2f33677f065a 100644
--- a/build/benchmark/render_pyright_history.py
+++ b/build/benchmark/render_pyright_history.py
@@ -10,6 +10,8 @@
from pathlib import Path
from typing import Any, Literal
+from benchmark_evidence import validate_evidence
+
CHECKERS = ("pyright", "pyright-pip")
PALETTE = (
@@ -32,6 +34,15 @@
"dependency_isolation",
"runs_per_package",
"warmup_runs",
+ "checker_config_version",
+ "uncounted_validation_runs_per_checker",
+ "installed_packages",
+ "platform",
+ "architecture",
+ "cpu_count",
+ "memory_measurement",
+ "timeout_s",
+ "import_path_overrides",
)
COMPARISON_PROFILE_FIELDS = tuple(
field for field in PROFILE_FIELDS if field not in {"runs_per_package", "warmup_runs"}
@@ -59,6 +70,7 @@ def _package_signature(data: dict[str, Any]) -> list[tuple[Any, ...]]:
package.get("commit"),
tuple(package.get("check_paths", [])),
tuple(package.get("exclude_directories", [])),
+ tuple((item["name"], item["version"]) for item in package.get("resolved_dependencies", [])),
)
for package in data.get("results", [])
)
@@ -71,8 +83,13 @@ def _package_corpus(data: dict[str, Any]) -> list[dict[str, Any]]:
"commit": commit,
"check_paths": list(check_paths),
"exclude_directories": list(exclude_directories),
+ **({
+ "resolved_dependencies": [
+ {"name": name, "version": version} for name, version in dependencies
+ ],
+ } if any("resolved_dependencies" in item for item in data.get("results", [])) else {}),
}
- for package_name, commit, check_paths, exclude_directories in _package_signature(
+ for package_name, commit, check_paths, exclude_directories, dependencies in _package_signature(
data
)
]
@@ -85,6 +102,7 @@ def _corpus_signature(corpus: list[dict[str, Any]]) -> list[tuple[Any, ...]]:
package.get("commit"),
tuple(package.get("check_paths", [])),
tuple(package.get("exclude_directories", [])),
+ tuple((item["name"], item["version"]) for item in package.get("resolved_dependencies", [])),
)
for package in corpus
)
@@ -119,9 +137,28 @@ def _package_metrics(
"execution_time_s": float(execution_time),
"peak_memory_mb": float(peak_memory),
}
+ if "files_selected" in metric:
+ package_metrics[str(name)]["files_selected"] = metric["files_selected"]
return package_metrics
+def _source_counts(data: dict[str, Any], checker: str) -> dict[str, int | None]:
+ return {
+ package["package_name"]: package.get("metrics", {}).get(checker, {}).get("files_selected")
+ for package in data.get("results", [])
+ }
+
+
+def _check_source_counts(
+ expected: dict[str, int | None], actual: dict[str, int | None], path: Path
+) -> None:
+ if expected.keys() != actual.keys() or any(
+ count is not None and actual[name] is not None and count != actual[name]
+ for name, count in expected.items()
+ ):
+ raise ValueError(f"{path} uses different native source counts")
+
+
def load_history(
paths: list[Path], manifest_path: Path | None = None
) -> dict[str, Any]:
@@ -140,9 +177,11 @@ def load_history(
versions: set[str] = set()
profile: dict[str, Any] | None = None
package_signature: list[tuple[Any, ...]] | None = None
+ source_counts: dict[str, int | None] | None = None
for path in paths:
data = json.loads(path.read_text(encoding="utf-8"))
+ validate_evidence(data)
checker = _checker(data, path)
version = str(
data.get("release_version")
@@ -164,6 +203,16 @@ def load_history(
package_signature = current_signature
elif current_signature != package_signature:
raise ValueError(f"{path} uses a different package corpus")
+ if data.get("checker_config_version", 0) >= 8:
+ current_counts = _source_counts(data, checker)
+ if source_counts is None:
+ source_counts = current_counts
+ else:
+ _check_source_counts(source_counts, current_counts, path)
+ source_counts = {
+ name: count if count is not None else current_counts[name]
+ for name, count in source_counts.items()
+ }
releases.append(
{
@@ -201,6 +250,7 @@ def load_history(
json.loads(paths[0].read_text(encoding="utf-8"))
),
"releases": releases,
+ **({"source_counts": source_counts} if source_counts is not None else {}),
}
@@ -222,6 +272,7 @@ def append_candidates(
candidate_paths, candidate_labels, strict=True
):
candidate = json.loads(candidate_path.read_text(encoding="utf-8"))
+ validate_evidence(candidate)
checker = _checker(candidate, candidate_path)
profile = {field: candidate.get(field) for field in PROFILE_FIELDS}
comparable_profile = {
@@ -233,6 +284,10 @@ def append_candidates(
}
if comparable_profile != history_profile:
raise ValueError(f"{candidate_path} uses a different benchmark profile")
+ if candidate.get("checker_config_version", 0) >= 8:
+ if "source_counts" not in history:
+ raise ValueError(f"{candidate_path} uses different native source counts")
+ _check_source_counts(history["source_counts"], _source_counts(candidate, checker), candidate_path)
if comparison_profile is None:
comparison_profile = profile
elif profile != comparison_profile:
diff --git a/build/benchmark/test_benchmark_capabilities.py b/build/benchmark/test_benchmark_capabilities.py
index 678c7679b0fe..1a53ee56ee9e 100644
--- a/build/benchmark/test_benchmark_capabilities.py
+++ b/build/benchmark/test_benchmark_capabilities.py
@@ -11,7 +11,8 @@ def _output(paths: list[Path], phase: str) -> capabilities.CapabilityOutput:
for path in paths:
for line, text in enumerate(path.read_text().splitlines(), 1):
if phase == "negative" and "# expect-error" in text:
- diagnostics.append(f"{path}:{line}:1: error: expected")
+ rule = "assert-type" if path.stem == "assert_type" else "assignment"
+ diagnostics.append(f"{path}:{line}:1: error: expected [{rule}]")
return {
"stdout": "\n".join(diagnostics),
"stderr": "",
@@ -24,21 +25,21 @@ def _output(paths: list[Path], phase: str) -> capabilities.CapabilityOutput:
class BenchmarkCapabilitiesTest(unittest.TestCase):
def test_requires_positive_precision_and_negative_diagnostics(self) -> None:
- def run(checker, root, paths):
+ def run(checker, root, paths, dependencies):
output = _output(paths, root.name)
if root.name == "positive":
- output["stdout"] = "generics.py:6:1: error: Any is not int"
+ output["stdout"] = "generics.py:6:1: error: Any is not int [assert-type]"
output["returncode"] = 1
else:
output["stdout"] = "\n".join(
line for line in output["stdout"].splitlines()
if "unannotated_return.py:" not in line
)
- output["stdout"] += "\nprotocols.py:4:1: error: unexpected"
+ output["stdout"] += "\nprotocols.py:4:1: error: unexpected [assignment]"
return output
- suite = capabilities.run_capability_suite(["checker"], run)
- result = suite["results"]["checker"]
+ suite = capabilities.run_capability_suite(["mypy"], run)
+ result = suite["results"]["mypy"]
self.assertTrue(result["ok"])
self.assertFalse(result["cases"]["generics"]["passed"])
self.assertFalse(result["cases"]["unannotated_return"]["passed"])
@@ -47,7 +48,7 @@ def run(checker, root, paths):
self.assertIn("Any is not int", result["evidence"]["positive"]["stdout"])
def test_missing_assert_type_control_cannot_pass_other_cases(self) -> None:
- def run(checker, root, paths):
+ def run(checker, root, paths, dependencies):
output = _output(paths, root.name)
output["stdout"] = "\n".join(
line for line in output["stdout"].splitlines()
@@ -55,14 +56,14 @@ def run(checker, root, paths):
)
return output
- suite = capabilities.run_capability_suite(["checker"], run)
- cases = suite["results"]["checker"]["cases"]
+ suite = capabilities.run_capability_suite(["mypy"], run)
+ cases = suite["results"]["mypy"]["cases"]
self.assertTrue(all(not case["passed"] for case in cases.values()))
def test_incomplete_or_unrecognized_runs_are_not_coverage(self) -> None:
for fault in ("count", "timeout", "fatal", "format", "outside"):
with self.subTest(fault=fault):
- def run(checker, root, paths):
+ def run(checker, root, paths, dependencies):
output = _output(paths, root.name)
if fault == "count":
output["files_selected"] = len(paths) - 1
@@ -75,33 +76,55 @@ def run(checker, root, paths):
output["stdout"] = "Unrecognized diagnostics"
else:
output["returncode"] = 1
- output["stdout"] = "capability_support.py:1: error: unexpected"
+ output["stdout"] = "capability_support.py:1: error: unexpected [assignment]"
return output
- result = capabilities.run_capability_suite(["checker"], run)["results"]["checker"]
+ result = capabilities.run_capability_suite(["mypy"], run)["results"]["mypy"]
self.assertFalse(result["ok"])
self.assertIn("error", result)
self.assertNotIn("cases", result)
def test_checker_start_failure_does_not_hide_other_checkers(self) -> None:
- def run(checker, root, paths):
+ def run(checker, root, paths, dependencies):
if checker == "missing":
raise FileNotFoundError("Not installed")
return _output(paths, root.name)
- results = capabilities.run_capability_suite(["missing", "installed"], run)["results"]
+ results = capabilities.run_capability_suite(["missing", "mypy"], run)["results"]
self.assertFalse(results["missing"]["ok"])
self.assertIn("Not installed", results["missing"]["error"])
- self.assertTrue(all(case["passed"] for case in results["installed"]["cases"].values()))
+ self.assertTrue(all(case["passed"] for case in results["mypy"]["cases"].values()))
+
+ def test_unrelated_same_line_errors_cannot_pass_assertion_control(self) -> None:
+ def run(checker, root, paths, dependencies):
+ output = _output(paths, root.name)
+ output["stdout"] = output["stdout"].replace("[assert-type]", "[unused-ignore]")
+ return output
+
+ result = capabilities.run_capability_suite(["mypy"], run)["results"]["mypy"]
+ self.assertTrue(result["ok"])
+ self.assertFalse(any(case["passed"] for case in result["cases"].values()))
+
+ def test_notes_and_source_excerpts_are_not_primary_diagnostics(self) -> None:
+ output = (
+ "probe.py:3: note: a message mentioning another.py:4:1\n"
+ "probe.py:5: error: wrong assignment [assignment]\n"
+ "probe.py:5: note: extra detail\n"
+ )
+ self.assertEqual(
+ capabilities.parse_diagnostics("mypy", output),
+ [{"file": "probe", "line": 5, "rule": "assignment", "category": "assignment"}],
+ )
@unittest.skipUnless(
os.environ.get("TYPECHECK_BENCHMARK_INTEGRATION") == "1",
"Set TYPECHECK_BENCHMARK_INTEGRATION=1 with all checkers installed",
)
def test_real_profiles_verify_precision_and_inference_difference(self) -> None:
- def run(checker, root, paths):
+ def run(checker, root, paths, dependencies):
result = benchmark._run_checker_process(
checker, root, paths, 60, 0, audit_files=True,
+ environment=benchmark._dependency_environment(dependencies, {}),
)
return {
**result,
@@ -118,8 +141,11 @@ def run(checker, root, paths):
self.assertTrue(result["cases"]["assert_type"]["passed"], result)
self.assertTrue(result["cases"]["generics"]["passed"], result)
self.assertTrue(result["cases"]["imported_types"]["passed"], result)
+ self.assertTrue(result["cases"]["installed_typed"]["passed"], result)
self.assertFalse(suite["results"]["ty"]["cases"]["unannotated_return"]["passed"])
self.assertTrue(suite["results"]["ty-infer"]["cases"]["unannotated_return"]["passed"])
+ self.assertFalse(suite["results"]["mypy"]["cases"]["installed_untyped"]["passed"])
+ self.assertTrue(suite["results"]["pyright"]["cases"]["installed_untyped"]["passed"])
if __name__ == "__main__":
diff --git a/build/benchmark/test_benchmark_evidence.py b/build/benchmark/test_benchmark_evidence.py
new file mode 100644
index 000000000000..9904cb159676
--- /dev/null
+++ b/build/benchmark/test_benchmark_evidence.py
@@ -0,0 +1,63 @@
+import copy
+import unittest
+
+from benchmark_evidence import validate_compatibility, validate_evidence
+
+
+def modern_result() -> dict:
+ return {
+ "checker_config_version": 8, "platform": "linux", "architecture": "x86_64",
+ "runner_class": "test", "runner_image": "ubuntu", "cpu_count": 4,
+ "python_version": "3.14.6", "memory_measurement": "/proc//status",
+ "memory_limit_mb": 8192, "dependency_isolation": "pip-target-per-package",
+ "uncounted_validation_runs_per_checker": 1, "runs_per_package": 3,
+ "warmup_runs": 1, "timeout_s": 1800, "installed_packages": [],
+ "import_path_overrides": {},
+ "type_checkers": ["pyright"],
+ "results": [{
+ "package_name": "example", "commit": "abc", "check_paths": ["source"],
+ "exclude_directories": [], "resolved_dependencies": [],
+ "repository_configs": [], "metrics": {
+ "pyright": {
+ "ok": True, "file_count_verified": True, "files_selected": 2,
+ "execution_time_s": 10.0, "peak_memory_mb": 100.0,
+ },
+ },
+ }],
+ }
+
+
+class BenchmarkEvidenceTest(unittest.TestCase):
+ def test_requires_native_count_and_preparation_evidence(self) -> None:
+ for field in ("file_count_verified", "files_selected"):
+ result = modern_result()
+ del result["results"][0]["metrics"]["pyright"][field]
+ with self.assertRaisesRegex(ValueError, "not verified"):
+ validate_evidence(result)
+ result = modern_result()
+ del result["results"][0]["resolved_dependencies"]
+ with self.assertRaisesRegex(ValueError, "resolved_dependencies"):
+ validate_evidence(result)
+
+ def test_rejects_mixed_environments_scope_and_dependencies(self) -> None:
+ baseline = modern_result()
+ for field, value in (
+ ("python_version", "3.11.0"), ("platform", "win32"), ("cpu_count", 8),
+ ("installed_packages", [{"name": "library", "version": "2"}]),
+ ):
+ candidate = copy.deepcopy(baseline)
+ candidate[field] = value
+ with self.subTest(field=field), self.assertRaisesRegex(ValueError, "ncompatible"):
+ validate_compatibility(baseline, candidate)
+ for field, value in (
+ ("commit", "def"), ("check_paths", ["other"]),
+ ("resolved_dependencies", [{"name": "library", "version": "2"}]),
+ ):
+ candidate = copy.deepcopy(baseline)
+ candidate["results"][0][field] = value
+ with self.subTest(field=field), self.assertRaisesRegex(ValueError, field):
+ validate_compatibility(baseline, candidate)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py
index 486052017693..3e7e561c5d1b 100644
--- a/build/benchmark/test_compare_benchmarks.py
+++ b/build/benchmark/test_compare_benchmarks.py
@@ -77,6 +77,14 @@ def _result(time: float, memory: float, ok: bool = True) -> dict:
class CompareBenchmarksTest(unittest.TestCase):
+ def test_missing_candidate_metric_is_reported_instead_of_crashing(self) -> None:
+ baseline = _result(10.0, 100.0)
+ candidate = _result(5.0, 100.0)
+ candidate["results"][0]["metrics"] = {}
+ failures, rows = compare_benchmarks._analyze(baseline, candidate, 10.0)
+ self.assertIn("example/pyright: candidate result failed or is missing", failures)
+ self.assertEqual(rows[0]["status"], "Missing")
+
def test_rejects_a_speedup_with_a_smaller_native_file_count(self) -> None:
baseline = _result(10.0, 100.0)
candidate = _result(5.0, 100.0)
@@ -572,7 +580,7 @@ def test_workflows_use_current_pnpm_setup(self) -> None:
{
"python-version": "${{ env.PYTHON_VERSION }}",
"cache": "pip",
- "cache-dependency-path": "build/benchmark/install_envs.json\n.github/workflows/typecheck_benchmark_weekly.yml\n",
+ "cache-dependency-path": "build/benchmark/install_envs.json\nbuild/benchmark/checker_requirements.txt\n.github/workflows/typecheck_benchmark_weekly.yml\n",
},
)
self.assertEqual(
@@ -601,8 +609,11 @@ def test_workflows_use_current_pnpm_setup(self) -> None:
},
)
self.assertEqual(
- weekly_benchmark_steps[3]["if"],
- "${{ matrix.checker != 'pyright' }}",
+ weekly_benchmark_steps[3],
+ {
+ "name": "Install shared checker environment",
+ "run": "python -m pip install -r build/benchmark/checker_requirements.txt",
+ },
)
self.assertEqual(
weekly_benchmark_steps[4]["if"],
diff --git a/build/benchmark/test_render_benchmark_html.py b/build/benchmark/test_render_benchmark_html.py
index 549a83a3eee1..a780ab6d2107 100644
--- a/build/benchmark/test_render_benchmark_html.py
+++ b/build/benchmark/test_render_benchmark_html.py
@@ -5,6 +5,7 @@
from pathlib import Path
import render_benchmark_html
+from test_benchmark_evidence import modern_result
def _result(checker: str, time: float, memory: float, ok: bool = True) -> dict:
@@ -69,6 +70,49 @@ def handle_data(self, data: str) -> None:
class RenderBenchmarkHtmlTest(unittest.TestCase):
+ def test_threading_speedup_requires_matching_pyright_version_and_heap_options(self) -> None:
+ single = _result("pyright", 2.0, 100.0)
+ threaded = _result("pyright-threads", 1.0, 100.0)
+ threaded["type_checker_versions"]["pyright-threads"] = "different"
+ with self.assertRaisesRegex(ValueError, "matching versions and Node options"):
+ render_benchmark_html.render_html({"pyright": single, "pyright-threads": threaded})
+ threaded["type_checker_versions"]["pyright-threads"] = "1.2.3"
+ threaded["node_options"] = "--max-old-space-size=1000"
+ with self.assertRaisesRegex(ValueError, "matching versions and Node options"):
+ render_benchmark_html.render_html({"pyright": single, "pyright-threads": threaded})
+
+ def test_current_reports_require_evidence_and_matching_source_environment(self) -> None:
+ import copy
+
+ pyright = modern_result()
+ ty = copy.deepcopy(pyright)
+ ty["type_checkers"] = ["ty"]
+ ty["results"][0]["metrics"] = {"ty": ty["results"][0]["metrics"]["pyright"]}
+ ty["results"][0]["commit"] = "different"
+ with self.assertRaisesRegex(ValueError, "commit"):
+ render_benchmark_html.render_html({"pyright": pyright, "ty": ty})
+ ty["results"][0]["commit"] = "abc"
+ del ty["results"][0]["metrics"]["ty"]["file_count_verified"]
+ with self.assertRaisesRegex(ValueError, "not verified"):
+ render_benchmark_html.render_html({"pyright": pyright, "ty": ty})
+
+ def test_shared_totals_exclude_failures_without_hiding_individual_work(self) -> None:
+ import copy
+
+ pyright = _result("pyright", 10.0, 100.0)
+ ty = _result("ty", 1.0, 50.0)
+ for checker, result in (("pyright", pyright), ("ty", ty)):
+ extra = copy.deepcopy(result["results"][0])
+ extra["package_name"] = "failed-for-ty"
+ extra["metrics"][checker]["execution_time_s"] = 20.0
+ if checker == "ty":
+ extra["metrics"][checker]["ok"] = False
+ result["results"].append(extra)
+ report = render_benchmark_html.render_html({"pyright": pyright, "ty": ty})
+ self.assertIn("30.0s", report)
+ self.assertIn("1 | 10.0s | ", report)
+ self.assertIn("1 | 1.0s | ", report)
+ self.assertIn("not an apples-to-apples speed ranking", report)
def test_native_count_mismatches_are_not_comparable_or_in_summary_totals(self) -> None:
pyright = _result("pyright", 2.0, 200.0)
mypy = _result("mypy", 1.0, 100.0)
@@ -247,11 +291,12 @@ def test_renders_comparison_semantics(self) -> None:
self.assertEqual(
parser.cells,
[
- "Checker", "Measured", "Failed", "Total time", "Median time", "Median RSS",
- "pyright", "1.2.3", "1", "0", "2.0s", "2.0s", "200 MB",
- "pyright (threads)", "1.2.3", "1", "0", "1.0s", "1.0s", "250 MB",
- "mypy", "1.2.3", "1", "0", "1.0s", "1.0s", "100 MB",
- "zuban", "1.2.3", "0", "1", "0.0s", "0.0s", "0 MB",
+ "Checker", "Measured", "Failed", "Individual total", "Shared packages", "Shared total",
+ "Median time", "Median RSS",
+ "pyright", "1.2.3", "1", "0", "2.0s", "0", "N/A", "2.0s", "200 MB",
+ "pyright (threads)", "1.2.3", "1", "0", "1.0s", "0", "N/A", "1.0s", "250 MB",
+ "mypy", "1.2.3", "1", "0", "1.0s", "0", "N/A", "1.0s", "100 MB",
+ "zuban", "1.2.3", "0", "1", "0.0s", "0", "N/A", "0.0s", "0 MB",
"Package", "pyright", "pyright (threads)", "mypy", "zuban",
"example", "2.00s", "200 MB", "1.00s", "250 MB", "1.00s",
"100 MB", "Not measured", "Package", "Single-thread",
diff --git a/build/benchmark/test_render_pyright_history.py b/build/benchmark/test_render_pyright_history.py
index 7a80b263f617..843c6a136ea1 100644
--- a/build/benchmark/test_render_pyright_history.py
+++ b/build/benchmark/test_render_pyright_history.py
@@ -6,6 +6,7 @@
from pathlib import Path
import render_pyright_history
+from test_benchmark_evidence import modern_result
REPO_ROOT = Path(__file__).resolve().parents[2]
@@ -104,6 +105,33 @@ def _result(version: str, published_at: str, scale: float = 1.0) -> dict:
class RenderPyrightHistoryTest(unittest.TestCase):
+ def test_current_history_rejects_mixed_config_versions_and_native_counts(self) -> None:
+ import copy
+
+ first = modern_result()
+ first["type_checker_versions"] = {"pyright": "1.1.413"}
+ first["timestamp"] = "2026-11-02T00:00:00+00:00"
+ second = copy.deepcopy(first)
+ second["type_checker_versions"] = {"pyright": "1.1.414"}
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ old_path, new_path = root / "old.json", root / "new.json"
+ old_path.write_text(json.dumps(first))
+ second["checker_config_version"] = 7
+ new_path.write_text(json.dumps(second))
+ with self.assertRaisesRegex(ValueError, "different benchmark profile"):
+ render_pyright_history.load_history([old_path, new_path])
+ second["checker_config_version"] = 8
+ second["results"][0]["metrics"]["pyright"]["files_selected"] = 3
+ new_path.write_text(json.dumps(second))
+ with self.assertRaisesRegex(ValueError, "different native source counts"):
+ render_pyright_history.load_history([old_path, new_path])
+ second["results"][0]["metrics"]["pyright"]["files_selected"] = 2
+ new_path.write_text(json.dumps(second))
+ history = render_pyright_history.load_history([old_path, new_path])
+ self.assertEqual(history["source_counts"], {"example": 2})
+ self.assertEqual(history["releases"][0]["packages"]["example"]["files_selected"], 2)
+
def test_appends_candidate_to_release_history(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
diff --git a/build/benchmark/test_typecheck_benchmark.py b/build/benchmark/test_typecheck_benchmark.py
index cb44dcbb9dd8..e91dd8555933 100644
--- a/build/benchmark/test_typecheck_benchmark.py
+++ b/build/benchmark/test_typecheck_benchmark.py
@@ -360,6 +360,7 @@ def test_load_install_envs_includes_source_only_packages(self) -> None:
self.assertEqual(packages[0]["name"], "click")
def test_install_failure_is_a_package_failure(self) -> None:
+ (self.root / "pyproject.toml").write_text('[tool.pyrefly]\npreset = "off"\n')
package = {
"name": "example",
"github_url": "https://example.com/example",
@@ -377,6 +378,8 @@ def test_install_failure_is_a_package_failure(self) -> None:
self.assertEqual(result["error"], "Dependency installation failed")
self.assertEqual(result["commit"], "abc123")
+ self.assertEqual(result["repository_configs"][0]["path"], "pyproject.toml")
+ self.assertIn("tool.pyrefly.preset", result["repository_configs"][0]["settings"])
install.assert_called_once_with(
self.root, package, self.root / "example-dependencies"
)
@@ -425,6 +428,8 @@ def test_packages_use_independent_dependency_environments(self) -> None:
{"name": "first", "github_url": "https://example.com/first"},
{"name": "second", "github_url": "https://example.com/second"},
]
+ for package in packages:
+ (self.root / package["name"]).mkdir()
with (
patch.object(
benchmark,
diff --git a/build/benchmark/typecheck_benchmark.py b/build/benchmark/typecheck_benchmark.py
index 105dde8b526d..31a84ba8bff4 100644
--- a/build/benchmark/typecheck_benchmark.py
+++ b/build/benchmark/typecheck_benchmark.py
@@ -21,6 +21,8 @@
import argparse
import configparser
+import hashlib
+import importlib.metadata
import json
import os
import platform
@@ -67,7 +69,7 @@
}
PYRIGHT_DEFAULT_EXCLUDES = ["**/node_modules", "**/__pycache__", "**/.*"]
DEPENDENCY_ISOLATION = "pip-target-per-package"
-CHECKER_CONFIG_VERSION = 7
+CHECKER_CONFIG_VERSION = 8
class BenchmarkError(Exception):
@@ -118,6 +120,7 @@ class PackageResult(TypedDict, total=False):
source_file_count: int
source_manifest_sha256: str
repository_configs: list[dict[str, Any]]
+ resolved_dependencies: list[dict[str, str]]
class AggregateStats(TypedDict):
@@ -550,6 +553,18 @@ def _dependency_environment(
return environment
+def _installed_packages(path: Path | None = None) -> list[dict[str, str]]:
+ distributions = (
+ importlib.metadata.distributions(path=[str(path)])
+ if path is not None else importlib.metadata.distributions()
+ )
+ return sorted(
+ ({"name": re.sub(r"[-_.]+", "-", item.metadata["Name"]).lower(), "version": item.version}
+ for item in distributions),
+ key=lambda item: (item["name"], item["version"]),
+ )
+
+
def install_deps(
package_path: Path, config: dict[str, Any], dependency_path: Path
) -> bool:
@@ -1196,6 +1211,7 @@ def _benchmark_directory(
local_path: str | None = None,
environment: dict[str, str] | None = None,
exclude_directories: list[str] | None = None,
+ repository_configs: list[dict[str, Any]] | None = None,
) -> PackageResult:
check_roots, missing = _resolve_check_paths(
package_path,
@@ -1213,7 +1229,8 @@ def _benchmark_directory(
"metrics": {},
}
try:
- repository_configs = _audit_repository_configs(package_path)
+ if repository_configs is None:
+ repository_configs = _audit_repository_configs(package_path)
except OSError as exc:
message = f"Repository configuration audit failed: {exc}"
print(f" {message}")
@@ -1378,11 +1395,13 @@ def _benchmark_directory(
result: PackageResult = {
"package_name": name,
"github_url": github_url,
+ "commit": get_package_commit(package_path) or "local-unversioned",
"error": None,
"metrics": metrics,
"check_paths": _relative_check_paths(package_path, check_roots) or [],
"exclude_directories": exclude_directories or [],
"repository_configs": repository_configs,
+ "resolved_dependencies": [],
}
if local_path:
result["local_path"] = local_path
@@ -1407,6 +1426,7 @@ def _benchmark_package(
"github_url": github_url,
"check_paths": raw_check_paths,
"exclude_directories": exclude_directories,
+ "commit": package.get("commit") or "unknown",
}
package_path = clone_package(
github_url, name, temp_path, package.get("commit")
@@ -1419,6 +1439,14 @@ def _benchmark_package(
}
commit = get_package_commit(package_path)
+ package_metadata["commit"] = commit or "unknown"
+ try:
+ package_metadata["repository_configs"] = _audit_repository_configs(package_path)
+ except OSError as exc:
+ return {
+ **package_metadata, "error": f"Repository configuration audit failed: {exc}",
+ "metrics": {},
+ }
dependency_path = temp_path / f"{name}-dependencies"
if not install_deps(package_path, package, dependency_path):
return {
@@ -1467,9 +1495,11 @@ def _benchmark_package(
memory_limit_mb=memory_limit_mb,
environment=_dependency_environment(dependency_path, package),
exclude_directories=exclude_directories,
+ repository_configs=package_metadata["repository_configs"],
)
result["commit"] = commit or "unknown"
result.update(package_metadata)
+ result["resolved_dependencies"] = _installed_packages(dependency_path)
return result
@@ -1515,6 +1545,11 @@ def _save_results(
"node_options": os.environ.get("NODE_OPTIONS", ""),
"dependency_isolation": DEPENDENCY_ISOLATION,
"checker_config_version": CHECKER_CONFIG_VERSION,
+ "installed_packages": _installed_packages(),
+ "import_path_overrides": {
+ name: hashlib.sha256(os.environ.get(name, "").encode("utf-8")).hexdigest()
+ for name in ("PYTHONPATH", "MYPYPATH")
+ },
"type_checkers": type_checkers,
"type_checker_versions": versions,
"package_count": len(results),
@@ -1664,10 +1699,11 @@ def run_benchmark(
print("\nUntimed behavioral capability audit:")
def run_capability(
- checker: str, root: Path, paths: list[Path]
+ checker: str, root: Path, paths: list[Path], dependencies: Path,
) -> CapabilityOutput:
result = _run_checker_process(
checker, root, paths, min(timeout, 60), memory_limit_mb, audit_files=True,
+ environment=_dependency_environment(dependencies, {}),
)
return {
**result,
From 454ef66305ca2af4d5811ca1c62f23f5b7cd3403 Mon Sep 17 00:00:00 2001
From: Bill Schnurr
Date: Fri, 20 Nov 2026 16:38:33 -0700
Subject: [PATCH 6/6] Test missing checker in benchmark comparisons
Classification: C (test coverage). No type precision changes.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
build/benchmark/test_compare_benchmarks.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py
index 3e7e561c5d1b..b7eaf4068aeb 100644
--- a/build/benchmark/test_compare_benchmarks.py
+++ b/build/benchmark/test_compare_benchmarks.py
@@ -85,6 +85,16 @@ def test_missing_candidate_metric_is_reported_instead_of_crashing(self) -> None:
self.assertIn("example/pyright: candidate result failed or is missing", failures)
self.assertEqual(rows[0]["status"], "Missing")
+ def test_missing_candidate_checker_is_reported_instead_of_crashing(self) -> None:
+ baseline = _result(10.0, 100.0)
+ candidate = _result(5.0, 100.0)
+ candidate["results"][0]["metrics"] = {
+ "mypy": dict(candidate["results"][0]["metrics"]["pyright"])
+ }
+ failures, rows = compare_benchmarks._analyze(baseline, candidate, 10.0)
+ self.assertIn("example/pyright: candidate result failed or is missing", failures)
+ self.assertEqual(rows[0]["status"], "Missing")
+
def test_rejects_a_speedup_with_a_smaller_native_file_count(self) -> None:
baseline = _result(10.0, 100.0)
candidate = _result(5.0, 100.0)