diff --git a/.github/workflows/typecheck_benchmark_weekly.yml b/.github/workflows/typecheck_benchmark_weekly.yml index 163f1408737e..ff39d56971c1 100644 --- a/.github/workflows/typecheck_benchmark_weekly.yml +++ b/.github/workflows/typecheck_benchmark_weekly.yml @@ -37,6 +37,7 @@ jobs: cache: 'pip' cache-dependency-path: | build/benchmark/install_envs.json + build/benchmark/checker_requirements.txt .github/workflows/typecheck_benchmark_weekly.yml - name: Verify build prerequisites @@ -45,9 +46,8 @@ jobs: g++ --version make --version - - name: Install ${{ matrix.checker }} - if: ${{ matrix.checker != 'pyright' }} - run: python -m pip install --upgrade '${{ matrix.checker }}' + - name: Install shared checker environment + run: python -m pip install -r build/benchmark/checker_requirements.txt - uses: pnpm/action-setup@f520eceda224fe1a4aed5a2a27a194379a409996 # v6 if: ${{ matrix.checker == 'pyright' }} @@ -92,6 +92,9 @@ jobs: checkers+=(pyright-threads) extra_args+=(--skip-pyright-build) fi + if [[ '${{ matrix.checker }}' == 'ty' ]]; then + checkers+=(ty-infer) + fi python build/benchmark/typecheck_benchmark.py \ -c "${checkers[@]}" -r 3 -w 1 -t 1800 \ --memory-limit-mb 8192 --os-name linux-x64 \ diff --git a/build/benchmark/README.md b/build/benchmark/README.md index 0ccec430ef5b..48bb4755db9c 100644 --- a/build/benchmark/README.md +++ b/build/benchmark/README.md @@ -8,7 +8,8 @@ The upstream copyright and MIT license are reproduced in [`UPSTREAM_LICENSE.txt`](UPSTREAM_LICENSE.txt). The local integration preserves repository-specific behavior: it builds and invokes the current checkout's Pyright production bundle rather than resolving Pyright from `PATH`. The benchmark -measures performance only, not diagnostic accuracy or type precision. +measures corpus performance, with a separate small behavioral audit rather than a comprehensive +diagnostic-accuracy or type-precision comparison. The checked corpus is a curated set of source repositories derived from [`lolpack/type_coverage_py`](https://github.com/lolpack/type_coverage_py/blob/main/included_packages.txt). @@ -16,10 +17,11 @@ Each entry in `install_envs.json` defines its repository, check paths, installat additional dependencies. Repository commits are pinned so pull-request comparisons analyze the same source revisions. -Entries can define `exclude_directories` to prune named directories while resolving their check -paths. The benchmark expands those entries to `.py` and `.pyi` files before invoking any checker, so -all checkers analyze the same scope. NumPy excludes its test directories; pandas includes its test -and `_testing` directories. Generated Pyright configurations explicitly retain +Entries can define `exclude_directories`, translated into exclusions in each benchmark config. +The benchmark passes configured roots to each checker and preserves native file discovery; it +does not select implementations over stubs or stubs over implementations. Native import resolution +normally prefers stubs where available. NumPy excludes its test directories; pandas includes its test +and `_testing` directories. Generated benchmark configurations explicitly retain its standard `**/node_modules`, `**/__pycache__`, and `**/.*` exclusions. ## Prerequisites @@ -92,8 +94,174 @@ it afterward. If Pyright is selected, the local Pyright production bundle is sti `--skip-pyright-build` is supplied. Checker configuration files use unique temporary names in the target directory and are removed after -each invocation. Existing `pyrightconfig.json`, mypy, ty, or Pyrefly configuration files are not -overwritten. The temporary configuration selects the full local directory. +each invocation. Every checker receives an explicit configuration, including Zuban, whose mode is +also explicitly `default`. Existing repository configuration files are not overwritten or extended. +The temporary configuration selects the full local directory. + +## Fairness and interpretation + +The benchmark is an end-to-end CLI throughput measurement, not an equal-work or quality-adjusted +ranking. A slower result does not by itself prove that a checker does more analysis. Different +inference algorithms, import graphs, bundled stubs, diagnostics, threading, startup, and diagnostic +formatting all contribute to wall time. + +All checkers target the running interpreter's Python major/minor version and receive its interpreter +path. ty and Pyrefly ignore `.gitignore` and `.ignore`; Zuban disables gitignore exclusions. +ty also merges user-level configuration even when `--config-file` is supplied, so both ty profiles +run with fresh empty `APPDATA`/`XDG_CONFIG_HOME` config directories, removed after each invocation. +ty's built-in directory exclusions are reset before applying Pyright's standard exclusions, and +Pyrefly's exclusion heuristics are disabled in favor of those same exclusions. This avoids silently +skipping selected `dist`, `venv`, or ignored source files, or checking hidden directories only in ty. +Imported dependencies may still be analyzed differently; matching entry-point scope does not imply +matching transitive work. + +Each checker discovers files natively within the configured roots. A missing configured path or +empty root selection is an error; neither falls back to checking the whole repository. +mypy and Zuban receive roots through TOML file lists to avoid command-line size limits and +Windows INI backslash escaping. ty's source selectors use gitignore-pattern separators even on +Windows, where native backslashes can silently match no files. + +Immediately after cloning, before dependency installation, an **untimed repository configuration search** +recursively inventories native +`pyrightconfig.json`, `ty.toml`, `pyrefly.toml`, `zuban.toml`, `mypy.ini`, and `.mypy.ini` candidates. +It also searches TOML files for checker-specific `[tool.*]` settings and `project.requires-python`, +and INI settings in `setup.cfg` and mypy configs. JSON records `repository_configs` per package; +console output and the weekly HTML show paths, sections/setting names, and that explicit benchmark +configs are passed instead. Configuration values are not published, and detected does not mean +applied. Invalid TOML/INI files produce visible audit parse errors rather than disappearing from +the inventory. The search skips VCS metadata, `node_modules`, `__pycache__`, and harness-generated +configs. Preparation failures retain the inventory. It does not modify or delete repository configs. +The real-binary suppression tests below, +not discovery alone, provide evidence that conflicting corpus configs cannot override the harness. + +Before any warmups or measured runs, each checker performs one **untimed file-count audit** using +the benchmark configuration and native discovery. Pyright reports checked files in `--stats`, +threaded Pyright reports files to analyze with `--verbose`, Pyrefly reports project modules with +`--summary=full`, ty logs checked file paths with `-vv`, and mypy/Zuban report checked source-file +counts in their summaries. Missing/zero counts and fatal errors reject that checker's measurements. +Verbose/debug audit output is not enabled for measured runs. +Counts in ordinary checker output are also checked during warmups and measured runs. +The audit is recorded separately as `uncounted_validation_runs_per_checker = 1`, including when +`--warmup 0` is selected; this is therefore not a cold-filesystem measurement. + +Each successful checker records its native `files_selected`, `file_count_verified`, and separate +`file_count_audit_time_s`. Different audited counts make that package **Not comparable**, including +across separate weekly checker jobs; its timings are excluded from report totals rather than +forcing the tools to discover the same files. Raw JSON retains the measurements and differences. +The regression comparator rejects changed native counts, roots, exclusions, or audit-run settings. +Configuration version 8 additionally requires successful results to carry positive audited counts, +source revisions, preparation evidence, and resolved dependency versions. The weekly renderer +rejects differing package sets/revisions, roots, exclusions, target dependencies, Python/platform/ +runner profiles, and ambient Python distribution versions. It also checks hashes of inherited +`PYTHONPATH`/`MYPYPATH` overrides without publishing their values. +Regenerate older baselines and release history rather than mixing pre-contract measurements. +History now retains native counts and rejects incompatible configuration, audit, import environment, +and successful native-scope evidence. +These are source entry-point counts, **not total imported or parsed files**. Equal counts do not +prove identical file identities, equal dependency work, or equal diagnostic coverage. +Repository and user configs are tested +separately with deliberately suppressed diagnostics, and source-level ignore comments retain each +tool's semantics. + +The `ty` profile retains ty's default return-type inference. The separately selectable `ty-infer` +profile uses the same binary and configuration except for +`analysis.return-type-inference = "always"`. Pyrefly explicitly uses its `all` preset, +`check-unannotated-defs = true`, and `infer-return-types = "checked"`. Pyright uses `basic` mode with +unannotated-function analysis enabled by default; mypy uses `check_untyped_defs = True`. +Pyrefly's `all` preset enables every error kind and strict callable subtyping; it is not equivalent +to Pyright's `basic` mode. Similar option names or presets do not imply equivalent inference. +Changing diagnostic severity does not necessarily change how much analysis a checker performs. +In particular, ty documents limitations for generator and asynchronous return inference even with `"always"`. + +```console +python build/benchmark/typecheck_benchmark.py -c ty ty-infer -r 3 -w 1 +``` + +After corpus timings finish, the harness runs an **untimed behavioral capability suite** with two +batch invocations per installed checker, using the same configurations and interpreter. +Ten paired probes cover an exact-type assertion +control, unannotated return inference, unannotated function body checking, generics, overloads, +protocols, narrowing, propagation from a local annotated import, and installed packages with and +without a `py.typed` marker. The installed-package fixtures use the same dependency-path mechanism +as the corpus. These probes do not model all third-party package or stub-resolution behavior. + +The positive examples must be accepted, including exact `typing.assert_type` checks; unexpected +`Any` or `Unknown` therefore cannot satisfy an `int` assertion. Negative examples must report a +diagnostic in the intended category at exactly the intended line, with no other primary diagnostics. +Checker-specific adapters read primary diagnostics and rule IDs; notes and source excerpts are +not diagnostics, and unrelated same-line lint errors cannot satisfy the assertion control. +The assertion control must pass before any case can match. Both batches require native counts covering all ten +probe files; crashes, timeouts, missing counts, and unrecognized output are **not evaluated**, +not zero capability or a successful result. + +JSON stores the versioned suite identity, per-case diagnostic categories/rule IDs/lines, and raw process evidence in +`capability_suite`. HTML adds a **Behavior matches** column and a per-case matrix. Mismatches can +mean a missed error, lost precision, or additional strictness diagnostics: for example, Pyrefly's +`all` preset also requires return annotations. Do not equate every mismatch with weaker inference. +Comparisons reject changed suite identities or behavioral outcomes rather than accepting a +speedup that silently changed these observed behaviors. +Package-shard merges require identical suite identities and outcomes but permit different raw +process evidence (such as temporary paths); the merged result retains the first shard's evidence. + +These are a few examples, not a weighted quality score, a measurement of internal work, or proof +that the timed corpus receives equivalent analysis. Runtime is not divided by matches, rules, or +diagnostic counts. Even passing every probe does not justify a quality-adjusted corpus speed ratio; +that requires a separately controlled workload with verified shared behavioral requirements. +Historical results without this audit have no verified behavior-match count. + +For defensible cross-checker comparisons, use the same machine, pinned source commits, dependency +versions, interpreter, target platform, source scope, and resource/thread budgets. The weekly jobs +use separate hosted runners. Every weekly job installs the same pinned checker requirements from +`checker_requirements.txt`; snapshots of ambient and per-project resolved distribution versions +detect differing environments, including unlocked transitive dependencies. This is not a shared +corpus dependency lock, identical physical hardware, or proof of equivalent inference. +Results remain independent trends rather than a controlled speed ranking. The summary distinguishes +individual successful-package totals from totals over the shared successful package intersection. +Never rank individual totals with differing coverage; even shared counts are not equal analysis. +Repeated fresh +processes after a warmup measure a warm-filesystem CLI workload, not editor incrementality or a +cold-disk workload. Alternate run order for paired experiments and report the individual samples +and dispersion, not only a speedup. + +Do not divide total runtime by enabled rule count or emitted diagnostic count. Rules differ in +granularity and share parsing, import resolution, inference, constraint solving, and caches. +Disabling a diagnostic often suppresses reporting without removing its prerequisite analysis. +For a feature-cost investigation, use focused workloads (for example overload resolution, +protocol matching, generics, narrowing, unannotated returns, and typed versus untyped imports), +verify expected diagnostics and inferred types outside the timed runs, and compare paired +feature-on/feature-off measurements within each checker where supported. Report these as marginal +costs on that workload, not additive or portable "time per rule". Report type-system conformance +and curated false-positive/false-negative cases separately from throughput; the +[Python typing conformance suite](https://github.com/python/typing/tree/main/conformance) is a useful +shared specification, but not a performance weighting scheme. + +Linux peak RSS and memory-limit enforcement currently use only `/proc//status`; +they do not sum child-process memory. +In particular, Pyright `--threads` RSS must not be interpreted as total worker memory or a memory +improvement over single-process Pyright. Windows RSS is unavailable. Neither limitation prevents +wall-time measurements, but neither supports cross-process memory rankings. + +Configuration changes are recorded as `checker_config_version` in result JSON. Comparisons reject +different versions, including pre-versioned results, so regenerate baselines rather than comparing +new file-selection or interpreter settings against old measurements. +Pyrefly's exit-code-1 "no files matched" error is a failed measurement, not an ordinary type error. + +The opt-in integration tests execute the real binaries against conflicting root/nested configs, +ignored files, directory-exclusion probes, and paired implementations/stubs. They verify every +checker's native count before and after the hostile configs, including clean files and standalone stubs, +and check that `ty-infer` catches an inferred-return +assignment that the default profile does not. Install all default checkers and build Pyright first, +then set `TYPECHECK_BENCHMARK_INTEGRATION=1` and run: + +```console +python -m unittest discover -s build/benchmark -p test_typecheck_benchmark.py +``` + +Configuration references: +[ty configuration](https://docs.astral.sh/ty/configuration/), +[ty inference and source options](https://docs.astral.sh/ty/reference/configuration/), +[Pyrefly configuration selection and presets](https://pyrefly.org/en/docs/configuration/), +and [Zuban modes and configuration](https://docs.zubanls.com/en/latest/usage.html). ## Developer workflow @@ -171,7 +339,7 @@ that first introduces them cannot trigger itself. | Flag | Description | | --- | --- | -| `-c, --checkers NAME [NAME ...]` | Checkers to run; choices are `pyright`, `pyright-threads`, `pyright-pip`, `pyrefly`, `ty`, `mypy`, and `zuban` | +| `-c, --checkers NAME [NAME ...]` | Checkers to run; choices are `pyright`, `pyright-threads`, `pyright-pip`, `pyrefly`, `ty`, `ty-infer`, `mypy`, and `zuban` | | `-r, --runs N` | Measured runs per checker (default: 5) | | `-w, --warmup N` | Warmup runs discarded before measurement (default: 1) | | `-t, --timeout SECONDS` | Timeout for each checker invocation (default: 300) | @@ -242,6 +410,8 @@ peak-memory charts. Reports and artifacts are published before a failed comparis unsuccessful. The weekly workflow runs Pyright, Pyrefly, ty, mypy, and Zuban in independent hosted-runner jobs. +The ty job runs `ty` and `ty-infer` sequentially against the same prepared corpus, with separate +labels and results; it does not replace ty's default behavior with a stricter configuration. The Pyright job measures single-threaded mode and `--threads` mode sequentially on the same runner and prepared package corpus. Each mode performs three measured runs after one warmup. The report includes a per-package Pyright threading chart with wall time, peak RSS, and speedup. The aggregate job stores diff --git a/build/benchmark/benchmark_capabilities.py b/build/benchmark/benchmark_capabilities.py new file mode 100644 index 000000000000..66ef6691b245 --- /dev/null +++ b/build/benchmark/benchmark_capabilities.py @@ -0,0 +1,302 @@ +"""Small behavioral probes, evaluated separately from corpus timings.""" + +from __future__ import annotations + +import hashlib +import json +import re +import tempfile +from pathlib import Path +from typing import Callable, TypedDict + + +CASES = { + "assert_type": ( + "Exact-type assertion control", + 'from typing import assert_type\n' + 'assert_type(int("1"), int)\n' + 'assert_type(int("1"), str) # expect-error\n', + ), + "unannotated_return": ( + "Unannotated return inference", + 'from typing import assert_type\n' + 'def inferred():\n' + ' return int("1")\n' + 'assert_type(inferred(), int)\n' + 'bad: str = inferred() # expect-error\n', + ), + "unannotated_body": ( + "Unannotated function body checking", + 'def checked():\n' + ' good: int = 1\n' + ' bad: str = 1 # expect-error\n', + ), + "generics": ( + "Generic type propagation", + 'from typing import TypeVar, assert_type\n' + 'T = TypeVar("T")\n' + 'def first(values: list[T]) -> T:\n' + ' return values[0]\n' + 'values: list[int] = [1]\n' + 'assert_type(first(values), int)\n' + 'bad: str = first(values) # expect-error\n', + ), + "overloads": ( + "Overload selection", + 'from typing import assert_type, overload\n' + '@overload\n' + 'def choose(value: int) -> int: ...\n' + '@overload\n' + 'def choose(value: str) -> str: ...\n' + 'def choose(value: int | str) -> int | str:\n' + ' return value\n' + 'assert_type(choose(int("1")), int)\n' + 'bad: str = choose(int("1")) # expect-error\n', + ), + "protocols": ( + "Structural protocol checking", + 'from typing import Protocol, assert_type\n' + 'class Reader(Protocol):\n' + ' def read(self) -> int: ...\n' + 'class Good:\n' + ' def read(self) -> int:\n' + ' return 1\n' + 'class Bad:\n' + ' def read(self) -> str:\n' + ' return "wrong"\n' + 'good: Reader = Good()\n' + 'assert_type(good.read(), int)\n' + 'bad: Reader = Bad() # expect-error\n', + ), + "narrowing": ( + "Union narrowing", + 'from typing import assert_type\n' + 'def narrow(value: int | str) -> None:\n' + ' if isinstance(value, int):\n' + ' assert_type(value, int)\n' + ' bad: str = value # expect-error\n' + ' else:\n' + ' assert_type(value, str)\n', + ), + "imported_types": ( + "Imported type propagation", + 'from typing import assert_type\n' + 'from capability_support import typed_value\n' + 'assert_type(typed_value(), int)\n' + 'bad: str = typed_value() # expect-error\n', + ), +} +SUPPORT_SOURCE = "def typed_value() -> int:\n return 1\n" +for _name, _label in ( + ("installed_typed", "Installed typed-package precision"), + ("installed_untyped", "Installed untyped-package precision"), +): + CASES[_name] = ( + _label, + "from typing import assert_type\n" + f"from capability_{_name} import typed_value\n" + "assert_type(typed_value(), int)\n" + "bad: str = typed_value() # expect-error\n", + ) +SUITE_VERSION = 2 +SUITE_SHA256 = hashlib.sha256( + json.dumps([SUITE_VERSION, CASES, SUPPORT_SOURCE], sort_keys=True).encode("utf-8") +).hexdigest() + + +class CapabilityOutput(TypedDict): + stdout: str + stderr: str + returncode: int + timed_out: bool + oom_killed: bool + files_selected: int | None + + +class Diagnostic(TypedDict): + file: str + line: int + rule: str + category: str + + +class CaseResult(TypedDict): + passed: bool + positive_diagnostic_lines: list[int] + negative_diagnostic_lines: list[int] + expected_error_line: int + positive_diagnostics: list[Diagnostic] + negative_diagnostics: list[Diagnostic] + + +class CheckerResult(TypedDict, total=False): + ok: bool + error: str + cases: dict[str, CaseResult] + evidence: dict[str, CapabilityOutput] + + +class CapabilitySuite(TypedDict): + version: int + sha256: str + cases: dict[str, str] + results: dict[str, CheckerResult] + + +def parse_diagnostics(checker: str, output: str) -> list[Diagnostic]: + """Read primary diagnostics and rule IDs, never source excerpts or notes.""" + diagnostics: list[Diagnostic] = [] + if checker in ("pyright", "pyright-threads", "pyright-pip"): + pattern = r"(?m)^\s+([^\n]+\.pyi?):(\d+):\d+ - (?:error|warning|information): " + matches = list(re.finditer(pattern, output)) + for index, match in enumerate(matches): + block = output[match.end():matches[index + 1].start() if index + 1 < len(matches) else len(output)] + rule = re.search(r"\((report\w+)\)", block) + if rule is None: + raise ValueError("Pyright diagnostic has no recognized rule ID") + diagnostics.append(_diagnostic(match[1], int(match[2]), rule[1])) + elif checker in ("ty", "ty-infer", "pyrefly"): + pattern = ( + r"(?m)^(?:error|warning|info)\[([\w-]+)\]:[^\n]*" + if checker in ("ty", "ty-infer") + else r"(?m)^(?:ERROR|WARN|INFO) [^\n]* \[([\w-]+)\]" + ) + matches = list(re.finditer(pattern, output)) + for index, match in enumerate(matches): + block = output[match.end():matches[index + 1].start() if index + 1 < len(matches) else len(output)] + location = re.search(r"(?m)^\s*-->\s+([^\n]+\.pyi?):(\d+):\d+", block) + if location is None: + raise ValueError(f"{checker} diagnostic has no primary source location") + diagnostics.append(_diagnostic(location[1], int(location[2]), match[1])) + elif checker in ("mypy", "zuban"): + for match in re.finditer(r"(?m)^([^\n]+\.pyi?):(\d+)(?::\d+)?: (?:error|warning): (.+)$", output): + rule = re.search(r"\[([\w-]+)\]\s*$", match[3]) + if rule is None: + raise ValueError(f"{checker} diagnostic has no recognized rule ID") + code = rule[1] + if checker == "zuban" and code == "misc" and match[3].startswith("Expression is of type "): + code = "assert-type" + diagnostics.append(_diagnostic(match[1], int(match[2]), code)) + else: + raise ValueError(f"No diagnostic adapter for {checker}") + return diagnostics + + +def _diagnostic(path: str, line: int, rule: str) -> Diagnostic: + categories = { + "reportAssertTypeFailure": "assert-type", "type-assertion-failure": "assert-type", + "assert-type": "assert-type", "reportAssignmentType": "assignment", + "invalid-assignment": "assignment", "bad-assignment": "assignment", + "assignment": "assignment", "arg-type": "argument", + } + return { + "file": Path(path.strip()).stem, "line": line, "rule": rule, + "category": categories.get(rule, f"other:{rule}"), + } + + +def run_capability_suite( + checkers: list[str], + run: Callable[[str, Path, list[Path], Path], CapabilityOutput], +) -> CapabilitySuite: + suite: CapabilitySuite = { + "version": SUITE_VERSION, + "sha256": SUITE_SHA256, + "cases": {name: label for name, (label, _) in CASES.items()}, + "results": {}, + } + with tempfile.TemporaryDirectory(prefix="typecheck-capabilities-") as directory: + root = Path(directory) + dependencies = root / "dependencies" + for name in ("installed_typed", "installed_untyped"): + package = dependencies / f"capability_{name}" + package.mkdir(parents=True) + (package / "__init__.py").write_text(SUPPORT_SOURCE, encoding="utf-8") + if name == "installed_typed": + (package / "py.typed").write_text("", encoding="utf-8") + paths: dict[str, list[Path]] = {} + expected: dict[str, int] = {} + for phase in ("positive", "negative"): + project = root / phase + project.mkdir() + (project / "capability_support.py").write_text(SUPPORT_SOURCE, encoding="utf-8") + paths[phase] = [] + for name, (_, source) in CASES.items(): + lines = source.splitlines(keepends=True) + expected[name] = next( + index for index, line in enumerate(lines, 1) if "# expect-error" in line + ) + if phase == "positive": + lines = [ + "\n" if "# expect-error" in line else line for line in lines + ] + path = project / f"{name}.py" + path.write_text("".join(lines), encoding="utf-8") + paths[phase].append(path) + + for checker in checkers: + evidence: dict[str, CapabilityOutput] = {} + locations: dict[str, list[Diagnostic]] = {} + error = None + for phase in ("positive", "negative"): + try: + output = run(checker, root / phase, paths[phase], dependencies) + except OSError as exc: + error = f"{phase}: unable to evaluate capabilities: {exc}" + break + evidence[phase] = output + if output["timed_out"] or output["oom_killed"]: + error = f"{phase}: capability checker timed out or exceeded memory limit" + elif output["returncode"] not in (0, 1): + error = f"{phase}: fatal checker exit {output['returncode']}" + elif output["files_selected"] != len(CASES): + error = f"{phase}: capability source-file count is not verified" + else: + try: + locations[phase] = parse_diagnostics(checker, output["stdout"]) + except ValueError as exc: + error = f"{phase}: {exc}" + break + if {item["file"] for item in locations[phase]} - CASES.keys(): + error = f"{phase}: unexpected diagnostics outside capability cases" + elif bool(locations[phase]) != (output["returncode"] == 1): + error = f"{phase}: unrecognized or inconsistent diagnostic output" + if error: + break + if error: + suite["results"][checker] = { + "ok": False, "error": error, "evidence": evidence, + } + continue + cases: dict[str, CaseResult] = {} + def case_diagnostics(phase: str, name: str) -> list[Diagnostic]: + return [item for item in locations[phase] if item["file"] == name] + + def expected_diagnostics(name: str, items: list[Diagnostic]) -> bool: + category = "assert-type" if name == "assert_type" else "assignment" + return ( + len(items) == 1 and items[0]["line"] == expected[name] + and ( + items[0]["category"] == category + or (name == "generics" and items[0]["category"] == "argument") + ) + ) + + control_passed = not case_diagnostics("positive", "assert_type") and expected_diagnostics( + "assert_type", case_diagnostics("negative", "assert_type") + ) + for name in CASES: + positive = case_diagnostics("positive", name) + negative = case_diagnostics("negative", name) + cases[name] = { + "passed": control_passed and not positive and expected_diagnostics(name, negative), + "positive_diagnostic_lines": sorted({item["line"] for item in positive}), + "negative_diagnostic_lines": sorted({item["line"] for item in negative}), + "expected_error_line": expected[name], + "positive_diagnostics": positive, + "negative_diagnostics": negative, + } + suite["results"][checker] = { + "ok": True, "cases": cases, "evidence": evidence, + } + return suite diff --git a/build/benchmark/benchmark_evidence.py b/build/benchmark/benchmark_evidence.py new file mode 100644 index 000000000000..05eea9c45d49 --- /dev/null +++ b/build/benchmark/benchmark_evidence.py @@ -0,0 +1,72 @@ +"""Compatibility checks shared by benchmark reports and regression history.""" + +from __future__ import annotations + +import math +from typing import Any + + +EVIDENCE_VERSION = 8 +ENVIRONMENT_FIELDS = ( + "platform", "architecture", "runner_class", "runner_image", "cpu_count", + "python_version", "memory_measurement", "memory_limit_mb", "dependency_isolation", + "checker_config_version", "uncounted_validation_runs_per_checker", + "runs_per_package", "warmup_runs", "timeout_s", + "import_path_overrides", +) +SCOPE_FIELDS = ("commit", "check_paths", "exclude_directories", "resolved_dependencies") + + +def validate_evidence(data: dict[str, Any]) -> None: + if data.get("checker_config_version", 0) < EVIDENCE_VERSION: + return + for field in (*ENVIRONMENT_FIELDS, "installed_packages"): + if field not in data: + raise ValueError(f"Missing benchmark evidence field: {field}") + names: set[str] = set() + for package in data.get("results", []): + name = package["package_name"] + if name in names: + raise ValueError(f"Duplicate package evidence: {name}") + names.add(name) + if not package.get("error"): + for field in (*SCOPE_FIELDS, "repository_configs"): + if field not in package: + raise ValueError(f"{name}: missing package evidence: {field}") + for checker, metric in package.get("metrics", {}).items(): + if not metric.get("ok"): + continue + count = metric.get("files_selected") + if ( + metric.get("file_count_verified") is not True + or not isinstance(count, int) or isinstance(count, bool) or count <= 0 + ): + raise ValueError(f"{name}/{checker}: source-file count is not verified") + for field in ("execution_time_s", "peak_memory_mb"): + value = metric.get(field) + if ( + not isinstance(value, (int, float)) or isinstance(value, bool) + or not math.isfinite(value) or value < 0 + ): + raise ValueError(f"{name}/{checker}: invalid {field}") + + +def validate_compatibility(left: dict[str, Any], right: dict[str, Any]) -> None: + validate_evidence(left) + validate_evidence(right) + for field in ENVIRONMENT_FIELDS: + if left.get(field) != right.get(field): + raise ValueError(f"Incompatible benchmark environment: {field}") + if left.get("installed_packages") != right.get("installed_packages"): + raise ValueError("Incompatible benchmark import environments: installed_packages") + left_packages = {item["package_name"]: item for item in left.get("results", [])} + right_packages = {item["package_name"]: item for item in right.get("results", [])} + if left_packages.keys() != right_packages.keys(): + raise ValueError("Incompatible benchmark package sets") + for name in left_packages: + old, new = left_packages[name], right_packages[name] + # A failed preparation may lack dependency evidence. It is not shared successful work. + fields = SCOPE_FIELDS if not old.get("error") and not new.get("error") else SCOPE_FIELDS[:3] + for field in fields: + if old.get(field) != new.get(field): + raise ValueError(f"{name}: incompatible package {field}") diff --git a/build/benchmark/checker_requirements.txt b/build/benchmark/checker_requirements.txt new file mode 100644 index 000000000000..6c8e5c3db0a0 --- /dev/null +++ b/build/benchmark/checker_requirements.txt @@ -0,0 +1,4 @@ +pyrefly==1.4.0 +ty==0.0.90 +mypy==2.4.0 +zuban==0.10.1 diff --git a/build/benchmark/compare_benchmarks.py b/build/benchmark/compare_benchmarks.py index bbc265e9a5a2..456f5db086e0 100644 --- a/build/benchmark/compare_benchmarks.py +++ b/build/benchmark/compare_benchmarks.py @@ -9,6 +9,8 @@ from pathlib import Path from typing import Any, Literal, TypedDict +from benchmark_evidence import validate_evidence + DEFAULT_THRESHOLD_PERCENT = 20.0 DEFAULT_TIME_NOISE_FLOOR_SECONDS = 1.0 DEFAULT_MEMORY_NOISE_FLOOR_MB = 100.0 @@ -146,6 +148,11 @@ def _analyze( rows: list[ComparisonRow] = [] if failures: return failures, rows + try: + validate_evidence(baseline) + validate_evidence(candidate) + except ValueError as exc: + return [str(exc)], rows for field in ( "platform", "architecture", @@ -157,7 +164,11 @@ def _analyze( "node_options", "runs_per_package", "warmup_runs", + "uncounted_validation_runs_per_checker", "dependency_isolation", + "checker_config_version", + "installed_packages", + "import_path_overrides", ): if baseline.get(field) != candidate.get(field): failures.append( @@ -168,6 +179,22 @@ def _analyze( candidate_metrics = _metrics_by_package(candidate) baseline_packages = _packages_by_name(baseline) candidate_packages = _packages_by_name(candidate) + old_suite = baseline.get("capability_suite") + new_suite = candidate.get("capability_suite") + if old_suite is not None or new_suite is not None: + if old_suite is None or new_suite is None or any( + old_suite.get(field) != new_suite.get(field) + for field in ("version", "sha256", "cases") + ): + failures.append("capability suite changed or was not evaluated") + else: + for checker, old_result in old_suite["results"].items(): + new_result = new_suite["results"].get(checker, {}) + if ( + old_result.get("ok") != new_result.get("ok") + or old_result.get("cases") != new_result.get("cases") + ): + failures.append(f"{checker}: behavioral capability outcomes changed") for key, old in sorted(baseline_metrics.items()): package, checker = key @@ -256,8 +283,19 @@ def _analyze( } ) continue + if not new: + failures.append(f"{package}/{checker}: candidate result failed or is missing") + rows.append({ + "package": package, "checker": checker, "execution_time_s": None, + "time_delta": None, "peak_memory_mb": None, "memory_delta": None, + "status": "Missing", + }) + continue scope_changed = False - for field in ("check_paths", "exclude_directories"): + if old.get("files_selected") != new.get("files_selected"): + failures.append(f"{package}/{checker}: native source-file count changed") + scope_changed = True + for field in ("check_paths", "exclude_directories", "source_manifest_sha256", "resolved_dependencies"): if old_package.get(field, []) != new_package.get(field, []): failures.append( f"{package}/{checker}: package {field} changed from " diff --git a/build/benchmark/merge_benchmark_results.py b/build/benchmark/merge_benchmark_results.py index bffe4282e1f1..ba5242a8cedf 100644 --- a/build/benchmark/merge_benchmark_results.py +++ b/build/benchmark/merge_benchmark_results.py @@ -9,6 +9,7 @@ from typing import Any import typecheck_benchmark +from benchmark_evidence import validate_evidence VARIABLE_FIELDS = { @@ -36,10 +37,22 @@ def merge_results(paths: list[Path]) -> dict[str, Any]: data = json.loads(path.read_text(encoding="utf-8")) if not isinstance(data, dict): raise ValueError(f"{path} must contain a JSON object") + validate_evidence(data) metadata = { key: value for key, value in data.items() if key not in VARIABLE_FIELDS } + if "capability_suite" in metadata: + suite = metadata["capability_suite"] + metadata["capability_suite"] = { + **suite, + "results": { + checker: { + key: value for key, value in result.items() if key != "evidence" + } + for checker, result in suite["results"].items() + }, + } if expected_metadata is None: expected_metadata = metadata merged = copy.deepcopy(data) diff --git a/build/benchmark/render_benchmark_html.py b/build/benchmark/render_benchmark_html.py index e06740b47a7d..8d9b32e14491 100644 --- a/build/benchmark/render_benchmark_html.py +++ b/build/benchmark/render_benchmark_html.py @@ -5,14 +5,19 @@ import argparse import html import json +import statistics from pathlib import Path from typing import Any +from benchmark_evidence import validate_compatibility, validate_evidence -CHECKER_ORDER = ["pyright", "pyright-threads", "pyrefly", "ty", "mypy", "zuban"] + +CHECKER_ORDER = ["pyright", "pyright-threads", "pyrefly", "ty", "ty-infer", "mypy", "zuban"] def _checker_label(checker: str) -> str: + if checker == "ty-infer": + return "ty (return inference)" return "pyright (threads)" if checker == "pyright-threads" else checker @@ -42,7 +47,7 @@ def _ordered_checkers(results: dict[str, dict[str, Any]]) -> list[str]: def _package_metrics(data: dict[str, Any], checker: str) -> dict[str, dict[str, Any]]: return { - package["package_name"]: package.get("metrics", {}).get(checker, {}) + package["package_name"]: package.get("metrics", {}).get(checker, {}).copy() for package in data.get("results", []) if package.get("package_name") } @@ -56,6 +61,21 @@ def _phase_time(phase_times: dict[str, Any], name: str) -> str: def render_html(results: dict[str, dict[str, Any]]) -> str: if not results: raise ValueError("At least one checker result is required") + config_version = next(iter(results.values())).get("checker_config_version") + if any(data.get("checker_config_version") != config_version for data in results.values()): + raise ValueError("Checker results use different benchmark configuration versions") + reference = next(iter(results.values())) + for data in results.values(): + validate_evidence(data) + validate_compatibility(reference, data) + if "pyright" in results and "pyright-threads" in results: + single, threaded = results["pyright"], results["pyright-threads"] + if ( + single.get("type_checker_versions", {}).get("pyright") + != threaded.get("type_checker_versions", {}).get("pyright-threads") + or single.get("node_options") != threaded.get("node_options") + ): + raise ValueError("Pyright threading comparison requires matching versions and Node options") checkers = _ordered_checkers(results) packages = sorted( { @@ -68,54 +88,241 @@ def render_html(results: dict[str, dict[str, Any]]) -> str: metrics = { checker: _package_metrics(results[checker], checker) for checker in checkers } + suites = { + checker: data["capability_suite"] + for checker, data in results.items() if "capability_suite" in data + } + identities = { + json.dumps( + [suite["version"], suite["sha256"], suite["cases"]], sort_keys=True + ) + for suite in suites.values() + } + if len(identities) > 1: + raise ValueError("Checker results use different capability suites") + capability_cases = next(iter(suites.values()))["cases"] if suites else {} + capabilities = { + checker: suite["results"].get(checker, {}) + for checker, suite in suites.items() + } + for checker, result in capabilities.items(): + if result.get("ok") and set(result["cases"]) != set(capability_cases): + raise ValueError(f"{checker}: incomplete capability results") + source_scopes: dict[str, tuple[int, str]] = {} + for checker in checkers: + for package in results[checker].get("results", []): + manifest_hash = package.get("source_manifest_sha256") + if manifest_hash is None: + continue + name = package["package_name"] + scope = (package["source_file_count"], manifest_hash) + if name in source_scopes and source_scopes[name] != scope: + raise ValueError(f"{name}: checker results use different source manifests") + source_scopes[name] = scope + metric = metrics[checker].get(name, {}) + if metric.get("ok") and ( + not metric.get("file_count_verified") + or metric.get("files_selected") != scope[0] + ): + raise ValueError(f"{name}/{checker}: source-file count is not verified") + mismatched_packages = set() + for package in packages: + native_counts = { + metric["files_selected"] + for checker in checkers + if (metric := metrics[checker].get(package, {})).get("file_count_verified") + } + if len(native_counts) > 1: + mismatched_packages.add(package) + for checker in checkers: + metric = metrics[checker].get(package, {}) + if metric.get("file_count_verified"): + metric["ok"] = False + metric["source_count_mismatch"] = True + aggregates = { + checker: results[checker].get("aggregate", {}).get(checker, {}) + for checker in checkers + } + for checker in checkers: + accepted = [metric for metric in metrics[checker].values() if metric.get("ok")] + aggregates[checker] = { + "packages_tested": len(accepted), + "packages_failed": len(packages) - len(accepted), + "total_execution_time_s": sum(metric["execution_time_s"] for metric in accepted), + "p50_execution_time_s": statistics.median( + [metric["execution_time_s"] for metric in accepted] + ) if accepted else 0, + "p50_peak_memory_mb": statistics.median( + [metric.get("peak_memory_mb", 0) for metric in accepted] + ) if accepted else 0, + } + shared_packages = [ + package for package in packages + if all(metrics[checker].get(package, {}).get("ok") for checker in checkers) + ] totals = { checker: float( - results[checker].get("aggregate", {}) - .get(checker, {}) - .get("total_execution_time_s", 0.0) + aggregates[checker].get("total_execution_time_s", 0.0) ) for checker in checkers } - max_total = max(totals.values(), default=0.0) timestamp = max(str(data.get("timestamp", "")) for data in results.values()) environment = next(iter(results.values())) summary_rows: list[str] = [] for checker in checkers: - aggregate = results[checker].get("aggregate", {}).get(checker, {}) + aggregate = aggregates[checker] tested = int(aggregate.get("packages_tested", 0)) failed = int(aggregate.get("packages_failed", 0)) total = totals[checker] - share = total / max_total * 100 if max_total else 0.0 + shared_total = sum(metrics[checker][package]["execution_time_s"] for package in shared_packages) + shared_text = f"{shared_total:.1f}s" if shared_packages else "N/A" version = results[checker].get("type_checker_versions", {}).get(checker, "unknown") + capability_cell = "" + if suites: + capability = capabilities.get(checker, {}) + text = ( + f'{sum(case["passed"] for case in capability["cases"].values())}' + f' / {len(capability_cases)}' + if capability.get("ok") else "Not evaluated" + ) + capability_cell = f"{text}" summary_rows.append( "" f"{html.escape(_checker_label(checker))}" f"{html.escape(str(version))}" f"{tested}{failed}" - f"
" - f"{total:.1f}s" + f"{total:.1f}s" + f"{len(shared_packages)}{shared_text}" f"{float(aggregate.get('p50_execution_time_s', 0.0)):.1f}s" f"{float(aggregate.get('p50_peak_memory_mb', 0.0)):.0f} MB" + + capability_cell + "" ) + capability_section = "" + if suites: + rows = [] + for name, label in capability_cases.items(): + cells = [] + for checker in checkers: + capability = capabilities.get(checker, {}) + if not capability.get("ok"): + error = html.escape(capability.get("error", "No capability audit recorded")) + cells.append(f'Not evaluated{error}') + continue + case = capability["cases"][name] + if case["passed"]: + cells.append('Match') + else: + details = ( + f'Positive diagnostic lines: {case["positive_diagnostic_lines"]}; ' + f'negative diagnostic lines: {case["negative_diagnostic_lines"]}; ' + f'expected negative line: {case["expected_error_line"]}.' + ) + if "negative_diagnostics" in case: + details += ( + f' Positive rules: {[item["rule"] for item in case["positive_diagnostics"]]};' + f' negative rules: {[item["rule"] for item in case["negative_diagnostics"]]}.' + ) + cells.append( + f'Mismatch{html.escape(details)}' + ) + rows.append( + f'{html.escape(label)}{"".join(cells)}' + ) + headers = "".join( + f'{html.escape(_checker_label(checker))}' for checker in checkers + ) + capability_section = ( + '

Behavioral capability probes

' + '

Untimed paired probes run after corpus timings with the same ' + 'checker configurations. A match requires accepting the positive example, preserving ' + 'asserted types, and reporting exactly the intended negative diagnostic category and line. ' + 'The assertion control must also pass. Extra strictness diagnostics can cause a mismatch; ' + 'a mismatch does not necessarily mean less inference. Installed-package probes separately ' + 'test imports with and without py.typed; matching project-file counts cannot establish ' + 'equal dependency analysis. Raw JSON retains rule IDs and diagnostic evidence.

' + '

These few examples are not a comprehensive conformance or quality score, ' + 'do not certify equivalent analysis on the timed projects, and are not a denominator ' + 'for normalizing runtime. Historical results without probes are not evaluated.

' + '
' + f'{headers}{"".join(rows)}
Behavior
' + ) + package_rows: list[str] = [] for package in packages: cells = [] for checker in checkers: metric = metrics[checker].get(package, {}) if not metric.get("ok"): - cells.append('Not measured') + cells.append( + 'Not comparable' + f'{metric["files_selected"]} native source files' + if metric.get("source_count_mismatch") + else 'Not measured' + ) continue cells.append( f'{float(metric["execution_time_s"]):.2f}s' - f'{float(metric.get("peak_memory_mb", 0.0)):.0f} MB' + f'{float(metric.get("peak_memory_mb", 0.0)):.0f} MB' + + ( + f'{metric["files_selected"]} source files verified' + if metric.get("file_count_verified") else "" + ) + '' ) + scope_label = ( + f'{source_scopes[package][0]} source files selected' + if package in source_scopes else "" + ) package_rows.append( - f'{html.escape(package)}{"".join(cells)}' + f'{html.escape(package)}{scope_label}{"".join(cells)}' ) + config_rows = [] + for package in packages: + records = { + json.dumps(record, sort_keys=True) + for data in results.values() + for item in data.get("results", []) + if item.get("package_name") == package + for record in item.get("repository_configs", []) + } + audited = any( + "repository_configs" in item + for data in results.values() for item in data.get("results", []) + if item.get("package_name") == package + ) + if not records and audited: + config_rows.append( + f'{html.escape(package)}' + 'No checker configs or TOML settings detected' + ) + for serialized in sorted(records): + record = json.loads(serialized) + detail = ", ".join( + record.get("settings") or record["sections"] or record["checkers"] + ) or "Unknown" + handling = ( + f'Scan parse error: {record["parse_error"]}' + if "parse_error" in record else record["disposition"] + ) + config_rows.append( + f'{html.escape(package)}{html.escape(record["path"])}' + f'{html.escape(detail)}{html.escape(handling)}' + ) + config_section = ( + '

Repository configuration audit

' + '

Untimed recursive search of each checkout for native checker configs, ' + 'TOML tool sections, and mypy INI settings. Detected does not mean applied: the harness ' + 'passes explicit benchmark configurations instead. Values are not published. Parse errors ' + 'are shown rather than treated as absence. The search skips VCS metadata, node_modules, ' + 'and __pycache__; source-level ignore comments retain their native semantics.

' + '
' + '' + f'{"".join(config_rows)}
PackageFileChecker / sectionsHandling
' + ) if config_rows else "" + pyright_stats_rows: list[str] = [] for package, metric in sorted(metrics.get("pyright", {}).items()): phase_times = metric.get("phase_times_s") @@ -232,10 +439,16 @@ def render_html(results: dict[str, dict[str, Any]]) -> str:

Python type checker benchmark

Weekly cross-checker performance on pinned open-source projects.

Run summary

Generated {html.escape(timestamp)} · Python {html.escape(str(environment.get('python_version', 'unknown')))} · {html.escape(str(environment.get('platform_details', environment.get('platform', 'unknown'))))}

-
{''.join(summary_rows)}
CheckerMeasuredFailedTotal timeMedian timeMedian RSS
+

Individual totals cover each checker's successful packages and must not be ranked against different coverage. Shared totals cover only packages successful for every displayed profile with matching audited counts. Shared source scope is not equivalent analysis; these remain independent CLI measurements, not an apples-to-apples speed ranking.

+
{'' if suites else ''}{''.join(summary_rows)}
CheckerMeasuredFailedIndividual totalShared packagesShared totalMedian timeMedian RSSBehavior matches

Package comparison

Different checker jobs run on separate hosted runners, so timings are independent trends rather than a fastest-checker ranking. The two Pyright modes run sequentially in one job. Cells show wall time and peak RSS.

+

Checker defaults, inference, and diagnostic coverage differ; these timings do not measure equivalent analysis or time per rule. Pyrefly uses its all preset; Pyright uses basic mode. These presets are not equivalent. The two ty modes run sequentially in one job: ty retains its default return inference, while ty (return inference) enables inference for unannotated returns. Neither mode guarantees Pyright-equivalent inference.

+

Linux RSS and the memory cap cover only the parent checker process, not its child processes; process-tree memory budgets and multi-process memory totals are not comparable.

+

Each checker discovers files natively within the configured roots; the harness does not choose .py over .pyi or vice versa. Untimed audits record native source-file counts, not imported dependencies. Different counts make a package not comparable and exclude its timings from report totals. Equal counts do not prove identical files or analysis. Historical results without audits are not verified.

{checker_headers}{''.join(package_rows)}
Package
{threading_section} +{capability_section} +{config_section}

Pyright release history

Track per-package execution time and peak memory across every stable Pyright release from the prior year.

View Pyright package performance by release

{pyright_stats_section}
diff --git a/build/benchmark/render_pyright_history.py b/build/benchmark/render_pyright_history.py index 72f4c5d8e631..2f33677f065a 100644 --- a/build/benchmark/render_pyright_history.py +++ b/build/benchmark/render_pyright_history.py @@ -10,6 +10,8 @@ from pathlib import Path from typing import Any, Literal +from benchmark_evidence import validate_evidence + CHECKERS = ("pyright", "pyright-pip") PALETTE = ( @@ -32,6 +34,15 @@ "dependency_isolation", "runs_per_package", "warmup_runs", + "checker_config_version", + "uncounted_validation_runs_per_checker", + "installed_packages", + "platform", + "architecture", + "cpu_count", + "memory_measurement", + "timeout_s", + "import_path_overrides", ) COMPARISON_PROFILE_FIELDS = tuple( field for field in PROFILE_FIELDS if field not in {"runs_per_package", "warmup_runs"} @@ -59,6 +70,7 @@ def _package_signature(data: dict[str, Any]) -> list[tuple[Any, ...]]: package.get("commit"), tuple(package.get("check_paths", [])), tuple(package.get("exclude_directories", [])), + tuple((item["name"], item["version"]) for item in package.get("resolved_dependencies", [])), ) for package in data.get("results", []) ) @@ -71,8 +83,13 @@ def _package_corpus(data: dict[str, Any]) -> list[dict[str, Any]]: "commit": commit, "check_paths": list(check_paths), "exclude_directories": list(exclude_directories), + **({ + "resolved_dependencies": [ + {"name": name, "version": version} for name, version in dependencies + ], + } if any("resolved_dependencies" in item for item in data.get("results", [])) else {}), } - for package_name, commit, check_paths, exclude_directories in _package_signature( + for package_name, commit, check_paths, exclude_directories, dependencies in _package_signature( data ) ] @@ -85,6 +102,7 @@ def _corpus_signature(corpus: list[dict[str, Any]]) -> list[tuple[Any, ...]]: package.get("commit"), tuple(package.get("check_paths", [])), tuple(package.get("exclude_directories", [])), + tuple((item["name"], item["version"]) for item in package.get("resolved_dependencies", [])), ) for package in corpus ) @@ -119,9 +137,28 @@ def _package_metrics( "execution_time_s": float(execution_time), "peak_memory_mb": float(peak_memory), } + if "files_selected" in metric: + package_metrics[str(name)]["files_selected"] = metric["files_selected"] return package_metrics +def _source_counts(data: dict[str, Any], checker: str) -> dict[str, int | None]: + return { + package["package_name"]: package.get("metrics", {}).get(checker, {}).get("files_selected") + for package in data.get("results", []) + } + + +def _check_source_counts( + expected: dict[str, int | None], actual: dict[str, int | None], path: Path +) -> None: + if expected.keys() != actual.keys() or any( + count is not None and actual[name] is not None and count != actual[name] + for name, count in expected.items() + ): + raise ValueError(f"{path} uses different native source counts") + + def load_history( paths: list[Path], manifest_path: Path | None = None ) -> dict[str, Any]: @@ -140,9 +177,11 @@ def load_history( versions: set[str] = set() profile: dict[str, Any] | None = None package_signature: list[tuple[Any, ...]] | None = None + source_counts: dict[str, int | None] | None = None for path in paths: data = json.loads(path.read_text(encoding="utf-8")) + validate_evidence(data) checker = _checker(data, path) version = str( data.get("release_version") @@ -164,6 +203,16 @@ def load_history( package_signature = current_signature elif current_signature != package_signature: raise ValueError(f"{path} uses a different package corpus") + if data.get("checker_config_version", 0) >= 8: + current_counts = _source_counts(data, checker) + if source_counts is None: + source_counts = current_counts + else: + _check_source_counts(source_counts, current_counts, path) + source_counts = { + name: count if count is not None else current_counts[name] + for name, count in source_counts.items() + } releases.append( { @@ -201,6 +250,7 @@ def load_history( json.loads(paths[0].read_text(encoding="utf-8")) ), "releases": releases, + **({"source_counts": source_counts} if source_counts is not None else {}), } @@ -222,6 +272,7 @@ def append_candidates( candidate_paths, candidate_labels, strict=True ): candidate = json.loads(candidate_path.read_text(encoding="utf-8")) + validate_evidence(candidate) checker = _checker(candidate, candidate_path) profile = {field: candidate.get(field) for field in PROFILE_FIELDS} comparable_profile = { @@ -233,6 +284,10 @@ def append_candidates( } if comparable_profile != history_profile: raise ValueError(f"{candidate_path} uses a different benchmark profile") + if candidate.get("checker_config_version", 0) >= 8: + if "source_counts" not in history: + raise ValueError(f"{candidate_path} uses different native source counts") + _check_source_counts(history["source_counts"], _source_counts(candidate, checker), candidate_path) if comparison_profile is None: comparison_profile = profile elif profile != comparison_profile: diff --git a/build/benchmark/test_benchmark_capabilities.py b/build/benchmark/test_benchmark_capabilities.py new file mode 100644 index 000000000000..1a53ee56ee9e --- /dev/null +++ b/build/benchmark/test_benchmark_capabilities.py @@ -0,0 +1,152 @@ +import os +import unittest +from pathlib import Path + +import benchmark_capabilities as capabilities +import typecheck_benchmark as benchmark + + +def _output(paths: list[Path], phase: str) -> capabilities.CapabilityOutput: + diagnostics = [] + for path in paths: + for line, text in enumerate(path.read_text().splitlines(), 1): + if phase == "negative" and "# expect-error" in text: + rule = "assert-type" if path.stem == "assert_type" else "assignment" + diagnostics.append(f"{path}:{line}:1: error: expected [{rule}]") + return { + "stdout": "\n".join(diagnostics), + "stderr": "", + "returncode": 1 if diagnostics else 0, + "timed_out": False, + "oom_killed": False, + "files_selected": len(paths), + } + + +class BenchmarkCapabilitiesTest(unittest.TestCase): + def test_requires_positive_precision_and_negative_diagnostics(self) -> None: + def run(checker, root, paths, dependencies): + output = _output(paths, root.name) + if root.name == "positive": + output["stdout"] = "generics.py:6:1: error: Any is not int [assert-type]" + output["returncode"] = 1 + else: + output["stdout"] = "\n".join( + line for line in output["stdout"].splitlines() + if "unannotated_return.py:" not in line + ) + output["stdout"] += "\nprotocols.py:4:1: error: unexpected [assignment]" + return output + + suite = capabilities.run_capability_suite(["mypy"], run) + result = suite["results"]["mypy"] + self.assertTrue(result["ok"]) + self.assertFalse(result["cases"]["generics"]["passed"]) + self.assertFalse(result["cases"]["unannotated_return"]["passed"]) + self.assertFalse(result["cases"]["protocols"]["passed"]) + self.assertTrue(result["cases"]["overloads"]["passed"]) + self.assertIn("Any is not int", result["evidence"]["positive"]["stdout"]) + + def test_missing_assert_type_control_cannot_pass_other_cases(self) -> None: + def run(checker, root, paths, dependencies): + output = _output(paths, root.name) + output["stdout"] = "\n".join( + line for line in output["stdout"].splitlines() + if "assert_type.py:" not in line + ) + return output + + suite = capabilities.run_capability_suite(["mypy"], run) + cases = suite["results"]["mypy"]["cases"] + self.assertTrue(all(not case["passed"] for case in cases.values())) + + def test_incomplete_or_unrecognized_runs_are_not_coverage(self) -> None: + for fault in ("count", "timeout", "fatal", "format", "outside"): + with self.subTest(fault=fault): + def run(checker, root, paths, dependencies): + output = _output(paths, root.name) + if fault == "count": + output["files_selected"] = len(paths) - 1 + elif fault == "timeout": + output["timed_out"] = True + elif fault == "fatal": + output["returncode"] = 2 + elif fault == "format": + output["returncode"] = 1 + output["stdout"] = "Unrecognized diagnostics" + else: + output["returncode"] = 1 + output["stdout"] = "capability_support.py:1: error: unexpected [assignment]" + return output + + result = capabilities.run_capability_suite(["mypy"], run)["results"]["mypy"] + self.assertFalse(result["ok"]) + self.assertIn("error", result) + self.assertNotIn("cases", result) + + def test_checker_start_failure_does_not_hide_other_checkers(self) -> None: + def run(checker, root, paths, dependencies): + if checker == "missing": + raise FileNotFoundError("Not installed") + return _output(paths, root.name) + + results = capabilities.run_capability_suite(["missing", "mypy"], run)["results"] + self.assertFalse(results["missing"]["ok"]) + self.assertIn("Not installed", results["missing"]["error"]) + self.assertTrue(all(case["passed"] for case in results["mypy"]["cases"].values())) + + def test_unrelated_same_line_errors_cannot_pass_assertion_control(self) -> None: + def run(checker, root, paths, dependencies): + output = _output(paths, root.name) + output["stdout"] = output["stdout"].replace("[assert-type]", "[unused-ignore]") + return output + + result = capabilities.run_capability_suite(["mypy"], run)["results"]["mypy"] + self.assertTrue(result["ok"]) + self.assertFalse(any(case["passed"] for case in result["cases"].values())) + + def test_notes_and_source_excerpts_are_not_primary_diagnostics(self) -> None: + output = ( + "probe.py:3: note: a message mentioning another.py:4:1\n" + "probe.py:5: error: wrong assignment [assignment]\n" + "probe.py:5: note: extra detail\n" + ) + self.assertEqual( + capabilities.parse_diagnostics("mypy", output), + [{"file": "probe", "line": 5, "rule": "assignment", "category": "assignment"}], + ) + + @unittest.skipUnless( + os.environ.get("TYPECHECK_BENCHMARK_INTEGRATION") == "1", + "Set TYPECHECK_BENCHMARK_INTEGRATION=1 with all checkers installed", + ) + def test_real_profiles_verify_precision_and_inference_difference(self) -> None: + def run(checker, root, paths, dependencies): + result = benchmark._run_checker_process( + checker, root, paths, 60, 0, audit_files=True, + environment=benchmark._dependency_environment(dependencies, {}), + ) + return { + **result, + "files_selected": benchmark._parse_source_file_count( + checker, result["stdout"] + "\n" + result["stderr"] + ), + } + + checkers = [*benchmark.DEFAULT_TYPE_CHECKERS, "ty-infer", "pyright-threads"] + suite = capabilities.run_capability_suite(checkers, run) + for checker, result in suite["results"].items(): + with self.subTest(checker=checker): + self.assertTrue(result["ok"], result) + self.assertTrue(result["cases"]["assert_type"]["passed"], result) + self.assertTrue(result["cases"]["generics"]["passed"], result) + self.assertTrue(result["cases"]["imported_types"]["passed"], result) + self.assertTrue(result["cases"]["installed_typed"]["passed"], result) + self.assertFalse(suite["results"]["ty"]["cases"]["unannotated_return"]["passed"]) + self.assertTrue(suite["results"]["ty-infer"]["cases"]["unannotated_return"]["passed"]) + self.assertFalse(suite["results"]["mypy"]["cases"]["installed_untyped"]["passed"]) + self.assertTrue(suite["results"]["pyright"]["cases"]["installed_untyped"]["passed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/build/benchmark/test_benchmark_evidence.py b/build/benchmark/test_benchmark_evidence.py new file mode 100644 index 000000000000..9904cb159676 --- /dev/null +++ b/build/benchmark/test_benchmark_evidence.py @@ -0,0 +1,63 @@ +import copy +import unittest + +from benchmark_evidence import validate_compatibility, validate_evidence + + +def modern_result() -> dict: + return { + "checker_config_version": 8, "platform": "linux", "architecture": "x86_64", + "runner_class": "test", "runner_image": "ubuntu", "cpu_count": 4, + "python_version": "3.14.6", "memory_measurement": "/proc//status", + "memory_limit_mb": 8192, "dependency_isolation": "pip-target-per-package", + "uncounted_validation_runs_per_checker": 1, "runs_per_package": 3, + "warmup_runs": 1, "timeout_s": 1800, "installed_packages": [], + "import_path_overrides": {}, + "type_checkers": ["pyright"], + "results": [{ + "package_name": "example", "commit": "abc", "check_paths": ["source"], + "exclude_directories": [], "resolved_dependencies": [], + "repository_configs": [], "metrics": { + "pyright": { + "ok": True, "file_count_verified": True, "files_selected": 2, + "execution_time_s": 10.0, "peak_memory_mb": 100.0, + }, + }, + }], + } + + +class BenchmarkEvidenceTest(unittest.TestCase): + def test_requires_native_count_and_preparation_evidence(self) -> None: + for field in ("file_count_verified", "files_selected"): + result = modern_result() + del result["results"][0]["metrics"]["pyright"][field] + with self.assertRaisesRegex(ValueError, "not verified"): + validate_evidence(result) + result = modern_result() + del result["results"][0]["resolved_dependencies"] + with self.assertRaisesRegex(ValueError, "resolved_dependencies"): + validate_evidence(result) + + def test_rejects_mixed_environments_scope_and_dependencies(self) -> None: + baseline = modern_result() + for field, value in ( + ("python_version", "3.11.0"), ("platform", "win32"), ("cpu_count", 8), + ("installed_packages", [{"name": "library", "version": "2"}]), + ): + candidate = copy.deepcopy(baseline) + candidate[field] = value + with self.subTest(field=field), self.assertRaisesRegex(ValueError, "ncompatible"): + validate_compatibility(baseline, candidate) + for field, value in ( + ("commit", "def"), ("check_paths", ["other"]), + ("resolved_dependencies", [{"name": "library", "version": "2"}]), + ): + candidate = copy.deepcopy(baseline) + candidate["results"][0][field] = value + with self.subTest(field=field), self.assertRaisesRegex(ValueError, field): + validate_compatibility(baseline, candidate) + + +if __name__ == "__main__": + unittest.main() diff --git a/build/benchmark/test_compare_benchmarks.py b/build/benchmark/test_compare_benchmarks.py index a1086b4e7971..b7eaf4068aeb 100644 --- a/build/benchmark/test_compare_benchmarks.py +++ b/build/benchmark/test_compare_benchmarks.py @@ -77,6 +77,50 @@ def _result(time: float, memory: float, ok: bool = True) -> dict: class CompareBenchmarksTest(unittest.TestCase): + def test_missing_candidate_metric_is_reported_instead_of_crashing(self) -> None: + baseline = _result(10.0, 100.0) + candidate = _result(5.0, 100.0) + candidate["results"][0]["metrics"] = {} + failures, rows = compare_benchmarks._analyze(baseline, candidate, 10.0) + self.assertIn("example/pyright: candidate result failed or is missing", failures) + self.assertEqual(rows[0]["status"], "Missing") + + def test_missing_candidate_checker_is_reported_instead_of_crashing(self) -> None: + baseline = _result(10.0, 100.0) + candidate = _result(5.0, 100.0) + candidate["results"][0]["metrics"] = { + "mypy": dict(candidate["results"][0]["metrics"]["pyright"]) + } + failures, rows = compare_benchmarks._analyze(baseline, candidate, 10.0) + self.assertIn("example/pyright: candidate result failed or is missing", failures) + self.assertEqual(rows[0]["status"], "Missing") + + def test_rejects_a_speedup_with_a_smaller_native_file_count(self) -> None: + baseline = _result(10.0, 100.0) + candidate = _result(5.0, 100.0) + baseline["results"][0]["metrics"]["pyright"]["files_selected"] = 2 + candidate["results"][0]["metrics"]["pyright"]["files_selected"] = 1 + failures, rows = compare_benchmarks._analyze(baseline, candidate, 10.0) + self.assertIn("example/pyright: native source-file count changed", failures) + self.assertEqual(rows, []) + + def test_rejects_faster_results_with_changed_capability_evidence(self) -> None: + baseline = _result(10.0, 100.0) + candidate = _result(5.0, 100.0) + for result in (baseline, candidate): + result["capability_suite"] = { + "version": 1, "sha256": "same", "cases": {"returns": "Return inference"}, + "results": { + "pyright": {"ok": True, "cases": {"returns": {"passed": True}}}, + }, + } + candidate["capability_suite"]["results"]["pyright"]["cases"]["returns"]["passed"] = False + failures, _ = compare_benchmarks._analyze(baseline, candidate, 10.0) + self.assertTrue(any("capability outcomes changed" in failure for failure in failures)) + candidate["capability_suite"]["sha256"] = "different" + failures, _ = compare_benchmarks._analyze(baseline, candidate, 10.0) + self.assertIn("capability suite changed or was not evaluated", failures) + def test_accepts_changes_within_threshold(self) -> None: with redirect_stdout(io.StringIO()): failures = compare_benchmarks.compare( @@ -238,6 +282,18 @@ def test_accepts_timeout_limit_change(self) -> None: self.assertEqual(failures, []) + def test_rejects_changed_checker_configuration(self) -> None: + candidate = _result(10.0, 100.0) + candidate["checker_config_version"] = 2 + with redirect_stdout(io.StringIO()): + failures = compare_benchmarks.compare( + _result(10.0, 100.0), candidate, 10.0 + ) + self.assertEqual( + failures, + ["environment mismatch for checker_config_version: None != 2"], + ) + def test_reports_success_without_failed_baseline(self) -> None: baseline = _result(0.0, 0.0, ok=False) candidate = _result(12.0, 345.0) @@ -326,7 +382,7 @@ def test_rejects_node_options_mismatch(self) -> None: ], ) - def test_allows_uncounted_validation_run_mismatch(self) -> None: + def test_rejects_uncounted_validation_run_mismatch(self) -> None: candidate = _result(10.0, 100.0) candidate["uncounted_validation_runs_per_checker"] = 1 @@ -335,7 +391,22 @@ def test_allows_uncounted_validation_run_mismatch(self) -> None: _result(10.0, 100.0), candidate, 10.0 ) - self.assertEqual(failures, []) + self.assertEqual( + failures, + ["environment mismatch for uncounted_validation_runs_per_checker: 0 != 1"], + ) + + def test_rejects_different_source_manifests(self) -> None: + baseline = _result(10.0, 100.0) + candidate = _result(10.0, 100.0) + baseline["results"][0]["source_manifest_sha256"] = "old" + candidate["results"][0]["source_manifest_sha256"] = "new" + with redirect_stdout(io.StringIO()): + failures = compare_benchmarks.compare(baseline, candidate, 10.0) + self.assertEqual( + failures, + ["example/pyright: package source_manifest_sha256 changed from old to new"], + ) def test_rejects_package_commit_mismatch(self) -> None: candidate = _result(10.0, 100.0) @@ -519,7 +590,7 @@ def test_workflows_use_current_pnpm_setup(self) -> None: { "python-version": "${{ env.PYTHON_VERSION }}", "cache": "pip", - "cache-dependency-path": "build/benchmark/install_envs.json\n.github/workflows/typecheck_benchmark_weekly.yml\n", + "cache-dependency-path": "build/benchmark/install_envs.json\nbuild/benchmark/checker_requirements.txt\n.github/workflows/typecheck_benchmark_weekly.yml\n", }, ) self.assertEqual( @@ -548,8 +619,11 @@ def test_workflows_use_current_pnpm_setup(self) -> None: }, ) self.assertEqual( - weekly_benchmark_steps[3]["if"], - "${{ matrix.checker != 'pyright' }}", + weekly_benchmark_steps[3], + { + "name": "Install shared checker environment", + "run": "python -m pip install -r build/benchmark/checker_requirements.txt", + }, ) self.assertEqual( weekly_benchmark_steps[4]["if"], @@ -579,7 +653,7 @@ def test_workflows_use_current_pnpm_setup(self) -> None: ) self.assertEqual( weekly_benchmark_steps[9]["run"], - "checkers=('${{ matrix.checker }}')\nextra_args=()\nif [[ '${{ matrix.checker }}' == 'pyright' ]]; then\n checkers+=(pyright-threads)\n extra_args+=(--skip-pyright-build)\nfi\npython build/benchmark/typecheck_benchmark.py \\\n -c \"${checkers[@]}\" -r 3 -w 1 -t 1800 \\\n --memory-limit-mb 8192 --os-name linux-x64 \\\n --output build/benchmark/results \\\n \"${extra_args[@]}\"\n", + "checkers=('${{ matrix.checker }}')\nextra_args=()\nif [[ '${{ matrix.checker }}' == 'pyright' ]]; then\n checkers+=(pyright-threads)\n extra_args+=(--skip-pyright-build)\nfi\nif [[ '${{ matrix.checker }}' == 'ty' ]]; then\n checkers+=(ty-infer)\nfi\npython build/benchmark/typecheck_benchmark.py \\\n -c \"${checkers[@]}\" -r 3 -w 1 -t 1800 \\\n --memory-limit-mb 8192 --os-name linux-x64 \\\n --output build/benchmark/results \\\n \"${extra_args[@]}\"\n", ) weekly_report_steps = weekly_workflow_data["jobs"]["report"]["steps"] self.assertEqual( diff --git a/build/benchmark/test_merge_benchmark_results.py b/build/benchmark/test_merge_benchmark_results.py index 132a0a199b75..915dca2fa06e 100644 --- a/build/benchmark/test_merge_benchmark_results.py +++ b/build/benchmark/test_merge_benchmark_results.py @@ -38,6 +38,35 @@ def _result(package: str, time: float) -> dict: class MergeBenchmarkResultsTest(unittest.TestCase): + def test_merges_variable_capability_evidence_but_rejects_changed_behavior(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + paths = [] + data = [] + for index, package in enumerate(("click", "numpy")): + result = _result(package, float(index + 1)) + result["capability_suite"] = { + "version": 1, "sha256": "same", "cases": {"returns": "Return inference"}, + "results": { + "pyright": { + "ok": True, "cases": {"returns": {"passed": True}}, + "evidence": {"stdout": f"Different temp path {index}", "time": index}, + }, + }, + } + path = root / f"{index}.json" + path.write_text(json.dumps(result), encoding="utf-8") + paths.append(path) + data.append(result) + merged = merge_benchmark_results.merge_results(paths) + self.assertEqual(merged["package_count"], 2) + self.assertEqual(merged["capability_suite"], data[0]["capability_suite"]) + + data[1]["capability_suite"]["results"]["pyright"]["cases"]["returns"]["passed"] = False + paths[1].write_text(json.dumps(data[1]), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "different benchmark metadata"): + merge_benchmark_results.merge_results(paths) + def test_merges_packages_and_recomputes_aggregate(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) diff --git a/build/benchmark/test_render_benchmark_html.py b/build/benchmark/test_render_benchmark_html.py index 4bd37c38627a..a780ab6d2107 100644 --- a/build/benchmark/test_render_benchmark_html.py +++ b/build/benchmark/test_render_benchmark_html.py @@ -5,6 +5,7 @@ from pathlib import Path import render_benchmark_html +from test_benchmark_evidence import modern_result def _result(checker: str, time: float, memory: float, ok: bool = True) -> dict: @@ -69,6 +70,171 @@ def handle_data(self, data: str) -> None: class RenderBenchmarkHtmlTest(unittest.TestCase): + def test_threading_speedup_requires_matching_pyright_version_and_heap_options(self) -> None: + single = _result("pyright", 2.0, 100.0) + threaded = _result("pyright-threads", 1.0, 100.0) + threaded["type_checker_versions"]["pyright-threads"] = "different" + with self.assertRaisesRegex(ValueError, "matching versions and Node options"): + render_benchmark_html.render_html({"pyright": single, "pyright-threads": threaded}) + threaded["type_checker_versions"]["pyright-threads"] = "1.2.3" + threaded["node_options"] = "--max-old-space-size=1000" + with self.assertRaisesRegex(ValueError, "matching versions and Node options"): + render_benchmark_html.render_html({"pyright": single, "pyright-threads": threaded}) + + def test_current_reports_require_evidence_and_matching_source_environment(self) -> None: + import copy + + pyright = modern_result() + ty = copy.deepcopy(pyright) + ty["type_checkers"] = ["ty"] + ty["results"][0]["metrics"] = {"ty": ty["results"][0]["metrics"]["pyright"]} + ty["results"][0]["commit"] = "different" + with self.assertRaisesRegex(ValueError, "commit"): + render_benchmark_html.render_html({"pyright": pyright, "ty": ty}) + ty["results"][0]["commit"] = "abc" + del ty["results"][0]["metrics"]["ty"]["file_count_verified"] + with self.assertRaisesRegex(ValueError, "not verified"): + render_benchmark_html.render_html({"pyright": pyright, "ty": ty}) + + def test_shared_totals_exclude_failures_without_hiding_individual_work(self) -> None: + import copy + + pyright = _result("pyright", 10.0, 100.0) + ty = _result("ty", 1.0, 50.0) + for checker, result in (("pyright", pyright), ("ty", ty)): + extra = copy.deepcopy(result["results"][0]) + extra["package_name"] = "failed-for-ty" + extra["metrics"][checker]["execution_time_s"] = 20.0 + if checker == "ty": + extra["metrics"][checker]["ok"] = False + result["results"].append(extra) + report = render_benchmark_html.render_html({"pyright": pyright, "ty": ty}) + self.assertIn("30.0s", report) + self.assertIn("110.0s", report) + self.assertIn("11.0s", report) + self.assertIn("not an apples-to-apples speed ranking", report) + def test_native_count_mismatches_are_not_comparable_or_in_summary_totals(self) -> None: + pyright = _result("pyright", 2.0, 200.0) + mypy = _result("mypy", 1.0, 100.0) + for checker, result, count in (("pyright", pyright, 2), ("mypy", mypy, 1)): + result["results"][0]["metrics"][checker].update( + files_selected=count, file_count_verified=True + ) + report = render_benchmark_html.render_html({"pyright": pyright, "mypy": mypy}) + self.assertEqual(report.count("Not comparable"), 2) + self.assertIn("2 native source files", report) + self.assertIn("1 native source files", report) + self.assertNotIn("2.00s", report) + self.assertNotIn("1.00s", report) + self.assertNotIn("2.0s", report) + self.assertTrue(pyright["results"][0]["metrics"]["pyright"]["ok"]) + + def test_repository_configuration_audit_is_visible_and_escaped(self) -> None: + result = _result("ty", 1.0, 100.0) + result["results"][0]["repository_configs"] = [ + { + "path": "nested/pyproject.toml", "checkers": ["ty"], + "sections": ["tool.ty"], "settings": ["tool.ty.rules.all"], + "disposition": "not applied: explicit benchmark config", + }, + { + "path": "broken.toml", "checkers": [], "sections": [], "settings": [], + "parse_error": "", + }, + ] + report = render_benchmark_html.render_html({"ty": result}) + self.assertIn("Repository configuration audit", report) + self.assertIn("nested/pyproject.toml", report) + self.assertIn("tool.ty.rules.all", report) + self.assertIn("not applied: explicit benchmark config", report) + self.assertIn("Scan parse error: <invalid TOML>", report) + self.assertNotIn("", report) + + def test_renders_behavior_matches_without_quality_adjusting_times(self) -> None: + pyright = _result("pyright", 2.0, 200.0) + ty = _result("ty", 1.0, 100.0) + for checker, result in (("pyright", pyright), ("ty", ty)): + result["capability_suite"] = { + "version": 1, + "sha256": "same", + "cases": {"returns": "Return inference"}, + "results": { + checker: { + "ok": True, + "cases": { + "returns": { + "passed": checker == "pyright", + "positive_diagnostic_lines": [] if checker == "pyright" else [3], + "negative_diagnostic_lines": [4] if checker == "pyright" else [], + "expected_error_line": 4, + } + }, + } + }, + } + report = render_benchmark_html.render_html({"pyright": pyright, "ty": ty}) + parser = _SemanticParser() + parser.feed(report) + self.assertIn("Behavior matches", parser.cells) + self.assertIn("1 / 1", parser.cells) + self.assertIn("0 / 1", parser.cells) + self.assertIn("2.00s", parser.cells) + self.assertIn("1.00s", parser.cells) + self.assertIn("Behavioral capability probes", parser.headings) + self.assertIn("Positive diagnostic lines: [3]", report) + self.assertIn("do not certify equivalent analysis", report) + + ty["capability_suite"]["sha256"] = "different" + with self.assertRaisesRegex(ValueError, "different capability suites"): + render_benchmark_html.render_html({"pyright": pyright, "ty": ty}) + ty["capability_suite"]["sha256"] = "same" + ty["capability_suite"]["results"]["ty"] = { + "ok": False, "error": "", + } + report = render_benchmark_html.render_html({"pyright": pyright, "ty": ty}) + self.assertIn("Not evaluated", report) + self.assertIn("<failed audit>", report) + self.assertNotIn("", report) + + def test_requires_matching_manifests_and_verified_native_counts(self) -> None: + pyright = _result("pyright", 2.0, 200.0) + ty = _result("ty", 1.0, 100.0) + for checker, result in (("pyright", pyright), ("ty", ty)): + result["checker_config_version"] = 4 + result["results"][0].update( + source_file_count=5, source_manifest_sha256="same" + ) + result["results"][0]["metrics"][checker].update( + files_selected=5, file_count_verified=True + ) + report = render_benchmark_html.render_html({"pyright": pyright, "ty": ty}) + self.assertIn("5 source files selected", report) + self.assertEqual(report.count("5 source files verified"), 2) + + ty["checker_config_version"] = 3 + with self.assertRaisesRegex(ValueError, "different benchmark configuration versions"): + render_benchmark_html.render_html({"pyright": pyright, "ty": ty}) + ty["checker_config_version"] = 4 + ty["results"][0]["source_manifest_sha256"] = "different" + with self.assertRaisesRegex(ValueError, "different source manifests"): + render_benchmark_html.render_html({"pyright": pyright, "ty": ty}) + ty["results"][0]["source_manifest_sha256"] = "same" + ty["results"][0]["metrics"]["ty"]["files_selected"] = 4 + with self.assertRaisesRegex(ValueError, "source-file count is not verified"): + render_benchmark_html.render_html({"pyright": pyright, "ty": ty}) + + def test_renders_ty_inference_as_a_separate_profile(self) -> None: + report = render_benchmark_html.render_html( + { + "ty-infer": _result("ty-infer", 2.0, 200.0), + "ty": _result("ty", 1.0, 100.0), + } + ) + parser = _SemanticParser() + parser.feed(report) + self.assertLess(parser.cells.index("ty"), parser.cells.index("ty (return inference)")) + self.assertIn("do not measure equivalent analysis or time per rule", report) + def test_loads_one_result_per_checker(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) @@ -125,11 +291,12 @@ def test_renders_comparison_semantics(self) -> None: self.assertEqual( parser.cells, [ - "Checker", "Measured", "Failed", "Total time", "Median time", "Median RSS", - "pyright", "1.2.3", "1", "0", "2.0s", "2.0s", "200 MB", - "pyright (threads)", "1.2.3", "1", "0", "1.0s", "1.0s", "250 MB", - "mypy", "1.2.3", "1", "0", "1.0s", "1.0s", "100 MB", - "zuban", "1.2.3", "0", "1", "0.0s", "0.0s", "0 MB", + "Checker", "Measured", "Failed", "Individual total", "Shared packages", "Shared total", + "Median time", "Median RSS", + "pyright", "1.2.3", "1", "0", "2.0s", "0", "N/A", "2.0s", "200 MB", + "pyright (threads)", "1.2.3", "1", "0", "1.0s", "0", "N/A", "1.0s", "250 MB", + "mypy", "1.2.3", "1", "0", "1.0s", "0", "N/A", "1.0s", "100 MB", + "zuban", "1.2.3", "0", "1", "0.0s", "0", "N/A", "0.0s", "0 MB", "Package", "pyright", "pyright (threads)", "mypy", "zuban", "example", "2.00s", "200 MB", "1.00s", "250 MB", "1.00s", "100 MB", "Not measured", "Package", "Single-thread", diff --git a/build/benchmark/test_render_pyright_history.py b/build/benchmark/test_render_pyright_history.py index 7a80b263f617..843c6a136ea1 100644 --- a/build/benchmark/test_render_pyright_history.py +++ b/build/benchmark/test_render_pyright_history.py @@ -6,6 +6,7 @@ from pathlib import Path import render_pyright_history +from test_benchmark_evidence import modern_result REPO_ROOT = Path(__file__).resolve().parents[2] @@ -104,6 +105,33 @@ def _result(version: str, published_at: str, scale: float = 1.0) -> dict: class RenderPyrightHistoryTest(unittest.TestCase): + def test_current_history_rejects_mixed_config_versions_and_native_counts(self) -> None: + import copy + + first = modern_result() + first["type_checker_versions"] = {"pyright": "1.1.413"} + first["timestamp"] = "2026-11-02T00:00:00+00:00" + second = copy.deepcopy(first) + second["type_checker_versions"] = {"pyright": "1.1.414"} + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + old_path, new_path = root / "old.json", root / "new.json" + old_path.write_text(json.dumps(first)) + second["checker_config_version"] = 7 + new_path.write_text(json.dumps(second)) + with self.assertRaisesRegex(ValueError, "different benchmark profile"): + render_pyright_history.load_history([old_path, new_path]) + second["checker_config_version"] = 8 + second["results"][0]["metrics"]["pyright"]["files_selected"] = 3 + new_path.write_text(json.dumps(second)) + with self.assertRaisesRegex(ValueError, "different native source counts"): + render_pyright_history.load_history([old_path, new_path]) + second["results"][0]["metrics"]["pyright"]["files_selected"] = 2 + new_path.write_text(json.dumps(second)) + history = render_pyright_history.load_history([old_path, new_path]) + self.assertEqual(history["source_counts"], {"example": 2}) + self.assertEqual(history["releases"][0]["packages"]["example"]["files_selected"], 2) + def test_appends_candidate_to_release_history(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) diff --git a/build/benchmark/test_typecheck_benchmark.py b/build/benchmark/test_typecheck_benchmark.py index 96be4b97e6a2..e91dd8555933 100644 --- a/build/benchmark/test_typecheck_benchmark.py +++ b/build/benchmark/test_typecheck_benchmark.py @@ -1,7 +1,9 @@ import io import json import os +import subprocess import tempfile +import tomllib import unittest from contextlib import redirect_stdout from pathlib import Path @@ -11,7 +13,88 @@ class TypecheckBenchmarkTest(unittest.TestCase): - def test_resolve_check_paths_excludes_named_directories(self) -> None: + def test_repository_config_audit_finds_native_files_and_toml_settings_without_values(self) -> None: + nested = self.root / "source" + nested.mkdir() + (self.root / "pyproject.toml").write_text( + '[project]\nrequires-python = ">=3.10"\n' + '[tool.ty.rules]\nall = "ignore"\n' + '[tool.pyrefly.errors]\nbad-assignment = false\n' + '[[tool.pyrefly.sub-config]]\nmatches = "**/*.py"\n' + '[tool.pyrefly.sub-config.errors]\nbad-return = false\n' + ) + (self.root / "pyrightconfig.json").write_text('{"typeCheckingMode":"off"}') + (nested / "ty.toml").write_text('[src]\nexclude = ["sentinel-value"]\n') + (nested / "pyrefly.toml").write_text('project-includes = ["sentinel-value"]\n') + (nested / "setup.cfg").write_text("[mypy]\nignore_errors = True\n") + (nested / "custom.toml").write_text('[tool.mypy]\nignore_errors = true\n') + (self.root / ".git").mkdir() + (self.root / ".git" / "ty.toml").write_text('[rules]\nall = "ignore"\n') + (self.root / ".typecheck-benchmark-ty-generated.toml").write_text("[rules]\n") + records = benchmark._audit_repository_configs(self.root) + by_path = {record["path"]: record for record in records} + self.assertEqual(set(by_path), { + "pyproject.toml", "pyrightconfig.json", "source/ty.toml", + "source/pyrefly.toml", "source/setup.cfg", "source/custom.toml", + }) + self.assertIn("tool.ty.rules.all", by_path["pyproject.toml"]["settings"]) + self.assertIn("tool.pyrefly.errors.bad-assignment", by_path["pyproject.toml"]["settings"]) + self.assertIn("tool.pyrefly.sub-config[0].errors.bad-return", by_path["pyproject.toml"]["settings"]) + self.assertIn("mypy.ignore_errors", by_path["source/setup.cfg"]["settings"]) + self.assertEqual(by_path["source/custom.toml"]["checkers"], ["mypy", "zuban"]) + self.assertIn("not applied", by_path["source/ty.toml"]["disposition"]) + self.assertNotIn("sentinel-value", json.dumps(records)) + + def test_repository_config_audit_surfaces_parse_errors_without_config_contents(self) -> None: + (self.root / "pyproject.toml").write_text('invalid = "sentinel-value') + (self.root / "mypy.ini").write_text("[mypy]\nsentinel-value\n") + records = benchmark._audit_repository_configs(self.root) + self.assertEqual(len(records), 2) + self.assertTrue(all("parse_error" in record for record in records)) + self.assertNotIn("sentinel-value", json.dumps(records)) + + def test_incomplete_repository_search_is_not_a_successful_empty_audit(self) -> None: + with ( + patch.object(benchmark, "_audit_repository_configs", side_effect=PermissionError("denied")), + patch.object(benchmark, "run_checker") as run, + redirect_stdout(io.StringIO()), + ): + result = benchmark._benchmark_directory( + name="local", github_url=None, package_path=self.root, resolved_paths=None, + type_checkers=["ty"], timeout=30, runs=1, warmup=0, memory_limit_mb=0, + ) + self.assertIn("Repository configuration audit failed", result["error"]) + self.assertNotIn("repository_configs", result) + run.assert_not_called() + + def test_different_native_counts_are_not_accepted_as_comparable_timings(self) -> None: + (self.root / "module.py").write_text("") + (self.root / "module.pyi").write_text("") + roots = [] + + def run(checker, package_path, check_paths, *args, **kwargs): + roots.append(check_paths) + return { + "ok": True, "execution_time_s": 0.1, "peak_memory_mb": 10.0, + "oom_killed": False, "files_selected": 2 if checker == "pyright" else 1, + } + + with ( + patch.object(benchmark, "_checker_command", return_value=["checker"]), + patch.object(benchmark, "run_checker", side_effect=run), + redirect_stdout(io.StringIO()), + ): + result = benchmark._benchmark_directory( + name="local", github_url=None, package_path=self.root, resolved_paths=None, + type_checkers=["pyright", "mypy"], timeout=30, runs=1, warmup=0, + memory_limit_mb=0, + ) + self.assertTrue(all(paths == [self.root] for paths in roots)) + self.assertTrue(all(not metric["ok"] for metric in result["metrics"].values())) + self.assertTrue(all(metric["source_count_mismatch"] for metric in result["metrics"].values())) + self.assertEqual(benchmark.compute_aggregate_stats([result], ["pyright"])["pyright"]["packages_tested"], 0) + + def test_resolve_check_paths_preserves_directories_for_native_discovery(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: package_path = Path(temp_dir) (package_path / "source" / "tests").mkdir(parents=True) @@ -21,15 +104,52 @@ def test_resolve_check_paths_excludes_named_directories(self) -> None: (package_path / "source" / "tests" / "test_module.py").write_text("") paths, missing = benchmark._resolve_check_paths( - package_path, ["source"], ["tests"] + package_path, ["source"] ) self.assertEqual(missing, []) self.assertEqual( - {path.relative_to(package_path).as_posix() for path in paths}, - {"source/module.py", "source/types.pyi"}, + paths, [package_path / "source"], ) + def test_resolve_check_paths_does_not_choose_implementations_or_stubs(self) -> None: + source = self.root / "source" + source.mkdir() + for name in ("module.py", "module.pyi", "stub.pyi", "clean.py", ".hidden.py"): + (source / name).write_text("") + for name in ("node_modules", "__pycache__", ".hidden", "tests"): + (source / name).mkdir() + (source / name / "ignored.py").write_text("") + paths, missing = benchmark._resolve_check_paths( + self.root, ["source", "source/module.py", "missing"] + ) + self.assertEqual(missing, ["missing"]) + self.assertEqual( + paths, [source, source / "module.py"], + ) + stub_only, _ = benchmark._resolve_check_paths( + self.root, ["source/module.pyi"] + ) + self.assertEqual(stub_only, [source / "module.pyi"]) + + def test_source_file_counts_do_not_include_imported_dependencies(self) -> None: + cases = [ + ("pyright", "Total files parsed and bound: 500\nTotal files checked: 5\n", 5), + ("pyright-threads", "Found 5 files to analyze\nFound 5 source files\n", 5), + ("pyrefly", " INFO 5 modules (500 dependent modules); 1000 lines\n", 5), + ("mypy", "Found 1 error in 1 file (checked 5 source files)\n", 5), + ("zuban", "Success: no issues found in 1 source file\n", 1), + ("ty", "DEBUG Checking file 'first.py'\nDEBUG Checking file 'first.py'\n" + "DEBUG Checking file 'second.pyi'\nDEBUG Checking all files took 0.1s\n", 2), + ("ty", "DEBUG Checking all files took 0.0s\n", 0), + ("ty", "All checks passed!\n", None), + ("pyrefly", " INFO 0 errors\n", None), + ("mypy", "errors prevented further checking\n", None), + ] + for checker, output, expected in cases: + with self.subTest(checker=checker, output=output): + self.assertEqual(benchmark._parse_source_file_count(checker, output), expected) + def setUp(self) -> None: self.temp_dir = tempfile.TemporaryDirectory( prefix=".typecheck-benchmark-test-", @@ -173,6 +293,8 @@ def test_pyright_config_and_command(self) -> None: config = json.loads(configs[0].read_text(encoding="utf-8")) self.assertEqual(config["include"], ["src"]) self.assertEqual(config["exclude"], benchmark.PYRIGHT_DEFAULT_EXCLUDES) + self.assertEqual(config["pythonVersion"], benchmark.TARGET_PYTHON_VERSION) + self.assertEqual(command[-3:-1], ["--pythonpath", benchmark.sys.executable]) self.assertIs(config["useLibraryCodeForTypes"], True) configs[0].unlink() @@ -193,6 +315,8 @@ def test_threaded_pyright_uses_runner_cpu_count_without_stats(self) -> None: "pyright.js", "--project", str(configs[0]), + "--pythonpath", + benchmark.sys.executable, "--threads", ], ) @@ -236,6 +360,7 @@ def test_load_install_envs_includes_source_only_packages(self) -> None: self.assertEqual(packages[0]["name"], "click") def test_install_failure_is_a_package_failure(self) -> None: + (self.root / "pyproject.toml").write_text('[tool.pyrefly]\npreset = "off"\n') package = { "name": "example", "github_url": "https://example.com/example", @@ -253,6 +378,8 @@ def test_install_failure_is_a_package_failure(self) -> None: self.assertEqual(result["error"], "Dependency installation failed") self.assertEqual(result["commit"], "abc123") + self.assertEqual(result["repository_configs"][0]["path"], "pyproject.toml") + self.assertIn("tool.pyrefly.preset", result["repository_configs"][0]["settings"]) install.assert_called_once_with( self.root, package, self.root / "example-dependencies" ) @@ -301,6 +428,8 @@ def test_packages_use_independent_dependency_environments(self) -> None: {"name": "first", "github_url": "https://example.com/first"}, {"name": "second", "github_url": "https://example.com/second"}, ] + for package in packages: + (self.root / package["name"]).mkdir() with ( patch.object( benchmark, @@ -336,16 +465,68 @@ def test_packages_use_independent_dependency_environments(self) -> None: ) self.assertEqual(checker_targets, install_targets) - def test_zuban_uses_positional_paths_without_config(self) -> None: - source_dir = self.root / "src" - source_dir.mkdir() + def test_zuban_uses_explicit_config_and_mode(self) -> None: + source_dir = self.root / "src" / "package" + source_dir.mkdir(parents=True) with patch.object(benchmark, "_checker_command", return_value=["zuban"]): command, configs = benchmark._build_checker_command( "zuban", self.root, [source_dir] ) - self.assertEqual(command, ["zuban", "check", "src"]) - self.assertEqual(configs, []) + self.assertEqual( + command, + [ + "zuban", "check", "--mode", "default", + "--config-file", str(configs[0]), + "--python-executable", benchmark.sys.executable, + ], + ) + config = tomllib.loads(configs[0].read_text())["tool"]["mypy"] + self.assertEqual(config["files"], [str(source_dir.relative_to(self.root))]) + self.assertIs(config["exclude_gitignore"], False) + configs[0].unlink() + + def test_ty_profiles_share_scope_but_not_return_inference(self) -> None: + with patch.object(benchmark, "_executable", return_value="ty") as executable: + self.assertEqual(benchmark._checker_command("ty-infer"), ["ty"]) + executable.assert_called_once_with("ty") + for checker in ("ty", "ty-infer"): + with self.subTest(checker=checker), patch.object( + benchmark, "_checker_command", return_value=["ty"] + ): + command, configs = benchmark._build_checker_command( + checker, self.root, [self.root / "source"] + ) + config = tomllib.loads(configs[0].read_text()) + self.assertEqual( + command, ["ty", "check", "--config-file", str(configs[0])] + ) + self.assertEqual( + config["src"], + { + "include": ["source"], + "respect-ignore-files": False, + "exclude": ["!**", *benchmark.PYRIGHT_DEFAULT_EXCLUDES], + }, + ) + self.assertEqual( + config["environment"]["python"], benchmark.sys.executable + ) + self.assertEqual( + config.get("analysis", {}).get("return-type-inference"), + "always" if checker == "ty-infer" else None, + ) + configs[0].unlink() + + def test_pyrefly_config_disables_ignore_files_and_pins_inference(self) -> None: + config_path = benchmark._write_pyrefly_config(self.root, ["source"]) + config = tomllib.loads(config_path.read_text()) + self.assertEqual(config["project-includes"], ["source"]) + self.assertIs(config["use-ignore-files"], False) + self.assertIs(config["check-unannotated-defs"], True) + self.assertEqual(config["infer-return-types"], "checked") + self.assertEqual(config["python-interpreter-path"], benchmark.sys.executable) + config_path.unlink() def test_mypy_config_preserves_testing_paths(self) -> None: testing_dir = self.root / "testing" @@ -360,8 +541,14 @@ def test_mypy_config_preserves_testing_paths(self) -> None: ["mypy", "--no-incremental", "--config-file", str(configs[0])], ) self.assertEqual( - configs[0].read_text(encoding="utf-8"), - "[mypy]\nfiles = testing\ncheck_untyped_defs = True\n", + tomllib.loads(configs[0].read_text(encoding="utf-8"))["tool"]["mypy"], + { + "files": ["testing"], + "check_untyped_defs": True, + "python_version": benchmark.TARGET_PYTHON_VERSION, + "python_executable": benchmark.sys.executable, + "exclude": [r"(^|[/\\])(?:node_modules|__pycache__|\.[^/\\]+)([/\\]|$)"], + }, ) configs[0].unlink() @@ -454,6 +641,26 @@ def test_uncaptured_run_rejects_fatal_exit_code(self) -> None: self.assertFalse(result["ok"]) self.assertEqual(result["error_message"], "Fatal error (exit code 2)") + def test_pyrefly_no_matching_files_is_not_a_successful_measurement(self) -> None: + process_result: benchmark.ProcessResult = { + "stdout": "", + "stderr": "No Python files matched pattern `source`\n", + "returncode": 1, + "timed_out": False, + "execution_time_s": 0.1, + "peak_memory_mb": 10.0, + "oom_killed": False, + } + with ( + patch.object(benchmark, "_checker_command", return_value=["pyrefly"]), + patch.object( + benchmark, "run_process_with_timeout", return_value=process_result + ), + ): + result = benchmark.run_checker("pyrefly", self.root, None, 30, 4096) + self.assertFalse(result["ok"]) + self.assertIn("No Python files matched", result["error_message"]) + def test_results_include_upstream_provenance_and_memory_limit(self) -> None: output_file = benchmark._save_results( [], @@ -472,11 +679,12 @@ def test_results_include_upstream_provenance_and_memory_limit(self) -> None: self.assertEqual(output["upstream_source"], benchmark.UPSTREAM_SOURCE) self.assertEqual(output["memory_limit_mb"], 2048) + self.assertEqual(output["checker_config_version"], benchmark.CHECKER_CONFIG_VERSION) self.assertEqual( output["node_options"], benchmark.os.environ.get("NODE_OPTIONS", "") ) self.assertEqual(output["warmup_runs"], 0) - self.assertEqual(output["uncounted_validation_runs_per_checker"], 0) + self.assertEqual(output["uncounted_validation_runs_per_checker"], 1) self.assertEqual(output["python_version"], benchmark.platform.python_version()) self.assertEqual(output["runner_image"], "local") self.assertEqual(output["architecture"], benchmark.platform.machine()) @@ -507,6 +715,7 @@ def test_local_mode_builds_pyright_without_clone_or_install(self) -> None: return_value={"pyright": "1.2.3"}, ), patch.object(benchmark, "_save_results", return_value=result_path), + patch.object(benchmark, "run_capability_suite", return_value={"results": {}}), patch.object(benchmark, "_print_summary"), patch.object(benchmark, "clone_package") as clone, patch.object(benchmark, "install_deps") as install, @@ -533,8 +742,9 @@ def test_local_mode_builds_pyright_without_clone_or_install(self) -> None: clone.assert_not_called() install.assert_not_called() - def test_zero_warmups_runs_only_measured_checks(self) -> None: + def test_zero_warmups_runs_an_audit_before_measured_checks(self) -> None: calls: list[bool] = [] + (self.root / "clean.py").write_text("") def run_checker( checker: str, @@ -544,13 +754,16 @@ def run_checker( memory_limit_mb: int, capture_output: bool = True, environment: dict[str, str] | None = None, + audit_files: bool = False, + exclude_directories: list[str] | None = None, ) -> benchmark.TimingMetrics: - calls.append(capture_output) + calls.append(audit_files) return { "ok": True, "execution_time_s": 1.0, "peak_memory_mb": 2.0, "oom_killed": False, + "files_selected": 1, } stdout = io.StringIO() @@ -571,9 +784,11 @@ def run_checker( memory_limit_mb=4096, ) - self.assertEqual(calls, [True, True]) + self.assertEqual(calls, [True, False, False]) self.assertEqual( stdout.getvalue(), + " Repository config audit: 0 candidate files\n" + " pyright source-file audit: 1 native files verified\n" " Running pyright (2 measured)...\n" " Run 1/2... 1.000s, 2.0 MB\n" " Run 2/2... 1.000s, 2.0 MB\n" @@ -582,6 +797,7 @@ def run_checker( def test_all_warmups_and_runs_capture_output(self) -> None: captures: list[bool] = [] + (self.root / "clean.py").write_text("") def run_checker( checker: str, @@ -591,6 +807,8 @@ def run_checker( memory_limit_mb: int, capture_output: bool = True, environment: dict[str, str] | None = None, + audit_files: bool = False, + exclude_directories: list[str] | None = None, ) -> benchmark.TimingMetrics: captures.append(capture_output) return { @@ -598,6 +816,7 @@ def run_checker( "execution_time_s": 1.0, "peak_memory_mb": 2.0, "oom_killed": False, + "files_selected": 1, } with ( @@ -617,7 +836,225 @@ def run_checker( memory_limit_mb=4096, ) - self.assertEqual(captures, [True, True, True]) + self.assertEqual(captures, [True, True, True, True]) + + def test_zero_native_file_count_prevents_timed_runs(self) -> None: + for name in ("clean.py", "second.py"): + (self.root / name).write_text("") + audit: benchmark.TimingMetrics = { + "ok": True, "execution_time_s": 0.1, "peak_memory_mb": 10.0, + "oom_killed": False, "files_selected": 0, + } + with ( + patch.object(benchmark, "_checker_command", return_value=["checker"]), + patch.object(benchmark, "run_checker", return_value=audit) as run, + redirect_stdout(io.StringIO()), + ): + result = benchmark._benchmark_directory( + name="local", github_url=None, package_path=self.root, + resolved_paths=None, type_checkers=["ty"], timeout=30, + runs=3, warmup=1, memory_limit_mb=4096, + ) + run.assert_called_once() + self.assertIs(run.call_args.kwargs["audit_files"], True) + self.assertEqual(result["check_paths"], ["."]) + self.assertFalse(result["metrics"]["ty"]["ok"]) + self.assertFalse(result["metrics"]["ty"]["file_count_verified"]) + self.assertIn("ty reported 0", result["metrics"]["ty"]["error_message"]) + + def test_empty_source_selection_is_not_a_full_repository_fallback(self) -> None: + with patch.object(benchmark, "run_checker") as run: + result = benchmark._benchmark_directory( + name="local", github_url=None, package_path=self.root, + resolved_paths=[], type_checkers=["ty"], timeout=30, + runs=1, warmup=0, memory_limit_mb=4096, + ) + run.assert_not_called() + self.assertEqual(result["error"], "No Python source files selected") + + def test_count_changes_after_the_audit_reject_measured_results(self) -> None: + (self.root / "clean.py").write_text("") + audit: benchmark.TimingMetrics = { + "ok": True, "execution_time_s": 0.1, "peak_memory_mb": 10.0, + "oom_killed": False, "files_selected": 1, + } + measured: benchmark.TimingMetrics = {**audit, "files_selected": 0} + with ( + patch.object(benchmark, "_checker_command", return_value=["checker"]), + patch.object(benchmark, "run_checker", side_effect=[audit, measured]), + redirect_stdout(io.StringIO()), + ): + result = benchmark._benchmark_directory( + name="local", github_url=None, package_path=self.root, + resolved_paths=None, type_checkers=["pyright"], timeout=30, + runs=1, warmup=0, memory_limit_mb=4096, + ) + self.assertFalse(result["metrics"]["pyright"]["ok"]) + self.assertIn("Source-file count changed", result["metrics"]["pyright"]["error_message"]) + + +@unittest.skipUnless( + os.environ.get("TYPECHECK_BENCHMARK_INTEGRATION") == "1", + "Set TYPECHECK_BENCHMARK_INTEGRATION=1 with all checkers installed", +) +class CheckerConfigIntegrationTest(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = tempfile.TemporaryDirectory(dir=benchmark.SCRIPT_DIR) + self.addCleanup(self.temp_dir.cleanup) + self.root = Path(self.temp_dir.name) + + def _run( + self, checker: str, check_paths: list[Path] | None = None, + exclude_directories: list[str] | None = None, + ) -> subprocess.CompletedProcess[str]: + command, configs = benchmark._build_checker_command( + checker, self.root, check_paths or [self.root / "source"], exclude_directories + ) + self.assertIsNotNone(command, f"{checker} must be installed for integration tests") + try: + return subprocess.run( + command, cwd=self.root, capture_output=True, text=True, timeout=60 + ) + finally: + for config in configs: + config.unlink() + + def test_native_configs_and_ignore_files_do_not_change_benchmark_scope(self) -> None: + source = self.root / "source" + source.mkdir() + (source / "selected.py").write_text('value: int = "wrong"\n') + (source / "ignored.py").write_text('value: int = "wrong"\n') + (self.root / "outside.py").write_text('value: int = "wrong"\n') + for directory in ("dist", "venv", "tests", "node_modules", ".hidden"): + path = source / directory + path.mkdir() + (path / f"{directory.strip('.')}_probe.py").write_text( + 'value: int = "wrong"\n' + ) + for name, content in ( + ("paired.py", 'value: int = "wrong"\n'), + ("paired.pyi", "value: MissingType\n"), + ("standalone.pyi", "value: int\n"), + ("clean.py", "value: int = 1\n"), + ): + (source / name).write_text(content) + (self.root / ".ignore").write_text("source/ignored.py\n") + (self.root / ".gitignore").write_text("source/ignored.py\n") + checkers = [*benchmark.DEFAULT_TYPE_CHECKERS, "ty-infer", "pyright-threads"] + baselines = {} + for checker in checkers: + baseline = benchmark.run_checker( + checker, self.root, [source], 60, 0, audit_files=True + ) + self.assertTrue(baseline["ok"], baseline) + baselines[checker] = baseline["files_selected"] + native_configs = { + "pyrightconfig.json": '{"typeCheckingMode":"off","exclude":["**"]}', + "ty.toml": '[rules]\nall = "ignore"\n', + "pyrefly.toml": 'project-includes = ["outside.py"]\n' + "[errors]\nbad-assignment = false\n", + "mypy.ini": "[mypy]\nignore_errors = True\n", + "pyproject.toml": '[project]\nname = "hostile"\nversion = "0"\n' + 'requires-python = ">=3.8"\n' + '[tool.ty.rules]\nall = "ignore"\n' + "[tool.pyrefly.errors]\nbad-assignment = false\n" + "[tool.mypy]\nignore_errors = true\n" + "[tool.zuban]\nignore_errors = true\n", + } + for directory in (self.root, source): + for name, content in native_configs.items(): + (directory / name).write_text(content) + + roots, missing = benchmark._resolve_check_paths(self.root, ["source"]) + self.assertEqual(missing, []) + self.assertEqual(roots, [source]) + for checker in checkers: + with self.subTest(checker=checker): + audit = benchmark.run_checker( + checker, self.root, roots, 60, 0, audit_files=True + ) + self.assertTrue(audit["ok"], audit) + self.assertEqual(audit["files_selected"], baselines[checker], audit) + result = self._run(checker, roots) + output = result.stdout + result.stderr + self.assertEqual(result.returncode, 1, output) + for name in ("selected", "ignored", "dist_probe", "venv_probe", "tests_probe"): + self.assertIn(f"{name}.py", output, output) + self.assertIn("paired.pyi", output, output) + for name in ("outside", "node_modules_probe", "hidden_probe"): + self.assertNotIn(f"{name}.py", output, output) + excluded = benchmark.run_checker( + checker, self.root, roots, 60, 0, audit_files=True, + exclude_directories=["tests"], + ) + self.assertTrue(excluded["ok"], excluded) + self.assertEqual(excluded["files_selected"], baselines[checker] - 1) + result = self._run(checker, roots, ["tests"]) + self.assertNotIn("tests_probe.py", result.stdout + result.stderr) + for directory in (self.root, source): + for name, content in native_configs.items(): + self.assertEqual((directory / name).read_text(), content) + + def test_pyrefly_enables_optional_diagnostics(self) -> None: + source = self.root / "source" + source.mkdir() + (source / "unused_ignore.py").write_text("value: int = 1 # type: ignore\n") + result = self._run("pyrefly") + output = result.stdout + result.stderr + self.assertEqual(result.returncode, 1, output) + self.assertIn("[unused-type-ignore]", output) + + def test_ty_inference_profile_changes_inferred_return_checking(self) -> None: + source = self.root / "source" + source.mkdir() + (source / "inferred.py").write_text( + 'def inferred():\n return "wrong"\n\nvalue: int = inferred()\n' + ) + default = self._run("ty") + inferred = self._run("ty-infer") + self.assertEqual(default.returncode, 0, default.stdout + default.stderr) + self.assertEqual(inferred.returncode, 1, inferred.stdout + inferred.stderr) + self.assertIn("invalid-assignment", inferred.stdout + inferred.stderr) + + def test_zuban_preserves_nested_source_paths(self) -> None: + source = self.root / "source" / "nested" + source.mkdir(parents=True) + (source / "selected.py").write_text('value: int = "wrong"\n') + result = self._run("zuban", [source]) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("selected.py", result.stdout + result.stderr) + + def test_ty_does_not_merge_user_level_rule_suppression(self) -> None: + source = self.root / "source" + source.mkdir() + selected = source / "selected.py" + selected.write_text('value: int = "wrong"\n') + user_config = self.root / "user-config" + (user_config / "ty").mkdir(parents=True) + (user_config / "ty" / "ty.toml").write_text('[rules]\nall = "ignore"\n') + outputs: list[benchmark.ProcessResult] = [] + original_run = benchmark.run_process_with_timeout + + def capture(*args, **kwargs): + output = original_run(*args, **kwargs) + outputs.append(output) + return output + + with ( + patch.dict(os.environ, { + "APPDATA": str(user_config), "XDG_CONFIG_HOME": str(user_config) + }), + patch.object(benchmark, "run_process_with_timeout", side_effect=capture), + ): + for checker in ("ty", "ty-infer"): + with self.subTest(checker=checker): + result = benchmark.run_checker( + checker, self.root, [selected], 60, 0, audit_files=True + ) + self.assertTrue(result["ok"], result) + self.assertEqual(result["files_selected"], 1) + self.assertEqual(outputs[-1]["returncode"], 1, outputs[-1]) + self.assertIn("invalid-assignment", outputs[-1]["stdout"]) if __name__ == "__main__": diff --git a/build/benchmark/typecheck_benchmark.py b/build/benchmark/typecheck_benchmark.py index 7d5d5c4e00a4..31a84ba8bff4 100644 --- a/build/benchmark/typecheck_benchmark.py +++ b/build/benchmark/typecheck_benchmark.py @@ -10,8 +10,9 @@ """Compare type-checker wall time and peak memory on configured Python projects. -This benchmark measures speed and peak RSS only. It does not compare diagnostics -or type-checking precision. Configured projects are cloned into a temporary +Corpus timings measure speed and peak RSS, with a separate small behavioral +capability audit that is not a comprehensive precision comparison. +Configured projects are cloned into a temporary directory and installed into package-specific dependency directories before they are checked; --local benchmarks an existing directory without clone or installation. """ @@ -19,6 +20,9 @@ from __future__ import annotations import argparse +import configparser +import hashlib +import importlib.metadata import json import os import platform @@ -31,10 +35,14 @@ import tempfile import threading import time +import tomllib +from contextlib import nullcontext from datetime import datetime, timezone from pathlib import Path from typing import Any, Sequence, TypedDict +from benchmark_capabilities import CapabilityOutput, CapabilitySuite, run_capability_suite + SCRIPT_DIR = Path(__file__).resolve().parent REPO_ROOT = SCRIPT_DIR.parents[1] PYRIGHT_PACKAGE_DIR = REPO_ROOT / "packages" / "pyright" @@ -43,7 +51,8 @@ PYRIGHT_ENTRY_POINT_ENV = "PYRIGHT_BENCHMARK_ENTRY_POINT" DEFAULT_TYPE_CHECKERS = ["pyright", "pyrefly", "ty", "mypy", "zuban"] -AVAILABLE_TYPE_CHECKERS = [*DEFAULT_TYPE_CHECKERS, "pyright-threads", "pyright-pip"] +AVAILABLE_TYPE_CHECKERS = [*DEFAULT_TYPE_CHECKERS, "pyright-threads", "pyright-pip", "ty-infer"] +TARGET_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" DEFAULT_TIMEOUT = 300 DEFAULT_MEMORY_LIMIT_MB = 4096 CLONE_TIMEOUT = 300 @@ -60,6 +69,7 @@ } PYRIGHT_DEFAULT_EXCLUDES = ["**/node_modules", "**/__pycache__", "**/.*"] DEPENDENCY_ISOLATION = "pip-target-per-package" +CHECKER_CONFIG_VERSION = 8 class BenchmarkError(Exception): @@ -91,6 +101,10 @@ class TimingMetrics(RequiredTimingMetrics, total=False): peak_memories_mb: list[float] execution_time_stats: RunStats peak_memory_stats: RunStats + files_selected: int + file_count_verified: bool + file_count_audit_time_s: float + source_count_mismatch: bool class PackageResult(TypedDict, total=False): @@ -102,6 +116,11 @@ class PackageResult(TypedDict, total=False): local_path: str error: str | None metrics: dict[str, TimingMetrics] + source_files: list[str] + source_file_count: int + source_manifest_sha256: str + repository_configs: list[dict[str, Any]] + resolved_dependencies: list[dict[str, str]] class AggregateStats(TypedDict): @@ -157,6 +176,8 @@ def _pyright_command() -> list[str] | None: def _checker_command(checker: str) -> list[str] | None: + if checker == "ty-infer": + checker = "ty" if checker in ("pyright", "pyright-threads"): return _pyright_command() if checker == "pyright-pip": @@ -532,6 +553,18 @@ def _dependency_environment( return environment +def _installed_packages(path: Path | None = None) -> list[dict[str, str]]: + distributions = ( + importlib.metadata.distributions(path=[str(path)]) + if path is not None else importlib.metadata.distributions() + ) + return sorted( + ({"name": re.sub(r"[-_.]+", "-", item.metadata["Name"]).lower(), "version": item.version} + for item in distributions), + key=lambda item: (item["name"], item["version"]), + ) + + def install_deps( package_path: Path, config: dict[str, Any], dependency_path: Path ) -> bool: @@ -596,7 +629,6 @@ def _relative_check_paths( def _resolve_check_paths( package_path: Path, check_paths: list[str], - exclude_directories: list[str], ) -> tuple[list[Path], list[str]]: configured_paths = [package_path / path for path in check_paths] missing = [ @@ -604,28 +636,110 @@ def _resolve_check_paths( for path in configured_paths if not path.exists() ] - existing_paths = [path for path in configured_paths if path.exists()] - if not exclude_directories: - return existing_paths, missing - - excluded_names = set(exclude_directories) - resolved_paths: list[Path] = [] - for path in existing_paths: - if path.is_file(): - resolved_paths.append(path) - continue - for root, directory_names, file_names in os.walk(path): - directory_names[:] = [ - name for name in directory_names if name not in excluded_names - ] - directory_names.sort() - root_path = Path(root) - resolved_paths.extend( - root_path / name - for name in sorted(file_names) - if Path(name).suffix in (".py", ".pyi") - ) - return resolved_paths, missing + return list(dict.fromkeys(path for path in configured_paths if path.exists())), missing + + +def _audit_repository_configs(package_path: Path) -> list[dict[str, Any]]: + native_names = { + "pyrightconfig.json": ["pyright"], + "ty.toml": ["ty"], + "pyrefly.toml": ["pyrefly"], + "mypy.ini": ["mypy", "zuban"], + ".mypy.ini": ["mypy", "zuban"], + "zuban.toml": ["zuban"], + } + tool_names = { + "pyright": ["pyright"], + "basedpyright": ["basedpyright"], + "ty": ["ty"], + "pyrefly": ["pyrefly"], + "mypy": ["mypy", "zuban"], + "zuban": ["zuban"], + } + records: list[dict[str, Any]] = [] + + def walk_error(error: OSError) -> None: + raise error + + for root, directories, files in os.walk(package_path, onerror=walk_error): + directories[:] = sorted( + name for name in directories + if name not in {".git", ".hg", ".svn", "node_modules", "__pycache__"} + ) + for name in sorted(files): + if name.startswith(".typecheck-benchmark-"): + continue + path = Path(root) / name + checkers = set(native_names.get(name, [])) + sections: list[str] = [] + settings: list[str] = [] + error = None + + def setting_names(data: dict[str, Any], prefix: str) -> None: + for key, value in data.items(): + setting = f"{prefix}.{key}" + settings.append(setting) + if isinstance(value, dict): + setting_names(value, setting) + elif isinstance(value, list): + for index, item in enumerate(value): + if isinstance(item, dict): + setting_names(item, f"{setting}[{index}]") + + if path.suffix == ".toml": + try: + data = tomllib.loads(path.read_text(encoding="utf-8-sig")) + except (tomllib.TOMLDecodeError, UnicodeError) as exc: + error = f"{type(exc).__name__}: unable to parse TOML" + else: + if name in native_names: + setting_names(data, path.stem) + tools = data.get("tool", {}) + if isinstance(tools, dict): + for tool, affected in tool_names.items(): + if tool in tools: + sections.append(f"tool.{tool}") + checkers.update(affected) + if isinstance(tools[tool], dict): + setting_names(tools[tool], f"tool.{tool}") + if isinstance(data.get("project"), dict) and "requires-python" in data["project"]: + sections.append("project.requires-python") + checkers.update(("ty", "pyrefly")) + settings.append("project.requires-python") + elif name == "setup.cfg" or name in ("mypy.ini", ".mypy.ini"): + parser = configparser.ConfigParser(interpolation=None) + try: + parser.read_string(path.read_text(encoding="utf-8-sig")) + except (configparser.Error, UnicodeError) as exc: + error = f"{type(exc).__name__}: unable to parse INI" + else: + sections = [ + section for section in parser.sections() + if section == "mypy" or section.startswith("mypy-") + ] + if sections: + checkers.update(("mypy", "zuban")) + settings.extend( + f"{section}.{key}" for section in sections for key in parser[section] + ) + if checkers or sections or error: + record: dict[str, Any] = { + "path": path.relative_to(package_path).as_posix(), + "checkers": sorted(checkers), + "sections": sections, + "settings": sorted(settings), + "disposition": "not applied: explicit benchmark config", + } + if error: + record["parse_error"] = error + records.append(record) + return sorted(records, key=lambda record: record["path"]) + + +def _glob_excludes(exclude_directories: list[str] | None) -> list[str]: + return [*PYRIGHT_DEFAULT_EXCLUDES, *( + f"**/{name}" for name in exclude_directories or [] + )] def _new_config_path(package_path: Path, checker: str, suffix: str) -> Path: @@ -638,46 +752,86 @@ def _new_config_path(package_path: Path, checker: str, suffix: str) -> Path: return Path(path) -def _write_pyright_config(package_path: Path, check_paths: list[str] | None) -> Path: +def _write_pyright_config( + package_path: Path, check_paths: list[str] | None, + exclude_directories: list[str] | None = None, +) -> Path: config_path = _new_config_path(package_path, "pyright", ".json") config = { "include": check_paths or ["."], - "exclude": PYRIGHT_DEFAULT_EXCLUDES, + "exclude": _glob_excludes(exclude_directories), "typeCheckingMode": "basic", "useLibraryCodeForTypes": True, + "pythonVersion": TARGET_PYTHON_VERSION, } config_path.write_text(json.dumps(config), encoding="utf-8") return config_path def _write_mypy_config( - package_path: Path, check_paths: list[str] | None, checker: str + package_path: Path, check_paths: list[str] | None, checker: str, + exclude_directories: list[str] | None = None, ) -> Path: - config_path = _new_config_path(package_path, checker, ".ini") - lines = ["[mypy]"] - if check_paths: - lines.append(f"files = {', '.join(check_paths)}") - lines.extend(["check_untyped_defs = True", ""]) + config_path = _new_config_path(package_path, checker, ".toml") + lines = [ + "[tool.mypy]", + f"files = {json.dumps(check_paths or ['.'])}", + "check_untyped_defs = true", + f"python_version = {json.dumps(TARGET_PYTHON_VERSION)}", + f"python_executable = {json.dumps(sys.executable)}", + "exclude = " + json.dumps([ + r"(^|[/\\])(?:node_modules|__pycache__|\.[^/\\]+)([/\\]|$)", + *( + rf"(^|[/\\]){re.escape(name)}([/\\]|$)" + for name in exclude_directories or [] + ), + ]), + ] + if checker == "zuban": + lines.append("exclude_gitignore = false") + lines.append("") config_path.write_text("\n".join(lines), encoding="utf-8") return config_path -def _write_ty_config(package_path: Path, check_paths: list[str] | None) -> Path: +def _write_ty_config( + package_path: Path, check_paths: list[str] | None, *, infer_returns: bool = False, + exclude_directories: list[str] | None = None, +) -> Path: config_path = _new_config_path(package_path, "ty", ".toml") - lines: list[str] = [] - if check_paths: - paths = ", ".join(json.dumps(path) for path in check_paths) - lines.extend(["[src]", f"include = [{paths}]"]) + lines = [ + "[environment]", + f"python = {json.dumps(sys.executable)}", + f"python-version = {json.dumps(TARGET_PYTHON_VERSION)}", + "", + "[src]", + "respect-ignore-files = false", + # These are gitignore-style patterns, not native Windows paths. + f"include = {json.dumps([Path(path).as_posix() for path in check_paths or ['.']])}", + f"exclude = {json.dumps(['!**', *_glob_excludes(exclude_directories)])}", + ] + if infer_returns: + lines.extend(["", "[analysis]", 'return-type-inference = "always"']) config_path.write_text("\n".join(lines) + "\n", encoding="utf-8") return config_path -def _write_pyrefly_config(package_path: Path, check_paths: list[str] | None) -> Path: +def _write_pyrefly_config( + package_path: Path, check_paths: list[str] | None, + exclude_directories: list[str] | None = None, +) -> Path: config_path = _new_config_path(package_path, "pyrefly", ".toml") - lines: list[str] = [] - if check_paths: - paths = ", ".join(json.dumps(path) for path in check_paths) - lines.append(f"project_includes = [{paths}]") + lines = [ + f"project-includes = {json.dumps(check_paths or ['.'])}", + "use-ignore-files = false", + "disable-project-excludes-heuristics = true", + f"project-excludes = {json.dumps(_glob_excludes(exclude_directories))}", + f"python-interpreter-path = {json.dumps(sys.executable)}", + f"python-version = {json.dumps(TARGET_PYTHON_VERSION)}", + 'preset = "all"', + "check-unannotated-defs = true", + 'infer-return-types = "checked"', + ] config_path.write_text("\n".join(lines) + "\n", encoding="utf-8") return config_path @@ -723,10 +877,30 @@ def _parse_pyright_stats( ) +def _parse_source_file_count(checker: str, output: str) -> int | None: + if checker in ("ty", "ty-infer"): + if not re.search(r"\bDEBUG Checking all files took ", output): + return None + return len(set(re.findall( + r"\bDEBUG Checking file '(.+)'$", output, re.MULTILINE + ))) + if checker in ("pyright", "pyright-pip"): + pattern = r"^Total files checked:\s+(\d+)$" + elif checker == "pyright-threads": + pattern = r"^Found (\d+) files? to analyze$" + elif checker == "pyrefly": + pattern = r"\bINFO (\d+) modules? \(\d+ dependent modules?\)" + else: + pattern = r"(?:checked |Success: no issues found in )(\d+) source files?\b" + counts = {int(count) for count in re.findall(pattern, output, re.MULTILINE)} + return counts.pop() if len(counts) == 1 else None + + def _build_checker_command( checker: str, package_path: Path, check_paths: list[Path] | None, + exclude_directories: list[str] | None = None, ) -> tuple[list[str] | None, list[Path]]: """Build one checker command and return temporary configs to remove.""" base_command = _checker_command(checker) @@ -735,11 +909,13 @@ def _build_checker_command( relative_paths = _relative_check_paths(package_path, check_paths) if checker in ("pyright", "pyright-threads", "pyright-pip"): - config_path = _write_pyright_config(package_path, relative_paths) + config_path = _write_pyright_config(package_path, relative_paths, exclude_directories) command = [ *base_command, "--project", str(config_path), + "--pythonpath", + sys.executable, ] if checker == "pyright-threads": command.append("--threads") @@ -747,15 +923,18 @@ def _build_checker_command( command.append("--stats") return command, [config_path] if checker == "pyrefly": - config_path = _write_pyrefly_config(package_path, relative_paths) + config_path = _write_pyrefly_config(package_path, relative_paths, exclude_directories) return [ *base_command, "check", "--config", str(config_path), ], [config_path] - if checker == "ty": - config_path = _write_ty_config(package_path, relative_paths) + if checker in ("ty", "ty-infer"): + config_path = _write_ty_config( + package_path, relative_paths, infer_returns=checker == "ty-infer", + exclude_directories=exclude_directories, + ) return [ *base_command, "check", @@ -763,22 +942,31 @@ def _build_checker_command( str(config_path), ], [config_path] if checker == "mypy": - config_path = _write_mypy_config(package_path, relative_paths, "mypy") + config_path = _write_mypy_config(package_path, relative_paths, "mypy", exclude_directories) command = [ *base_command, "--no-incremental", "--config-file", str(config_path), ] - if not relative_paths: - command.append(".") return command, [config_path] if checker == "zuban": - return [*base_command, "check", *(relative_paths or ["."])], [] + config_path = _write_mypy_config(package_path, relative_paths, "zuban", exclude_directories) + command = [ + *base_command, + "check", + "--mode", + "default", + "--config-file", + str(config_path), + "--python-executable", + sys.executable, + ] + return command, [config_path] return None, [] -def run_checker( +def _run_checker_process( checker: str, package_path: Path, check_paths: list[Path] | None, @@ -786,10 +974,12 @@ def run_checker( memory_limit_mb: int, capture_output: bool = True, environment: dict[str, str] | None = None, -) -> TimingMetrics: - """Run one checker once using a minimal checker-specific configuration.""" + audit_files: bool = False, + exclude_directories: list[str] | None = None, +) -> ProcessResult: + """Execute a checker with the shared benchmark configuration and isolation.""" command, config_paths = _build_checker_command( - checker, package_path, check_paths + checker, package_path, check_paths, exclude_directories ) if not command: error_message = ( @@ -797,36 +987,72 @@ def run_checker( if checker in AVAILABLE_TYPE_CHECKERS else f"Unknown checker: {checker}" ) - return { - "ok": False, - "execution_time_s": 0.0, - "peak_memory_mb": 0.0, - "oom_killed": False, - "error_message": error_message, - } + raise FileNotFoundError(error_message) + + if audit_files: + if checker == "pyright-threads": + command.append("--verbose") + elif checker == "pyrefly": + command.append("--summary=full") + elif checker in ("ty", "ty-infer"): + command.append("-vv") try: - try: - result = run_process_with_timeout( + # ty merges user configuration even with an explicit project config. + with ( + tempfile.TemporaryDirectory( + prefix=".typecheck-benchmark-user-", dir=package_path + ) + if checker in ("ty", "ty-infer") + else nullcontext(None) + ) as config_home: + checker_environment = environment + if config_home is not None: + checker_environment = ( + environment if environment is not None else os.environ + ).copy() + checker_environment["APPDATA"] = config_home + checker_environment["XDG_CONFIG_HOME"] = config_home + return run_process_with_timeout( command, cwd=package_path, timeout=timeout, memory_limit_mb=memory_limit_mb, capture_output=capture_output, - environment=environment, + environment=checker_environment, ) - except OSError as exc: - return { - "ok": False, - "execution_time_s": 0.0, - "peak_memory_mb": 0.0, - "oom_killed": False, - "error_message": f"Unable to start {checker}: {exc}", - } finally: for config_path in config_paths: config_path.unlink(missing_ok=True) + +def run_checker( + checker: str, + package_path: Path, + check_paths: list[Path] | None, + timeout: int, + memory_limit_mb: int, + capture_output: bool = True, + environment: dict[str, str] | None = None, + audit_files: bool = False, + exclude_directories: list[str] | None = None, +) -> TimingMetrics: + """Run one checker once using a minimal checker-specific configuration.""" + try: + result = _run_checker_process( + checker, package_path, check_paths, timeout, memory_limit_mb, + capture_output=capture_output, environment=environment, audit_files=audit_files, + exclude_directories=exclude_directories, + ) + except OSError as exc: + return { + "ok": False, + "execution_time_s": 0.0, + "peak_memory_mb": 0.0, + "oom_killed": False, + "error_message": f"Unable to start {checker}: {exc}", + } + if result["oom_killed"]: memory = result["peak_memory_mb"] memory_text = f" at {memory:.1f} MB" if memory > 0 else "" @@ -849,13 +1075,25 @@ def run_checker( } combined_output = f"{result['stderr']}\n{result['stdout']}" + if checker == "pyrefly" and re.search( + r"^No Python files matched pattern", combined_output, re.MULTILINE + ): + return { + "ok": False, + "execution_time_s": result["execution_time_s"], + "peak_memory_mb": result["peak_memory_mb"], + "oom_killed": False, + "error_message": "No Python files matched the configured include patterns", + } if capture_output and "errors prevented further checking" in combined_output: return { "ok": False, "execution_time_s": result["execution_time_s"], "peak_memory_mb": result["peak_memory_mb"], "oom_killed": False, - "error_message": "Fatal: errors prevented further checking", + "error_message": ( + "Fatal: errors prevented further checking; " + _failure_message(result) + ), } # All supported checkers use exit code 1 for ordinary type errors. @@ -882,6 +1120,12 @@ def run_checker( metrics["files_checked"] = files_checked if phase_times: metrics["phase_times_s"] = phase_times + files_selected = _parse_source_file_count(checker, combined_output) + if files_selected is not None: + metrics["files_selected"] = files_selected + elif audit_files: + metrics["ok"] = False + metrics["error_message"] = f"{checker} did not report a verifiable source-file count" return metrics @@ -966,7 +1210,39 @@ def _benchmark_directory( memory_limit_mb: int, local_path: str | None = None, environment: dict[str, str] | None = None, + exclude_directories: list[str] | None = None, + repository_configs: list[dict[str, Any]] | None = None, ) -> PackageResult: + check_roots, missing = _resolve_check_paths( + package_path, + (_relative_check_paths(package_path, resolved_paths) or []) + if resolved_paths is not None else ["."], + ) + if missing or not check_roots: + return { + "package_name": name, + "github_url": github_url, + "error": ( + f"Missing configured source paths: {', '.join(missing)}" + if missing else "No Python source files selected" + ), + "metrics": {}, + } + try: + if repository_configs is None: + repository_configs = _audit_repository_configs(package_path) + except OSError as exc: + message = f"Repository configuration audit failed: {exc}" + print(f" {message}") + return { + "package_name": name, "github_url": github_url, + "error": message, "metrics": {}, + } + print(f" Repository config audit: {len(repository_configs)} candidate files") + for config in repository_configs: + detail = ", ".join(config["sections"] or config["checkers"]) or "unparsed TOML" + status = "parse error" if "parse_error" in config else config["disposition"] + print(f" {config['path']}: {detail} ({status})") metrics: dict[str, TimingMetrics] = {} for checker in type_checkers: if not _checker_command(checker): @@ -980,6 +1256,27 @@ def _benchmark_directory( } continue + audit = run_checker( + checker, package_path, check_roots, timeout, memory_limit_mb, + environment=environment, audit_files=True, + exclude_directories=exclude_directories, + ) + selected_count = audit.get("files_selected") + if not audit["ok"] or selected_count is None or selected_count == 0: + audit["ok"] = False + audit["file_count_verified"] = False + if not audit.get("error_message"): + audit["error_message"] = ( + f"No verifiable nonzero native source-file count: {checker} reported {selected_count}" + ) + audit["file_count_audit_time_s"] = audit["execution_time_s"] + audit["execution_time_s"] = 0.0 + audit["peak_memory_mb"] = 0.0 + metrics[checker] = audit + print(f" {checker} source-file audit failed: {audit['error_message']}") + continue + print(f" {checker} source-file audit: {selected_count} native files verified") + if warmup == 0: run_description = f"{runs} measured" else: @@ -1002,11 +1299,19 @@ def _benchmark_directory( metric = run_checker( checker, package_path, - resolved_paths, + check_roots, timeout, memory_limit_mb, environment=environment, + exclude_directories=exclude_directories, ) + run_count = metric.get("files_selected") + if metric["ok"] and run_count is not None and run_count != selected_count: + metric["ok"] = False + metric["error_message"] = ( + f"Source-file count changed: expected {selected_count}, " + f"{checker} reported {run_count}" + ) if not metric.get("ok"): print(f"failed: {metric.get('error_message', 'Unknown error')}") failure = metric @@ -1050,6 +1355,9 @@ def _benchmark_directory( "peak_memories_mb": peak_memories, "execution_time_stats": compute_run_stats(execution_times), "peak_memory_stats": compute_run_stats(peak_memories), + "files_selected": selected_count, + "file_count_verified": True, + "file_count_audit_time_s": audit["execution_time_s"], } if parsed_file_counts: result_metric["files_parsed"] = parsed_file_counts[0] @@ -1072,11 +1380,28 @@ def _benchmark_directory( f"(stddev: {result_metric['execution_time_stats']['stddev']:.3f}s)" ) + native_counts = { + checker: metric["files_selected"] + for checker, metric in metrics.items() if metric.get("file_count_verified") + } + if len(set(native_counts.values())) > 1: + message = f"Native source-file counts differ: {native_counts}; timings are not comparable" + print(f" {message}") + for checker in native_counts: + metrics[checker]["ok"] = False + metrics[checker]["source_count_mismatch"] = True + metrics[checker]["error_message"] = message + result: PackageResult = { "package_name": name, "github_url": github_url, + "commit": get_package_commit(package_path) or "local-unversioned", "error": None, "metrics": metrics, + "check_paths": _relative_check_paths(package_path, check_roots) or [], + "exclude_directories": exclude_directories or [], + "repository_configs": repository_configs, + "resolved_dependencies": [], } if local_path: result["local_path"] = local_path @@ -1101,6 +1426,7 @@ def _benchmark_package( "github_url": github_url, "check_paths": raw_check_paths, "exclude_directories": exclude_directories, + "commit": package.get("commit") or "unknown", } package_path = clone_package( github_url, name, temp_path, package.get("commit") @@ -1113,6 +1439,14 @@ def _benchmark_package( } commit = get_package_commit(package_path) + package_metadata["commit"] = commit or "unknown" + try: + package_metadata["repository_configs"] = _audit_repository_configs(package_path) + except OSError as exc: + return { + **package_metadata, "error": f"Repository configuration audit failed: {exc}", + "metrics": {}, + } dependency_path = temp_path / f"{name}-dependencies" if not install_deps(package_path, package, dependency_path): return { @@ -1125,10 +1459,15 @@ def _benchmark_package( resolved_paths: list[Path] | None = None if raw_check_paths: resolved_paths, missing = _resolve_check_paths( - package_path, raw_check_paths, exclude_directories + package_path, raw_check_paths ) if missing: - print(f" Warning: missing configured check paths: {', '.join(missing)}") + return { + **package_metadata, + "commit": commit or "unknown", + "error": f"Missing configured check paths: {', '.join(missing)}", + "metrics": {}, + } if resolved_paths: check_text = ", ".join(raw_check_paths) if exclude_directories: @@ -1137,7 +1476,12 @@ def _benchmark_package( ) + ")" print(f" Checking: {check_text}") else: - print(" Warning: configured check paths do not exist; checking full repo") + return { + **package_metadata, + "commit": commit or "unknown", + "error": "No Python source files selected", + "metrics": {}, + } result = _benchmark_directory( name=name, @@ -1150,9 +1494,12 @@ def _benchmark_package( warmup=warmup, memory_limit_mb=memory_limit_mb, environment=_dependency_environment(dependency_path, package), + exclude_directories=exclude_directories, + repository_configs=package_metadata["repository_configs"], ) result["commit"] = commit or "unknown" result.update(package_metadata) + result["resolved_dependencies"] = _installed_packages(dependency_path) return result @@ -1168,6 +1515,7 @@ def _save_results( memory_limit_mb: int, os_name: str | None, local_dir: Path | None, + capability_suite: CapabilitySuite | None = None, ) -> Path: timestamp = datetime.now(timezone.utc) date = timestamp.strftime("%Y-%m-%d") @@ -1196,12 +1544,18 @@ def _save_results( "memory_limit_mb": memory_limit_mb, "node_options": os.environ.get("NODE_OPTIONS", ""), "dependency_isolation": DEPENDENCY_ISOLATION, + "checker_config_version": CHECKER_CONFIG_VERSION, + "installed_packages": _installed_packages(), + "import_path_overrides": { + name: hashlib.sha256(os.environ.get(name, "").encode("utf-8")).hexdigest() + for name in ("PYTHONPATH", "MYPYPATH") + }, "type_checkers": type_checkers, "type_checker_versions": versions, "package_count": len(results), "runs_per_package": runs, "warmup_runs": warmup, - "uncounted_validation_runs_per_checker": 0, + "uncounted_validation_runs_per_checker": 1, "timeout_s": timeout, "aggregate": aggregate, "results": results, @@ -1210,6 +1564,8 @@ def _save_results( output["os"] = os_name if local_dir: output["local_directory"] = str(local_dir) + if capability_suite is not None: + output["capability_suite"] = capability_suite serialized = json.dumps(output, indent=2) + "\n" dated_file.write_text(serialized, encoding="utf-8") @@ -1340,6 +1696,30 @@ def run_benchmark( ) ) + print("\nUntimed behavioral capability audit:") + + def run_capability( + checker: str, root: Path, paths: list[Path], dependencies: Path, + ) -> CapabilityOutput: + result = _run_checker_process( + checker, root, paths, min(timeout, 60), memory_limit_mb, audit_files=True, + environment=_dependency_environment(dependencies, {}), + ) + return { + **result, + "files_selected": _parse_source_file_count( + checker, result["stdout"] + "\n" + result["stderr"] + ), + } + + capability_suite = run_capability_suite(type_checkers, run_capability) + for checker, result in capability_suite["results"].items(): + if result["ok"]: + passed = sum(case["passed"] for case in result["cases"].values()) + print(f" {checker}: {passed}/{len(capability_suite['cases'])} cases passed") + else: + print(f" {checker}: not evaluated: {result['error']}") + aggregate = compute_aggregate_stats(results, type_checkers) dated_file = _save_results( results, @@ -1353,6 +1733,7 @@ def run_benchmark( memory_limit_mb, os_name, resolved_local_dir, + capability_suite, ) print("\nBenchmark complete.")