diff --git a/.github/scripts/gate_suites.py b/.github/scripts/gate_suites.py index 105afdd9..4340122b 100644 --- a/.github/scripts/gate_suites.py +++ b/.github/scripts/gate_suites.py @@ -158,6 +158,7 @@ "gate/test_split_probe.py": BOTH, "gate/test_tmpleak.py": BOTH, "gate/test_toolguard_run_preflight.py": LOCAL_ONLY, + "golden/test_x86pins.py": BOTH, "models/test_cbpath.py": BOTH, "models/test_cbwalls.py": BOTH, "models/test_higgs.py": BOTH, diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de52cb31..d4c6636c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -189,6 +189,20 @@ jobs: cat "$RUNNER_TEMP/gate.log" exit $rc + # Issue #69: DOCWATCH runs compare.py for its counts and prints only the + # number that failed, so the x86_64 golden failures had no names. This + # runs it once more where its own FAIL lines are visible, against the + # artefact the gate staged, so each divergence can be pinned by name. + - name: golden failures by name (x86_64) + if: always() + run: | + set +e + TORCH_C_ARTEFACT="$TORCH_C_STAGE/_C.abi3.so" \ + python tests/golden/compare.py > "$RUNNER_TEMP/golden.log" 2>&1 + echo "EXIT=$?" >> "$RUNNER_TEMP/golden.log" + grep -E '^(FAIL|SUMMARY|EXIT=)' "$RUNNER_TEMP/golden.log" + exit 0 + - name: upload the gate log if: always() uses: actions/upload-artifact@v4 @@ -196,6 +210,7 @@ jobs: name: gate-log-linux path: | ${{ runner.temp }}/gate.log + ${{ runner.temp }}/golden.log ${{ runner.temp }}/gate-stage/suite-logs.*/ if-no-files-found: warn retention-days: 14 diff --git a/docs/verification/GOLDEN.md b/docs/verification/GOLDEN.md index 96d1b93f..139624fc 100644 --- a/docs/verification/GOLDEN.md +++ b/docs/verification/GOLDEN.md @@ -300,3 +300,23 @@ $PY tests/golden/compare.py # 2843/2843, ops covered=119, exit 0 $PY tests/golden/compare.py --self-test # PASS, exit 0 PYTHON=$PY sh tests/run.sh # 177 ok / 34 pre-existing unrelated (ยง7), exit 1 ``` + +## 9. x86_64 Linux: the named pins (issue #69) + +On the x86_64 Linux CI runner `compare.py` reported **47 to 54** failing cases +out of 12103, and the count moved between runs on the same base: some of these +cases are run-to-run unstable on x86. Every suite passed, but the +`golden_cases_failed eq 0` marker turned every PR's Linux CI red. + +The 54 case names of the CI run on PR #90 are pinned by exact name in +`compare.py`'s `KNOWN_X86_64_LINUX`, each with its #40 class (or #69 / #81) and +a reason: weighted `nll_loss_forward` 24, `matmul` 12, `kaiser_window` 8 (A4), +`gelu` erf form on `+inf` 4, the 80-tie `sort`/`argsort` 3 (#81), reduced-float +`sdpa_flash_cpu` logsumexp 2 (A3), bf16 `add` with alpha 1 (A2). Only on x86_64 +Linux, and only when such a case fails, it prints `X86KNOWN` and counts in +`x86_known` on the SUMMARY line, not in `failed`; DOCWATCH reads `failed` and +exposes the other as the count `golden_x86_known`. Elsewhere the table is never +read and `--self-test` never reads it. A case that fails on x86 in a later run +and is not among the 54 stays a FAIL, by design: add it by name. +`tests/golden/test_x86pins.py` checks that every pin names a case `cases.py` +builds and that the classification is platform-gated. diff --git a/tests/docwatch/check_docs.py b/tests/docwatch/check_docs.py index 8bf510dd..329b2499 100644 --- a/tests/docwatch/check_docs.py +++ b/tests/docwatch/check_docs.py @@ -270,7 +270,8 @@ def golden(self) -> dict: env=self.env, timeout=LIVE_TIMEOUT_S) m = re.search( r"SUMMARY:\s*(\d+)/(\d+) cases passed, (\d+) failed, " - r"ops covered=(\d+), pending case builders=(\d+)", + r"ops covered=(\d+), pending case builders=(\d+)" + r"(?:, x86_known=(\d+))?", proc.stdout, ) if not m: @@ -278,13 +279,18 @@ def golden(self) -> dict: f"compare.py exit={proc.returncode}, no SUMMARY line found; " f"stderr={proc.stderr[-400:]}" ) - passed, total, failed, ops, pending = (int(x) for x in m.groups()) + # `failed` excludes the x86_64-Linux cases pinned by name in + # compare.py's KNOWN_X86_64_LINUX; those are `x86_known` (0 off + # x86_64 Linux, where the table is never consulted). + passed, total, failed, ops, pending = (int(x) for x in m.groups()[:5]) + x86_known = int(m.group(6) or 0) self._golden_cache = { "golden_cases_passed": passed, "golden_cases_total": total, "golden_cases_failed": failed, "golden_ops_covered": ops, "golden_pending": pending, + "golden_x86_known": x86_known, } return self._golden_cache @@ -516,6 +522,7 @@ def skip_lines_visible(self) -> int: "golden_cases_total": lambda lf: lf.golden()["golden_cases_total"], "golden_ops_covered": lambda lf: lf.golden()["golden_ops_covered"], "golden_pending": lambda lf: lf.golden()["golden_pending"], + "golden_x86_known": lambda lf: lf.golden()["golden_x86_known"], "schema_entries_matched": lambda lf: lf.schema()["schema_entries_matched"], "schema_entries_total": lambda lf: lf.schema()["schema_entries_total"], "decomp_implemented": lambda lf: lf.decomp()["decomp_implemented"], diff --git a/tests/golden/compare.py b/tests/golden/compare.py index cd26abdd..cc3240a8 100644 --- a/tests/golden/compare.py +++ b/tests/golden/compare.py @@ -74,6 +74,135 @@ class Outcome: fault_applied: bool = False +# KNOWN x86_64 LINUX DIVERGENCE -- the golden twin of `_skip.known_x86_64_linux_divergence` +# ---------------------------------------------------------------------------- +# On the x86_64 Linux CI runner this harness reports 47 to 54 failing cases of +# 12103 (the count moves between runs on the same base: some of these cases are +# run-to-run unstable on x86). The shim reproduces the arm64 upstream build's +# arithmetic; the x86_64 wheel's capability-dispatched kernels give other bits +# (issue #40's A1..A5 split, issue #69). They are not fixed, so they are pinned +# here BY EXACT CASE NAME (the 54 names of the CI run on PR #90), never by op. +# +# Only on x86_64 Linux, and only for a FAILING case, a pin turns FAIL into +# `X86KNOWN` and the case is counted in `x86_known`, not in `failed`. A pinned +# case that passes is a plain pass. A failure that matches no pin stays FAIL. +# Off x86_64 Linux this table is never consulted, and `--self-test` never +# consults it. What it cannot see: a case that fails on x86 in some other run +# but was not among the 54 stays FAIL (that is the point: add it by name). +# tests/golden/test_x86pins.py checks that every pin names a real case. +_A2 = ("#40 split A2", "bf16 add with alpha: x86 fuses alpha in the 2*V vector body, arm64 narrows first") +_A3 = ("#40 split A3", "reduced-float SDPA logsumexp: x86 flash kernel's exp_u20 and GEMM route") +_A4 = ("#40 split A4", "kaiser_window float32: x86 capability-dispatched kernel contracts to FMA") +_TIES = ("#81 (#40)", "80 tied elements: upstream's tie order on Linux is libstdc++'s, the shim keeps the stable order") +_GEMM = ("#69 (#40 class: x86 GEMM route)", "matmul float32/float64 bit-exactness: x86 upstream's GEMM accumulates in another order") +_NLL = ("#69 (#40 class A5: vectorised accumulation)", "weighted nll_loss sum/mean: x86 upstream's vectorised accumulation rounds the weighted terms differently") +_GELU = ("#69", "gelu erf form, float32 and bfloat16: x86 upstream returns NaN for gelu(+inf) where arm64 upstream and the shim return +inf") + +KNOWN_X86_64_LINUX: dict[str, list[tuple[str, str, str]]] = { + 'aten._scaled_dot_product_flash_attention_for_cpu.default': [ + ('sdpa_flash_cpu(dtype=bfloat16, q_len=33, kv_len=33, mask=False, is_causal=True) [second query block, causal]', *_A3), + ('sdpa_flash_cpu(dtype=float16, q_len=33, kv_len=33, mask=False, is_causal=True) [second query block, causal]', *_A3), + ], + 'aten.add.Tensor': [ + ('add(dtype=bfloat16, shape=(64,), alpha=3.0) [exact narrowing: 64 elements, alpha=3 (alpha is applied in opmath too)]', *_A2), + ], + 'aten.argsort.default': [ + ('aten.argsort.default(float32, 80 identical elements)', *_TIES), + ], + 'aten.argsort.stable': [ + ('aten.argsort.stable(float32, 80 identical elements)', *_TIES), + ], + 'aten.gelu.default': [ + ("gelu(dtype=bfloat16, shape=(3,)) [nan/+-inf -- +inf passes through, -inf gives NaN; explicit 'none' -- must equal the unqualified call]", *_GELU), + ("gelu(dtype=bfloat16, shape=(3,)) [nan/+-inf -- +inf passes through, -inf gives NaN; unqualified -- upstream's schema default is 'none', the erf form]", *_GELU), + ("gelu(dtype=float32, shape=(3,)) [nan/+-inf -- +inf passes through, -inf gives NaN; explicit 'none' -- must equal the unqualified call]", *_GELU), + ("gelu(dtype=float32, shape=(3,)) [nan/+-inf -- +inf passes through, -inf gives NaN; unqualified -- upstream's schema default is 'none', the erf form]", *_GELU), + ], + 'aten.kaiser_window.beta': [ + ('kaiser_window(16, False, 12.0)', *_A4), + ('kaiser_window(16, False, beta=14.0)', *_A4), + ('kaiser_window(16, False, beta=38.0)', *_A4), + ('kaiser_window(16, False, beta=5.0)', *_A4), + ('kaiser_window(16, False, beta=9.0)', *_A4), + ('kaiser_window(64, False, 12.0)', *_A4), + ], + 'aten.kaiser_window.periodic': [ + ('kaiser_window(16, False)', *_A4), + ('kaiser_window(64, False)', *_A4), + ], + 'aten.matmul.default': [ + ("matmul(dtype=float32, a_shape=(1, 6, 4), b_shape=(4, 5)) [3D x 2D, batch=1 -- F.linear's own activation rank, still folds]", *_GEMM), + ("matmul(dtype=float32, a_shape=(1, 6, 4), w_shape=(5, 4) transposed view -> (4, 5)) [rhs = t(weight), batch=1 activation -- F.linear's exact shape]", *_GEMM), + ('matmul(dtype=float32, a_shape=(2, 2, 3, 4), b_shape=(4, 5)) [4D x 2D -- the fold branch at rank 4]', *_GEMM), + ("matmul(dtype=float32, a_shape=(2, 3, 4), b_shape=(4, 5)) [3D x 2D, batch=2 -- the fold branch (batched_matmul's first arm)]", *_GEMM), + ('matmul(dtype=float32, a_shape=(2,3,4,5) transpose(0,1) -> (3,2,4,5) swapped-batch-axis, w_shape=(3,2,5,6))', *_GEMM), + ('matmul(dtype=float32, a_shape=(3, 6, 4), w_shape=(5, 4) transposed view -> (4, 5)) [rhs = t(weight), batch=3 activation]', *_GEMM), + ('matmul(dtype=float32, a_shape=(4, 3) transposed view -> (3, 4), w_shape=(4, 5)) [lhs is the transposed view instead -- gemm_with_layout_fallback treats both operands the same, checked directly rather than assumed symmetric]', *_GEMM), + ('matmul(dtype=float32, a_shape=(4,16) sliced [:, ::2] -> (4,8) strided, w_shape=(8,5))', *_GEMM), + ("matmul(dtype=float64, a_shape=(1, 6, 4), b_shape=(4, 5)) [3D x 2D, batch=1 -- F.linear's own activation rank, still folds]", *_GEMM), + ("matmul(dtype=float64, a_shape=(1, 6, 4), w_shape=(5, 4) transposed view -> (4, 5)) [rhs = t(weight), batch=1 activation -- F.linear's exact shape]", *_GEMM), + ("matmul(dtype=float64, a_shape=(2, 3, 4), b_shape=(4, 5)) [3D x 2D, batch=2 -- the fold branch (batched_matmul's first arm)]", *_GEMM), + ('matmul(dtype=float64, a_shape=(3, 6, 4), w_shape=(5, 4) transposed view -> (4, 5)) [rhs = t(weight), batch=3 activation]', *_GEMM), + ], + 'aten.nll_loss_forward.default': [ + ('nll_loss_forward(dtype=float32, n=16, reduction=mean, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float32, n=16, reduction=sum, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float32, n=17, reduction=mean, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float32, n=17, reduction=sum, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float32, n=300, reduction=mean, weight=True, ignore_index=3)', *_NLL), + ('nll_loss_forward(dtype=float32, n=300, reduction=sum, weight=True, ignore_index=3)', *_NLL), + ('nll_loss_forward(dtype=float32, n=64, reduction=mean, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float32, n=64, reduction=mean, weight=True, ignore_index=3)', *_NLL), + ('nll_loss_forward(dtype=float32, n=64, reduction=sum, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float32, n=64, reduction=sum, weight=True, ignore_index=3)', *_NLL), + ('nll_loss_forward(dtype=float32, n=65, reduction=mean, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float32, n=65, reduction=mean, weight=True, ignore_index=3)', *_NLL), + ('nll_loss_forward(dtype=float32, n=65, reduction=sum, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float32, n=65, reduction=sum, weight=True, ignore_index=3)', *_NLL), + ('nll_loss_forward(dtype=float64, n=16, reduction=mean, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float64, n=16, reduction=mean, weight=True, ignore_index=3)', *_NLL), + ('nll_loss_forward(dtype=float64, n=16, reduction=sum, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float64, n=16, reduction=sum, weight=True, ignore_index=3)', *_NLL), + ('nll_loss_forward(dtype=float64, n=17, reduction=mean, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float64, n=17, reduction=mean, weight=True, ignore_index=3)', *_NLL), + ('nll_loss_forward(dtype=float64, n=17, reduction=sum, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float64, n=17, reduction=sum, weight=True, ignore_index=3)', *_NLL), + ('nll_loss_forward(dtype=float64, n=65, reduction=mean, weight=True, ignore_index=-100)', *_NLL), + ('nll_loss_forward(dtype=float64, n=65, reduction=sum, weight=True, ignore_index=-100)', *_NLL), + ], + 'aten.sort.default': [ + ('sort(float32, 80 elements all tied two ways, descending) [stability at scale]', *_TIES), + ], +} + + +def _on_x86_64_linux() -> bool: + """The same predicate as `tests/_support/_skip.on_x86_64_linux`.""" + sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "_support")) + import _skip + + return _skip.on_x86_64_linux() + + +def x86_pin(op: str, name: str, on_x86_64_linux: bool) -> tuple[str, str] | None: + """(issue, reason) of the pin naming this case, or None. Never consults + the table unless `on_x86_64_linux` is true.""" + if not on_x86_64_linux: + return None + for pinned, issue, reason in KNOWN_X86_64_LINUX.get(op, ()): + if pinned == name: + return issue, reason + return None + + +def classify(op: str, name: str, passed: bool, on_x86_64_linux: bool) -> str: + """"PASS", "FAIL" or "X86KNOWN". Only a failing pinned case on x86_64 + Linux is X86KNOWN; tests pass `on_x86_64_linux` explicitly.""" + if passed: + return "PASS" + return "X86KNOWN" if x86_pin(op, name, on_x86_64_linux) else "FAIL" + + def _flatten(x) -> list: if isinstance(x, list): out: list = [] @@ -689,6 +818,9 @@ def run(artefact: str | None, verbose: bool, inject_fault: str | None) -> int: _DTYPE_OWNERS[:] = [torch, c_module] + on_x86 = _on_x86_64_linux() + x86_known: list[Outcome] = [] + implemented = list(c_module._aten_implemented()) print(f"target={c_module._shim_target()} implemented={implemented}") if dt.EXCLUDED_DTYPES: @@ -751,8 +883,12 @@ def run(artefact: str | None, verbose: bool, inject_fault: str | None) -> int: if fault_for_this_case and outcome.fault_applied: injected[cmp_name] = (case, outcome) all_outcomes.append(outcome) - if verbose or not outcome.passed: - status = "PASS" if outcome.passed else "FAIL" + status = classify(case.op, case.name, outcome.passed, on_x86) + if status == "X86KNOWN": + x86_known.append(outcome) + issue, _why = x86_pin(case.op, case.name, on_x86) + print(f"X86KNOWN {case.op} :: {case.name} -- {issue}: {outcome.detail}") + elif verbose or status == "FAIL": print(f"{status} {case.op} :: {case.name} -- {outcome.detail}") for op_name in missing_builders: @@ -763,8 +899,8 @@ def run(artefact: str | None, verbose: bool, inject_fault: str | None) -> int: ) total = len(all_outcomes) + len(missing_builders) - failed = sum(1 for o in all_outcomes if not o.passed) + len(missing_builders) - passed = total - failed + failed = sum(1 for o in all_outcomes if not o.passed) + len(missing_builders) - len(x86_known) + passed = total - failed - len(x86_known) if inject_fault: _print_injection_verdict(inject_fault, seen_comparators, injected) @@ -800,7 +936,8 @@ def run(artefact: str | None, verbose: bool, inject_fault: str | None) -> int: print( f"SUMMARY: {passed}/{total} cases passed, {failed} failed, " - f"ops covered={len(implemented)}, pending case builders={len(pending_builders)}" + f"ops covered={len(implemented)}, pending case builders={len(pending_builders)}, " + f"x86_known={len(x86_known)}" ) return 1 if failed else 0 diff --git a/tests/golden/test_x86pins.py b/tests/golden/test_x86pins.py new file mode 100644 index 00000000..78611361 --- /dev/null +++ b/tests/golden/test_x86pins.py @@ -0,0 +1,130 @@ +"""The golden harness's x86_64 Linux pins (issue #69): named, live, and narrow. + +`tests/golden/compare.py`'s `KNOWN_X86_64_LINUX` demotes a failing golden case +to `X86KNOWN` (counted in `x86_known`, not in `failed`) only on x86_64 Linux and +only when the case is named in the table. This suite checks three things: + + (a) every pin names a case `cases.py` really builds (no dead pins: a renamed + case would otherwise leave its pin matching nothing, and the real case + would go back to FAIL with nobody told why the pin stopped working); + (b) with the platform forced true through `classify`'s argument (not an env + var production could set), a failing pinned case is X86KNOWN, a failing + unpinned case stays FAIL, and a passing pinned case is a plain PASS; + (c) with the platform false, the same pinned failure stays FAIL. + +What this cannot find: whether the pins are the *right* set on the runner (only +a CI run of compare.py measures that), and whether a pinned case would fail for +a new, different reason on x86 (the pin is by name, so it would be hidden as +X86KNOWN; the detail is still printed on the X86KNOWN line). +""" + +import json +import os +import pathlib +import subprocess +import sys + +import _skip + +HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import compare # noqa: E402 + +_NAMES_SCRIPT = r""" +import json, sys +sys.path.insert(0, sys.argv[1]) +import compare +from loader import load_shim, resolve_torch_overload +import torch +c = load_shim(None) +compare._DTYPE_OWNERS[:] = [torch, c] +out = {} +for op in compare.KNOWN_X86_64_LINUX: + builder = compare.CASE_BUILDERS[op] + out[op] = [case.name for case in builder(torch, c, resolve_torch_overload(torch, op))] +print("NAMES " + json.dumps(out)) +""" + + +def _staged_artefact(): + if os.environ.get("TORCH_C_ARTEFACT"): + return os.environ["TORCH_C_ARTEFACT"] + for entry in os.environ.get("PYTHONPATH", "").split(os.pathsep): + cand = pathlib.Path(entry) / "_C.abi3.so" + if entry and cand.is_file(): + return str(cand) + return None + + +def _case_names(): + # A subprocess without PYTHONPATH: the gate's PYTHONPATH puts the vendored + # shim `torch` first, and the builders need upstream torch, as compare.py + # itself runs. + env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"} + art = _staged_artefact() + if art: + env["TORCH_C_ARTEFACT"] = art + proc = subprocess.run( + [sys.executable, "-c", _NAMES_SCRIPT, str(HERE)], + capture_output=True, text=True, env=env, timeout=600, + ) + lines = [l for l in proc.stdout.splitlines() if l.startswith("NAMES ")] + assert proc.returncode == 0 and lines, ( + f"could not build the golden case names (exit {proc.returncode}): {proc.stderr[-800:]}" + ) + return json.loads(lines[0][len("NAMES "):]) + + +def test_every_pin_names_a_case_cases_py_builds(): + names = _case_names() + dead = [] + total = 0 + for op, pins in compare.KNOWN_X86_64_LINUX.items(): + for name, issue, reason in pins: + total += 1 + assert issue and reason, (op, name) + if name not in names[op]: + dead.append((op, name)) + assert not dead, f"{len(dead)} pin(s) name no case cases.py builds: {dead}" + assert total == 54, f"expected the 54 pinned names of the PR #90 CI run, found {total}" + + +def test_no_pin_is_a_whole_op_or_a_duplicate(): + for op, pins in compare.KNOWN_X86_64_LINUX.items(): + names = [p[0] for p in pins] + assert len(names) == len(set(names)), f"{op}: duplicate pin" + assert op not in names and "*" not in "".join(names), f"{op}: pin is not an exact case name" + + +def _one_pin(): + op, pins = next(iter(compare.KNOWN_X86_64_LINUX.items())) + return op, pins[0][0] + + +def test_on_x86_a_failing_pinned_case_is_x86known_and_an_unpinned_one_stays_failed(): + op, name = _one_pin() + assert compare.classify(op, name, passed=False, on_x86_64_linux=True) == "X86KNOWN" + assert compare.classify(op, name + " (not pinned)", passed=False, on_x86_64_linux=True) == "FAIL" + assert compare.classify("aten.not_pinned.default", name, passed=False, on_x86_64_linux=True) == "FAIL" + assert compare.classify(op, name, passed=True, on_x86_64_linux=True) == "PASS" + assert compare.x86_pin(op, name, True)[0].startswith("#") + + +def test_off_x86_the_same_pinned_failure_stays_failed(): + op, name = _one_pin() + assert compare.classify(op, name, passed=False, on_x86_64_linux=False) == "FAIL" + assert compare.x86_pin(op, name, False) is None + + +def test_the_production_predicate_is_skips_own(): + assert compare._on_x86_64_linux() == _skip.on_x86_64_linux() + + +if __name__ == "__main__": + raise SystemExit( + _skip.run_tests( + [(n, f) for n, f in sorted(globals().items()) if n.startswith("test_") and callable(f)], + suite="test_x86pins", + ) + )