From 5a8a23a7e1babb7c4822dbe04e1e8a8a44f38edf Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Mon, 26 Oct 2026 22:05:12 +0000 Subject: [PATCH 1/2] Add SM120 inference kernels and a fit+predict benchmark for KumoTabular Fused exact SM120 kernels swapped in through an Inductor post-grad FX pass, BF16 and opt-in 8-bit ICL attention, a speed benchmark, and accuracy/determinism harnesses with tests and docs. Signed-off-by: Vedaant Jain --- benchmark/attention/__init__.py | 2 + benchmark/attention/accuracy.py | 159 ++++++ benchmark/attention/compare.py | 469 ++++++++++++++++ benchmark/attention/compare_data.py | 110 ++++ benchmark/attention/datasets.py | 102 ++++ benchmark/attention/determinism.py | 72 +++ benchmark/attention/regression.py | 288 ++++++++++ benchmark/inference/README.md | 137 +++++ benchmark/inference/__init__.py | 2 + benchmark/inference/__main__.py | 43 ++ benchmark/inference/config.py | 202 +++++++ benchmark/inference/data.py | 83 +++ benchmark/inference/kernels/__init__.py | 11 + benchmark/inference/kernels/build.py | 160 ++++++ .../inference/kernels/csrc/.clang-format | 5 + .../kernels/csrc/common/bf16_wmma.cuh | 39 ++ .../kernels/csrc/common/bias_gelu.cuh | 27 + .../kernels/csrc/common/head_rms.cuh | 105 ++++ .../inference/kernels/csrc/common/rms_ptx.cuh | 30 + .../kernels/csrc/common/row_source.cuh | 35 ++ .../kernels/csrc/common/triton_gelu.cuh | 63 +++ .../kernels/csrc/gating_sm120/gating.cu | 211 +++++++ .../icl_attention_int8.cu | 530 ++++++++++++++++++ .../csrc/icl_attention_sm120/icl_attention.cu | 468 ++++++++++++++++ .../kernels/csrc/icl_qk_sm120/icl_qk.cu | 156 ++++++ .../kernels/csrc/kv_rms_sm120/kv_rms.cu | 53 ++ .../csrc/linear_gelu_sm120/linear_gelu.cu | 141 +++++ .../projection_rms_sm120/projection_rms.cu | 202 +++++++ .../kernels/csrc/query_rms_sm120/query_rms.cu | 42 ++ .../kernels/csrc/rope_rms_sm120/rope_rms.cu | 218 +++++++ .../inference/kernels/fusions/__init__.py | 20 + benchmark/inference/kernels/fusions/gating.py | 244 ++++++++ .../kernels/fusions/icl_attention.py | 289 ++++++++++ benchmark/inference/kernels/fusions/icl_qk.py | 417 ++++++++++++++ benchmark/inference/kernels/fusions/kv_rms.py | 209 +++++++ .../inference/kernels/fusions/linear_gelu.py | 153 +++++ .../kernels/fusions/projection_rms.py | 315 +++++++++++ .../inference/kernels/fusions/query_rms.py | 143 +++++ .../inference/kernels/fusions/rope_rms.py | 367 ++++++++++++ benchmark/inference/kernels/fx_pass.py | 160 ++++++ benchmark/inference/kernels/integration.py | 126 +++++ benchmark/inference/kernels/registry.py | 239 ++++++++ benchmark/inference/kernels/sm120.py | 45 ++ benchmark/inference/report.py | 224 ++++++++ benchmark/inference/runner.py | 503 +++++++++++++++++ benchmark/inference/schedule.py | 101 ++++ benchmark/inference/snapshot.py | 106 ++++ benchmark/inference/sweep.py | 358 ++++++++++++ docs/inference-kernel-interface.md | 109 ++++ docs/inference-kernel-results.md | 235 ++++++++ docs/inference-kernels.md | 195 +++++++ test/benchmark/inference/conftest.py | 175 ++++++ test/benchmark/inference/test_build.py | 44 ++ test/benchmark/inference/test_config.py | 87 +++ test/benchmark/inference/test_fusions.py | 438 +++++++++++++++ test/benchmark/inference/test_fx_pass.py | 158 ++++++ .../benchmark/inference/test_icl_attention.py | 208 +++++++ test/benchmark/inference/test_icl_qk.py | 94 ++++ test/benchmark/inference/test_measure.py | 319 +++++++++++ test/benchmark/inference/test_registry.py | 133 +++++ test/benchmark/inference/test_schedule.py | 81 +++ test/benchmark/inference/test_snapshot.py | 76 +++ 62 files changed, 10536 insertions(+) create mode 100644 benchmark/attention/__init__.py create mode 100644 benchmark/attention/accuracy.py create mode 100644 benchmark/attention/compare.py create mode 100644 benchmark/attention/compare_data.py create mode 100644 benchmark/attention/datasets.py create mode 100644 benchmark/attention/determinism.py create mode 100644 benchmark/attention/regression.py create mode 100644 benchmark/inference/README.md create mode 100644 benchmark/inference/__init__.py create mode 100644 benchmark/inference/__main__.py create mode 100644 benchmark/inference/config.py create mode 100644 benchmark/inference/data.py create mode 100644 benchmark/inference/kernels/__init__.py create mode 100644 benchmark/inference/kernels/build.py create mode 100644 benchmark/inference/kernels/csrc/.clang-format create mode 100644 benchmark/inference/kernels/csrc/common/bf16_wmma.cuh create mode 100644 benchmark/inference/kernels/csrc/common/bias_gelu.cuh create mode 100644 benchmark/inference/kernels/csrc/common/head_rms.cuh create mode 100644 benchmark/inference/kernels/csrc/common/rms_ptx.cuh create mode 100644 benchmark/inference/kernels/csrc/common/row_source.cuh create mode 100644 benchmark/inference/kernels/csrc/common/triton_gelu.cuh create mode 100644 benchmark/inference/kernels/csrc/gating_sm120/gating.cu create mode 100644 benchmark/inference/kernels/csrc/icl_attention_int8_sm120/icl_attention_int8.cu create mode 100644 benchmark/inference/kernels/csrc/icl_attention_sm120/icl_attention.cu create mode 100644 benchmark/inference/kernels/csrc/icl_qk_sm120/icl_qk.cu create mode 100644 benchmark/inference/kernels/csrc/kv_rms_sm120/kv_rms.cu create mode 100644 benchmark/inference/kernels/csrc/linear_gelu_sm120/linear_gelu.cu create mode 100644 benchmark/inference/kernels/csrc/projection_rms_sm120/projection_rms.cu create mode 100644 benchmark/inference/kernels/csrc/query_rms_sm120/query_rms.cu create mode 100644 benchmark/inference/kernels/csrc/rope_rms_sm120/rope_rms.cu create mode 100644 benchmark/inference/kernels/fusions/__init__.py create mode 100644 benchmark/inference/kernels/fusions/gating.py create mode 100644 benchmark/inference/kernels/fusions/icl_attention.py create mode 100644 benchmark/inference/kernels/fusions/icl_qk.py create mode 100644 benchmark/inference/kernels/fusions/kv_rms.py create mode 100644 benchmark/inference/kernels/fusions/linear_gelu.py create mode 100644 benchmark/inference/kernels/fusions/projection_rms.py create mode 100644 benchmark/inference/kernels/fusions/query_rms.py create mode 100644 benchmark/inference/kernels/fusions/rope_rms.py create mode 100644 benchmark/inference/kernels/fx_pass.py create mode 100644 benchmark/inference/kernels/integration.py create mode 100644 benchmark/inference/kernels/registry.py create mode 100644 benchmark/inference/kernels/sm120.py create mode 100644 benchmark/inference/report.py create mode 100644 benchmark/inference/runner.py create mode 100644 benchmark/inference/schedule.py create mode 100644 benchmark/inference/snapshot.py create mode 100644 benchmark/inference/sweep.py create mode 100644 docs/inference-kernel-interface.md create mode 100644 docs/inference-kernel-results.md create mode 100644 docs/inference-kernels.md create mode 100644 test/benchmark/inference/conftest.py create mode 100644 test/benchmark/inference/test_build.py create mode 100644 test/benchmark/inference/test_config.py create mode 100644 test/benchmark/inference/test_fusions.py create mode 100644 test/benchmark/inference/test_fx_pass.py create mode 100644 test/benchmark/inference/test_icl_attention.py create mode 100644 test/benchmark/inference/test_icl_qk.py create mode 100644 test/benchmark/inference/test_measure.py create mode 100644 test/benchmark/inference/test_registry.py create mode 100644 test/benchmark/inference/test_schedule.py create mode 100644 test/benchmark/inference/test_snapshot.py diff --git a/benchmark/attention/__init__.py b/benchmark/attention/__init__.py new file mode 100644 index 0000000000..d51c4fe1e0 --- /dev/null +++ b/benchmark/attention/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmark/attention/accuracy.py b/benchmark/attention/accuracy.py new file mode 100644 index 0000000000..2b87ea17e3 --- /dev/null +++ b/benchmark/attention/accuracy.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Classification accuracy of the ICL attention kernel against compiled SDM. + + python -m benchmark.attention.accuracy --data-dir DIR --out DIR \ + [--context-rows 10000] [--sizes large medium] [--query-rows 16 1024] \ + [--seed 0] [--datasets covertype adult ...] + +Runs ``python -m benchmark.inference --qualify-only`` (arms eager, compiled +and optimized with every kernel; no timing) per combination. The other +kernels are bitwise, so optimized differs from compiled only through ICL +attention. The environment is passed through, so ``SDM_ICL_ATTENTION=fa2`` +measures FlashAttention 2 instead. ``--data-dir`` is where +``benchmark.attention.datasets`` wrote its files. Writes ``accuracy.md`` and +``accuracy.json`` to ``--out``. +""" + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +COVERTYPE = { + "dataset": "covertype", + "rows": 581012, + "features": 54, + "classes": 7, +} + + +def catalog(directory: Path, names: list[str] | None) -> list[dict]: + path = directory / "manifest.json" + manifest = json.loads(path.read_text()) if path.is_file() else {} + entries = [{**COVERTYPE, "file": None, "target": None}] + [ + {"dataset": name, **info} + for name, info in manifest.items() + if info["task"] == "classification" + ] + return [e for e in entries if names is None or e["dataset"] in names] + + +def run( + entry: dict, size: str, queries: int, args: argparse.Namespace +) -> dict: + out = ( + args.out + / "runs" + / f"{entry['dataset']}-{size}-q{queries}-s{args.seed}" + ) + command = [ + *(sys.executable, "-m", "benchmark.inference"), + *("--model-size", size), + *("--context-rows", str(args.context_rows)), + *("--query-rows", str(queries)), + *("--arms", "eager", "compiled", "optimized"), + "--qualify-only", + *("--seed", str(args.seed)), + *("--output", str(out)), + ] + if entry["file"] is not None: + command += ["--data", str(args.data_dir / entry["file"])] + command += ["--target", entry["target"]] + subprocess.run(command, check=False) + row = { + "dataset": entry["dataset"], + "rows": entry["rows"], + "features": entry["features"], + "classes": entry["classes"], + "size": size, + "seed": args.seed, + "query_rows": queries, + "pass": False, + } + report_path = out / "report.json" + if not report_path.is_file(): + return row + report = json.loads(report_path.read_text()) + qualification = report["qualification"] + chosen = {d["kernel"] for d in report["kernels"]["decisions"]} + row["kernels"] = sorted(chosen - {"native"}) + if "optimized_vs_compiled_distance" in qualification: + compiled = qualification["compiled_vs_eager"] + optimized = qualification["optimized_vs_compiled_distance"] + row.update( + compiled_vs_eager_pmax=compiled["probability_max_abs_diff"], + compiled_vs_eager_agree=compiled["argmax_agreement_pct"], + optimized_vs_compiled_pmax=optimized["probability_max_abs_diff"], + optimized_vs_compiled_agree=optimized["argmax_agreement_pct"], + ) + row["status"] = report["status"] + row["launches"] = report["kernels"].get("icl_attention_launches") + row["pass"] = bool( + qualification["passed"] + and row["kernels"] + and report["status"] == "qualified" + ) + return row + + +COLUMNS = ( + ("dataset", "dataset"), + ("rows", "rows"), + ("features", "features"), + ("classes", "classes"), + ("size", "size"), + ("seed", "seed"), + ("query_rows", "query rows"), + ("compiled_vs_eager_pmax", "compiled vs eager max |dp|"), + ("compiled_vs_eager_agree", "compiled vs eager argmax agree %"), + ("optimized_vs_compiled_pmax", "optimized vs compiled max |dp|"), + ("optimized_vs_compiled_agree", "optimized vs compiled argmax agree %"), + ("pass", "pass"), +) + + +def markdown(rows: list[dict]) -> str: + def cell(value: object) -> str: + return f"{value:.4g}" if isinstance(value, float) else str(value) + + lines = [ + "| " + " | ".join(h for _, h in COLUMNS) + " |", + "|" + "---|" * len(COLUMNS), + ] + for row in rows: + lines.append( + "| " + " | ".join(cell(row.get(k, "")) for k, _ in COLUMNS) + " |" + ) + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--data-dir", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--context-rows", type=int, default=10000) + parser.add_argument("--sizes", nargs="+", default=["large"]) + parser.add_argument( + "--query-rows", nargs="+", type=int, default=[16, 1024] + ) + parser.add_argument("--datasets", nargs="+") + parser.add_argument("--seed", type=int, default=0) + args = parser.parse_args(argv) + args.out.mkdir(parents=True, exist_ok=True) + rows = [ + run(entry, size, queries, args) + for entry in catalog(args.data_dir, args.datasets) + for size in args.sizes + for queries in args.query_rows + ] + (args.out / "accuracy.json").write_text(json.dumps(rows, indent=2)) + text = markdown(rows) + (args.out / "accuracy.md").write_text(text) + print(text) + + +if __name__ == "__main__": + main() diff --git a/benchmark/attention/compare.py b/benchmark/attention/compare.py new file mode 100644 index 0000000000..19ff0a100b --- /dev/null +++ b/benchmark/attention/compare.py @@ -0,0 +1,469 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Task metrics of ICL attention settings against compiled SDM. + + python -m benchmark.attention.compare run --data-dir D --out R \ + --settings sm120 int8 int8+ends --contexts 50000 --queries 16 1024 \ + --seeds 0 1 2 + python -m benchmark.attention.compare tabulate --results R + +Each setting arm is compiled SDM plus only the ``icl_attention`` op under +that ``SDM_ICL_ATTENTION`` value. One JSON (and one npz of predictions) per +dataset, model size, context, query rows and seed, with the eager, compiled +and every setting arm. ``--data-dir`` is where +``benchmark.attention.compare_data`` wrote its tables. +""" + +import argparse +import gc +import json +import time +from collections import defaultdict +from contextlib import ExitStack +from pathlib import Path + +import numpy as np +import pandas as pd +import torch +from sklearn.metrics import log_loss, r2_score, roc_auc_score + +import sdm +from benchmark.inference import runner +from benchmark.inference.kernels import integration, registry + +KERNEL_OF_BACKEND = { + "sm120": "sm120_icl_attention_d64", + "fa2": "fa2_icl_attention_d64", + "int8": "sm120_int8_icl", +} +CLIP = 1e-7 +# More classes than this run ECOC codes as an ICL batch above 1, which the +# 8-bit kernel does not take (batch 1 only). +MAX_INT8_CLASSES = 10 +# LEVELS[i] is the (i + 1) / 1000 quantile. +LEVELS = (np.arange(999) + 1) / 1000 +MEDIAN = 499 +BANDS = { + "low_tail": LEVELS <= 0.05, + "median": (LEVELS >= 0.45) & (LEVELS <= 0.55), + "high_tail": LEVELS >= 0.95, +} + + +def build_tables( + df: pd.DataFrame, target: str, regression: bool, device: str, context: int +) -> tuple[sdm.TableTensor, sdm.TableTensor, sdm.TableTensor]: + overrides = {} if regression else {target: "categorical"} + stypes = sdm.infer_stypes(df, overrides=overrides) + table = sdm.TableTensor.from_pandas(df, stypes=stypes) + features, labels = table.drop_columns(target), table[:, target] + return ( + features[:context].to(device), + labels[:context].to(device), + features[context:].to(device), + ) + + +class Arms: + def __init__(self, model, task, settings, device): + self.model, self.device = model, device + self.expected = { + s: KERNEL_OF_BACKEND.get(registry.parse_attention(s)[0]) + for s in settings + } + self.inner = model.models[task] + self.forwards = { + "compiled": torch.compile( + self.inner.forward, **integration.COMPILE + ) + } + for setting in settings: + self.forwards[setting] = integration.compile_with_kernels( + self.inner.forward, ops=("icl_attention",), attention=setting + ) + + def predict(self, arm, tables, seed): + x, y, query = tables + self.model.clear() + generator = torch.Generator(self.device).manual_seed(seed) + with ExitStack() as stack: + if arm != "eager": + self.inner.forward = self.forwards[arm] + stack.callback(vars(self.inner).pop, "forward") + stack.enter_context(torch.inference_mode()) + stack.enter_context(torch.autocast("cuda", dtype=torch.bfloat16)) + gc.collect() + torch.cuda.synchronize(self.device) + before = dict(registry.census()["launches"]) + start = time.perf_counter() + self.model.fit(x=x, y=y, num_estimators=1, generator=generator) + result = self.model.predict(query) + torch.cuda.synchronize(self.device) + seconds = time.perf_counter() - start + after = registry.census()["launches"] + launches = { + k: v - before.get(k, 0) + for k, v in after.items() + if v != before.get(k, 0) + } + expected = self.expected.get(arm) + if set(launches) != ({expected} if expected else set()): + raise RuntimeError( + f"arm {arm!r} launched {launches}, expected {expected}" + ) + out = result.numerical.detach().float().cpu().numpy() + names = result.columns[sdm.Stype.numerical] + self.model.clear() + return out, names, seconds, launches + + +def classification_metrics(prob, classes, truth, num_classes): + full = np.zeros((len(prob), num_classes)) + full[:, classes] = prob + full /= full.sum(1, keepdims=True) + full = np.clip(full, CLIP, 1) + full /= full.sum(1, keepdims=True) + labels = np.arange(num_classes) + metrics = { + "accuracy": float((full.argmax(1) == truth).mean()), + "log_loss": float(log_loss(truth, full, labels=labels)), + "auc": None, + } + present = np.unique(truth) + if num_classes == 2 and len(present) == 2: + metrics["auc"] = float(roc_auc_score(truth, full[:, 1])) + elif num_classes > 2 and len(present) == num_classes: + metrics["auc"] = float( + roc_auc_score( + truth, full, multi_class="ovr", average="macro", labels=labels + ) + ) + return metrics + + +def regression_metrics(quantiles, truth, scale): + quantiles = np.sort(quantiles.astype(np.float64), axis=1) + mean, median = quantiles.mean(1), quantiles[:, MEDIAN] + err = truth[:, None] - quantiles + pinball = np.maximum(LEVELS * err, (LEVELS - 1) * err) + metrics = { + "rmse": float(np.sqrt(((mean - truth) ** 2).mean())), + "r2": float(r2_score(truth, mean)), + "rmse_median": float(np.sqrt(((median - truth) ** 2).mean())), + "pinball": float(pinball.mean()), + "pinball_q05": float(pinball[:, 49].mean()), + "pinball_q50": float(pinball[:, MEDIAN].mean()), + "pinball_q95": float(pinball[:, 949].mean()), + } + for name, lo, hi in (("80", 99, 899), ("90", 49, 949)): + inside = (truth >= quantiles[:, lo]) & (truth <= quantiles[:, hi]) + metrics[f"coverage_{name}"] = float(inside.mean()) + metrics[f"width_{name}"] = float( + (quantiles[:, hi] - quantiles[:, lo]).mean() / scale + ) + return metrics + + +def distance(reference, candidate, regression, scale): + ref, cand = reference.astype(np.float64), candidate.astype(np.float64) + diff = np.abs(ref - cand) + if regression: + diff = diff / scale + out = { + "max_abs_diff": float(diff.max()), + "mean_abs_diff": float(diff.mean()), + } + for band, mask in BANDS.items(): + out[f"{band}_max"] = float(diff[:, mask].max()) + out[f"{band}_mean"] = float(diff[:, mask].mean()) + point = np.abs(ref.mean(1) - cand.mean(1)) / scale + out.update( + point_max=float(point.max()), point_mean=float(point.mean()) + ) + return out + flips = int((ref.argmax(1) != cand.argmax(1)).sum()) + return { + "max_abs_diff": float(diff.max()), + "mean_abs_diff": float(diff.mean()), + "flips": flips, + "flip_rate": flips / len(ref), + } + + +def run_group(args, name, info, df, size, context, model, task): + regression = info["task"] == "regression" + settings = [ + s + for s in args.settings + if (info["classes"] or 0) <= MAX_INT8_CLASSES + or registry.parse_attention(s)[0] != "int8" + ] + arms = ["eager", "compiled", *settings] + torch._dynamo.reset() + forwards = Arms(model, task, settings, args.device) + for query_rows in args.queries: + for seed in args.seeds: + path = ( + args.out + / f"{name}_{size}_c{context}_q{query_rows}_s{seed}.json" + ) + if path.exists(): + continue + rows = np.random.default_rng(seed).permutation(len(df))[ + : context + query_rows + ] + sub = df.iloc[rows].reset_index(drop=True) + tables = build_tables( + sub, info["target"], regression, args.device, context + ) + targets = sub[info["target"]].to_numpy() + truth = targets[context:] + scale = float(targets[:context].std()) + preds, seconds, columns, launches = {}, {}, {}, {} + for arm in arms: + preds[arm], columns[arm], seconds[arm], launches[arm] = ( + forwards.predict(arm, tables, seed) + ) + record = { + "dataset": name, + "task": info["task"], + "model_size": size, + "context_rows": context, + "query_rows": query_rows, + "seed": seed, + "features": info["features"], + "classes": info["classes"], + "target_std": scale, + "seconds": seconds, + "launches": launches, + "arms": {}, + } + for arm in arms: + if regression: + metrics = regression_metrics(preds[arm], truth, scale) + else: + classes = np.array([int(c) for c in columns[arm]]) + metrics = classification_metrics( + preds[arm], classes, truth, info["classes"] + ) + entry = {"metrics": metrics} + if arm != "compiled": + entry["vs_compiled"] = distance( + preds["compiled"], preds[arm], regression, scale + ) + record["arms"][arm] = entry + path.write_text(json.dumps(record, indent=1)) + np.savez_compressed(path.with_suffix(".npz"), truth=truth, **preds) + gap = record["arms"]["eager"]["vs_compiled"]["max_abs_diff"] + times = {k: round(v, 1) for k, v in seconds.items()} + print( + f"{path.name} eager_gap={gap:.3e} seconds={times}", flush=True + ) + del tables + torch.cuda.empty_cache() + + +def run(args): + args.out.mkdir(parents=True, exist_ok=True) + manifest = json.loads((args.data_dir / "manifest.json").read_text()) + torch._dynamo.config.cache_size_limit = 256 + torch._dynamo.config.accumulated_cache_size_limit = 4096 + runner.set_precision("ieee") + for name in args.datasets or manifest: + info = manifest[name] + df = pd.read_parquet(args.data_dir / info["file"]) + task = ( + sdm.Task.regression + if info["task"] == "regression" + else sdm.Task.classification + ) + for size in args.model_sizes: + model = sdm.models.KumoTabular( + task=task, size=size, device=args.device + ) + top = max(args.queries) + contexts = sorted( + {min(c, info["rows"] - top) for c in args.contexts} + ) + for context in contexts: + run_group(args, name, info, df, size, context, model, task) + del model + gc.collect() + torch.cuda.empty_cache() + + +def _cell(values, fmt): + return f"{fmt % np.mean(values)} / {fmt % max(values)}" + + +def _metric(records, arm, key): + values = [r["arms"][arm]["metrics"][key] for r in records] + if None in values: + return "-" + base = [r["arms"]["compiled"]["metrics"][key] for r in records] + return f"{np.mean(values):.4f} ({np.mean(values) - np.mean(base):+.4f})" + + +def _stat(records, arm, key): + return [r["arms"][arm]["vs_compiled"][key] for r in records] + + +def _table(title, head, rows): + out = [ + f"### {title}\n", + "| " + " | ".join(head) + " |", + "|" + "---|" * len(head), + ] + out.extend("| " + " | ".join(r) + " |" for r in rows) + return [*out, ""] + + +def tabulate(args): + groups = defaultdict(list) + for path in sorted(args.results.glob("*.json")): + record = json.loads(path.read_text()) + key = ( + record["task"], + record["model_size"], + record["context_rows"], + record["query_rows"], + record["dataset"], + ) + groups[key].append(record) + sections = defaultdict(lambda: ([], [])) + for (task, size, context, query_rows, name), records in sorted( + groups.items() + ): + for arm in records[0]["arms"]: + base = [name, arm, str(len(records))] + moved = arm != "compiled" + if task == "regression": + bands = ["low_tail", "median", "high_tail"] + metric = [ + _metric(records, arm, k) + for k in ( + "rmse", + "r2", + "pinball", + "coverage_80", + "width_80", + "coverage_90", + "width_90", + ) + ] + move = ( + [ + _cell(_stat(records, arm, f"{b}_{m}"), "%.4f") + for b in bands + for m in ("max", "mean") + ] + + [_cell(_stat(records, arm, "point_max"), "%.4f")] + if moved + else ["-"] * 7 + ) + else: + metric = [ + _metric(records, arm, k) + for k in ("accuracy", "auc", "log_loss") + ] + move = ( + [ + _cell(_stat(records, arm, "max_abs_diff"), "%.4f"), + _cell(_stat(records, arm, "mean_abs_diff"), "%.5f"), + _cell(_stat(records, arm, "flips"), "%.1f"), + "%.2f%%" + % (100 * np.mean(_stat(records, arm, "flip_rate"))), + ] + if moved + else ["-"] * 4 + ) + sections[(task, size, context, query_rows)][0].append( + base + metric + ) + sections[(task, size, context, query_rows)][1].append(base + move) + out = [] + for (task, size, context, query_rows), ( + metric_rows, + move_rows, + ) in sections.items(): + label = f"{task}, {size}, context {context}, Q{query_rows}" + head = ["dataset", "arm", "seeds"] + if task == "regression": + out += _table( + f"{label}: task metrics, mean over seeds (change vs compiled)", + [ + *head, + "RMSE", + "R2", + "pinball", + "coverage 80", + "width 80/std", + "coverage 90", + "width 90/std", + ], + metric_rows, + ) + out += _table( + f"{label}: quantile movement vs compiled, |d|/target std, " + "mean / worst over seeds", + [ + *head, + "low tail max", + "low tail mean", + "median max", + "median mean", + "high tail max", + "high tail mean", + "mean-pred max", + ], + move_rows, + ) + else: + out += _table( + f"{label}: task metrics, mean over seeds (change vs compiled)", + [*head, "accuracy", "AUC", "log-loss"], + metric_rows, + ) + out += _table( + f"{label}: probability movement vs compiled, " + "mean / worst over seeds", + [*head, "max |dp|", "mean |dp|", "flips", "flip rate"], + move_rows, + ) + text = "\n".join(out) + print(text) + if args.output: + args.output.write_text(text) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + r = sub.add_parser("run") + r.add_argument("--data-dir", type=Path, required=True) + r.add_argument("--out", type=Path, required=True) + r.add_argument("--datasets", nargs="+") + r.add_argument("--settings", nargs="*", default=[]) + r.add_argument( + "--model-sizes", + nargs="+", + default=["large"], + choices=["small", "medium", "large"], + ) + r.add_argument("--contexts", nargs="+", type=int, default=[50000]) + r.add_argument("--queries", nargs="+", type=int, default=[16, 1024]) + r.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2]) + r.add_argument("--device", default="cuda:0") + t = sub.add_parser("tabulate") + t.add_argument("--results", type=Path, required=True) + t.add_argument("--output", type=Path) + args = parser.parse_args() + if args.command == "run": + run(args) + else: + tabulate(args) + + +if __name__ == "__main__": + main() diff --git a/benchmark/attention/compare_data.py b/benchmark/attention/compare_data.py new file mode 100644 index 0000000000..fea6641ec7 --- /dev/null +++ b/benchmark/attention/compare_data.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Download the tables of ``benchmark.attention.compare`` as parquet files. + + python -m benchmark.attention.compare_data --out DIR [--names adult] + +Each table gets a ``target`` column: integer class codes, or the numeric +response for the regression tables, which are those of +``benchmark.attention.datasets``. ``manifest.json`` records rows, features +and classes. +""" + +import argparse +import json +from pathlib import Path + +import pandas as pd +from sklearn.datasets import fetch_covtype, fetch_openml + +from benchmark.attention.datasets import ( + REGRESSION, + categoricals_as_objects, + regression_table, +) + +TARGET = "target" +CLASSIFICATION = { + "adult": 1590, + "higgs": 23512, + "volkert": 41166, + "letter": 6, + "bank_marketing": 1461, +} + + +def _covertype() -> pd.DataFrame: + data = fetch_covtype(as_frame=True) + df = data.data.copy() + for column in df.columns[10:]: + df[column] = df[column].astype(int).astype(str) + df[TARGET] = data.target.to_numpy().astype(int) - 1 + return df + + +def _classification(name: str) -> pd.DataFrame: + data = fetch_openml( + data_id=CLASSIFICATION[name], + as_frame=True, + parser="auto", + target_column="default-target", + ) + df = categoricals_as_objects(data.data) + df[TARGET] = pd.factorize(data.target.astype(str), sort=True)[0] + return df + + +def build(name: str) -> pd.DataFrame: + if name == "covertype": + return _covertype() + if name in REGRESSION: + return regression_table(name) + return _classification(name) + + +def describe(name: str, df: pd.DataFrame) -> dict: + features = df.drop(columns=TARGET) + categorical = [ + c + for c in features.columns + if not pd.api.types.is_numeric_dtype(features[c]) + ] + task = "regression" if name in REGRESSION else "classification" + return { + "rows": len(df), + "features": features.shape[1], + "categorical_features": len(categorical), + "task": task, + "classes": int(df[TARGET].nunique()) + if task == "classification" + else None, + "target": TARGET, + "file": f"{name}.parquet", + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument( + "--names", + nargs="+", + default=["covertype", *CLASSIFICATION, *REGRESSION], + ) + args = parser.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + manifest_path = args.out / "manifest.json" + manifest = ( + json.loads(manifest_path.read_text()) if manifest_path.exists() else {} + ) + for name in args.names: + df = build(name) + df.to_parquet(args.out / f"{name}.parquet", index=False) + manifest[name] = describe(name, df) + print(name, manifest[name], flush=True) + manifest_path.write_text(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/attention/datasets.py b/benchmark/attention/datasets.py new file mode 100644 index 0000000000..607b187e50 --- /dev/null +++ b/benchmark/attention/datasets.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Download the ICL attention accuracy-suite tables as parquet files. + + python -m benchmark.attention.datasets DIR [--names adult diamonds ...] + +Classification tables keep their OpenML columns, with the target last as a +string column ``class``; Covertype is the benchmark's built-in dataset and is +not written. Regression tables are TabArena v0.1 regression tasks with a +numeric ``target`` column. ``DIR/manifest.json`` records each table's task, +file, target column, rows, features and classes. OpenML data is cached under +``$SCIKIT_LEARN_DATA``. +""" + +import argparse +import json +from pathlib import Path + +import pandas as pd +from sklearn.datasets import fetch_openml + +CLASSIFICATION = { + "adult": 1590, + "letter": 6, + "electricity": 151, + "magic_telescope": 1120, +} +REGRESSION = { + "diamonds": (46923, "price"), + "physiochemical_protein": (46949, "ResidualSize"), + "superconductivity": (46961, "critical_temp"), + "houses": (46934, "LnMedianHouseValue"), + "food_delivery_time": (46928, "Time_taken(min)"), + "qsar_tid_11": (46953, "MEDIAN_PXC50"), +} + + +def categoricals_as_objects(df: pd.DataFrame) -> pd.DataFrame: + """Copy of ``df`` with categorical columns as objects, missing as None.""" + df = df.copy() + for column in df.columns: + if isinstance(df[column].dtype, pd.CategoricalDtype): + values = df[column].astype(object) + df[column] = values.where(df[column].notna(), None) + return df + + +def _classification(name: str) -> pd.DataFrame: + frame = fetch_openml(data_id=CLASSIFICATION[name], as_frame=True).frame + frame = frame.rename(columns={frame.columns[-1]: "class"}) + frame["class"] = frame["class"].astype(str) + return frame + + +def regression_table(name: str) -> pd.DataFrame: + """Features of a ``REGRESSION`` table and its response as ``target``.""" + data_id, column = REGRESSION[name] + data = fetch_openml( + data_id=data_id, as_frame=True, parser="auto", target_column=column + ) + df = categoricals_as_objects(data.data) + df["target"] = data.target.astype(float).to_numpy() + return df + + +def fetch(name: str, directory: Path) -> dict: + regression = name in REGRESSION + if regression: + df, target = regression_table(name), "target" + else: + df, target = _classification(name), "class" + path = directory / f"{name}.parquet" + df.to_parquet(path, index=not regression) + return { + "task": "regression" if regression else "classification", + "file": path.name, + "target": target, + "rows": len(df), + "features": df.shape[1] - 1, + "classes": None if regression else int(df[target].nunique()), + } + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + parser.add_argument( + "--names", nargs="+", default=[*CLASSIFICATION, *REGRESSION] + ) + args = parser.parse_args(argv) + args.directory.mkdir(parents=True, exist_ok=True) + path = args.directory / "manifest.json" + manifest = json.loads(path.read_text()) if path.is_file() else {} + for name in args.names: + manifest[name] = fetch(name, args.directory) + print(name, manifest[name], flush=True) + path.write_text(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/attention/determinism.py b/benchmark/attention/determinism.py new file mode 100644 index 0000000000..c32e8778dc --- /dev/null +++ b/benchmark/attention/determinism.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run-to-run determinism of fresh fit + predict, per arm. + + python -m benchmark.attention.determinism --out FILE [--size large] \ + [--rows 50000] [--arms compiled optimized] [--runs 10] \ + [--attention sm120] + +Repeats fit + predict on identical inputs and compares every snapshot (the +fitted cache and the probabilities) bitwise with the arm's first run. The +optimized arm runs only the ``icl_attention`` op, with ``--attention`` +(default ``$SDM_ICL_ATTENTION``, else ``sm120``). The kernel-level check is +``test_split_merge_is_deterministic`` in ``test/benchmark/inference``. +""" + +import argparse +import json +import os +from pathlib import Path + +from benchmark.inference.config import Config +from benchmark.inference.kernels import registry +from benchmark.inference.runner import CudaBench +from benchmark.inference.snapshot import compare + +WARMUP_CALLS = 3 + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--size", default="large") + parser.add_argument("--rows", type=int, default=50000) + parser.add_argument("--arms", nargs="+", default=["compiled", "optimized"]) + parser.add_argument("--runs", type=int, default=10) + parser.add_argument( + "--attention", default=os.environ.get(registry.ATTENTION_ENV, "sm120") + ) + args = parser.parse_args() + config = Config( + model_size=args.size, + context_rows=args.rows, + output=args.out, + arms=tuple(args.arms), + ops=("icl_attention",), + attention=args.attention, + ) + bench = CudaBench(config) + report = {} + for arm in args.arms: + for _ in range(WARMUP_CALLS): + bench.seconds(arm) + first = bench.snapshot(arm) + differing = [ + run + for run in range(1, args.runs) + if not compare(first, bench.snapshot(arm))["bitwise_equal"] + ] + report[arm] = { + "runs": args.runs, + "leaves": len(first), + "differing_runs": differing, + } + counts = f"{len(differing)}/{args.runs - 1}" + print(f"{arm:>10} {counts} runs differ", flush=True) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/benchmark/attention/regression.py b/benchmark/attention/regression.py new file mode 100644 index 0000000000..3b407e40d8 --- /dev/null +++ b/benchmark/attention/regression.py @@ -0,0 +1,288 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Regression accuracy of the ICL attention kernel against compiled SDM. + + python -m benchmark.attention.regression run --data-dir DIR --out RES \ + [--settings sm120] [--contexts 50000] [--queries 16 1024] \ + [--seeds 0 1 2] + python -m benchmark.attention.regression table --results RES --out FILE.md + +``run`` writes one JSON (and an npz of the 999 predicted quantiles) per +table, context, query rows and seed, with the eager and compiled arms and +compiled plus only the ``icl_attention`` op for each setting. Every arm is +checked with the op census and the CUDA profiler: eager and compiled launch +no ICL attention kernel, a setting arm launches its kernel once per fit call. +Differences are |delta| / context target std, over all quantiles and on their +mean (the point prediction that RMSE and R2 score). +""" + +import argparse +import gc +import json +import os +from collections import defaultdict +from contextlib import ExitStack +from pathlib import Path + +import numpy as np +import pandas as pd +import torch +from sklearn.metrics import r2_score +from torch.profiler import ProfilerActivity, profile + +import sdm +from benchmark.attention.compare import LEVELS, build_tables +from benchmark.inference import runner +from benchmark.inference.kernels import integration, registry + +EXPECTED = { + "sm120": "sm120_icl_attention_d64", + "fa2": "fa2_icl_attention_d64", +} + + +class Arms: + def __init__(self, model, settings, device): + self.model, self.device = model, device + self.expected = {"eager": None, "compiled": None} + self.expected.update({s: EXPECTED[s] for s in settings}) + self.inner = model.models[sdm.Task.regression] + self.calls = len(self.inner.icl_block.layers) - 1 + self.forwards = { + "compiled": torch.compile( + self.inner.forward, **integration.COMPILE + ) + } + for setting in settings: + self.forwards[setting] = integration.compile_with_kernels( + self.inner.forward, ops=("icl_attention",) + ) + + def predict(self, arm, tables, seed): + x, y, query = tables + os.environ[registry.ATTENTION_ENV] = arm if arm in EXPECTED else "off" + self.model.clear() + registry.reset_census() + generator = torch.Generator(self.device).manual_seed(seed) + with ExitStack() as stack: + if arm != "eager": + self.inner.forward = self.forwards[arm] + stack.callback(vars(self.inner).pop, "forward") + stack.enter_context(torch.inference_mode()) + stack.enter_context(torch.autocast("cuda", dtype=torch.bfloat16)) + prof = stack.enter_context( + profile(activities=[ProfilerActivity.CUDA]) + ) + self.model.fit(x=x, y=y, num_estimators=1, generator=generator) + result = self.model.predict(query) + torch.cuda.synchronize(self.device) + out = result.numerical.detach().float().cpu().numpy() + self.model.clear() + launches = runner.icl_attention_launches( + registry.census()["launches"], + runner.kernel_counts(prof), + self.expected[arm], + ) + runner.verify_icl_attention(launches, self.expected[arm], self.calls) + return out, launches + + +def metrics(quantiles, truth): + quantiles = np.sort(quantiles.astype(np.float64), axis=1) + mean = quantiles.mean(1) + err = truth[:, None] - quantiles + pinball = np.maximum(LEVELS * err, (LEVELS - 1) * err) + out = { + "rmse": float(np.sqrt(((mean - truth) ** 2).mean())), + "r2": float(r2_score(truth, mean)), + "pinball": float(pinball.mean()), + } + for name, lo, hi in (("80", 99, 899), ("90", 49, 949)): + inside = (truth >= quantiles[:, lo]) & (truth <= quantiles[:, hi]) + out[f"coverage_{name}"] = float(inside.mean()) + return out + + +def distance(reference, candidate, scale): + ref, cand = reference.astype(np.float64), candidate.astype(np.float64) + diff = np.abs(ref - cand) / scale + point = np.abs(ref.mean(1) - cand.mean(1)) / scale + return { + "max_abs_diff": float(diff.max()), + "mean_abs_diff": float(diff.mean()), + "p999_abs_diff": float(np.quantile(diff, 0.999)), + "point_max": float(point.max()), + "point_mean": float(point.mean()), + } + + +def run(args): + args.out.mkdir(parents=True, exist_ok=True) + manifest = json.loads((args.data_dir / "manifest.json").read_text()) + names = args.datasets or [ + n for n, i in manifest.items() if i["task"] == "regression" + ] + integration.isolate_compiles() + runner.set_precision("ieee") + arms = ["eager", "compiled", *args.settings] + for name in names: + info = manifest[name] + df = pd.read_parquet(args.data_dir / info["file"]) + model = sdm.models.KumoTabular( + task=sdm.Task.regression, size=args.model_size, device=args.device + ) + top = max(args.queries) + for context in sorted( + {min(c, info["rows"] - top) for c in args.contexts} + ): + torch._dynamo.reset() + forwards = Arms(model, args.settings, args.device) + for query_rows in args.queries: + for seed in args.seeds: + path = ( + args.out + / f"{name}_c{context}_q{query_rows}_s{seed}.json" + ) + if path.exists(): + continue + rows = np.random.default_rng(seed).permutation(len(df)) + sub = df.iloc[rows[: context + query_rows]] + sub = sub.reset_index(drop=True) + tables = build_tables( + sub, info["target"], True, args.device, context + ) + targets = sub[info["target"]].to_numpy() + truth, scale = ( + targets[context:], + float(targets[:context].std()), + ) + preds, launches = {}, {} + for arm in arms: + preds[arm], launches[arm] = forwards.predict( + arm, tables, seed + ) + record = { + "dataset": name, + "model_size": args.model_size, + "context_rows": context, + "query_rows": query_rows, + "seed": seed, + "target_std": scale, + "icl_attention_launches": launches, + "arms": { + arm: { + "metrics": metrics(preds[arm], truth), + **( + {} + if arm == "compiled" + else { + "vs_compiled": distance( + preds["compiled"], + preds[arm], + scale, + ) + } + ), + } + for arm in arms + }, + } + path.write_text(json.dumps(record, indent=1)) + np.savez_compressed( + path.with_suffix(".npz"), truth=truth, **preds + ) + print(path.name, flush=True) + del tables + torch.cuda.empty_cache() + del model + gc.collect() + torch.cuda.empty_cache() + + +def _pair(records, key, setting): + return " / ".join( + f"{max(r['arms'][a]['vs_compiled'][key] for r in records):.3f}" + for a in (setting, "eager") + ) + + +def _metric(records, key, setting): + return " / ".join( + f"{np.mean([r['arms'][a]['metrics'][key] for r in records]):.4g}" + for a in ("eager", "compiled", setting) + ) + + +def table(args): + groups = defaultdict(list) + for p in sorted(args.results.glob("*.json")): + r = json.loads(p.read_text()) + groups[(r["dataset"], r["query_rows"])].append(r) + setting = args.setting + head = [ + "dataset", + "query rows", + "seeds", + "runs with kernel diff above eager", + "max quantile diff, kernel / eager", + "99.9th pct quantile diff, kernel / eager", + "max mean-prediction diff, kernel / eager", + "RMSE eager / compiled / kernel", + "R2 eager / compiled / kernel", + ] + lines = ["| " + " | ".join(head) + " |", "|" + "---|" * len(head)] + for (name, q), rs in sorted(groups.items()): + over = sum( + r["arms"][setting]["vs_compiled"]["max_abs_diff"] + > r["arms"]["eager"]["vs_compiled"]["max_abs_diff"] + for r in rs + ) + cells = [ + name, + q, + len(rs), + f"{over} of {len(rs)}", + _pair(rs, "max_abs_diff", setting), + _pair(rs, "p999_abs_diff", setting), + _pair(rs, "point_max", setting), + _metric(rs, "rmse", setting), + _metric(rs, "r2", setting), + ] + lines.append("| " + " | ".join(map(str, cells)) + " |") + text = "\n".join(lines) + "\n" + print(text) + if args.out: + args.out.write_text(text) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + r = sub.add_parser("run") + r.add_argument("--data-dir", type=Path, required=True) + r.add_argument("--out", type=Path, required=True) + r.add_argument("--datasets", nargs="+") + r.add_argument( + "--settings", nargs="+", default=["sm120"], choices=sorted(EXPECTED) + ) + r.add_argument( + "--model-size", default="large", choices=["small", "medium", "large"] + ) + r.add_argument("--contexts", nargs="+", type=int, default=[50000]) + r.add_argument("--queries", nargs="+", type=int, default=[16, 1024]) + r.add_argument("--seeds", nargs="+", type=int, default=[0, 1, 2]) + r.add_argument("--device", default="cuda:0") + t = sub.add_parser("table") + t.add_argument("--results", type=Path, required=True) + t.add_argument("--setting", default="sm120", choices=sorted(EXPECTED)) + t.add_argument("--out", type=Path) + args = parser.parse_args() + if args.command == "run": + run(args) + else: + table(args) + + +if __name__ == "__main__": + main() diff --git a/benchmark/inference/README.md b/benchmark/inference/README.md new file mode 100644 index 0000000000..e55e7894e9 --- /dev/null +++ b/benchmark/inference/README.md @@ -0,0 +1,137 @@ +# KumoTabular inference latency + +Measures the latency of `fit(context, labels)` + `predict(query_rows)` with a fresh context on a KumoTabular classifier, on one CUDA GPU. + +## Prerequisites + +- An NVIDIA GPU with a CUDA build of PyTorch and Triton, and SDM installed from this checkout, plus `scikit-learn`, `pandas` and `ninja`: + + ```bash + python -m venv .venv && . .venv/bin/activate + pip install torch==2.14.1 --index-url https://download.pytorch.org/whl/cu130 + pip install -e . scikit-learn pandas ninja + ``` + + Results were measured with torch 2.14.1+cu130. Compiled code, and so the timings and the bitwise status of a run, depend on the torch version. + +- For the `optimized` arm: an SM 12.0 (Blackwell) GPU, `ninja`, a CUDA toolkit whose `nvcc` matches torch's CUDA version (set `CUDA_HOME`, e.g. `/usr/local/cuda-13.0`, if it is not the first `nvcc` on `PATH`), and `git` to fetch CUTLASS at a pinned commit (or set `CUTLASS_ROOT` to a checkout). The first optimized run compiles the kernels in `benchmark/inference/kernels/csrc/`, which takes several minutes; later runs reuse them. + +- Run from the repository root. scikit-learn downloads Covertype on first use. Without `--checkpoint`, the pretrained weights are downloaded from Hugging Face. + +## Run + +```bash +python -m benchmark.inference --model-size large --context-rows 50000 --output runs/large-50k +``` + +| Flag | Default | Meaning | +| -------------------- | ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--model-size` | required | `small`, `medium` or `large` | +| `--context-rows` | required | context rows per call | +| `--output` | required | new directory; existing directories are refused | +| `--checkpoint` | pretrained download | classifier state dict (`/classifier.pt`) | +| `--kernels` | `$SDM_KERNEL_CACHE` or `~/.cache/sdm-inference-kernels/-` | kernel module directory for `optimized`; missing or stale modules are built into it, also when a directory is given | +| `--ops` | all | limit `optimized` to these named ops, for cumulative measurements; the names are listed in [`docs/inference-kernel-interface.md`](../../docs/inference-kernel-interface.md#ops) | +| `--attention` | `$SDM_ICL_ATTENTION` or `sm120` | ICL attention kernel in `optimized`: `sm120` (BF16), `fa2` (FlashAttention 2, if installed), `off` (native), or the lossy opt-in `int8` or `int8+ends`; `sm120+uncapped` lifts the BF16 row cap, for diagnosis only | +| `--shadow` | off | run each selected kernel beside native on native's inputs and report per op whether its outputs are bitwise equal; no timing (status `shadow_bitwise` or `shadow_mismatch`) | +| `--data`, `--target` | Covertype | a `.parquet` or `.csv` table and its categorical target column | +| `--query-rows` | 16 | query rows per call | +| `--device` | `cuda:0` | CUDA device | +| `--arms` | all three | any of `eager compiled optimized`, plus `optimized:` arms (see Arms); optimized arms need `compiled` | +| `--blocks` | 12 | timing blocks; a multiple of the arm count (every order is used when the blocks are a multiple of their count) | +| `--warmup` | 3 | untimed calls per arm; the first includes compilation | +| `--precision` | `ieee` | `ieee` disables TF32 and reduced-precision BF16 reductions; `default` leaves PyTorch defaults | +| `--seed` | 0 | seed of the row permutation and of the per-call CUDA generator | +| `--allow-mismatch` | off | time `optimized` even if it fails its gate against `compiled` | +| `--qualify-only` | off | warm up and qualify the arms, then stop before timing (status `qualified`) | + +Rows come from a seeded permutation: the context is its first rows and the query the rows right after. + +The exit code is 0 when the run completed, 2 when it completed without `optimized` (skipped on a non-SM120 GPU, or when no kernel admits any region), and 1 otherwise (qualification failed, completed with an allowed mismatch, or a kernel build error). + +## Arms + +- `eager`: SDM as is, no compilation. +- `compiled`: SDM with the classifier's `forward` replaced by `torch.compile(forward, fullgraph=False, dynamic=False, options={"triton.cudagraphs": False})` for every call in fit and predict. +- `optimized`: `compiled` plus an Inductor post-grad pass that routes matched regions to the registered kernels (see [`docs/inference-kernel-interface.md`](../../docs/inference-kernel-interface.md)). SDM source is unchanged. By default these are the ops that are bitwise equal to compiled SDM plus the ICL attention kernel of `--attention`; `--ops` limits the pass to some named ops, and `--arms eager compiled` turns the kernels off. +- `optimized:`: like `optimized`, with its own ICL attention setting, compiled separately. Several of them are timed in one run, for example `--arms compiled optimized:off optimized:sm120 optimized:int8`. + +Typical comparisons, as separate runs or as arms of one run (`compiled` is in every run): + +| Optimized arm | Flags | Gate | +| ----------------------------------- | ----------------------------------- | --------- | +| exact kernels only | `--attention off` | bitwise | +| exact kernels + BF16 ICL attention | (default) | tolerance | +| exact kernels + 8-bit ICL attention | `--attention int8` (or `int8+ends`) | opt-in | +| BF16 ICL attention only | `--ops icl_attention` | tolerance | + +Regions no kernel admits (other shapes, other model sizes) stay native. On a non-Blackwell GPU, or when no kernel admits any region, the optimized arm is skipped with a warning, the native arms are still measured, and the reason is in `summary.md` and `report.json` (`optimized_unavailable`). A failed kernel build is an error. + +To use the kernels outside the benchmark, compile an SDM model with them in one call: + +```python +from benchmark.inference import kernels + +kernels.enable(model) # torch.compile every task model with the kernels on +... +kernels.disable(model) +``` + +`kernels.enable(model, exact=True)` keeps only the kernels that are bitwise equal to compiled SDM end to end (ICL attention stays native). `kernels.enable(model, attention="int8")` (or `SDM_ICL_ATTENTION=int8`) opts in to 8-bit ICL attention. + +## Supported scope + +- **Contexts.** Compiled SDM runs any context: SDPA splits flash calls above 65,535 batch entries and attention and cell embedding chunk under `torch.compile`. The 100k runs here used `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`. +- **What runs where.** Chunking splits the row blocks into context chunks and the column blocks into feature chunks once the context is above roughly 7,000 rows. The row-local kernels and the GEMM epilogues match every chunk; the column table kernels do not take a feature range, so at large contexts those regions stay native. See `docs/inference-kernels.md` (Coverage and limits). +- **Wider tables.** The default recipe allows up to 500 features. Beyond a kernel's checked size, those regions run native and the gain shrinks. +- **How the bounds were set.** Each exact kernel's `supports` admits rows up to the largest size checked bitwise against compiled native. `--shadow` re-checks every kernel on the real model. +- **ICL attention is not bitwise.** The BF16 `icl_attention` kernel is on by default and switches the gate to the tolerance rule below; `--attention off` (or `SDM_ICL_ATTENTION=off`) gates the run bitwise. Above 65,535 context rows, the largest size its accuracy was checked at, it stays native. 8-bit attention (`--attention int8`) is lossy, opt-in and has no row cap. + +## What is timed + +Outside the clock, before each call: `model.clear()`, `gc.collect()` and a fresh, seeded CUDA generator; Python garbage collection is disabled during the timed call (same for every arm). Inputs are already on the GPU. Inside the clock: CUDA synchronize, then `fit` + `predict` under BF16 autocast and `torch.inference_mode()`, then CUDA synchronize. Compilation and warmup are reported separately and never timed; a recompile during timing raises an error instead of being timed. Compiled code depends on the environment (torch, Triton, CUDA, GPU) and on Inductor's cache: a run that reuses cached compiled code (for example a repeat on the same machine) does not recompile, and a different environment can compile differently. The reported initial and repeat runs shared one Inductor cache. Arms are interleaved: each block runs an order of the arms followed by its reverse. When the blocks are a multiple of the number of permutations every permutation is used equally often; otherwise the orders are the rows of a Williams Latin square (each arm once in each position). The peak allocated and reserved GPU memory of each arm over its timed calls is in `report.json` (`peak_memory`) and `summary.md`. Every sample is kept. + +## Correctness policy + +Before timing, each arm runs once from identical state. The run captures the predicted probabilities and every tensor reachable through the fitted cache's dicts, lists and dataclasses. The preprocessing recipe object in the cache runs outside the compiled model and is not compared. + +- `optimized` vs `compiled` must be bitwise equal on every tensor when every selected kernel is bitwise end to end (the default ops with `--attention off`). If they differ, nothing is timed (unless `--allow-mismatch` is passed), and the report names the first differing tensor. +- When a selected kernel is not bitwise (BF16 `icl_attention`), the gate becomes a tolerance gate: the optimized-vs-compiled maximum probability difference must not exceed the compiled-vs-eager one from the same run, and the predicted classes must all agree. Eager is run once for this, untimed when it is not an arm (`qualification.eager_timed`). `report.json` records which gate ran (`qualification.gate`). +- With a lossy opt-in kernel (`--attention int8` or `int8+ends`) the gate is `opt-in`: the distance to compiled (and compiled vs eager, with the eager arm) is recorded, the run is timed, and `summary.md` says it is lossy. +- `--shadow` is a per-kernel check: each kernel runs beside native on native's inputs and its outputs are compared bit for bit (`report.json`, `kernels.shadow`; status `shadow_bitwise` when all are equal). +- With `--qualify-only`, every arm is also profiled once and must launch the expected ICL attention kernel exactly once per fit call (none for eager and compiled; two fewer with `+ends`), or the status is `launch_verification_failed`; the counts are in `report.json` (`kernels.icl_attention_launches`). +- `compiled` vs `eager` is characterized, not gated: probability differences, argmax agreement, and the maximum difference per cache tensor. + +## Outputs + +- `samples.csv`: every timed call (`block`, `position`, `arm`, `seconds`). +- `report.json`: configuration, environment (SDM and benchmark git revisions, versions, GPU, precision flags, compile options), warmup times, qualification, statistics, and the kernel report. +- `summary.md`: a short human summary. + +Statistics are per-arm mean/median/min/stdev. Each arm pair also gets the reduction in mean latency (in %), the per-block paired effects, the median block effect, and the number of blocks where the candidate was faster. + +The PyTorch profiler prints `USDT ... profiler_start/stop` lines for the profiled calls; they are harmless, and no environment variable silences them. + +The kernel report has CUDA kernel names, counts and times from one profiled call per arm. It also has the registry's selection decisions and the kernel launch counts for one optimized call. `kernels_verified` checks that every selected kernel shows its `cuda_symbols` in the optimized profile and not in the compiled one. + +## Sweeps + +```bash +python -m benchmark.inference.sweep run OUT # Large, Medium, Small x 5k..100k; resumable +python -m benchmark.inference.sweep run OUT --dry-run # list cells, arms and blocks +python -m benchmark.inference.sweep table OUT # OUT/tables.md and OUT/table.csv +``` + +Each cell is one run with every arm interleaved: compiled, `optimized:off`, `optimized:sm120` (up to its 65,535-row cap), `optimized:int8`, eager at 5k and 50k, and at Large `int8+ends` (50k, 100k) and `sm120+uncapped` (100k), plus a repeat of Large 50k and 100k. Blocks are rounded up to a multiple of the cell's arm count. Runs pass `--allow-mismatch`, so a failed gate is timed and flagged in the table. A cell that runs out of memory is retried once with `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`, then recorded as OOM; failures go to `OUT/failures.log`. Each attempt keeps its benchmark output, log and an `nvidia-smi` sample (`smi.csv`); the table reports median power and SM clock over the timed calls. Arguments after `--` go to every run. + +## Building kernels separately + +```bash +python -m benchmark.inference.kernels.build --out DIR [--cutlass PATH] +``` + +Builds every module in `kernels/build.py` into `DIR/.so`; modules whose sources and flags are unchanged are skipped. + +## Adding a kernel + +See [`docs/inference-kernel-interface.md`](../../docs/inference-kernel-interface.md). diff --git a/benchmark/inference/__init__.py b/benchmark/inference/__init__.py new file mode 100644 index 0000000000..d51c4fe1e0 --- /dev/null +++ b/benchmark/inference/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/benchmark/inference/__main__.py b/benchmark/inference/__main__.py new file mode 100644 index 0000000000..6dcb3c10b7 --- /dev/null +++ b/benchmark/inference/__main__.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import hashlib + +from benchmark.inference.config import parse +from benchmark.inference.kernels import integration +from benchmark.inference.report import environment, summary, write +from benchmark.inference.runner import ( + CudaBench, + checkpoint_path, + measure, + precision_flags, +) + +EXIT_CODES = { + "completed": 0, + "qualified": 0, + "shadow_bitwise": 0, + "completed_without_optimized": 2, +} + + +def main(argv: list[str] | None = None) -> int: + config = parse(argv) + bench = CudaBench(config) + report = measure(config, bench) + with checkpoint_path(config).open("rb") as checkpoint: + digest = hashlib.file_digest(checkpoint, "sha256").hexdigest() + report["environment"] = { + **environment(bench.device), + "checkpoint_sha256": digest, + "precision": precision_flags(), + "compile": integration.COMPILE, + } + write(config.output, report) + print(summary(report)) + print(f"Wrote {config.output}") + return EXIT_CODES.get(report["status"], 1) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/inference/config.py b/benchmark/inference/config.py new file mode 100644 index 0000000000..6f461406a0 --- /dev/null +++ b/benchmark/inference/config.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import os +from dataclasses import asdict, dataclass +from pathlib import Path + +from benchmark.inference.kernels import registry +from benchmark.inference.schedule import is_optimized, orders + +ARMS = ("eager", "compiled", "optimized") + + +@dataclass(frozen=True) +class Config: + model_size: str + context_rows: int + output: Path + checkpoint: Path | None = None + kernels: Path | None = None + ops: tuple[str, ...] | None = None + attention: str = "sm120" + data: Path | None = None + target: str | None = None + query_rows: int = 16 + device: str = "cuda:0" + arms: tuple[str, ...] = ARMS + blocks: int = 12 + warmup: int = 3 + precision: str = "ieee" + seed: int = 0 + allow_mismatch: bool = False + qualify_only: bool = False + shadow: bool = False + + def arm_attention(self, arm: str) -> str: + return arm_attention(arm, self.attention) + + @property + def dataset(self) -> str: + return "covertype" if self.data is None else self.data.name + + def record(self) -> dict: + record = asdict(self) + # Only file names: local directory layouts are not part of a result. + for key in ("output", "checkpoint", "kernels", "data"): + path = record[key] + record[key] = None if path is None else path.name + record["dataset"] = self.dataset + return record + + +def arm_attention(arm: str, default: str) -> str: + """ICL attention setting of ``optimized`` or ``optimized:``.""" + if arm == "optimized": + return default + _, setting, *rest = arm.split(":") + if rest: + raise ValueError(f"arm {arm!r}: expected optimized:") + registry.parse_attention(setting) + return setting + + +def _arm(text: str) -> str: + if text in ARMS: + return text + if not is_optimized(text): + raise argparse.ArgumentTypeError( + f"arm must be one of {ARMS} or optimized:, got {text}" + ) + try: + arm_attention(text, "sm120") + except ValueError as error: + raise argparse.ArgumentTypeError(str(error)) from error + return text + + +def _positive(text: str) -> int: + value = int(text) + if value <= 0: + raise argparse.ArgumentTypeError( + f"expected a positive integer, got {text}" + ) + return value + + +def parse(argv: list[str] | None = None) -> Config: + parser = argparse.ArgumentParser( + prog="python -m benchmark.inference", + description="Latency of fresh-context KumoTabular fit + predict.", + ) + parser.add_argument( + "--model-size", + required=True, + choices=("small", "medium", "large"), + ) + parser.add_argument("--context-rows", required=True, type=_positive) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument( + "--checkpoint", + type=Path, + help="classifier state dict; default downloads the pretrained one", + ) + parser.add_argument( + "--kernels", + type=Path, + help="kernel module directory for the optimized arms; missing " + "modules are built into it (default: a cache directory)", + ) + parser.add_argument( + "--ops", + nargs="+", + help="limit the optimized arm to these named ops (default: all)", + ) + parser.add_argument( + "--attention", + default=os.environ.get(registry.ATTENTION_ENV, "sm120"), + help="ICL attention kernel of the optimized arm: sm120 (BF16), fa2, " + "off (native, bitwise), or the lossy opt-in int8[+ends] " + "(default: $SDM_ICL_ATTENTION, else sm120)", + ) + parser.add_argument( + "--data", + type=Path, + help=".parquet or .csv table; default is Covertype", + ) + parser.add_argument("--target", help="target column of --data") + parser.add_argument("--query-rows", type=_positive, default=16) + parser.add_argument("--device", default="cuda:0") + parser.add_argument( + "--arms", + nargs="+", + type=_arm, + default=ARMS, + help="eager, compiled, optimized, and optimized: arms, " + "each compiled with its own ICL attention setting", + ) + parser.add_argument("--blocks", type=_positive, default=12) + parser.add_argument( + "--warmup", + type=_positive, + default=3, + help="untimed calls per arm; the first includes compilation", + ) + parser.add_argument( + "--precision", + choices=("ieee", "default"), + default="ieee", + help="ieee disables TF32 and reduced-precision BF16 reductions", + ) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument( + "--allow-mismatch", + action="store_true", + help="time optimized arms even if they fail their gate", + ) + parser.add_argument( + "--qualify-only", + action="store_true", + help="warm up and qualify the arms, then stop before timing", + ) + parser.add_argument( + "--shadow", + action="store_true", + help="run each selected kernel beside native on native's inputs and " + "report per op whether its outputs are bitwise equal; no timing", + ) + args = parser.parse_args(argv) + args.arms = tuple(args.arms) + args.ops = None if args.ops is None else tuple(args.ops) + + if (args.data is None) != (args.target is None): + parser.error("--data and --target must be given together") + if args.data is not None and args.data.suffix not in (".parquet", ".csv"): + parser.error("--data must be a .parquet or .csv file") + if args.checkpoint is not None and not args.checkpoint.is_file(): + parser.error(f"checkpoint {args.checkpoint} does not exist") + try: + registry.parse_attention(args.attention) + except ValueError as error: + parser.error(f"--attention: {error}") + if len(set(args.arms)) != len(args.arms): + parser.error("--arms must not repeat an arm") + if ( + any(is_optimized(arm) for arm in args.arms) + and "compiled" not in args.arms + ): + parser.error( + "--arms: an optimized arm requires compiled, which it is " + "qualified against" + ) + try: + orders(args.arms, args.blocks) + except ValueError as error: + parser.error( + f"--blocks: {error}, so every arm appears equally often in every " + "position" + ) + if args.output.exists(): + parser.error(f"output {args.output} already exists; pick a new one") + return Config(**vars(args)) diff --git a/benchmark/inference/data.py b/benchmark/inference/data.py new file mode 100644 index 0000000000..576ac66e6b --- /dev/null +++ b/benchmark/inference/data.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np +import pandas as pd +import torch +from sklearn.datasets import fetch_covtype + +import sdm +from benchmark.inference.config import Config + +Tables = tuple[sdm.TableTensor, sdm.TableTensor, sdm.TableTensor] + + +def _rows(config: Config, available: int) -> np.ndarray: + rows = config.context_rows + config.query_rows + if rows > available: + raise SystemExit( + f"{config.dataset} has {available} rows; --context-rows " + f"{config.context_rows} plus --query-rows {config.query_rows} " + "do not fit" + ) + return np.random.default_rng(config.seed).permutation(available)[:rows] + + +def _covertype(config: Config) -> tuple[sdm.TableTensor, sdm.TableTensor]: + dataset = fetch_covtype() + rows = _rows(config, len(dataset.target)) + x = torch.from_numpy(dataset.data[rows]).float() # [R, 54] + names = list(dataset.feature_names) + # 10 numerical columns, then 44 binary indicators as categoricals: + features = sdm.TableTensor( + columns={ + sdm.Stype.numerical: names[:10], + sdm.Stype.categorical: names[10:], + }, + numerical=x[:, :10].contiguous(), + categorical=sdm.CategoricalTensor( + code=x[:, 10:].long(), + categories=tuple(torch.tensor([0, 1]) for _ in range(44)), + ), + ) + labels = torch.from_numpy(dataset.target[rows]).long() + target = sdm.TableTensor( + columns={sdm.Stype.categorical: ["cover_type"]}, + categorical=sdm.CategoricalTensor( + code=(labels - 1).unsqueeze(-1), + categories=(torch.arange(1, 8),), + ), + ) + return features, target + + +def _table(config: Config) -> tuple[sdm.TableTensor, sdm.TableTensor]: + assert config.data is not None + assert config.target is not None + if config.data.suffix == ".parquet": + df = pd.read_parquet(config.data) + else: + df = pd.read_csv(config.data) + if config.target not in df.columns: + raise SystemExit(f"--target {config.target!r} is not a column") + df = df.iloc[_rows(config, len(df))].reset_index(drop=True) + stypes = sdm.infer_stypes(df, overrides={config.target: "categorical"}) + table = sdm.TableTensor.from_pandas(df, stypes=stypes) + return table.drop_columns(config.target), table[:, config.target] + + +def tables(config: Config) -> Tables: + """Context features, context labels and query features on the device. + + Rows are a fixed-seed permutation: context is its first rows, query + the rows right after. + """ + features, target = ( + _covertype(config) if config.data is None else _table(config) + ) + context, device = config.context_rows, config.device + return ( + features[:context].to(device), + target[:context].to(device), + features[context:].to(device), + ) diff --git a/benchmark/inference/kernels/__init__.py b/benchmark/inference/kernels/__init__.py new file mode 100644 index 0000000000..df980c084d --- /dev/null +++ b/benchmark/inference/kernels/__init__.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from benchmark.inference.kernels.integration import ( + Unsupported, + compile_with_kernels, + disable, + enable, +) + +__all__ = ["Unsupported", "compile_with_kernels", "disable", "enable"] diff --git a/benchmark/inference/kernels/build.py b/benchmark/inference/kernels/build.py new file mode 100644 index 0000000000..9891348150 --- /dev/null +++ b/benchmark/inference/kernels/build.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build the SM120 kernel modules from ``csrc/`` into ``/.so``. + +Headers shared by several modules live in ``csrc/common/`` and are included +as ``"common/
"``. +""" + +import argparse +import hashlib +import os +import shutil +import subprocess +from collections.abc import Iterable +from pathlib import Path + +import torch +from torch.utils import cpp_extension + +CSRC = Path(__file__).parent / "csrc" +CUTLASS_URL = "https://github.com/NVIDIA/cutlass" +CUTLASS_COMMIT = "da5e086dab31d63815acafdac9a9c5893b1c69e2" + +_GENCODE = ["-gencode=arch=compute_120,code=sm_120"] +# nvcc's defaults, pinned: the exact kernels' rounding depends on them. +_IEEE = ["--fmad=true", "--ftz=false", "--prec-div=true", "--prec-sqrt=true"] +_EXACT = (["-O3", *_GENCODE, *_IEEE], ["-O3"]) +_ATTENTION = (["-O3", *_GENCODE], ["-O3"]) + +# module -> (source file, (nvcc flags, host flags), needs CUTLASS) +MODULES = { + "linear_gelu_sm120": ("linear_gelu.cu", _EXACT, True), + "gating_sm120": ("gating.cu", _EXACT, True), + "rope_rms_sm120": ("rope_rms.cu", _EXACT, False), + "kv_rms_sm120": ("kv_rms.cu", _EXACT, True), + "query_rms_sm120": ("query_rms.cu", _EXACT, True), + "projection_rms_sm120": ("projection_rms.cu", _EXACT, True), + "icl_qk_sm120": ("icl_qk.cu", _EXACT, False), + "icl_attention_sm120": ("icl_attention.cu", _ATTENTION, False), + "icl_attention_int8_sm120": ("icl_attention_int8.cu", _EXACT, False), +} + + +def stamp(module: str) -> str: + digest = hashlib.sha256() + paths = sorted((CSRC / module).iterdir()) + if (CSRC / "common").is_dir(): + paths += sorted((CSRC / "common").iterdir()) + for path in paths: + digest.update(path.name.encode() + path.read_bytes()) + _, flags, _ = MODULES[module] + digest.update( + repr( + (flags, CUTLASS_COMMIT, torch.__version__, torch.version.cuda) + ).encode() + ) + return digest.hexdigest() + + +def check_toolchain() -> None: + cuda_home = cpp_extension.CUDA_HOME + if cuda_home is None or not (Path(cuda_home) / "bin" / "nvcc").is_file(): + raise RuntimeError( + "building kernels needs the CUDA toolkit: set CUDA_HOME to a " + f"toolkit with bin/nvcc matching torch's CUDA {torch.version.cuda}" + ) + if shutil.which("ninja") is None: + raise RuntimeError("building kernels needs ninja (pip install ninja)") + if not torch.cuda.is_available() or torch.cuda.get_device_capability() != ( + 12, + 0, + ): + raise RuntimeError("building kernels needs a visible SM 12.0 GPU") + + +def fetch_cutlass(out_dir: Path, cutlass: Path | None) -> Path: + root = cutlass or ( + Path(os.environ["CUTLASS_ROOT"]) + if "CUTLASS_ROOT" in os.environ + else None + ) + root = root or out_dir / "cutlass" + if not (root / "include" / "cutlass" / "cutlass.h").is_file(): + if cutlass is not None or "CUTLASS_ROOT" in os.environ: + raise RuntimeError(f"no CUTLASS headers under {root}/include") + root.mkdir(parents=True, exist_ok=True) + for command in ( + ["init", "-q"], + ["fetch", "-q", "--depth", "1", CUTLASS_URL, CUTLASS_COMMIT], + ["checkout", "-q", "FETCH_HEAD"], + ): + subprocess.run(["git", "-C", str(root), *command], check=True) + return root + + +def build( + out_dir: Path, + modules: Iterable[str] | None = None, + cutlass: Path | None = None, + verbose: bool = False, +) -> Path: + """Build ``modules`` (default all) into ``out_dir``; skip current ones.""" + out_dir = Path(out_dir) + modules = list(MODULES) if modules is None else list(modules) + todo = [] + for module in modules: + marker = out_dir / f"{module}.sha256" + current = (out_dir / f"{module}.so").is_file() and marker.is_file() + if not current or marker.read_text() != stamp(module): + todo.append(module) + if not todo: + return out_dir + check_toolchain() + os.environ["TORCH_CUDA_ARCH_LIST"] = "12.0" + root = ( + fetch_cutlass(out_dir, cutlass) + if any(MODULES[m][2] for m in todo) + else None + ) + for module in todo: + source, (cuda_flags, host_flags), needs_cutlass = MODULES[module] + build_dir = out_dir / "build" / module + build_dir.mkdir(parents=True, exist_ok=True) + extra = [str(CSRC / module), str(CSRC)] + ( + [str(root / "include")] if needs_cutlass else [] + ) + cpp_extension.load( + name=module, + sources=[str(CSRC / module / source)], + extra_include_paths=extra, + extra_cflags=host_flags, + extra_cuda_cflags=cuda_flags, + build_directory=str(build_dir), + is_python_module=False, + verbose=verbose, + ) + shutil.copyfile(build_dir / f"{module}.so", out_dir / f"{module}.so") + (out_dir / f"{module}.sha256").write_text(stamp(module)) + return out_dir + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser( + prog="python -m benchmark.inference.kernels.build" + ) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument( + "--cutlass", + type=Path, + help="CUTLASS checkout; default $CUTLASS_ROOT or a pinned fetch", + ) + parser.add_argument("--modules", nargs="+", choices=tuple(MODULES)) + parser.add_argument("--quiet", action="store_true") + args = parser.parse_args(argv) + build(args.out, args.modules, args.cutlass, verbose=not args.quiet) + + +if __name__ == "__main__": + main() diff --git a/benchmark/inference/kernels/csrc/.clang-format b/benchmark/inference/kernels/csrc/.clang-format new file mode 100644 index 0000000000..9cf418a401 --- /dev/null +++ b/benchmark/inference/kernels/csrc/.clang-format @@ -0,0 +1,5 @@ +BasedOnStyle: Google +ColumnLimit: 120 +DerivePointerAlignment: false +PointerAlignment: Left +SortIncludes: Never diff --git a/benchmark/inference/kernels/csrc/common/bf16_wmma.cuh b/benchmark/inference/kernels/csrc/common/bf16_wmma.cuh new file mode 100644 index 0000000000..dd393c1457 --- /dev/null +++ b/benchmark/inference/kernels/csrc/common/bf16_wmma.cuh @@ -0,0 +1,39 @@ +#pragma once +#include +#include + +// CUTLASS da5e086 has no BF16 Wmma operator (and maps BF16 types only in the device pass). +// This is the m16n16k16 BF16 mma_sync with FP32 accumulation that cuBLAS's wmma kernels use. +namespace cutlass { +namespace arch { +#if !defined(__CUDA_ARCH__) +template <> +struct CutlassToWmmaDataType { + using Type = __nv_bfloat16; +}; +template <> +struct WmmaToCutlassDataType<__nv_bfloat16> { + using Type = bfloat16_t; +}; +#endif +template <> +struct Wmma, bfloat16_t, layout::RowMajor, bfloat16_t, layout::ColumnMajor, float, + layout::RowMajor, OpMultiplyAdd> { + using Shape = gemm::GemmShape<16, 16, 16>; + using ElementA = bfloat16_t; + using ElementB = bfloat16_t; + using ElementC = float; + using LayoutA = layout::RowMajor; + using LayoutB = layout::ColumnMajor; + using LayoutC = layout::RowMajor; + using Operator = OpMultiplyAdd; + using ArchTag = Sm80; + using FragmentA = nvcuda::wmma::fragment; + using FragmentB = nvcuda::wmma::fragment; + using FragmentC = nvcuda::wmma::fragment; + CUTLASS_DEVICE void operator()(FragmentC& d, FragmentA const& a, FragmentB const& b, FragmentC const& c) const { + nvcuda::wmma::mma_sync(d, a, b, c); + } +}; +} // namespace arch +} // namespace cutlass diff --git a/benchmark/inference/kernels/csrc/common/bias_gelu.cuh b/benchmark/inference/kernels/csrc/common/bias_gelu.cuh new file mode 100644 index 0000000000..7bceb965c1 --- /dev/null +++ b/benchmark/inference/kernels/csrc/common/bias_gelu.cuh @@ -0,0 +1,27 @@ +// cuBLAS addmm's epilogue (bias added in FP32, RN to BF16) followed by Inductor's Triton GELU on the BF16 value. +#pragma once +#include +#include "common/triton_gelu.cuh" + +namespace bias_gelu { +using Linear = cutlass::epilogue::thread::LinearCombination; + +// Each module derives its own named op from Op, so its kernel symbol names the module. +struct Op : Linear { + using Linear::Linear; + CUTLASS_DEVICE static FragmentOutput gelu(FragmentOutput x) { + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < kCount; ++i) { + const cutlass::bfloat16_t v = x[i]; + x[i] = cutlass::bfloat16_t::bitcast(__bfloat16_as_ushort(triton_gelu_bf16(__ushort_as_bfloat16(v.raw())))); + } + return x; + } + CUTLASS_DEVICE FragmentOutput operator()(FragmentAccumulator const& acc, FragmentSource const& bias) const { + return gelu(Linear::operator()(acc, bias)); + } + CUTLASS_DEVICE FragmentOutput operator()(FragmentAccumulator const& acc) const { + return gelu(Linear::operator()(acc)); + } +}; +} // namespace bias_gelu diff --git a/benchmark/inference/kernels/csrc/common/head_rms.cuh b/benchmark/inference/kernels/csrc/common/head_rms.cuh new file mode 100644 index 0000000000..5c9725a461 --- /dev/null +++ b/benchmark/inference/kernels/csrc/common/head_rms.cuh @@ -0,0 +1,105 @@ +// D-wide projection with SDM's unweighted per-head RMS fused into the epilogue. SM120, BF16. +// +// out = rms(bf16(a @ w.T + bias)) per head of Head channels, eps 1e-6, no weight +// +// a [M, D] (lda D), w [D, D] row-major (the leading D rows of a larger weight are fine), bias [D], +// out [M, D] with row stride ld_out. One N tile spans all D columns, so a CTA has read all of its A rows +// before it stores: out may be exactly a. Without the RMS (Rms = false) this is the plain addmm. +// Bytes match cuBLAS's addmm (CUTLASS 128x256x32 tile, no split-K) followed by SDM's Triton rmsnorm_cast +// at channels = Head (32 or 64). Site is an empty tag type that only names the caller in the kernel symbol. +#pragma once +#include +#include +#include +#include +#include +#include "common/rms_ptx.cuh" + +namespace head_rms { +using BF = cutlass::bfloat16_t; +using Bias = cutlass::epilogue::thread::LinearCombination; +using rms_ptx::add; +using rms_ptx::inv_rms; +using rms_ptx::mul; +using rms_ptx::square_add; + +// Each lane holds 8 adjacent channels (two rmsnorm_cast groups of 4) and Head / 8 adjacent lanes hold one head. +// rmsnorm_cast squares each group of 4 in channel order, then reduces the Head / 4 group sums by an XOR +// butterfly with group shifts Head/8, ..., 2, 1. Lane shifts Head/16, ..., 1 are group shifts +// Head/8, ..., 2; the in-lane add of the two groups comes last as group shift 1. +template +struct HeadRmsEpilogue : Base { + using Iterator = typename Base::OutputTileIterator; + using Fragment = typename Iterator::Fragment; + using Map = typename Iterator::ThreadMap; + static constexpr int kLanes = Head / 8; + static_assert(Iterator::kElementsPerAccess == 8 && Head % 16 == 0 && kLanes <= 32, "8 channels per lane"); + static_assert(Map::Detail::kAccessWidth % kLanes == 0 && Map::Delta::kColumn % Head == 0, + "aligned groups of Head/8 lanes each cover one head of one row"); + using Base::Base; + + struct Source : Base::SourceAspectNeeded { + using Base::SourceAspectNeeded::SourceAspectNeeded; + CUTLASS_DEVICE void apply_output_operator(Fragment& out, Bias const& op, + typename Base::SharedLoadIterator::Fragment const& acc) { + Base::SourceAspectNeeded::apply_output_operator(out, op, acc, this->source_fragment); + auto* v = reinterpret_cast*>(&out); + CUTLASS_PRAGMA_UNROLL + for (int j = 0; j < Fragment::kElements / 8; ++j) { + float s0 = 0.f, s1 = 0.f; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < 4; ++i) { + s0 = square_add(BF(v[j][i]).raw(), s0); + s1 = square_add(BF(v[j][i + 4]).raw(), s1); + } + CUTLASS_PRAGMA_UNROLL + for (int shift = kLanes / 2; shift >= 1; shift >>= 1) { + s0 = add(s0, __shfl_xor_sync(0xffffffffu, s0, shift, kLanes)); + s1 = add(s1, __shfl_xor_sync(0xffffffffu, s1, shift, kLanes)); + } + const float scale = inv_rms(add(s0, s1)); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < 8; ++i) v[j][i] = BF(mul(float(v[j][i]), scale)); + } + } + }; + + CUTLASS_DEVICE void operator()(Bias const& op, Iterator destination, typename Base::AccumulatorTile const& acc, + Iterator bias) { + Base::operator()(op, destination, acc, Source(bias)); + } +}; + +template +using Gemm = cutlass::gemm::device::Gemm< + BF, cutlass::layout::RowMajor, BF, cutlass::layout::ColumnMajor, BF, cutlass::layout::RowMajor, float, + cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80, cutlass::gemm::GemmShape<128, D, 32>, + cutlass::gemm::GemmShape<64, 64, 32>, cutlass::gemm::GemmShape<16, 8, 16>, Bias, + cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<1>, 3, 8, 8, false, cutlass::arch::OpMultiplyAdd>; + +template +using Kernel = cutlass::gemm::kernel::Gemm< + typename Gemm::GemmKernel::Mma, + std::conditional_t::GemmKernel::Epilogue, Head, Site>, + typename Gemm::GemmKernel::Epilogue>, + typename Gemm::GemmKernel::ThreadblockSwizzle, false>; + +constexpr int64_t kMaxM = INT_MAX - 128; // the swizzle computes M + tile_m in int32 + +template +void launch(BF const* a, BF const* w, BF const* bias, BF* out, int ld_out, int m, cudaStream_t stream) { + using G = Gemm; + using K = Kernel; + static_assert(sizeof(typename K::SharedStorage) == sizeof(typename G::GemmKernel::SharedStorage), "no extra smem"); + typename G::Arguments args({m, D, D}, {a, D}, {w, D}, {bias, 0}, {out, ld_out}, {1.f, 1.f}, 1); + TORCH_CHECK(G::can_implement(args) == cutlass::Status::kSuccess, "head_rms: CUTLASS cannot implement this GEMM"); + typename G::ThreadblockSwizzle swizzle; + auto tiled = swizzle.get_tiled_shape(args.problem_size, {128, D, 32}, 1); + typename K::Params params(args.problem_size, tiled, args.ref_A.non_const_ref(), args.ref_B.non_const_ref(), + args.ref_C.non_const_ref(), args.ref_D, args.epilogue, nullptr); + constexpr int smem = sizeof(typename K::SharedStorage); + C10_CUDA_CHECK(cudaFuncSetAttribute(cutlass::Kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem)); + cutlass::Kernel<<>>(params); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} +} // namespace head_rms diff --git a/benchmark/inference/kernels/csrc/common/rms_ptx.cuh b/benchmark/inference/kernels/csrc/common/rms_ptx.cuh new file mode 100644 index 0000000000..225d0e9388 --- /dev/null +++ b/benchmark/inference/kernels/csrc/common/rms_ptx.cuh @@ -0,0 +1,30 @@ +// The FP32 steps of SDM's Triton rmsnorm_cast at channels = Head, as Triton lowers them to PTX. +#pragma once + +namespace rms_ptx { +__device__ __forceinline__ float add(float a, float b) { + float v; + asm("add.rn.f32 %0,%1,%2;" : "=f"(v) : "f"(a), "f"(b)); + return v; +} +__device__ __forceinline__ float mul(float a, float b) { + float v; + asm("mul.rn.f32 %0,%1,%2;" : "=f"(v) : "f"(a), "f"(b)); + return v; +} +// sum + x * x for the BF16 x with bits `x`, one rounding. +__device__ __forceinline__ float square_add(unsigned short x, float sum) { + float v; + asm("fma.rn.f32.bf16 %0,%1,%1,%2;" : "=f"(v) : "h"(x), "f"(sum)); + return v; +} +// rsqrt(sum / Head + 1e-6) exactly as Triton emits it: div.full, add.rn, rsqrt.approx.ftz. +template +__device__ __forceinline__ float inv_rms(float sum) { + float mean, shifted, v; + asm("div.full.f32 %0,%1,%2;" : "=f"(mean) : "f"(sum), "f"(float(Head))); + asm("add.rn.f32 %0,%1,0f358637BD;" : "=f"(shifted) : "f"(mean)); + asm("rsqrt.approx.ftz.f32 %0,%1;" : "=f"(v) : "f"(shifted)); + return v; +} +} // namespace rms_ptx diff --git a/benchmark/inference/kernels/csrc/common/row_source.cuh b/benchmark/inference/kernels/csrc/common/row_source.cuh new file mode 100644 index 0000000000..b048d29e17 --- /dev/null +++ b/benchmark/inference/kernels/csrc/common/row_source.cuh @@ -0,0 +1,35 @@ +// Source aspect for CUTLASS 2.x epilogues whose operator needs the output coordinates. Like the stock aspect it +// streams the C operand (here the bias, stride 0); it also records where each output fragment starts. +#pragma once +#include +#include + +namespace row_source { +template +struct BiasSource { + Iterator bias_iterator, coordinate_iterator; + typename Iterator::Fragment bias; + cutlass::MatrixCoord start; + CUTLASS_DEVICE void load() { + bias_iterator.load(bias); + start = coordinate_iterator.thread_start(); + ++bias_iterator; + ++coordinate_iterator; + } +}; + +// Calls f(row, fragment_row) for each output row of the fragment that starts at `start`, in fragment order. +template +CUTLASS_DEVICE void for_each_row(cutlass::MatrixCoord start, F&& f) { + CUTLASS_PRAGMA_UNROLL + for (int cluster = 0; cluster < Map::Iterations::kCluster; ++cluster) { + CUTLASS_PRAGMA_UNROLL + for (int group = 0; group < Map::Iterations::kGroup; ++group) { + CUTLASS_PRAGMA_UNROLL + for (int r = 0; r < Map::Iterations::kRow; ++r) + f(start.row() + r * Map::Delta::kRow + group * Map::Delta::kGroup + cluster * Map::Delta::kCluster, + r + Map::Iterations::kRow * (group + Map::Iterations::kGroup * cluster)); + } + } +} +} // namespace row_source diff --git a/benchmark/inference/kernels/csrc/common/triton_gelu.cuh b/benchmark/inference/kernels/csrc/common/triton_gelu.cuh new file mode 100644 index 0000000000..2a0177e165 --- /dev/null +++ b/benchmark/inference/kernels/csrc/common/triton_gelu.cuh @@ -0,0 +1,63 @@ +#pragma once +#include + +// Inductor's Triton GELU (erf form) as lowered to PTX on CUDA 13: same constants, same op order, +// same FTZ flags. Not the toolkit erff. +namespace triton_gelu_ptx { +__device__ __forceinline__ float bits(unsigned x) { return __uint_as_float(x); } +__device__ __forceinline__ float mul(float a, float b) { + float r; + asm("mul.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +__device__ __forceinline__ float add(float a, float b) { + float r; + asm("add.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +__device__ __forceinline__ float sub(float a, float b) { + float r; + asm("sub.rn.f32 %0, %1, %2;" : "=f"(r) : "f"(a), "f"(b)); + return r; +} +__device__ __forceinline__ float abs_ftz(float a) { + float r; + asm("abs.ftz.f32 %0, %1;" : "=f"(r) : "f"(a)); + return r; +} +__device__ __forceinline__ float fma_ftz(float a, float b, float c) { + float r; + asm("fma.rn.ftz.f32 %0, %1, %2, %3;" : "=f"(r) : "f"(a), "f"(b), "f"(c)); + return r; +} +__device__ __forceinline__ float exp2_ftz(float a) { + float r; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(r) : "f"(a)); + return r; +} +__device__ __forceinline__ float erf_observed(float s) { + const float a = abs_ftz(s); + const bool large = a >= bits(0x3f8060feu); + const float t = large ? a : mul(s, s); + float p = bits(large ? 0x3789ca3cu : 0x38b1e96au); + p = fma_ftz(p, t, bits(large ? 0xb9f560b9u : 0xba574d20u)); + p = fma_ftz(p, t, bits(large ? 0x3bac840bu : 0x3baad5eau)); + p = fma_ftz(p, t, bits(large ? 0xbd0c8162u : 0xbcdc1be7u)); + p = fma_ftz(p, t, bits(large ? 0x3e1cf906u : 0x3de718afu)); + p = fma_ftz(p, t, bits(large ? 0x3f6a937eu : 0xbec093acu)); + p = fma_ftz(p, t, bits(large ? 0x3f20d842u : 0x3e0375d3u)); + const float u = large ? -t : s; + const float v = fma_ftz(p, u, u); + if (!large) return v; + const float r = sub(1.0f, exp2_ftz(v)); + return bits(__float_as_uint(r) | (__float_as_uint(s) & 0x80000000u)); +} +} // namespace triton_gelu_ptx + +__device__ __forceinline__ __nv_bfloat16 triton_gelu_bf16(__nv_bfloat16 input) { + using namespace triton_gelu_ptx; + const float z = __bfloat162float(input); + const float s = mul(z, bits(0x3f3504f3u)); + const float e = erf_observed(s); + return __float2bfloat16_rn(mul(mul(z, 0.5f), add(e, 1.0f))); +} diff --git a/benchmark/inference/kernels/csrc/gating_sm120/gating.cu b/benchmark/inference/kernels/csrc/gating_sm120/gating.cu new file mode 100644 index 0000000000..4d56c40cb5 --- /dev/null +++ b/benchmark/inference/kernels/csrc/gating_sm120/gating.cu @@ -0,0 +1,211 @@ +// Row-block GatedLogScale gate MLP for head width H (H = 64 Large/Medium, 32 Small), SM120, BF16. +// first: h = gelu(bf16(q @ w1.T + b1)) q[M,H] w1[64,H] h[M,64] +// second: out = bf16(q * logscale * scale[row%4] * (1 + tanh(bf16(h @ w2.T + b2)))) w2[H,64] out[M,H] +// Both GEMMs are the CUTLASS form of the kernel cuBLAS picks for them at model sizes, including K32 +// (cutlass_80_wmma_tensorop_bf16_s161616gemm_bf16_32x32_64x1_tn_align8): one 32x32x64 CTA tile, K16 BF16 +// mma_sync groups accumulated in ascending K in FP32, bias added in FP32, rounded to BF16 before the +// activation. At K32 the K64 tile reads K 32..63 as predicated zeros, as the cuBLAS kernel does. (At tiny M +// cuBLAS picks its 32x32_128x2 variant for the second GEMM; that gives the same bytes, see the tests.) +#include +#include +#include +#include +#include "common/bf16_wmma.cuh" +#include +#include +#include "common/bias_gelu.cuh" + +namespace gating { +using BF = cutlass::bfloat16_t; +using Linear = cutlass::epilogue::thread::LinearCombination; +constexpr int kHidden = 64, kHeads = 4; +constexpr int64_t kMaxM = INT_MAX - 32; // the swizzle computes M + 31 in int32 + +template +using Gemm = cutlass::gemm::device::Gemm< + BF, cutlass::layout::RowMajor, BF, cutlass::layout::ColumnMajor, BF, cutlass::layout::RowMajor, float, + cutlass::arch::OpClassWmmaTensorOp, cutlass::arch::Sm80, cutlass::gemm::GemmShape<32, 32, 64>, + cutlass::gemm::GemmShape<16, 16, 64>, cutlass::gemm::GemmShape<16, 16, 16>, Op, + cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<2>, 1, 8, 8, false, cutlass::arch::OpMultiplyAdd>; + +struct BiasGelu : bias_gelu::Op { // names this module in the kernel symbol + using bias_gelu::Op::Op; +}; + +// Inductor's libdevice tanh as emitted in the native gate kernel's PTX (explicit FTZ where it has FTZ). +__device__ __forceinline__ float native_tanh(float x) { + float result; + asm("{ .reg .pred p; .reg .f32 a,y,t,e,d,r,v; .reg .b32 s,b;\n" + "abs.ftz.f32 a,%1;\n" + "setp.ltu.f32 p,a,0f3F19999A;\n" + "@!p bra large;\n" + "mul.f32 y,%1,%1;\n" + "fma.rn.ftz.f32 t,0f3C80F082,y,0fBD563CAE;\n" + "fma.rn.ftz.f32 t,t,y,0f3E085941;\n" + "fma.rn.ftz.f32 t,t,y,0fBEAAA9ED;\n" + "fma.rn.ftz.f32 t,t,y,0f00000000;\n" + "fma.rn.ftz.f32 %0,t,%1,%1;\n" + "bra done;\n" + "large:\n" + "mul.f32 e,a,0f4038AA3B;\n" + "ex2.approx.ftz.f32 e,e;\n" + "add.f32 d,e,0f3F800000;\n" + "rcp.approx.ftz.f32 r,d;\n" + "fma.rn.ftz.f32 v,r,0fC0000000,0f3F800000;\n" + "setp.ge.f32 p,a,0f41102CB4;\n" + "selp.f32 v,0f3F800000,v,p;\n" + "mov.b32 s,%1; and.b32 s,s,0x80000000;\n" + "mov.b32 b,v; or.b32 b,b,s; mov.b32 %0,b;\n" + "done: }\n" + : "=f"(result) + : "f"(x)); + return result; +} + +// ((q * logscale) * scale) * (tanh(z) + 1), each an RN FP32 op without contraction, then RN BF16. +__device__ __forceinline__ BF gate(BF z, BF q, float scale, float logscale) { + float a, b, c, r; + const float t = native_tanh(float(z)); + asm("mul.rn.f32 %0,%1,%2;" : "=f"(a) : "f"(float(q)), "f"(logscale)); + asm("mul.rn.f32 %0,%1,%2;" : "=f"(b) : "f"(a), "f"(scale)); + asm("add.rn.f32 %0,%1,0f3F800000;" : "=f"(c) : "f"(t)); + asm("mul.rn.f32 %0,%1,%2;" : "=f"(r) : "f"(b), "f"(c)); + return BF(r); +} + +struct GateOp : Linear { + struct Params : Linear::Params { + BF const* q; + float const* scale; + int rows; + float logscale; + CUTLASS_HOST_DEVICE Params(BF const* q = nullptr, float const* scale = nullptr, int rows = 0, float logscale = 0) + : Linear::Params(1.0f, 1.0f), q(q), scale(scale), rows(rows), logscale(logscale) {} + }; + Params p; + CUTLASS_HOST_DEVICE GateOp(Params const& p) : Linear(p), p(p) {} +}; + +// The stock epilogue (accumulator exchange, bias load, predicated store) with a source aspect that also +// loads Q at the destination coordinates, so the operator sees z, Q and the output row together. +template +struct GateEpilogue : Base { + using Iterator = typename Base::OutputTileIterator; + using Fragment = typename Iterator::Fragment; + using SharedFragment = typename Base::SharedLoadIterator::Fragment; + using ThreadMap = typename Iterator::ThreadMap; + static_assert(Iterator::kIterations == 1 && Fragment::kElements == 8 && SharedFragment::kElements == 8 && + ThreadMap::Iterations::kCount == 1, + "one eight-element output vector per thread"); + int thread; + CUTLASS_DEVICE GateEpilogue(typename Base::SharedStorage& s, int t, int w, int lane) + : Base(s, t, w, lane), thread(t) {} + + struct Source { + Iterator bias_it, q_it; + Fragment bias, q; + int row; + CUTLASS_DEVICE void load() { + bias_it.load(bias); + q_it.load(q); + ++bias_it; + ++q_it; + } + CUTLASS_DEVICE void apply_output_operator(Fragment& out, GateOp const& op, SharedFragment const& acc) { + auto const& values = *reinterpret_cast(&acc); + auto const& source = *reinterpret_cast(&bias); + const typename Linear::FragmentOutput z = static_cast(op)(values, source); + const float scale = op.p.scale[row % kHeads]; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < 8; ++i) out[i] = gate(BF(z[i]), BF(q[i]), scale, op.p.logscale); + } + }; + + CUTLASS_DEVICE void operator()(GateOp const& op, Iterator destination, typename Base::AccumulatorTile const& acc, + Iterator bias) { + typename Iterator::Params q_params{cutlass::layout::RowMajor(N)}; + auto offset = destination.thread_start() - ThreadMap::initial_offset(thread); + Iterator q_it(q_params, const_cast(op.p.q), {op.p.rows, N}, thread, offset); + Source aspect{bias, q_it, {}, {}, destination.thread_start().row()}; + Base::template operator()(op, destination, acc, aspect); + } +}; + +template +struct Head { + static_assert(H % 16 == 0 && H <= 64, "head width"); + using First = Gemm; + using SecondBase = typename Gemm::GemmKernel; + using Second = cutlass::gemm::kernel::Gemm, + typename SecondBase::ThreadblockSwizzle, false>; + static_assert(sizeof(typename Second::SharedStorage) < 48 * 1024, "static shared launch"); + + static void first(at::Tensor const& q, at::Tensor const& w, at::Tensor const& b, at::Tensor const& out) { + typename First::Arguments args({int(q.size(0)), kHidden, H}, {ptr(q), H}, {ptr(w), H}, {ptr(b), 0}, + {ptr(out), kHidden}, {1.0f, 1.0f}, 1); + First gemm; + auto stream = at::cuda::getCurrentCUDAStream(); + TORCH_CHECK(gemm.can_implement(args) == cutlass::Status::kSuccess, "gating: first layer can_implement failed"); + TORCH_CHECK(gemm.initialize(args, nullptr, stream) == cutlass::Status::kSuccess, + "gating: first layer initialize failed"); + TORCH_CHECK(gemm.run(stream) == cutlass::Status::kSuccess, "gating: first layer run failed"); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + } + + static void second(at::Tensor const& h, at::Tensor const& w, at::Tensor const& b, at::Tensor const& q, + at::Tensor const& scale, at::Tensor const& out, double logscale) { + const int m = int(h.size(0)); + const cutlass::gemm::GemmCoord size(m, H, kHidden); + typename SecondBase::ThreadblockSwizzle swizzle; + auto tiled = swizzle.get_tiled_shape(size, {32, 32, kHidden}, 1); + typename Second::Params params(size, tiled, {ptr(h), kHidden}, {ptr(w), kHidden}, {ptr(b), 0}, {ptr(out), H}, + GateOp::Params(ptr(q), scale.data_ptr(), m, float(logscale)), nullptr); + cutlass::Kernel<<>>(params); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + } + + static BF* ptr(at::Tensor const& t) { return reinterpret_cast(t.data_ptr()); } +}; + +void check(std::initializer_list ts) { + for (auto const* t : ts) + TORCH_CHECK(t->is_cuda() && t->scalar_type() == at::kBFloat16 && t->is_contiguous() && + reinterpret_cast(t->data_ptr()) % 16 == 0, + "gating: operands must be contiguous, 16-byte aligned CUDA BF16 tensors"); +} + +// The Python supports() is the contract; these checks only keep a bad call from touching memory. +void first(at::Tensor const& q, at::Tensor const& w, at::Tensor const& b, at::Tensor const& out) { + check({&q, &w, &b, &out}); + const int64_t m = q.size(0), k = q.size(1); + TORCH_CHECK(q.dim() == 2 && m >= 1 && m <= kMaxM && w.dim() == 2 && w.size(0) == kHidden && w.size(1) == k && + b.dim() == 1 && b.size(0) == kHidden && out.dim() == 2 && out.size(0) == m && out.size(1) == kHidden, + "gating: expected q [M, H], w1 [64, H], b1 [64], h [M, 64]"); + c10::cuda::CUDAGuard guard(q.device()); + if (k == 64) return Head<64>::first(q, w, b, out); + if (k == 32) return Head<32>::first(q, w, b, out); + TORCH_CHECK(false, "gating: no first-layer instance for head ", k); +} + +void second(at::Tensor const& h, at::Tensor const& w, at::Tensor const& b, at::Tensor const& q, at::Tensor const& scale, + at::Tensor const& out, double logscale) { + check({&h, &w, &b, &q, &out}); + TORCH_CHECK(scale.is_cuda() && scale.scalar_type() == at::kFloat && scale.is_contiguous() && scale.numel() == kHeads, + "gating: scale must be a contiguous CUDA float32 tensor of 4 heads"); + const int64_t m = h.size(0), n = q.size(1); + TORCH_CHECK(h.dim() == 2 && m >= 1 && m <= kMaxM && h.size(1) == kHidden && w.dim() == 2 && w.size(0) == n && + w.size(1) == kHidden && b.dim() == 1 && b.size(0) == n && q.dim() == 2 && q.size(0) == m && + out.dim() == 2 && out.size(0) == m && out.size(1) == n, + "gating: expected h [M, 64], w2 [H, 64], b2 [H], q [M, H], out [M, H]"); + c10::cuda::CUDAGuard guard(h.device()); + if (n == 64) return Head<64>::second(h, w, b, q, scale, out, logscale); + if (n == 32) return Head<32>::second(h, w, b, q, scale, out, logscale); + TORCH_CHECK(false, "gating: no second-layer instance for head ", n); +} +} // namespace gating + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("first", &gating::first); + m.def("second", &gating::second); +} diff --git a/benchmark/inference/kernels/csrc/icl_attention_int8_sm120/icl_attention_int8.cu b/benchmark/inference/kernels/csrc/icl_attention_int8_sm120/icl_attention_int8.cu new file mode 100644 index 0000000000..e0803c8593 --- /dev/null +++ b/benchmark/inference/kernels/csrc/icl_attention_int8_sm120/icl_attention_int8.cu @@ -0,0 +1,530 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// 8-bit self-attention for SM120, head dim 64, no mask. +// +// prep: K - mean_s(K) quantized to INT8 per 64-key block; V quantized to FP8 E4M3 per channel and stored +// transposed [D, S] with keys permuted inside every 32-key group to match the P fragment layout. +// attend: Q quantized to INT8 per row in the prologue, S = Q K^T on the INT8 MMA, online softmax in FP32, +// P (scaled by 448) as FP8 times V on the FP8 MMA with FP32 accumulation. + +#include +#include +#include +#include +#include +#include +#include + +namespace icl_attention_int8 { + +constexpr int D = 64; +constexpr int BN = 64; +constexpr float kLog2P = 8.807354922057604f; // log2(448) +constexpr float kLog2e = 1.4426950408889634f; +constexpr int kMagic = 0x4B400000; // 1.5 * 2^23 +constexpr float kMagicF = 12582912.f; + +struct Strided { // bf16 or fp16 bits + const uint16_t* p; + int64_t sb, ss, sh; // batch, token, head strides (elements) + __device__ const uint16_t* at(int b, int s, int h) const { return p + b * sb + s * ss + h * sh; } +}; + +__device__ __forceinline__ uint32_t smem_u32(const void* p) { + return static_cast(__cvta_generic_to_shared(p)); +} +__device__ __forceinline__ void cp_async16(uint32_t dst, const void* src) { + asm volatile("cp.async.cg.shared.global [%0], [%1], 16;" ::"r"(dst), "l"(src)); +} +__device__ __forceinline__ void cp_commit() { asm volatile("cp.async.commit_group;"); } +template +__device__ __forceinline__ void cp_wait() { + asm volatile("cp.async.wait_group %0;" ::"n"(N)); +} + +__device__ __forceinline__ void ldsm_x4(uint32_t (&r)[4], uint32_t addr) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) + : "r"(addr)); +} + +__device__ __forceinline__ void mma_s8(int (&c)[4], const uint32_t (&a)[4], uint32_t b0, uint32_t b1) { + asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};" + : "+r"(c[0]), "+r"(c[1]), "+r"(c[2]), "+r"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1)); +} +// d = a b + {c, c, c, c} +__device__ __forceinline__ void mma_s8_init(int (&d)[4], const uint32_t (&a)[4], uint32_t b0, uint32_t b1, int c) { + asm("mma.sync.aligned.m16n8k32.row.col.s32.s8.s8.s32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%10,%10,%10};" + : "=r"(d[0]), "=r"(d[1]), "=r"(d[2]), "=r"(d[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1), "r"(c)); +} +__device__ __forceinline__ void mma_e4m3(float (&c)[4], const uint32_t (&a)[4], uint32_t b0, uint32_t b1) { + asm("mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};" + : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1)); +} + +__device__ __forceinline__ float ex2(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); + return y; +} +// byte i of the result = x_i +__device__ __forceinline__ uint32_t pack_e4m3(float x0, float x1, float x2, float x3) { + uint32_t r; + asm("{ .reg .b16 l, h;\n" + "cvt.rn.satfinite.e4m3x2.f32 l, %2, %1;\n" + "cvt.rn.satfinite.e4m3x2.f32 h, %4, %3;\n" + "mov.b32 %0, {l, h}; }" + : "=r"(r) + : "f"(x0), "f"(x1), "f"(x2), "f"(x3)); + return r; +} +template +__device__ __forceinline__ float tof(uint16_t x) { + if constexpr (F16) + return __half2float(__ushort_as_half(x)); + else + return __bfloat162float(__ushort_as_bfloat16(x)); +} +template +__device__ __forceinline__ uint32_t pack16(float lo, float hi) { + if constexpr (F16) { + __half2 v = __floats2half2_rn(lo, hi); + return *reinterpret_cast(&v); + } else { + __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi); + return *reinterpret_cast(&v); + } +} +__device__ __forceinline__ int quant8(float x) { return max(-127, min(127, __float2int_rn(x))); } +__device__ __forceinline__ uint32_t pack_s8(int a, int b, int c, int d) { + return (a & 0xff) | ((b & 0xff) << 8) | ((c & 0xff) << 16) | ((uint32_t)(d & 0xff) << 24); +} + +// 64-byte rows, 16-byte chunks XOR-swizzled so ldmatrix rows hit distinct banks. +__device__ __forceinline__ int swz64(int row, int chunk) { return row * 64 + ((chunk ^ ((row >> 1) & 3)) << 4); } + +// Logical position p in a 32-key group holds key sigma(p), so that a thread's +// QK^T accumulators are its P A-fragment bytes in order. +__device__ __forceinline__ int sigma(int p) { + int h = p >> 4, t = (p >> 2) & 3, i = p & 3; + return 16 * h + 2 * t + (i & 1) + 8 * (i >> 1); +} + +// Per (b, h, chunk): kpart[d] = sum_s K over the chunk, vmax[d] = max_s |V| (as uint bits). +constexpr int kStatTokens = 1024; +template +__global__ void __launch_bounds__(256) stats_kernel(Strided k, Strided v, int S, int H, float* kpart, unsigned* vmax) { + int b = blockIdx.z, h = blockIdx.y, s0 = blockIdx.x * kStatTokens; + int lane8 = threadIdx.x & 7, row = threadIdx.x >> 3; // 32 token rows per pass + float sum[8] = {}, mx[8] = {}; + int s1 = min(S, s0 + kStatTokens); + for (int s = s0 + row; s < s1; s += 32) { + uint4 ku = *reinterpret_cast(k.at(b, s, h) + lane8 * 8); + const uint16_t* ke = reinterpret_cast(&ku); +#pragma unroll + for (int i = 0; i < 8; ++i) sum[i] += tof(ke[i]); + uint4 vu = *reinterpret_cast(v.at(b, s, h) + lane8 * 8); + const uint16_t* ve = reinterpret_cast(&vu); +#pragma unroll + for (int i = 0; i < 8; ++i) mx[i] = fmaxf(mx[i], fabsf(tof(ve[i]))); + } + __shared__ float red[2][32][65]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + red[0][row][lane8 * 8 + i] = sum[i]; + red[1][row][lane8 * 8 + i] = mx[i]; + } + __syncthreads(); + if (threadIdx.x < 64) { + float s = 0.f, m = 0.f; + for (int r = 0; r < 32; ++r) { + s += red[0][r][threadIdx.x]; + m = fmaxf(m, red[1][r][threadIdx.x]); + } + kpart[((int64_t)(b * H + h) * gridDim.x + blockIdx.x) * D + threadIdx.x] = s; + atomicMax(vmax + (b * H + h) * D + threadIdx.x, __float_as_uint(m)); + } +} + +// Fixed-order sum of the chunk partials, so the K mean is bitwise reproducible. +__global__ void __launch_bounds__(D) ksum_kernel(const float* kpart, int chunks, float* ksum) { + const float* src = kpart + (int64_t)blockIdx.x * chunks * D + threadIdx.x; + float s = 0.f; + for (int c = 0; c < chunks; ++c) s += src[c * D]; + ksum[blockIdx.x * D + threadIdx.x] = s; +} + +// One block per (64-key block, h, b), 256 threads. +template +__global__ void __launch_bounds__(256) quant_kernel(Strided k, Strided v, int S, int S_pad, int H, const float* ksum, + const unsigned* vmax, int8_t* kq, float* kscale, uint8_t* vt) { + int b = blockIdx.z, h = blockIdx.y, blk = blockIdx.x, s0 = blk * BN; + int bh = b * H + h; + __shared__ float vs[BN][D + 1]; + __shared__ float wmax[8]; + int lane8 = threadIdx.x & 7, row = threadIdx.x >> 3; // rows row, row + 32 + float kv[2][8]; + float amax = 0.f, tmax[2] = {0.f, 0.f}; + float inv_n = 1.f / S; +#pragma unroll + for (int r = 0; r < 2; ++r) { + int s = s0 + row + 32 * r; +#pragma unroll + for (int i = 0; i < 8; ++i) kv[r][i] = 0.f; + if (s < S) { + uint4 u = *reinterpret_cast(k.at(b, s, h) + lane8 * 8); + const uint16_t* e = reinterpret_cast(&u); +#pragma unroll + for (int i = 0; i < 8; ++i) { + float x = tof(e[i]); + x -= ksum[bh * D + lane8 * 8 + i] * inv_n; + kv[r][i] = x; + tmax[r] = fmaxf(tmax[r], fabsf(x)); + } + } +#pragma unroll + for (int o = 4; o; o >>= 1) tmax[r] = fmaxf(tmax[r], __shfl_xor_sync(0xffffffff, tmax[r], o)); + amax = fmaxf(amax, tmax[r]); + } +#pragma unroll + for (int o = 16; o; o >>= 1) amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, o)); + if ((threadIdx.x & 31) == 0) wmax[threadIdx.x >> 5] = amax; +#pragma unroll + for (int r = 0; r < 2; ++r) { + int s = s0 + row + 32 * r; + float f[8] = {}; + if (s < S) { + uint4 u = *reinterpret_cast(v.at(b, s, h) + lane8 * 8); + const uint16_t* e = reinterpret_cast(&u); +#pragma unroll + for (int i = 0; i < 8; ++i) { + float m = __uint_as_float(vmax[bh * D + lane8 * 8 + i]); + f[i] = m > 0.f ? tof(e[i]) * (448.f / m) : 0.f; + } + } +#pragma unroll + for (int i = 0; i < 8; ++i) vs[row + 32 * r][lane8 * 8 + i] = f[i]; + } + __syncthreads(); + amax = 0.f; +#pragma unroll + for (int w = 0; w < 8; ++w) amax = fmaxf(amax, wmax[w]); + if (threadIdx.x == 0) kscale[bh * (S_pad / BN) + blk] = amax / 127.f; +#pragma unroll + for (int r = 0; r < 2; ++r) { + int s = s0 + row + 32 * r; + float inv = amax > 0.f ? 127.f / amax : 0.f; + uint2 out; + out.x = pack_s8(quant8(kv[r][0] * inv), quant8(kv[r][1] * inv), quant8(kv[r][2] * inv), quant8(kv[r][3] * inv)); + out.y = pack_s8(quant8(kv[r][4] * inv), quant8(kv[r][5] * inv), quant8(kv[r][6] * inv), quant8(kv[r][7] * inv)); + *reinterpret_cast(kq + ((int64_t)bh * S_pad + s) * D + lane8 * 8) = out; + } + // 64 channels x 64 keys; each thread writes 16 consecutive logical keys of one channel. + int c = threadIdx.x >> 2, q = threadIdx.x & 3; + float x[16]; +#pragma unroll + for (int i = 0; i < 16; ++i) { + int p = q * 16 + i; + x[i] = vs[(p & ~31) + sigma(p & 31)][c]; + } + uint4 o; + o.x = pack_e4m3(x[0], x[1], x[2], x[3]); + o.y = pack_e4m3(x[4], x[5], x[6], x[7]); + o.z = pack_e4m3(x[8], x[9], x[10], x[11]); + o.w = pack_e4m3(x[12], x[13], x[14], x[15]); + *reinterpret_cast(vt + ((int64_t)bh * D + c) * S_pad + s0 + q * 16) = o; +} + +// WM m16 tiles of query rows per warp, NW warps, a STAGES-deep cp.async ring of 64-key K/V tiles. +constexpr int WM = 2, NW = 4, STAGES = 3; +constexpr int BM = 16 * WM * NW; +constexpr int NT = NW * 32; +constexpr int KBYTES = BN * D, VBYTES = BN * D; +constexpr int STAGE = KBYTES + VBYTES; +constexpr int SMEM = STAGES * STAGE; +constexpr int NK = KBYTES / 16 / NT, NV = VBYTES / 16 / NT; + +template +__global__ void __launch_bounds__(NT, 1) + attend_kernel(Strided q, const int8_t* __restrict__ kq, const float* __restrict__ kscale, + const uint8_t* __restrict__ vt, const unsigned* __restrict__ vmax, uint16_t* out, int64_t osb, + int64_t oss, int64_t osh, int S, int S_pad, int H, float sm_log2) { + extern __shared__ __align__(128) uint8_t smem[]; + const int b = blockIdx.z, h = blockIdx.y, bh = b * H + h; + const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31, g = lane >> 2, t = lane & 3; + const int m0 = blockIdx.x * BM + warp * 16 * WM; + const int ntiles = S_pad / BN; + const float* ks = kscale + (int64_t)bh * ntiles; + const uint32_t sbase = smem_u32(smem); + + const int8_t* kg = kq + (int64_t)bh * S_pad * D + tid * 16; + const uint8_t* vg = vt + ((int64_t)bh * D + (tid >> 2)) * S_pad + (tid & 3) * 16; + const int64_t vstep = (int64_t)(NT / 4) * S_pad; + const uint32_t sw = swz64(tid >> 2, tid & 3); + auto load = [&](int tile, int stage) { + const uint32_t st = sbase + stage * STAGE; +#pragma unroll + for (int i = 0; i < NK; ++i) cp_async16(st + sw + i * NT * 16, kg + (int64_t)tile * BN * D + i * NT * 16); +#pragma unroll + for (int i = 0; i < NV; ++i) cp_async16(st + KBYTES + sw + i * NT * 16, vg + i * vstep + tile * BN); + }; + +#pragma unroll + for (int i = 0; i < STAGES - 1; ++i) { + if (i < ntiles) load(i, i); + cp_commit(); + } + + // Q prologue: per-row INT8 quantization straight into A fragments. + uint32_t qa[WM][2][4]; + float qs[WM][2]; +#pragma unroll + for (int i = 0; i < WM; ++i) { +#pragma unroll + for (int hr = 0; hr < 2; ++hr) { + int row = m0 + 16 * i + g + 8 * hr; + float x[16]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + uint2 u = make_uint2(0, 0); + if (row < S) u = *reinterpret_cast(q.at(b, row, h) + 16 * j + 4 * t); + const uint16_t* e = reinterpret_cast(&u); +#pragma unroll + for (int e4 = 0; e4 < 4; ++e4) x[4 * j + e4] = tof(e[e4]); + } + float amax = 0.f; +#pragma unroll + for (int j = 0; j < 16; ++j) amax = fmaxf(amax, fabsf(x[j])); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 1)); + amax = fmaxf(amax, __shfl_xor_sync(0xffffffff, amax, 2)); + float inv = amax > 0.f ? 127.f / amax : 0.f; + qs[i][hr] = amax / 127.f * sm_log2; +#pragma unroll + for (int s = 0; s < 2; ++s) +#pragma unroll + for (int half = 0; half < 2; ++half) { + const float* y = x + 4 * (2 * s + half); + qa[i][s][2 * half + hr] = + pack_s8(quant8(y[0] * inv), quant8(y[1] * inv), quant8(y[2] * inv), quant8(y[3] * inv)); + } + } + } + + float o[WM][8][4]; + float mrow[WM][2], lrow[WM][2]; +#pragma unroll + for (int i = 0; i < WM; ++i) { +#pragma unroll + for (int j = 0; j < 8; ++j) +#pragma unroll + for (int e = 0; e < 4; ++e) o[i][j][e] = 0.f; + mrow[i][0] = mrow[i][1] = -INFINITY; + lrow[i][0] = lrow[i][1] = 0.f; + } + + const int lrow8 = lane & 7, lmat = lane >> 3; + float ksc = ks[0]; + int rd = 0, wr = STAGES - 1; + + auto step = [&](int tile, auto masked_t) { + constexpr bool masked = decltype(masked_t)::value; + cp_wait(); + __syncthreads(); + if (tile + STAGES - 1 < ntiles) load(tile + STAGES - 1, wr); + cp_commit(); + wr = wr == STAGES - 1 ? 0 : wr + 1; + const float ksn = tile + 1 < ntiles ? ks[tile + 1] : 0.f; + const uint32_t kst = sbase + rd * STAGE, vst = kst + KBYTES; + + // Scores land as the bits of 1.5 * 2^23 + s (a float, exact for |s| < 2^22). + int sc[WM][8][4]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + uint32_t kb[4]; + ldsm_x4(kb, kst + swz64(8 * j + lrow8, lmat)); +#pragma unroll + for (int i = 0; i < WM; ++i) { + mma_s8_init(sc[i][j], qa[i][0], kb[0], kb[1], kMagic); + mma_s8(sc[i][j], qa[i][1], kb[2], kb[3]); + } + } + + if (masked) { +#pragma unroll + for (int i = 0; i < WM; ++i) +#pragma unroll + for (int j = 0; j < 8; ++j) +#pragma unroll + for (int e = 0; e < 4; ++e) + if (tile * BN + 8 * j + 2 * t + (e & 1) >= S) sc[i][j][e] = 0x4B000000; // s = -2^22 + } + + float cs[WM][2], mnew[WM][2]; + bool grow = false; +#pragma unroll + for (int i = 0; i < WM; ++i) { +#pragma unroll + for (int hr = 0; hr < 2; ++hr) { + int mx = sc[i][0][2 * hr]; +#pragma unroll + for (int j = 0; j < 8; ++j) mx = max(mx, max(sc[i][j][2 * hr], sc[i][j][2 * hr + 1])); + mx = max(mx, __shfl_xor_sync(0xffffffff, mx, 1)); + mx = max(mx, __shfl_xor_sync(0xffffffff, mx, 2)); + const float mxf = __int_as_float(mx) - kMagicF; + cs[i][hr] = qs[i][hr] * ksc; + mnew[i][hr] = fmaxf(mrow[i][hr], mxf * cs[i][hr]); + grow |= mnew[i][hr] > mrow[i][hr]; + } + } + if (__any_sync(0xffffffff, grow)) { +#pragma unroll + for (int i = 0; i < WM; ++i) +#pragma unroll + for (int hr = 0; hr < 2; ++hr) { + const float alpha = ex2(mrow[i][hr] - mnew[i][hr]); + lrow[i][hr] *= alpha; +#pragma unroll + for (int j = 0; j < 8; ++j) { + o[i][j][2 * hr] *= alpha; + o[i][j][2 * hr + 1] *= alpha; + } + mrow[i][hr] = mnew[i][hr]; + } + } + uint32_t pa[WM][2][4]; +#pragma unroll + for (int i = 0; i < WM; ++i) { +#pragma unroll + for (int hr = 0; hr < 2; ++hr) { + const float bias = kLog2P - mrow[i][hr] - kMagicF * cs[i][hr]; + float p[8][2]; +#pragma unroll + for (int j = 0; j < 8; ++j) +#pragma unroll + for (int e = 0; e < 2; ++e) { + const float x = fmaf(__int_as_float(sc[i][j][2 * hr + e]), cs[i][hr], bias); + float z = ex2(x); + if (masked && tile * BN + 8 * j + 2 * t + e >= S) z = 0.f; + p[j][e] = z; + } + float s4[4]; +#pragma unroll + for (int u = 0; u < 4; ++u) s4[u] = (p[2 * u][0] + p[2 * u][1]) + (p[2 * u + 1][0] + p[2 * u + 1][1]); + lrow[i][hr] += (s4[0] + s4[1]) + (s4[2] + s4[3]); +#pragma unroll + for (int s = 0; s < 2; ++s) { + pa[i][s][hr] = pack_e4m3(p[4 * s][0], p[4 * s][1], p[4 * s + 1][0], p[4 * s + 1][1]); + pa[i][s][2 + hr] = pack_e4m3(p[4 * s + 2][0], p[4 * s + 2][1], p[4 * s + 3][0], p[4 * s + 3][1]); + } + } + } +#pragma unroll + for (int j = 0; j < 8; ++j) { + uint32_t vb[4]; + ldsm_x4(vb, vst + swz64(8 * j + lrow8, lmat)); +#pragma unroll + for (int i = 0; i < WM; ++i) { + mma_e4m3(o[i][j], pa[i][0], vb[0], vb[1]); + mma_e4m3(o[i][j], pa[i][1], vb[2], vb[3]); + } + } + ksc = ksn; + rd = rd == STAGES - 1 ? 0 : rd + 1; + }; + for (int tile = 0; tile < ntiles - 1; ++tile) step(tile, std::false_type{}); + if (S % BN) + step(ntiles - 1, std::true_type{}); + else + step(ntiles - 1, std::false_type{}); + cp_wait<0>(); + +#pragma unroll + for (int i = 0; i < WM; ++i) { +#pragma unroll + for (int hr = 0; hr < 2; ++hr) { + float l = lrow[i][hr]; + l += __shfl_xor_sync(0xffffffff, l, 1); + l += __shfl_xor_sync(0xffffffff, l, 2); + const float inv = 1.f / l; + int row = m0 + 16 * i + g + 8 * hr; + if (row >= S) continue; + uint16_t* dst = out + b * osb + row * oss + h * osh; +#pragma unroll + for (int j = 0; j < 8; ++j) { + int d = 8 * j + 2 * t; + float f0 = inv, f1 = inv; + f0 *= __uint_as_float(vmax[bh * D + d]) * (1.f / 448.f); + f1 *= __uint_as_float(vmax[bh * D + d + 1]) * (1.f / 448.f); + *reinterpret_cast(dst + d) = pack16(o[i][j][2 * hr] * f0, o[i][j][2 * hr + 1] * f1); + } + } + } +} + +Strided strided(const at::Tensor& x) { // x: [B, H, S, D] logical + return {reinterpret_cast(x.data_ptr()), x.stride(0), x.stride(2), x.stride(1)}; +} + +void check(const at::Tensor& x, const char* name) { + TORCH_CHECK(x.is_cuda() && (x.scalar_type() == at::kBFloat16 || x.scalar_type() == at::kHalf) && x.dim() == 4 && + x.size(3) == D && x.stride(3) == 1, + "icl_attention_int8: ", name, " must be a CUDA bf16 or fp16 [B, H, S, 64] tensor with unit last stride"); + TORCH_CHECK(x.stride(0) % 8 == 0 && x.stride(1) % 8 == 0 && x.stride(2) % 8 == 0 && + reinterpret_cast(x.data_ptr()) % 16 == 0, + "icl_attention_int8: ", name, " rows must be 16-byte aligned"); +} + +// q, k, v, out: [B, H, S, 64] views of one dtype, any strides with a unit last stride. +void run(const at::Tensor& q, const at::Tensor& k, const at::Tensor& v, at::Tensor& out, double sm_scale) { + check(q, "q"); + check(k, "k"); + check(v, "v"); + TORCH_CHECK(k.scalar_type() == q.scalar_type() && v.scalar_type() == q.scalar_type(), + "icl_attention_int8: k and v must have the dtype of q"); + TORCH_CHECK(out.scalar_type() == q.scalar_type() && out.sizes() == q.sizes() && out.stride(3) == 1 && + out.stride(0) % 2 == 0 && out.stride(1) % 2 == 0 && out.stride(2) % 2 == 0, + "icl_attention_int8: out must have the dtype and shape of q, a unit last stride and even strides"); + TORCH_CHECK(k.sizes() == q.sizes() && v.sizes() == q.sizes(), "icl_attention_int8: k and v must have the shape of q"); + int B = q.size(0), H = q.size(1), S = q.size(2); + TORCH_CHECK(S > 0 && (int64_t)S * std::max(q.stride(2), std::max(k.stride(2), v.stride(2))) < (1ll << 31) && + (int64_t)B * H * S * D < (1ll << 31), + "icl_attention_int8: S must be positive and element offsets must fit in int32"); + c10::cuda::CUDAGuard guard(q.device()); + auto stream = at::cuda::getCurrentCUDAStream(); + int S_pad = (S + BN - 1) / BN * BN; + auto f32 = q.options().dtype(at::kFloat); + auto ws = at::zeros({2, B * H * D}, f32); + float* ksum = ws.data_ptr(); + unsigned* vmax = reinterpret_cast(ksum + B * H * D); + auto kq = at::empty({B, H, S_pad, D}, q.options().dtype(at::kChar)); + auto kscale = at::empty({B, H, S_pad / BN}, f32); + auto vt = at::empty({B, H, D, S_pad}, q.options().dtype(at::kByte)); + const int chunks = (S + kStatTokens - 1) / kStatTokens; + auto kpart = at::empty({B * H, chunks, D}, f32); + auto launch = [&](auto f16_t) { + constexpr bool F16 = decltype(f16_t)::value; + stats_kernel + <<>>(strided(k), strided(v), S, H, kpart.data_ptr(), vmax); + ksum_kernel<<>>(kpart.data_ptr(), chunks, ksum); + quant_kernel<<>>(strided(k), strided(v), S, S_pad, H, ksum, vmax, + kq.data_ptr(), kscale.data_ptr(), + vt.data_ptr()); + attend_kernel<<>>( + strided(q), kq.data_ptr(), kscale.data_ptr(), vt.data_ptr(), vmax, + reinterpret_cast(out.data_ptr()), out.stride(0), out.stride(2), out.stride(1), S, S_pad, H, + float(sm_scale * kLog2e)); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + }; + if (q.scalar_type() == at::kHalf) + launch(std::true_type{}); + else + launch(std::false_type{}); +} + +} // namespace icl_attention_int8 + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("run", &icl_attention_int8::run); } diff --git a/benchmark/inference/kernels/csrc/icl_attention_sm120/icl_attention.cu b/benchmark/inference/kernels/csrc/icl_attention_sm120/icl_attention.cu new file mode 100644 index 0000000000..7bc59fb7d0 --- /dev/null +++ b/benchmark/inference/kernels/csrc/icl_attention_sm120/icl_attention.cu @@ -0,0 +1,468 @@ +#include +#include +#include +#include + +#include + +namespace icl_attention { + +constexpr int D = 64, BN = 64, TILE = BN * D; + +struct Params { + CUtensorMap tq, tk, tv; + __nv_bfloat16* o; + float* ws; + int* counts; + int64_t ob, os, oh; + int sq, skv, heads, nblk, tiles, whole, splits; + float scale_log2; +}; + +__device__ __forceinline__ uint32_t smem_u32(const void* p) { + return static_cast(__cvta_generic_to_shared(p)); +} + +// Rows of 64 bf16 = 8 chunks of 16 B, chunk index XOR (row & 7): the TMA +// 128B swizzle, conflict-free for ldmatrix. +__device__ __forceinline__ int swz(int row, int chunk) { return row * D + ((chunk ^ (row & 7)) << 3); } + +__device__ __forceinline__ void bar_init(uint32_t bar, int count) { + asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" ::"r"(bar), "r"(count)); +} +__device__ __forceinline__ void bar_arrive(uint32_t bar) { + asm volatile("mbarrier.arrive.shared::cta.b64 _, [%0];\n" ::"r"(bar) : "memory"); +} +__device__ __forceinline__ void bar_wait(uint32_t bar, uint32_t parity) { + asm volatile( + "{\n.reg .pred p;\nW: mbarrier.try_wait.parity.shared::cta.b64 p, [%0], " + "%1;\n@!p bra W;\n}\n" ::"r"(bar), + "r"(parity) + : "memory"); +} +__device__ __forceinline__ void bar_expect(uint32_t bar, uint32_t bytes) { + asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;\n" ::"r"(bar), "r"(bytes) : "memory"); +} +// Rows [row, row + box) of head h, batch b -> swizzled smem tile. +__device__ __forceinline__ void tma(uint32_t dst, const CUtensorMap* map, int row, int h, int b, uint32_t bar) { + asm volatile( + "cp.async.bulk.tensor.4d.shared::cta.global.tile.mbarrier::" + "complete_tx::bytes [%0], [%1, {0, %2, %3, %4}], [%5];\n" ::"r"(dst), + "l"(map), "r"(h), "r"(row), "r"(b), "r"(bar) + : "memory"); +} + +__device__ __forceinline__ void ldsm4(uint32_t* r, uint32_t addr) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) + : "r"(addr)); +} +__device__ __forceinline__ void ldsm4t(uint32_t* r, uint32_t addr) { + asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n" + : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) + : "r"(addr)); +} + +__device__ __forceinline__ void mma(float* c, const uint32_t* a, uint32_t b0, uint32_t b1) { + asm volatile( + "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " + "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};\n" + : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3]) + : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1)); +} + +__device__ __forceinline__ float ex2(float x) { + float y; + asm("ex2.approx.ftz.f32 %0, %1;\n" : "=f"(y) : "f"(x)); + return y; +} + +__device__ __forceinline__ uint32_t pack_bf16(float a, float b) { + __nv_bfloat162 h = __floats2bfloat162_rn(a, b); + return *reinterpret_cast(&h); +} + +// WARPS warps, each owning MT m16 tiles of query rows; a STAGES-deep TMA +// ring of 64-key K/V tiles with full/empty mbarriers. CTAs below `whole` own +// a query block; the blocks after them (the last wave) are split into +// `splits` key ranges, merged by the last CTA to finish the block. +template +struct Kernel { + static constexpr int BM = WARPS * 16 * MT; + static constexpr int THREADS = WARPS * 32; + static constexpr int NT = BN / 8; + static constexpr int SMEM_Q = BM * D * 2; + static constexpr int SMEM = SMEM_Q + STAGES * 2 * TILE * 2 + 8 * (2 * STAGES + 1); + static constexpr int PART = MT * 8 * 4 + MT * 4; // floats per thread + + struct State { + uint32_t qf[MT][4][4]; + float o[MT][8][4]; + float mx[MT][2], msc[MT][2], l[MT][2]; + }; + + template + static __device__ __forceinline__ void step(State& st, uint32_t kt, uint32_t vt, int col0, int skv, float sl2) { + const int lane = threadIdx.x & 31; + const int rx = lane & 7; + float s[MT][NT][4]; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) +#pragma unroll + for (int j = 0; j < NT; ++j) +#pragma unroll + for (int e = 0; e < 4; ++e) s[mt][j][e] = 0.f; + const uint32_t koff = (((lane >> 4) << 3) + rx) * D * 2; + const int kx = (lane >> 3) & 1; +#pragma unroll + for (int kk = 0; kk < 4; ++kk) +#pragma unroll + for (int jp = 0; jp < NT / 2; ++jp) { + uint32_t bf[4]; + ldsm4(bf, kt + koff + jp * 16 * D * 2 + (((kk * 2 + kx) ^ rx) << 4)); +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + mma(s[mt][2 * jp], st.qf[mt][kk], bf[0], bf[1]); + mma(s[mt][2 * jp + 1], st.qf[mt][kk], bf[2], bf[3]); + } + } + + if (MASK) { + const int c = col0 + 2 * (lane & 3); +#pragma unroll + for (int mt = 0; mt < MT; ++mt) +#pragma unroll + for (int j = 0; j < NT; ++j) +#pragma unroll + for (int e = 0; e < 4; ++e) + if (c + j * 8 + (e & 1) >= skv) s[mt][j][e] = -INFINITY; + } + + // Exact online softmax; the warp-wide max and O rescale only run when + // some row of the warp has a new maximum. + float tm[MT][2]; + bool grew = false; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) +#pragma unroll + for (int r = 0; r < 2; ++r) { + float v[NT]; +#pragma unroll + for (int j = 0; j < NT; ++j) v[j] = fmaxf(s[mt][j][2 * r], s[mt][j][2 * r + 1]); +#pragma unroll + for (int w = NT / 2; w > 0; w /= 2) +#pragma unroll + for (int j = 0; j < w; ++j) v[j] = fmaxf(v[j], v[j + w]); + tm[mt][r] = v[0]; + grew |= v[0] > st.mx[mt][r]; + } + if (__any_sync(0xffffffff, grew)) { +#pragma unroll + for (int mt = 0; mt < MT; ++mt) +#pragma unroll + for (int r = 0; r < 2; ++r) { + float m = tm[mt][r]; + m = fmaxf(m, __shfl_xor_sync(0xffffffff, m, 1)); + m = fmaxf(m, __shfl_xor_sync(0xffffffff, m, 2)); + const float nm = fmaxf(st.mx[mt][r], m); + const float f = ex2((st.mx[mt][r] - nm) * sl2); + st.mx[mt][r] = nm; + st.msc[mt][r] = nm * sl2; + st.l[mt][r] *= f; +#pragma unroll + for (int j = 0; j < 8; ++j) { + st.o[mt][j][2 * r] *= f; + st.o[mt][j][2 * r + 1] *= f; + } + } + } + + uint32_t pf[MT][NT / 2][4]; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) +#pragma unroll + for (int j = 0; j < NT; ++j) { + float e[4]; +#pragma unroll + for (int x = 0; x < 4; ++x) e[x] = ex2(fmaf(s[mt][j][x], sl2, -st.msc[mt][x >> 1])); + st.l[mt][0] += e[0] + e[1]; + st.l[mt][1] += e[2] + e[3]; + pf[mt][j >> 1][(j & 1) * 2] = pack_bf16(e[0], e[1]); + pf[mt][j >> 1][(j & 1) * 2 + 1] = pack_bf16(e[2], e[3]); + } + + const uint32_t voff = (lane & 15) * D * 2; + const int vx = lane >> 4; +#pragma unroll + for (int kk = 0; kk < NT / 2; ++kk) +#pragma unroll + for (int dp = 0; dp < 4; ++dp) { + uint32_t bf[4]; + ldsm4t(bf, vt + voff + kk * 16 * D * 2 + (((dp * 2 + vx) ^ rx) << 4)); +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + mma(st.o[mt][2 * dp], pf[mt][kk], bf[0], bf[1]); + mma(st.o[mt][2 * dp + 1], pf[mt][kk], bf[2], bf[3]); + } + } + } + + static __device__ __forceinline__ float* part(const Params& p, int slot) { + return p.ws + ((int64_t)slot * THREADS + threadIdx.x) * PART; + } + + static __device__ void store(const Params& p, State& st, int blk) { + const int lane = threadIdx.x & 31, warp = threadIdx.x >> 5; + const int mb = blk % p.nblk, bh = blk / p.nblk; + const int h = bh % p.heads, b = bh / p.heads; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) +#pragma unroll + for (int r = 0; r < 2; ++r) { + float s = st.l[mt][r]; + s += __shfl_xor_sync(0xffffffff, s, 1); + s += __shfl_xor_sync(0xffffffff, s, 2); + const float inv = 1.f / s; + const int row = mb * BM + (warp * MT + mt) * 16 + (lane >> 2) + r * 8; + if (row < p.sq) { + __nv_bfloat16* out = p.o + b * p.ob + h * p.oh + row * p.os; +#pragma unroll + for (int j = 0; j < 8; ++j) + *reinterpret_cast(out + j * 8 + 2 * (lane & 3)) = + pack_bf16(st.o[mt][j][2 * r] * inv, st.o[mt][j][2 * r + 1] * inv); + } + } + } + + // Partial result of a split block: publish it; the last split to finish + // merges all parts in split order and writes the output. + static __device__ void merge(const Params& p, State& st, int blk, int split, float sl2) { + __shared__ int last; + const int first = (blk - p.whole) * p.splits; + float4* w = reinterpret_cast(part(p, first + split)); +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { +#pragma unroll + for (int j = 0; j < 8; ++j) + w[mt * 8 + j] = make_float4(st.o[mt][j][0], st.o[mt][j][1], st.o[mt][j][2], st.o[mt][j][3]); + w[MT * 8 + mt] = make_float4(st.mx[mt][0], st.mx[mt][1], st.l[mt][0], st.l[mt][1]); + } + __threadfence(); + __syncthreads(); + if (threadIdx.x == 0) last = atomicAdd(p.counts + blk, 1) == p.splits - 1; + __syncthreads(); + if (!last) return; + __threadfence(); + const float4* x0 = reinterpret_cast(part(p, first)); +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { +#pragma unroll + for (int j = 0; j < 8; ++j) { + const float4 v = __ldcg(x0 + mt * 8 + j); + st.o[mt][j][0] = v.x, st.o[mt][j][1] = v.y; + st.o[mt][j][2] = v.z, st.o[mt][j][3] = v.w; + } + const float4 ml = __ldcg(x0 + MT * 8 + mt); + st.mx[mt][0] = ml.x, st.mx[mt][1] = ml.y; + st.l[mt][0] = ml.z, st.l[mt][1] = ml.w; + } + for (int sp = 1; sp < p.splits; ++sp) { + const float4* x = reinterpret_cast(part(p, first + sp)); +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + const float4 ml = __ldcg(x + MT * 8 + mt); + float f[2], g[2]; + const float om[2] = {ml.x, ml.y}, ol[2] = {ml.z, ml.w}; +#pragma unroll + for (int r = 0; r < 2; ++r) { + const float nm = fmaxf(st.mx[mt][r], om[r]); + f[r] = ex2((st.mx[mt][r] - nm) * sl2); + g[r] = ex2((om[r] - nm) * sl2); + st.mx[mt][r] = nm; + st.l[mt][r] = st.l[mt][r] * f[r] + ol[r] * g[r]; + } +#pragma unroll + for (int j = 0; j < 8; ++j) { + const float4 v = __ldcg(x + mt * 8 + j); + st.o[mt][j][0] = st.o[mt][j][0] * f[0] + v.x * g[0]; + st.o[mt][j][1] = st.o[mt][j][1] * f[0] + v.y * g[0]; + st.o[mt][j][2] = st.o[mt][j][2] * f[1] + v.z * g[1]; + st.o[mt][j][3] = st.o[mt][j][3] * f[1] + v.w * g[1]; + } + } + } + store(p, st, blk); + } + + static __device__ void run(const Params& p) { + extern __shared__ __align__(1024) __nv_bfloat16 smem[]; + const int tid = threadIdx.x, warp = tid >> 5, lane = tid & 31; + int blk = blockIdx.x, split = 0, t0 = 0, t1 = p.tiles; + if (blk >= p.whole) { + const int j = blk - p.whole; + blk = p.whole + j / p.splits; + split = j % p.splits; + t0 = split * p.tiles / p.splits; + t1 = (split + 1) * p.tiles / p.splits; + } + const int bh = blk / p.nblk, h = bh % p.heads, b = bh / p.heads; + const uint32_t qs = smem_u32(smem), ks = qs + SMEM_Q; + const uint32_t vs = ks + STAGES * TILE * 2, bars = vs + STAGES * TILE * 2; + const uint32_t qbar = bars + 16 * STAGES; + const float sl2 = p.scale_log2; + const int n = t1 - t0; + + auto load = [&](int i) { + if (i >= n) return; + const int slot = i % STAGES; + const uint32_t full = bars + 8 * slot; + if (i >= STAGES) bar_wait(bars + 8 * (STAGES + slot), ((i / STAGES) - 1) & 1); + bar_expect(full, 2 * TILE * 2); + tma(ks + slot * TILE * 2, &p.tk, (t0 + i) * BN, h, b, full); + tma(vs + slot * TILE * 2, &p.tv, (t0 + i) * BN, h, b, full); + }; + if (tid == 0) { + for (int s = 0; s < STAGES; ++s) { + bar_init(bars + 8 * s, 1); + bar_init(bars + 8 * (STAGES + s), WARPS); + } + bar_init(qbar, 1); + asm volatile("fence.mbarrier_init.release.cluster;\n" ::: "memory"); + bar_expect(qbar, BM * D * 2); + tma(qs, &p.tq, (blk % p.nblk) * BM, h, b, qbar); + for (int s = 0; s < STAGES - 1; ++s) load(s); + } + __syncthreads(); + + State st; + bar_wait(qbar, 0); +#pragma unroll + for (int mt = 0; mt < MT; ++mt) +#pragma unroll + for (int kk = 0; kk < 4; ++kk) { + const int row = (warp * MT + mt) * 16 + (lane & 15); + ldsm4(st.qf[mt][kk], qs + 2 * swz(row, kk * 2 + (lane >> 4))); + } +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { +#pragma unroll + for (int j = 0; j < 8; ++j) +#pragma unroll + for (int e = 0; e < 4; ++e) st.o[mt][j][e] = 0.f; + st.mx[mt][0] = st.mx[mt][1] = -INFINITY; + st.l[mt][0] = st.l[mt][1] = 0.f; + } + + auto tile = [&](int i, auto mask) { + const int slot = i % STAGES; + bar_wait(bars + 8 * slot, (i / STAGES) & 1); + if (tid == 0) load(i + STAGES - 1); + step(st, ks + slot * TILE * 2, vs + slot * TILE * 2, (t0 + i) * BN, p.skv, sl2); + __syncwarp(); + if (lane == 0) bar_arrive(bars + 8 * (STAGES + slot)); + }; + const int open = t1 == p.tiles ? n - 1 : n; + for (int i = 0; i < open; ++i) tile(i, std::false_type{}); + if (open < n) tile(open, std::true_type{}); + if (blk < p.whole) + store(p, st, blk); + else + merge(p, st, blk, split, sl2); + } +}; + +template +__global__ void __launch_bounds__(W * 32, W > 4 ? 1 : 2) icl_attention_kernel(const __grid_constant__ Params p) { + Kernel::run(p); +} + +// 8 warps x 2 m16 tiles = 256 query rows per CTA, 2-stage K/V ring, up to 4 key splits in the last wave. +constexpr int kWarps = 8, kTiles = 2, kStages = 2, kMaxSplits = 4; +using Cta = Kernel; + +using Encode = CUresult (*)(CUtensorMap*, CUtensorMapDataType, cuuint32_t, void*, const cuuint64_t*, const cuuint64_t*, + const cuuint32_t*, const cuuint32_t*, CUtensorMapInterleave, CUtensorMapSwizzle, + CUtensorMapL2promotion, CUtensorMapFloatOOBfill); + +// [B, S, H, 64] -> boxes of `rows` x 64 for one (head, batch), 128B swizzle. +CUtensorMap tensor_map(const at::Tensor& t, int rows) { + static Encode encode = [] { + void* fn = nullptr; + cudaDriverEntryPointQueryResult q; + const cudaError_t e = cudaGetDriverEntryPointByVersion("cuTensorMapEncodeTiled", &fn, 12000, cudaEnableDefault, &q); + TORCH_CHECK(e == cudaSuccess && fn, "icl_attention: cuTensorMapEncodeTiled is unavailable"); + return reinterpret_cast(fn); + }(); + CUtensorMap map; + cuuint64_t dims[4] = {D, (cuuint64_t)t.size(2), (cuuint64_t)t.size(1), (cuuint64_t)t.size(0)}; + cuuint64_t strides[3] = {(cuuint64_t)t.stride(2) * 2, (cuuint64_t)t.stride(1) * 2, (cuuint64_t)t.stride(0) * 2}; + cuuint32_t box[4] = {D, 1, (cuuint32_t)rows, 1}, unit[4] = {1, 1, 1, 1}; + CUresult r = encode(&map, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 4, t.data_ptr(), dims, strides, box, unit, + CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, CU_TENSOR_MAP_L2_PROMOTION_L2_256B, + CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE); + TORCH_CHECK(r == CUDA_SUCCESS, "icl_attention: cuTensorMapEncodeTiled failed with ", (int)r); + return map; +} + +void check(const at::Tensor& t, const char* name) { + TORCH_CHECK(t.is_cuda() && t.dtype() == at::kBFloat16 && t.dim() == 4, "icl_attention: ", name, + " must be a 4-d CUDA bf16 tensor"); + TORCH_CHECK(t.size(3) == D && t.stride(3) == 1, "icl_attention: ", name, " must have head dim 64 with unit stride"); + TORCH_CHECK(t.stride(0) % 8 == 0 && t.stride(1) % 8 == 0 && t.stride(2) % 8 == 0 && + reinterpret_cast(t.data_ptr()) % 16 == 0, + "icl_attention: ", name, " rows must be 16-byte aligned"); +} + +// q [B, Sq, H, 64], k/v [B, Skv, H, 64], out [B, Sq, H, 64]. +at::Tensor run(const at::Tensor& q, const at::Tensor& k, const at::Tensor& v, at::Tensor out, double scale) { + check(q, "q"); + check(k, "k"); + check(v, "v"); + check(out, "out"); + TORCH_CHECK(k.sizes() == v.sizes() && q.size(0) == k.size(0) && q.size(2) == k.size(2) && out.sizes() == q.sizes(), + "icl_attention: k and v must match, q must share their batch and heads, out must match q"); + const auto kernel = icl_attention_kernel; + Params p{}; + p.o = reinterpret_cast<__nv_bfloat16*>(out.data_ptr()); + p.ob = out.stride(0), p.os = out.stride(1), p.oh = out.stride(2); + p.sq = q.size(1), p.skv = k.size(1); + p.heads = q.size(2); + p.scale_log2 = static_cast(scale * 1.4426950408889634); + if (p.sq == 0 || q.size(0) == 0 || p.heads == 0) return out; + TORCH_CHECK(p.skv > 0, "icl_attention: k must have at least one row"); + p.nblk = (p.sq + Cta::BM - 1) / Cta::BM; + p.tiles = (p.skv + BN - 1) / BN; + const int64_t blocks = q.size(0) * p.heads * (int64_t)p.nblk; + TORCH_CHECK(blocks * p.tiles < (1ll << 31), "icl_attention: blocks x key tiles must fit in int32"); + p.tq = tensor_map(q, Cta::BM); + p.tk = tensor_map(k, BN); + p.tv = tensor_map(v, BN); + cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, Cta::SMEM); + // The last wave plus the remainder run as `splits` key ranges each, so + // the tail of the launch is filled with short CTAs. + int per_sm = 0; + cudaOccupancyMaxActiveBlocksPerMultiprocessor(&per_sm, kernel, Cta::THREADS, Cta::SMEM); + const int64_t wave = (int64_t)std::max(per_sm, 1) * at::cuda::getCurrentDeviceProperties()->multiProcessorCount; + p.splits = std::max(1, std::min(kMaxSplits, p.tiles / 96)); + p.whole = blocks; + at::Tensor ws, counts; + if (p.splits > 1) { + p.whole = blocks % wave ? std::max(blocks - blocks % wave - wave, 0) : blocks; + const int64_t parts = (blocks - p.whole) * p.splits; + if (parts) { + ws = at::empty({parts * Cta::THREADS * Cta::PART}, q.options().dtype(at::kFloat)); + p.ws = ws.data_ptr(); + counts = at::zeros({blocks}, q.options().dtype(at::kInt)); + p.counts = counts.data_ptr(); + } + } + const int64_t grid = p.whole + (blocks - p.whole) * p.splits; + TORCH_CHECK(grid < (1ll << 31), "icl_attention: grid must fit in int32"); + kernel<<(grid), Cta::THREADS, Cta::SMEM, at::cuda::getCurrentCUDAStream()>>>(p); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return out; +} + +} // namespace icl_attention + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("run", &icl_attention::run); } diff --git a/benchmark/inference/kernels/csrc/icl_qk_sm120/icl_qk.cu b/benchmark/inference/kernels/csrc/icl_qk_sm120/icl_qk.cu new file mode 100644 index 0000000000..4a45239bc6 --- /dev/null +++ b/benchmark/inference/kernels/csrc/icl_qk_sm120/icl_qk.cu @@ -0,0 +1,156 @@ +// Per-head RMS of the ICL attention query and key, with the cache copies fused in. SM120, BF16, head width 64. +// +// query: out[m, h, :] = bf16(bf16(rms(q[m, h, :])) * log_n * head_scale[h]) (FP32 products, one rounding) +// key: k[m, h, :] = rms(kv[m, h * 64 : (h + 1) * 64]) for h < key_heads +// k_cache[m, h, :] = rms(kv[m, h * 64 : ...]) for h < cache +// v_cache[m, h, :] = kv[m, (heads + h) * 64 : ...] for h < cache +// +// q [M, H, 64] with strides (ld, 64, 1); kv [M, 2 * heads * 64] with row stride ld (packed K | V of the KV addmm, +// read in place). head_scale FP32 [H]. Outputs contiguous. eps 1e-6, no weight. +// Bytes match SDM's Triton rmsnorm_cast at channels = 64, then Inductor's fused LogScale multiply (FP32 between +// the two products, head_scale read as FP32) and the cache head-slice copies. +#include +#include +#include +#include "common/rms_ptx.cuh" + +namespace icl_qk { +using rms_ptx::add; +using rms_ptx::inv_rms; +using rms_ptx::mul; +using rms_ptx::square_add; +constexpr int kHead = 64, kLanes = 8, kThreads = 256, kUnits = kThreads / kLanes; +using u16 = unsigned short; + +__device__ __forceinline__ float f32(u16 a) { + float x; + asm("cvt.f32.bf16 %0,%1;" : "=f"(x) : "h"(a)); + return x; +} +__device__ __forceinline__ u16 bf16(float a) { + u16 x; + asm("cvt.rn.bf16.f32 %0,%1;" : "=h"(x) : "f"(a)); + return x; +} +__device__ __forceinline__ u16 element(uint4 w, int i) { + const unsigned word = i < 2 ? w.x : i < 4 ? w.y : i < 6 ? w.z : w.w; + return u16(word >> (16 * (i & 1))); +} +__device__ __forceinline__ uint4 pack(const u16 (&a)[8]) { + return make_uint4(a[0] | unsigned(a[1]) << 16, a[2] | unsigned(a[3]) << 16, a[4] | unsigned(a[5]) << 16, + a[6] | unsigned(a[7]) << 16); +} + +// Each lane holds 8 adjacent channels (two rmsnorm_cast groups of 4); 8 adjacent lanes hold one head. +// rmsnorm_cast squares each group in channel order and reduces the 16 group sums by an XOR butterfly with group +// shifts 8, 4, 2, 1: lane shifts 4, 2, 1 are group shifts 8, 4, 2 and the in-lane add of the two groups is group +// shift 1. All lanes, including those of masked units, take part in every shuffle. +__device__ __forceinline__ void head_rms(uint4 w, u16 (&out)[8]) { + float s0 = 0.f, s1 = 0.f; +#pragma unroll + for (int i = 0; i < 4; ++i) { + s0 = square_add(element(w, i), s0); + s1 = square_add(element(w, i + 4), s1); + } +#pragma unroll + for (int shift = kLanes / 2; shift >= 1; shift >>= 1) { + s0 = add(s0, __shfl_xor_sync(~0u, s0, shift, kLanes)); + s1 = add(s1, __shfl_xor_sync(~0u, s1, shift, kLanes)); + } + const float scale = inv_rms(add(s0, s1)); +#pragma unroll + for (int i = 0; i < 8; ++i) out[i] = bf16(mul(f32(element(w, i)), scale)); +} + +__global__ void __launch_bounds__(kThreads) + query_kernel(const u16* __restrict__ q, int64_t ld, int heads, const float* __restrict__ head_scale, float log_n, + u16* __restrict__ out, int64_t units) { + const int lane = threadIdx.x % kLanes; + const int64_t unit = int64_t(blockIdx.x) * kUnits + threadIdx.x / kLanes; + const bool valid = unit < units; + const int64_t row = unit / heads; + const int h = int(unit % heads); + const uint4 w = valid ? *reinterpret_cast(q + row * ld + h * kHead + lane * 8) : make_uint4(0, 0, 0, 0); + u16 r[8]; + head_rms(w, r); + if (!valid) return; + const float a = head_scale[h]; +#pragma unroll + for (int i = 0; i < 8; ++i) r[i] = bf16(mul(mul(f32(r[i]), log_n), a)); + *reinterpret_cast(out + unit * kHead + lane * 8) = pack(r); +} + +// Units cover heads [0, span) of each row: span = heads when K is stored, else cache. +__global__ void __launch_bounds__(kThreads) + key_kernel(const u16* __restrict__ kv, int64_t ld, int heads, int span, bool key, int cache, + u16* __restrict__ k_out, u16* __restrict__ k_cache, u16* __restrict__ v_cache, int64_t units) { + const int lane = threadIdx.x % kLanes; + const int64_t unit = int64_t(blockIdx.x) * kUnits + threadIdx.x / kLanes; + const bool valid = unit < units; + const int64_t row = unit / span; + const int h = int(unit % span); + const u16* src = kv + row * ld + h * kHead + lane * 8; + const uint4 w = valid ? *reinterpret_cast(src) : make_uint4(0, 0, 0, 0); + u16 r[8]; + head_rms(w, r); + if (!valid) return; + const uint4 packed = pack(r); + if (key) *reinterpret_cast(k_out + (row * heads + h) * kHead + lane * 8) = packed; + if (h < cache) { + const int64_t at = (row * cache + h) * kHead + lane * 8; + *reinterpret_cast(k_cache + at) = packed; + *reinterpret_cast(v_cache + at) = *reinterpret_cast(src + heads * kHead); + } +} + +u16* ptr(at::Tensor const& t) { return reinterpret_cast(t.data_ptr()); } +int64_t blocks(int64_t units) { return (units + kUnits - 1) / kUnits; } + +// The Python supports() decides eligibility; these checks only keep a bad call from touching memory. +void check_rows(at::Tensor const& t, int64_t width, const char* what) { + TORCH_CHECK(t.is_cuda() && t.scalar_type() == at::kBFloat16 && t.stride(-1) == 1 && t.size(0) > 0 && + t.stride(0) % 8 == 0 && t.stride(0) >= width && reinterpret_cast(t.data_ptr()) % 16 == 0, + "icl_qk: ", what, " must be CUDA bf16 rows of at least ", width, " elements, 16-byte aligned"); +} + +at::Tensor query(at::Tensor const& q, at::Tensor const& head_scale, double log_n) { + TORCH_CHECK(q.dim() == 3 && q.size(2) == kHead && q.stride(1) == kHead, + "icl_qk: q must be [M, H, 64] with head stride 64"); + check_rows(q, q.size(1) * kHead, "q"); + TORCH_CHECK(head_scale.is_cuda() && head_scale.scalar_type() == at::kFloat && head_scale.is_contiguous() && + head_scale.numel() == q.size(1), + "icl_qk: head_scale must be a contiguous CUDA fp32 [H] tensor"); + c10::cuda::CUDAGuard guard(q.device()); + auto out = at::empty(q.sizes(), q.options().memory_format(at::MemoryFormat::Contiguous)); + const int64_t units = q.size(0) * q.size(1); + TORCH_CHECK(blocks(units) <= INT_MAX, "icl_qk: q has too many rows for an int32 grid"); + query_kernel<<>>( + ptr(q), q.stride(0), int(q.size(1)), head_scale.data_ptr(), float(log_n), ptr(out), units); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return out; +} + +std::vector key(at::Tensor const& kv, int64_t heads, bool with_key, int64_t cache) { + TORCH_CHECK(kv.dim() == 2 && kv.size(1) == 2 * heads * kHead, "icl_qk: kv must be [M, 2 * heads * 64]"); + TORCH_CHECK(0 <= cache && cache <= heads && (with_key || cache > 0), + "icl_qk: cache must be in [0, heads] and nonzero without with_key"); + check_rows(kv, kv.size(1), "kv"); + c10::cuda::CUDAGuard guard(kv.device()); + const int64_t m = kv.size(0); + auto opts = kv.options().memory_format(at::MemoryFormat::Contiguous); + auto k_out = at::empty({m, with_key ? heads : 0, kHead}, opts); + auto k_cache = at::empty({m, cache, kHead}, opts), v_cache = at::empty({m, cache, kHead}, opts); + const int span = int(with_key ? heads : cache); + const int64_t units = m * span; + TORCH_CHECK(blocks(units) <= INT_MAX, "icl_qk: kv has too many rows for an int32 grid"); + key_kernel<<>>( + ptr(kv), kv.stride(0), int(heads), span, with_key, int(cache), ptr(k_out), ptr(k_cache), ptr(v_cache), units); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + return {k_out, k_cache, v_cache}; +} +} // namespace icl_qk + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("query", &icl_qk::query); + m.def("key", &icl_qk::key); +} diff --git a/benchmark/inference/kernels/csrc/kv_rms_sm120/kv_rms.cu b/benchmark/inference/kernels/csrc/kv_rms_sm120/kv_rms.cu new file mode 100644 index 0000000000..0e4bccb55a --- /dev/null +++ b/benchmark/inference/kernels/csrc/kv_rms_sm120/kv_rms.cu @@ -0,0 +1,53 @@ +// Column KV projection with the key RMS fused into the K epilogue; raw K is never stored. SM120, BF16. +// +// packed[:, D:] = bf16(a @ w[D:].T + bias[D:]) (V) +// key = rms(bf16(a @ w[:D].T + bias[:D])) per head of Head (normalized K, no weight, eps 1e-6) +// +// a [M, D], w [2D, D], bias [2D], packed [M, 2D] (packed[:, :D] is left untouched), key [M, D]. +// key may be exactly a (the compiled graph reuses A's buffer for normalized K); nothing else may overlap. +// Two launches on one stream, each a single N tile of width D: V first, then K. A K CTA overwrites only +// the rows of A it alone has read, so the alias is safe without inter-CTA synchronization. +// Bytes match cuBLAS addmm followed by SDM's Triton rmsnorm_cast at channels = Head (32 or 64). +#include +#include +#include +#include +#include "common/head_rms.cuh" + +namespace kv_rms { +using head_rms::BF; +using head_rms::kMaxM; +using head_rms::launch; +struct Key {}; // names the K launch in the kernel symbol + +template +void run_instance(at::Tensor const& a, at::Tensor const& w, at::Tensor const& bias, at::Tensor const& packed, + at::Tensor const& key) { + auto ptr = [](at::Tensor const& t) { return reinterpret_cast(t.data_ptr()); }; + const int m = int(a.size(0)); + auto stream = at::cuda::getCurrentCUDAStream(); + launch(ptr(a), ptr(w) + D * D, ptr(bias) + D, ptr(packed) + D, 2 * D, m, stream); + launch(ptr(a), ptr(w), ptr(bias), ptr(key), D, m, stream); +} + +// The Python supports() decides eligibility (including aliasing); these checks only keep a bad call from touching +// memory. +void run(at::Tensor const& a, at::Tensor const& w, at::Tensor const& bias, at::Tensor const& packed, + at::Tensor const& key, int64_t head) { + for (auto const* t : {&a, &w, &bias, &packed, &key}) + TORCH_CHECK(t->is_cuda() && t->scalar_type() == at::kBFloat16 && t->is_contiguous() && + reinterpret_cast(t->data_ptr()) % 16 == 0, + "kv_rms: a, w, bias, packed and key must be contiguous 16-byte aligned CUDA bf16"); + const int64_t m = a.size(0), d = a.size(1); + TORCH_CHECK(a.dim() == 2 && m >= 1 && m <= kMaxM && w.dim() == 2 && w.size(0) == 2 * d && w.size(1) == d && + bias.dim() == 1 && bias.size(0) == 2 * d && packed.dim() == 2 && packed.size(0) == m && + packed.size(1) == 2 * d && key.sizes() == a.sizes(), + "kv_rms: need a [M, D], w [2D, D], bias [2D], packed [M, 2D] and key [M, D]"); + c10::cuda::CUDAGuard guard(a.device()); + if (d == 256 && head == 64) return run_instance<256, 64>(a, w, bias, packed, key); + if (d == 128 && head == 32) return run_instance<128, 32>(a, w, bias, packed, key); + TORCH_CHECK(false, "kv_rms: (D, head) must be (256, 64) or (128, 32), got (", d, ", ", head, ")"); +} +} // namespace kv_rms + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("run", &kv_rms::run); } diff --git a/benchmark/inference/kernels/csrc/linear_gelu_sm120/linear_gelu.cu b/benchmark/inference/kernels/csrc/linear_gelu_sm120/linear_gelu.cu new file mode 100644 index 0000000000..d021369ad3 --- /dev/null +++ b/benchmark/inference/kernels/csrc/linear_gelu_sm120/linear_gelu.cu @@ -0,0 +1,141 @@ +// out = gelu(bf16(a @ w.T + bias)) for BF16 a[M,K], w[N,K], bias[N], out[M,N]; SM120. +// Matches cuBLAS addmm (FP32 accumulate, bias added in FP32, RN to BF16) followed by Inductor's GELU. +#include +#include +#include +#include +#include +#include +#include "common/bias_gelu.cuh" + +namespace linear_gelu { +using BF = cutlass::bfloat16_t; + +struct BiasGelu : bias_gelu::Op { // names this module in the kernel symbol + using bias_gelu::Op::Op; +}; + +template +using Gemm = cutlass::gemm::device::Gemm< + BF, cutlass::layout::RowMajor, BF, cutlass::layout::ColumnMajor, BF, cutlass::layout::RowMajor, float, + cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80, cutlass::gemm::GemmShape<128, TileN, 32>, + cutlass::gemm::GemmShape<64, 64, 32>, cutlass::gemm::GemmShape<16, 8, 16>, Op, + cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<1>, 3, 8, 8, false, cutlass::arch::OpMultiplyAdd>; + +constexpr int64_t kMaxM = INT_MAX - 128; // the swizzle computes M + tile_m in int32 + +BF* ptr(at::Tensor const& t) { return reinterpret_cast(t.data_ptr()); } + +template +void launch(at::Tensor const& a, at::Tensor const& w, at::Tensor const& bias, at::Tensor const& out) { + static_assert(K % 8 == 0 && N % 8 == 0, "16-byte vector access"); + using G = Gemm; + typename G::Arguments args({int(a.size(0)), N, K}, {ptr(a), K}, {ptr(w), K}, {ptr(bias), 0}, {ptr(out), N}, + {1.0f, 1.0f}, 1); + G gemm; + TORCH_CHECK(gemm.can_implement(args) == cutlass::Status::kSuccess, "linear_gelu: can_implement failed"); + auto stream = at::cuda::getCurrentCUDAStream(); + TORCH_CHECK(gemm.initialize(args, nullptr, stream) == cutlass::Status::kSuccess, "linear_gelu: initialize failed"); + TORCH_CHECK(gemm.run(stream) == cutlass::Status::kSuccess, "linear_gelu: run failed"); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +// K1024/N2048 (dataset FFN): the same GEMM with an epilogue that loads the bias through the source aspect and +// applies bias and GELU to each 8-element vector of the accumulator exchange. +using Bias = cutlass::epilogue::thread::LinearCombination; +struct FfnOp : Bias { + struct Params : Bias::Params { + CUTLASS_HOST_DEVICE Params() : Bias::Params(1.f, 1.f) {} + }; + CUTLASS_HOST_DEVICE FfnOp(Params const& p) : Bias(p) {} +}; + +template +struct GeluEpilogue : Base { + using Iterator = typename Base::OutputTileIterator; + using Fragment = typename Iterator::Fragment; + using SharedFragment = typename Base::SharedLoadIterator::Fragment; + static_assert(Iterator::kElementsPerAccess == 8, "8-element vectors"); + static_assert(Fragment::kElements == SharedFragment::kElements, "one output fragment per exchanged fragment"); + CUTLASS_DEVICE GeluEpilogue(typename Base::SharedStorage& s, int t, int w, int l) : Base(s, t, w, l) {} + + struct Source { + Iterator bias_iterator; + Fragment bias; + CUTLASS_DEVICE void load() { + bias_iterator.load(bias); + ++bias_iterator; + } + CUTLASS_DEVICE void apply_output_operator(Fragment& out, FfnOp const& op, SharedFragment const& acc) { + auto const* av = reinterpret_cast(&acc); + auto const* bv = reinterpret_cast(&bias); + auto* ov = reinterpret_cast(&out); + CUTLASS_PRAGMA_UNROLL + for (int idx = 0; idx < Fragment::kElements / 8; ++idx) { + auto rounded = static_cast(op)(av[idx], bv[idx]); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < 8; ++i) { + BF scalar = rounded[i]; + auto value = __ushort_as_bfloat16(scalar.raw()); + ov[idx][i] = BF::bitcast(__bfloat16_as_ushort(triton_gelu_bf16(value))); + } + } + } + }; + + CUTLASS_DEVICE void operator()(FfnOp const& op, Iterator destination, typename Base::AccumulatorTile const& acc, + Iterator source) { + Source aspect{source, {}}; + Base::template operator()(op, destination, acc, aspect); + } +}; + +using FfnGemm = Gemm<256, FfnOp>; +using FfnBase = FfnGemm::GemmKernel; +using FfnKernel = cutlass::gemm::kernel::Gemm, + typename FfnBase::ThreadblockSwizzle, false>; +static_assert(sizeof(FfnKernel::SharedStorage) == sizeof(FfnBase::SharedStorage), "no extra shared memory"); + +void launch_ffn(at::Tensor const& a, at::Tensor const& w, at::Tensor const& bias, at::Tensor const& out) { + FfnGemm::Arguments args({int(a.size(0)), 2048, 1024}, {ptr(a), 1024}, {ptr(w), 1024}, {ptr(bias), 0}, + {ptr(out), 2048}, FfnOp::Params(), 1); + TORCH_CHECK(FfnGemm::can_implement(args) == cutlass::Status::kSuccess, "linear_gelu: can_implement failed"); + typename FfnBase::ThreadblockSwizzle swizzle; + auto tiled = swizzle.get_tiled_shape(args.problem_size, {128, 256, 32}, 1); + FfnKernel::Params params(args.problem_size, tiled, args.ref_A.non_const_ref(), args.ref_B.non_const_ref(), + args.ref_C.non_const_ref(), args.ref_D, args.epilogue, nullptr); + constexpr int smem = sizeof(FfnKernel::SharedStorage); + C10_CUDA_CHECK(cudaFuncSetAttribute(cutlass::Kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem)); + cutlass::Kernel + <<>>(params); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +struct Instance { + int64_t k, n; + void (*fn)(at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&); +}; +constexpr Instance kInstances[] = { + {128, 256, launch<128, 256, 128>}, // embedding MLPs, Small + {256, 512, launch<256, 512, 256>}, // embedding MLPs, Large and Medium + {1024, 2048, launch_ffn}, // dataset FFN, Large +}; + +// The Python supports() is the contract; these checks only keep a bad call from touching memory. +void run(at::Tensor const& a, at::Tensor const& w, at::Tensor const& bias, at::Tensor const& out) { + for (auto const* t : {&a, &w, &bias, &out}) + TORCH_CHECK(t->is_cuda() && t->scalar_type() == at::kBFloat16 && t->is_contiguous() && + reinterpret_cast(t->data_ptr()) % 16 == 0, + "linear_gelu: operands must be contiguous, 16-byte aligned CUDA BF16 tensors"); + const int64_t m = a.size(0), k = a.size(1), n = w.size(0); + TORCH_CHECK(a.dim() == 2 && w.dim() == 2 && w.size(1) == k && bias.dim() == 1 && bias.size(0) == n && + out.dim() == 2 && out.size(0) == m && out.size(1) == n && m >= 1 && m <= kMaxM, + "linear_gelu: expected a [M, K], w [N, K], bias [N], out [M, N]"); + c10::cuda::CUDAGuard guard(a.device()); + for (auto const& i : kInstances) + if (i.k == k && i.n == n) return i.fn(a, w, bias, out); + TORCH_CHECK(false, "linear_gelu: no instance for K=", k, " N=", n); +} +} // namespace linear_gelu + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("run", &linear_gelu::run); } diff --git a/benchmark/inference/kernels/csrc/projection_rms_sm120/projection_rms.cu b/benchmark/inference/kernels/csrc/projection_rms_sm120/projection_rms.cu new file mode 100644 index 0000000000..d9270d7ef0 --- /dev/null +++ b/benchmark/inference/kernels/csrc/projection_rms_sm120/projection_rms.cu @@ -0,0 +1,202 @@ +// Column attention out-projection + residual + weighted RMSNorm + BF16 cast in one CUTLASS GEMM +// (SM120, BF16, D in {128, 256}). Rows r = f*C + c of [F, C, D]: +// +// raw[r] = RN_bf16(a[r] @ w.T + bias) cuBLAS's addmm: 128 x D x 32 CTA tile (one N tile per row), +// 64x64x32 warps, mma 16x8x16, 3 stages, no split-K +// x[r] = RN_bf16(raw[r] + table[c, P + f]) one FP32 add +// norm[r] = RN_bf16(RN(x[r] * inv) * weight) inv = rsqrt.approx.ftz(div.full(sum_c x_c^2, D) + eps) +// +// The GEMM destination receives raw, which stays live for the column-down residual. +// +// Reduction order is SDM's Triton rmsnorm_cast: group g = channels 4g..4g+3, fma from 0 in channel order, +// butterfly 16,8,4,2,1 over each run of 32 groups (one 128-channel half), halves added last. In the +// epilogue 16 lanes own a row, lane l holds channels 8l..8l+7 of each half: groups 2l (s0) and 2l+1 (s1). +// Lane XOR 8,4,2,1 is group XOR 16,8,4,2 and add(s0, s1) is group XOR 1. +#include +#include +#include +#include +#include +#include +#include "common/rms_ptx.cuh" +#include "common/row_source.cuh" + +namespace projection_rms { +using BF = cutlass::bfloat16_t; +using Vector = cutlass::AlignedArray; +using BiasOp = cutlass::epilogue::thread::LinearCombination; + +struct Op : BiasOp { + struct Params : BiasOp::Params { + BF const* table = nullptr; + float const* weight = nullptr; + BF* norm = nullptr; + int64_t rows = 0, contexts = 1, row_features = 0, prefix = 0; + float eps = 0.f; + CUTLASS_HOST_DEVICE Params() : BiasOp::Params(1.f, 1.f) {} + }; + Params parameters; + CUTLASS_HOST_DEVICE Op(Params const& p) : BiasOp(p), parameters(p) {} +}; + +using rms_ptx::add; +using rms_ptx::mul; +using rms_ptx::square_add; +CUTLASS_DEVICE float inverse(float sum, float divisor, float eps) { + float v; + asm volatile("div.full.f32 %0,%1,%2;" : "=f"(v) : "f"(sum), "f"(divisor)); + v = add(v, eps); + asm volatile("rsqrt.approx.ftz.f32 %0,%1;" : "=f"(v) : "f"(v)); + return v; +} +// Squares of one 128-channel half; result is identical in all 16 lanes of the row. +CUTLASS_DEVICE float half_sum(Vector const& z) { + float s0 = 0.f, s1 = 0.f; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < 4; ++i) { + s0 = square_add(BF(z[i]).raw(), s0); + s1 = square_add(BF(z[i + 4]).raw(), s1); + } + CUTLASS_PRAGMA_UNROLL + for (int shift = 8; shift >= 1; shift >>= 1) { + s0 = add(s0, __shfl_xor_sync(0xffffffffu, s0, shift, 16)); + s1 = add(s1, __shfl_xor_sync(0xffffffffu, s1, shift, 16)); + } + return add(s0, s1); +} + +// Named for the kernel symbol profiles match on. +template +struct ColumnProjectionRms : Base { + using Iterator = typename Base::OutputTileIterator; + using Fragment = typename Iterator::Fragment; + using SharedFragment = typename Base::SharedLoadIterator::Fragment; + using Map = typename Iterator::ThreadMap; + static constexpr int kHalves = D / 128; + static_assert(Iterator::kElementsPerAccess == 8 && Fragment::kElements == SharedFragment::kElements, + "8-channel vectors"); + static_assert(Map::Detail::kAccessWidth == 16 && Map::Iterations::kColumn == kHalves && + (kHalves == 1 || Map::Delta::kColumn == 128), + "16 lanes per row, one 128-channel half per vector"); + + CUTLASS_DEVICE ColumnProjectionRms(typename Base::SharedStorage& s, int t, int w, int l) : Base(s, t, w, l) {} + + struct Source : row_source::BiasSource { + CUTLASS_DEVICE void apply_output_operator(Fragment& out, Op const& op, SharedFragment const& acc) { + auto const* av = reinterpret_cast(&acc); + auto const* bv = reinterpret_cast(&this->bias); + auto* ov = reinterpret_cast(&out); + auto const& p = op.parameters; + const int start_column = this->start.column(); + row_source::for_each_row(this->start, [&](int64_t row, int fr) { + const bool valid = row < p.rows; + const int64_t f = row / p.contexts, c = row - f * p.contexts; + const int64_t residual = (c * p.row_features + p.prefix + f) * D; + Vector x[kHalves]; + CUTLASS_PRAGMA_UNROLL + for (int h = 0; h < kHalves; ++h) { + const int idx = fr * kHalves + h, col = start_column + h * 128; + const auto raw = static_cast(op)(av[idx], bv[idx]); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < 8; ++i) x[h][i] = BF(0.f); + if (valid) { + Vector t; + cutlass::arch::global_load(t, p.table + residual + col, true); + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < 8; ++i) x[h][i] = BF(add(float(raw[i]), float(t[i]))); + } + ov[idx] = raw; + } + // Tail rows (valid == false) still join every full-mask shuffle. + float total = half_sum(x[0]); + CUTLASS_PRAGMA_UNROLL + for (int h = 1; h < kHalves; ++h) total = add(total, half_sum(x[h])); + const float inv = inverse(total, float(D), p.eps); + CUTLASS_PRAGMA_UNROLL + for (int h = 0; h < kHalves; ++h) { + const int col = start_column + h * 128; + Vector y; + CUTLASS_PRAGMA_UNROLL + for (int i = 0; i < 8; ++i) y[i] = BF(mul(mul(float(x[h][i]), inv), p.weight[col + i])); + if (valid) cutlass::arch::global_store(y, p.norm + row * D + col, true); + } + }); + } + }; + + CUTLASS_DEVICE void operator()(Op const& op, Iterator destination, typename Base::AccumulatorTile const& acc, + Iterator source) { + Source aspect{{source, destination, {}, {}}}; + Base::template operator()(op, destination, acc, aspect); + } +}; + +template +using Gemm = cutlass::gemm::device::Gemm< + BF, cutlass::layout::RowMajor, BF, cutlass::layout::ColumnMajor, BF, cutlass::layout::RowMajor, float, + cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80, cutlass::gemm::GemmShape<128, D, 32>, + cutlass::gemm::GemmShape<64, 64, 32>, cutlass::gemm::GemmShape<16, 8, 16>, Op, + cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<1>, 3, 8, 8, false, cutlass::arch::OpMultiplyAdd>; + +template +using Kernel = cutlass::gemm::kernel::Gemm::GemmKernel::Mma, + ColumnProjectionRms::GemmKernel::Epilogue>, + typename Gemm::GemmKernel::ThreadblockSwizzle, false>; + +constexpr int64_t kMaxM = INT_MAX - 128; // the swizzle computes M + tile_m in int32 + +template +void launch(at::Tensor const& a, at::Tensor const& w, at::Tensor const& bias, at::Tensor const& dest, Op::Params p) { + using G = Gemm; + using K = Kernel; + static_assert(sizeof(typename K::SharedStorage) == sizeof(typename G::GemmKernel::SharedStorage), "no extra smem"); + auto bf = [](at::Tensor const& t) { return reinterpret_cast(t.data_ptr()); }; + typename G::Arguments args({int(a.size(0)), D, D}, {bf(a), D}, {bf(w), D}, {bf(bias), 0}, {bf(dest), D}, p, 1); + TORCH_CHECK(G().can_implement(args) == cutlass::Status::kSuccess, + "projection_rms: CUTLASS cannot implement this GEMM"); + typename G::GemmKernel::ThreadblockSwizzle swizzle; + const auto tiled = swizzle.get_tiled_shape(args.problem_size, {128, D, 32}, 1); + typename K::Params params(args.problem_size, tiled, args.ref_A.non_const_ref(), args.ref_B.non_const_ref(), + args.ref_C.non_const_ref(), args.ref_D, args.epilogue, nullptr); + constexpr int smem = sizeof(typename K::SharedStorage); + C10_CUDA_CHECK(cudaFuncSetAttribute(cutlass::Kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem)); + cutlass::Kernel + <<>>(params); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +// The Python supports() decides eligibility; these checks only keep a bad call from touching memory. +void run(at::Tensor const& a, at::Tensor const& w, at::Tensor const& bias, at::Tensor const& table, + at::Tensor const& weight, at::Tensor const& dest, at::Tensor const& norm, double eps) { + const int64_t m = a.size(0), d = a.size(1); + TORCH_CHECK(a.dim() == 2 && (d == 128 || d == 256) && m >= 1 && m <= kMaxM, + "projection_rms: a must be [M, 128] or [M, 256] with 1 <= M <= INT_MAX - 128"); + TORCH_CHECK(table.dim() == 3 && table.size(2) == d && w.sizes() == at::IntArrayRef({d, d}) && bias.size(0) == d && + weight.size(0) == d && dest.numel() == m * d && norm.numel() == m * d, + "projection_rms: w, bias, weight and table must match the width of a; dest and norm its size"); + for (auto const* t : {&a, &w, &bias, &table, &dest, &norm}) + TORCH_CHECK(t->is_cuda() && t->scalar_type() == at::kBFloat16 && t->is_contiguous() && + t->data_ptr() && reinterpret_cast(t->data_ptr()) % 16 == 0, + "projection_rms: a, w, bias, table, dest and norm must be contiguous 16-byte aligned CUDA bf16"); + TORCH_CHECK(weight.is_cuda() && weight.scalar_type() == at::kFloat && weight.is_contiguous(), + "projection_rms: weight must be a contiguous CUDA fp32 tensor"); + Op::Params p; + p.table = reinterpret_cast(table.data_ptr()); + p.weight = weight.data_ptr(); + p.norm = reinterpret_cast(norm.data_ptr()); + p.rows = m; + p.contexts = table.size(0); + p.row_features = table.size(1); + p.prefix = table.size(1) - m / table.size(0); + p.eps = float(eps); + TORCH_CHECK(m % table.size(0) == 0 && p.prefix >= 0, + "projection_rms: a must have C * F rows with F <= table.size(1)"); + const c10::cuda::CUDAGuard guard(a.device()); + if (d == 128) + launch<128>(a, w, bias, dest, p); + else + launch<256>(a, w, bias, dest, p); +} +} // namespace projection_rms + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("run", &projection_rms::run); } diff --git a/benchmark/inference/kernels/csrc/query_rms_sm120/query_rms.cu b/benchmark/inference/kernels/csrc/query_rms_sm120/query_rms.cu new file mode 100644 index 0000000000..d4fe45ed4e --- /dev/null +++ b/benchmark/inference/kernels/csrc/query_rms_sm120/query_rms.cu @@ -0,0 +1,42 @@ +// Column query projection with SDM's unweighted per-head RMS fused into the epilogue; the raw query is +// never stored. SM120, BF16. +// +// out = rms(bf16(a @ w.T + bias)) per head of Head channels, eps 1e-6, no weight +// +// a [M, D], w [D, D], bias [D], out [M, D]; out may be exactly a, nothing else may overlap it. +// One launch of common/head_rms.cuh with a single N tile of width D. +#include +#include +#include +#include "common/head_rms.cuh" + +namespace query_rms { +using head_rms::BF; +struct Query {}; // names this launch in the kernel symbol + +template +void run_instance(at::Tensor const& a, at::Tensor const& w, at::Tensor const& bias, at::Tensor const& out) { + auto ptr = [](at::Tensor const& t) { return reinterpret_cast(t.data_ptr()); }; + head_rms::launch(ptr(a), ptr(w), ptr(bias), ptr(out), D, int(a.size(0)), + at::cuda::getCurrentCUDAStream()); +} + +// The Python supports() decides eligibility (including aliasing); these checks only keep a bad call from touching +// memory. +void run(at::Tensor const& a, at::Tensor const& w, at::Tensor const& bias, at::Tensor const& out, int64_t head) { + for (auto const* t : {&a, &w, &bias, &out}) + TORCH_CHECK(t->is_cuda() && t->scalar_type() == at::kBFloat16 && t->is_contiguous() && + reinterpret_cast(t->data_ptr()) % 16 == 0, + "query_rms: a, w, bias and out must be contiguous 16-byte aligned CUDA bf16"); + const int64_t m = a.size(0), d = a.size(1); + TORCH_CHECK(a.dim() == 2 && m >= 1 && m <= head_rms::kMaxM && w.dim() == 2 && w.size(0) == d && w.size(1) == d && + bias.dim() == 1 && bias.size(0) == d && out.sizes() == a.sizes(), + "query_rms: need a [M, D], w [D, D], bias [D] and out [M, D]"); + c10::cuda::CUDAGuard guard(a.device()); + if (d == 256 && head == 64) return run_instance<256, 64>(a, w, bias, out); + if (d == 128 && head == 32) return run_instance<128, 32>(a, w, bias, out); + TORCH_CHECK(false, "query_rms: (D, head) must be (256, 64) or (128, 32), got (", d, ", ", head, ")"); +} +} // namespace query_rms + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("run", &query_rms::run); } diff --git a/benchmark/inference/kernels/csrc/rope_rms_sm120/rope_rms.cu b/benchmark/inference/kernels/csrc/rope_rms_sm120/rope_rms.cu new file mode 100644 index 0000000000..2e465a8b60 --- /dev/null +++ b/benchmark/inference/kernels/csrc/rope_rms_sm120/rope_rms.cu @@ -0,0 +1,218 @@ +// Split-half RoPE followed by the unweighted per-head RMS of Q and of K, one launch. SM120, BF16. +// +// q_out[c, s, h, :] = rms(rope(q[c, s, h, :], s)), k_out likewise from k (eps 1e-6, no weight) +// rope(x, s)[j] = bf16(fma(x[j], cos(a), -x[j + Head/2] * sin(a))) for j < Head/2 +// bf16(fma(x[j], cos(a), x[j - Head/2] * sin(a))) for j >= Head/2 +// a = s * inv_freq[j % (Head/2)] +// +// q, k: [C, S, 4, Head] BF16 with strides (S * ld, ld, Head, 1); ld may differ between q and k (K is a slice of +// packed KV). inv_freq: [Head / 2] FP32. q_out, k_out: contiguous [C, S, 4, Head], disjoint from the inputs. +// Bytes match Inductor's fused rope pointwise (FP32 between ops, libdevice sin/cos, one BF16 rounding) followed +// by SDM's Triton rmsnorm_cast at channels = Head. +#include +#include +#include +#include "common/rms_ptx.cuh" + +namespace rope_rms { +using rms_ptx::add; +using rms_ptx::inv_rms; +using rms_ptx::mul; +using rms_ptx::square_add; +constexpr int kHeads = 4, kThreads = 128; + +__device__ __forceinline__ float bf16_to_f32(unsigned short a) { + float x; + asm("cvt.f32.bf16 %0,%1;" : "=f"(x) : "h"(a)); + return x; +} +__device__ __forceinline__ unsigned short f32_to_bf16(float a) { + unsigned short x; + asm("cvt.rn.bf16.f32 %0,%1;" : "=h"(x) : "f"(a)); + return x; +} +__device__ __forceinline__ float fma_rn(float a, float b, float c) { + float x; + asm("fma.rn.f32 %0,%1,%2,%3;" : "=f"(x) : "f"(a), "f"(b), "f"(c)); + return x; +} +// Triton's libdevice sin/cos run with FTZ, so a subnormal angle is first flushed to a signed zero. +// Checked bitwise against Inductor's tl_math.sin/cos over every FP32 input. +__device__ __forceinline__ void trig(float a, float& s, float& c) { + asm("mul.ftz.f32 %0,%0,0f3F800000;" : "+f"(a)); + s = sinf(a); + c = cosf(a); +} +// Element i of 4 BF16 values packed in w. +__device__ __forceinline__ float element(uint2 w, int i) { return bf16_to_f32((i < 2 ? w.x : w.y) >> (16 * (i & 1))); } +__device__ __forceinline__ uint2 pack(const unsigned short (&a)[4]) { + return make_uint2(a[0] | unsigned(a[1]) << 16, a[2] | unsigned(a[3]) << 16); +} + +// One head per Head/4 adjacent lanes, 4 adjacent channels per lane (one rmsnorm_cast group). The rotation partner +// (channel +- Head/2) is lane XOR Head/8 in the same head. rmsnorm_cast squares each group in channel order, then +// reduces the Head/4 group sums by an XOR butterfly with shifts Head/8, ..., 1. +// All lanes, including those of masked tail heads, take part in every shuffle. +template +__global__ void __launch_bounds__(kThreads) + rope_rms_kernel(const unsigned short* __restrict__ q, int64_t q_ld, const unsigned short* __restrict__ k, + int64_t k_ld, const float* __restrict__ inv_freq, unsigned short* __restrict__ q_out, + unsigned short* __restrict__ k_out, int64_t heads, int64_t seq) { + constexpr int kLanes = Head / 4, kHalf = Head / 2; + const int lane = threadIdx.x % kLanes, channel = 4 * lane; + const int64_t head = int64_t(blockIdx.x) * (kThreads / kLanes) + threadIdx.x / kLanes; + const bool valid = head < heads; + const int64_t token = head / kHeads, offset = (head % kHeads) * Head + channel; + const float position = float(token % seq); + const uint2 qv = valid ? *reinterpret_cast(q + token * q_ld + offset) : make_uint2(0, 0); + const uint2 kv = valid ? *reinterpret_cast(k + token * k_ld + offset) : make_uint2(0, 0); + const uint2 qp = make_uint2(__shfl_xor_sync(~0u, qv.x, kLanes / 2), __shfl_xor_sync(~0u, qv.y, kLanes / 2)); + const uint2 kp = make_uint2(__shfl_xor_sync(~0u, kv.x, kLanes / 2), __shfl_xor_sync(~0u, kv.y, kLanes / 2)); + unsigned short rq[4], rk[4]; + float sum_q = 0.f, sum_k = 0.f; +#pragma unroll + for (int i = 0; i < 4; ++i) { + float s, c; + trig(mul(position, inv_freq[(channel + i) % kHalf]), s, c); + float pq = mul(element(qp, i), s), pk = mul(element(kp, i), s); + if (channel < kHalf) pq = -pq, pk = -pk; + rq[i] = f32_to_bf16(fma_rn(element(qv, i), c, pq)); + rk[i] = f32_to_bf16(fma_rn(element(kv, i), c, pk)); + sum_q = square_add(rq[i], sum_q); + sum_k = square_add(rk[i], sum_k); + } +#pragma unroll + for (int shift = kLanes / 2; shift >= 1; shift >>= 1) { + sum_q = add(sum_q, __shfl_xor_sync(~0u, sum_q, shift, kLanes)); + sum_k = add(sum_k, __shfl_xor_sync(~0u, sum_k, shift, kLanes)); + } + const float fq = inv_rms(sum_q), fk = inv_rms(sum_k); + unsigned short oq[4], ok[4]; +#pragma unroll + for (int i = 0; i < 4; ++i) { + oq[i] = f32_to_bf16(mul(bf16_to_f32(rq[i]), fq)); + ok[i] = f32_to_bf16(mul(bf16_to_f32(rk[i]), fk)); + } + if (valid) { + *reinterpret_cast(q_out + head * Head + channel) = pack(oq); + *reinterpret_cast(k_out + head * Head + channel) = pack(ok); + } +} + +unsigned short* u16(at::Tensor const& t) { return reinterpret_cast(t.data_ptr()); } + +template +void launch(at::Tensor const& q, at::Tensor const& k, at::Tensor const& inv_freq, at::Tensor& q_out, + at::Tensor& k_out) { + const int64_t heads = q.size(0) * q.size(1) * kHeads, per_block = kThreads / (Head / 4); + rope_rms_kernel<<<(heads + per_block - 1) / per_block, kThreads, 0, at::cuda::getCurrentCUDAStream()>>>( + u16(q), q.stride(1), u16(k), k.stride(1), inv_freq.data_ptr(), u16(q_out), u16(k_out), heads, q.size(1)); + C10_CUDA_KERNEL_LAUNCH_CHECK(); +} + +void check_inv_freq(at::Tensor const& inv_freq, int64_t head) { + TORCH_CHECK(inv_freq.is_cuda() && inv_freq.scalar_type() == at::kFloat && inv_freq.is_contiguous() && + inv_freq.numel() == head / 2, + "rope_rms: inv_freq must be a contiguous CUDA fp32 [head / 2] tensor"); +} + +// The Python supports() decides eligibility (including overlap); these checks only keep a bad call from touching +// memory. +std::vector run(at::Tensor const& q, at::Tensor const& k, at::Tensor const& inv_freq) { + const int64_t head = q.size(-1); + for (auto const* t : {&q, &k}) + TORCH_CHECK(t->is_cuda() && t->scalar_type() == at::kBFloat16 && t->dim() == 4 && t->sizes() == q.sizes() && + t->size(2) == kHeads && t->stride(3) == 1 && t->stride(2) == head && + t->stride(0) == t->size(1) * t->stride(1) && t->stride(1) % 4 == 0 && + reinterpret_cast(t->data_ptr()) % 8 == 0, + "rope_rms: q and k must be CUDA bf16 [C, S, 4, head] of one shape with strides (S * ld, ld, head, 1)"); + check_inv_freq(inv_freq, head); + TORCH_CHECK(q.numel() > 0 && q.size(0) * q.size(1) * kHeads <= int64_t(INT_MAX) * 8, + "rope_rms: q must be nonempty with few enough heads for an int32 grid"); + c10::cuda::CUDAGuard guard(q.device()); + auto q_out = at::empty(q.sizes(), q.options().memory_format(at::MemoryFormat::Contiguous)); + auto k_out = at::empty_like(q_out); + if (head == 64) + launch<64>(q, k, inv_freq, q_out, k_out); + else if (head == 32) + launch<32>(q, k, inv_freq, q_out, k_out); + else + TORCH_CHECK(false, "rope_rms: head must be 32 or 64, got ", head); + return {q_out, k_out}; +} + +// Q and K of different lengths (the last row layer's 4 readout queries): the same per-head math as +// rope_rms_kernel on one tensor; blocks below q_blocks take Q, the rest K. +template +__global__ void __launch_bounds__(kThreads) + rope_rms_uneven_kernel(const unsigned short* __restrict__ q, int64_t q_ld, int64_t q_seq, int64_t q_heads, + unsigned short* __restrict__ q_out, const unsigned short* __restrict__ k, int64_t k_ld, + int64_t k_seq, int64_t k_heads, unsigned short* __restrict__ k_out, + const float* __restrict__ inv_freq, int64_t q_blocks) { + constexpr int kLanes = Head / 4, kHalf = Head / 2; + const bool is_k = blockIdx.x >= q_blocks; + const unsigned short* x = is_k ? k : q; + unsigned short* out = is_k ? k_out : q_out; + const int64_t ld = is_k ? k_ld : q_ld, seq = is_k ? k_seq : q_seq, heads = is_k ? k_heads : q_heads; + const int lane = threadIdx.x % kLanes, channel = 4 * lane; + const int64_t head = (blockIdx.x - (is_k ? q_blocks : 0)) * (kThreads / kLanes) + threadIdx.x / kLanes; + const bool valid = head < heads; + const int64_t token = head / kHeads, offset = (head % kHeads) * Head + channel; + const float position = float(token % seq); + const uint2 v = valid ? *reinterpret_cast(x + token * ld + offset) : make_uint2(0, 0); + const uint2 p = make_uint2(__shfl_xor_sync(~0u, v.x, kLanes / 2), __shfl_xor_sync(~0u, v.y, kLanes / 2)); + unsigned short r[4]; + float sum = 0.f; +#pragma unroll + for (int i = 0; i < 4; ++i) { + float s, c; + trig(mul(position, inv_freq[(channel + i) % kHalf]), s, c); + float rotated = mul(element(p, i), s); + if (channel < kHalf) rotated = -rotated; + r[i] = f32_to_bf16(fma_rn(element(v, i), c, rotated)); + sum = square_add(r[i], sum); + } +#pragma unroll + for (int shift = kLanes / 2; shift >= 1; shift >>= 1) sum = add(sum, __shfl_xor_sync(~0u, sum, shift, kLanes)); + const float f = inv_rms(sum); + unsigned short o[4]; +#pragma unroll + for (int i = 0; i < 4; ++i) o[i] = f32_to_bf16(mul(bf16_to_f32(r[i]), f)); + if (valid) *reinterpret_cast(out + head * Head + channel) = pack(o); +} + +std::vector run_uneven(at::Tensor const& q, at::Tensor const& k, at::Tensor const& inv_freq) { + const int64_t head = q.size(-1); + for (auto const* t : {&q, &k}) + TORCH_CHECK(t->is_cuda() && t->scalar_type() == at::kBFloat16 && t->dim() == 4 && t->size(0) == q.size(0) && + t->size(2) == kHeads && t->size(3) == head && t->stride(3) == 1 && t->stride(2) == head && + t->stride(0) == t->size(1) * t->stride(1) && t->stride(1) % 4 == 0 && t->numel() > 0 && + reinterpret_cast(t->data_ptr()) % 8 == 0, + "rope_rms: q and k must be nonempty CUDA bf16 [C, S, 4, head] with strides (S * ld, ld, head, 1)"); + check_inv_freq(inv_freq, head); + c10::cuda::CUDAGuard guard(q.device()); + auto q_out = at::empty(q.sizes(), q.options().memory_format(at::MemoryFormat::Contiguous)); + auto k_out = at::empty(k.sizes(), k.options().memory_format(at::MemoryFormat::Contiguous)); + auto go = [&](auto kernel, int64_t per_block) { + const int64_t q_heads = q.size(0) * q.size(1) * kHeads, k_heads = k.size(0) * k.size(1) * kHeads; + const int64_t q_blocks = (q_heads + per_block - 1) / per_block, k_blocks = (k_heads + per_block - 1) / per_block; + TORCH_CHECK(q_blocks + k_blocks <= INT_MAX, "rope_rms: q and k have too many heads for an int32 grid"); + kernel<<>>( + u16(q), q.stride(1), q.size(1), q_heads, u16(q_out), u16(k), k.stride(1), k.size(1), k_heads, u16(k_out), + inv_freq.data_ptr(), q_blocks); + C10_CUDA_KERNEL_LAUNCH_CHECK(); + }; + if (head == 64) + go(rope_rms_uneven_kernel<64>, kThreads / 16); + else if (head == 32) + go(rope_rms_uneven_kernel<32>, kThreads / 8); + else + TORCH_CHECK(false, "rope_rms: head must be 32 or 64, got ", head); + return {q_out, k_out}; +} +} // namespace rope_rms + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("run", &rope_rms::run); + m.def("run_uneven", &rope_rms::run_uneven); +} diff --git a/benchmark/inference/kernels/fusions/__init__.py b/benchmark/inference/kernels/fusions/__init__.py new file mode 100644 index 0000000000..32d232dd62 --- /dev/null +++ b/benchmark/inference/kernels/fusions/__init__.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""One module per kernel family. + +Importing a module defines its ops and graph rules; ``register(directory)`` +registers its kernels from the built modules in ``directory``. +""" + +import importlib +import pkgutil +from pathlib import Path + + +def load(directory: Path) -> None: + names = sorted(info.name for info in pkgutil.iter_modules(__path__)) + # Import every module first: gating.py registers a linear_gelu kernel. + modules = [importlib.import_module(f"{__name__}.{n}") for n in names] + for module in modules: + module.register(directory) diff --git a/benchmark/inference/kernels/fusions/gating.py b/benchmark/inference/kernels/fusions/gating.py new file mode 100644 index 0000000000..a3003d7e19 --- /dev/null +++ b/benchmark/inference/kernels/fusions/gating.py @@ -0,0 +1,244 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Gate MLP of the row block's ``GatedLogScale`` query scaling. + +- First layer: ``linear_gelu`` with K = head (32 or 64), N = 64. +- Second layer: ``gated_log_scale``, the second linear plus the log-scale/tanh + query multiply, as one GEMM with the gate in its epilogue. + +Both match cuBLAS's wmma 32x32_64x1 kernel plus Inductor's pointwise kernel +bitwise. +""" + +from pathlib import Path + +import torch +from torch import Tensor +from torch.fx import Graph, Node + +from benchmark.inference.kernels import fx_pass, registry, sm120 +from benchmark.inference.kernels.fusions.kv_rms import is_view +from benchmark.inference.kernels.fx_pass import is_call, val + +aten, prims = torch.ops.aten, torch.ops.prims +MODULE = "gating_sm120" +HEADS = (32, 64) +HIDDEN = 64 +MIN_M = sm120.MIN_M +# Head width -> largest M (query rows) checked bitwise against compiled native. +FIRST_MAX_M = {32: 132_118_560, 64: 40_000_000} +SECOND_MAX_M = {32: 80_937_920, 64: 49_583_864} + + +def _reference( + x: Tensor, + weight: Tensor, + bias: Tensor, + query: Tensor, + head_scale: Tensor, + logscale: float, +) -> Tensor: + z = torch.addmm(bias, x, weight.t()).float() + scale = head_scale.float().repeat(query.size(0) // 4 + 1)[: query.size(0)] + q = query.float() * logscale * scale[:, None] + return (q * (torch.tanh(z) + 1)).to(query.dtype) + + +registry.define_op( + name="gated_log_scale", + reference=_reference, + doc="query * logscale * head_scale[row % 4] * " + "(1 + tanh(linear(x, weight, bias)))", +) + + +@torch.library.custom_op( + "sdm_inference::gated_log_scale", + mutates_args=(), + tags=registry.EXACT_STRIDES, +) +def gated_log_scale( + x: Tensor, + weight: Tensor, + bias: Tensor, + query: Tensor, + head_scale: Tensor, + logscale: float, + kernel: str, +) -> Tensor: + return registry.launch( + registry.kernel(kernel), x, weight, bias, query, head_scale, logscale + ) + + +@gated_log_scale.register_fake +def _(x, weight, bias, query, head_scale, logscale, kernel): + return query.new_empty(query.shape) + + +def _operands(mul: Node, test) -> tuple[Node, Node] | None: + """(matching, other) operands of a binary node, in either order.""" + a, b = mul.args[:2] + return (a, b) if test(a) else (b, a) if test(b) else None + + +def _gated_log_scale(graph: Graph, tanh: Node) -> bool: + """Match ``(q * logscale * head_scale) * (1 + tanh(addmm(...)))``.""" + shaped = tanh.args[0] + if not is_view(shaped): + return False + addmm = shaped.args[0] + if not fx_pass.plain_addmm(addmm): + return False + bias, x, transposed = addmm.args + if not is_call(transposed, aten.permute.default, [1, 0]): + return False + users = list(tanh.users) + if len(users) != 1 or not is_call(users[0], aten.add.Tensor, 1): + return False + shifted = users[0] + if len(shifted.users) != 1 or not is_call( + final := next(iter(shifted.users)), aten.mul.Tensor + ): + return False + pair = _operands(final, lambda n: n is shifted) + scaled = pair and pair[1] + if not is_call(scaled, aten.mul.Tensor) or len(scaled.users) != 1: + return False + pair = _operands(scaled, is_view) + if pair is None: + return False + head_view, logscaled = pair + convert = head_view.args[0] + if not ( + is_call(convert, prims.convert_element_type.default, torch.bfloat16) + and is_call(logscaled, aten.mul.Tensor) + and len(logscaled.users) == 1 + ): + return False + pair = _operands(logscaled, lambda n: is_call(n, aten.full.default)) + if pair is None: + return False + full, query = pair + head_scale = convert.args[0] + # The kernel scales row r by head_scale[r % 4]: heads on dim -2. + heads = [1] * (val(query).dim() - 2) + [4, 1] + if not ( + isinstance(full.args[1], float) + and val(full).numel() == 1 + and val(head_scale).dtype == torch.float32 + and val(head_scale).dim() == 1 + and list(val(head_view).shape) == heads + and val(query).shape[-2] == 4 + and len(addmm.users) == 1 + and len(shaped.users) == 1 + ): + return False + head = val(query).size(-1) + if ( + val(query).shape != val(shaped).shape + or val(final).shape != val(query).shape + ): + return False + q2 = val(query).reshape(-1, head) + weight = transposed.args[0] + logscale = float(full.args[1]) + kernel = registry.select( + "gated_log_scale", + val(x), + val(weight), + val(bias), + q2, + val(head_scale), + logscale, + ) + if kernel is None: + return False + with graph.inserting_before(final): + flat = graph.call_function(aten.reshape.default, (query, [-1, head])) + flat.meta["val"] = q2 + fused = graph.call_function( + torch.ops.sdm_inference.gated_log_scale.default, + (x, weight, bias, flat, head_scale, logscale, kernel.name), + ) + fused.meta["val"] = q2.new_empty(q2.shape) + out = graph.call_function( + aten.reshape.default, (fused, list(val(final).shape)) + ) + out.meta["val"] = val(final) + fx_pass.replace(final, out) + return True + + +fx_pass.add_rule(aten.tanh.default, "gated_log_scale", _gated_log_scale) + + +def _first(module, head: int) -> registry.Kernel: + def supports(x: Tensor, weight: Tensor, bias: Tensor) -> str | None: + if ( + x.dim() != 2 + or x.size(1) != head + or not MIN_M <= x.size(0) <= FIRST_MAX_M[head] + ): + return ( + f"needs x of shape [{MIN_M}<=M<={FIRST_MAX_M[head]}, {head}]" + ) + if weight.shape != (HIDDEN, head) or bias.shape != (HIDDEN,): + return f"needs a {HIDDEN}x{head} weight and bias" + return sm120.policy(x, weight, bias) + + def impl(x: Tensor, weight: Tensor, bias: Tensor) -> Tensor: + out = x.new_empty(x.size(0), HIDDEN) + module.first(x, weight, bias, out) + return out + + return registry.Kernel( + op="linear_gelu", + name=f"sm120_linear_gelu_k{head}_n{HIDDEN}", + supports=supports, + impl=impl, + cuda_symbols=("6gating8BiasGelu",), + ) + + +def _second_supports( + x, weight, bias, query, head_scale, logscale +) -> str | None: + if query.dim() != 2 or query.size(1) not in HEADS: + return f"needs query [M, head] with head in {HEADS}" + m, head = query.shape + if not MIN_M <= m <= SECOND_MAX_M[head]: + return f"needs {MIN_M}<=M<={SECOND_MAX_M[head]}" + if ( + x.shape != (m, HIDDEN) + or weight.shape != (head, HIDDEN) + or bias.shape != (head,) + ): + return f"needs x [M, {HIDDEN}], a {head}x{HIDDEN} weight and bias" + if head_scale.shape != (4,) or head_scale.dtype != torch.float32: + return "needs a float32 head scale of 4 heads" + if head_scale.device != query.device or not head_scale.is_contiguous(): + return "needs a contiguous head scale on the query device" + return sm120.policy(x, weight, bias, query) + + +def register(directory: Path) -> None: + module = sm120.load_module(directory, MODULE) + + def second(x, weight, bias, query, head_scale, logscale): + out = torch.empty_like(query) + module.second(x, weight, bias, query, head_scale, out, logscale) + return out + + for head in HEADS: + registry.register(_first(module, head)) + registry.register( + registry.Kernel( + op="gated_log_scale", + name="sm120_gated_log_scale", + supports=_second_supports, + impl=second, + cuda_symbols=("6gating12GateEpilogue",), + ) + ) diff --git a/benchmark/inference/kernels/fusions/icl_attention.py b/benchmark/inference/kernels/fusions/icl_attention.py new file mode 100644 index 0000000000..f1bafc77fc --- /dev/null +++ b/benchmark/inference/kernels/fusions/icl_attention.py @@ -0,0 +1,289 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Self-attention over the context rows (the ICL fit site). + +``icl_attention(q, k, v, scale)`` is ``F.scaled_dot_product_attention`` on +``[B, S, H, 64]`` memory (the ``[B, H, S, D]`` graph tensors transposed), +returned as ``[B, S, H, 64]``. The pass replaces PyTorch's flash node with +this op followed by the permute that restores the flash output strides. + +None of these kernels is bitwise equal to PyTorch. ``$SDM_ICL_ATTENTION`` (or +the ``attention`` of :func:`registry.enabled`) picks one: + +- ``sm120`` (default): BF16 SM120 kernel, qualified by the tolerance gate; +- ``fa2``: FlashAttention 2, if installed; +- ``int8[+ends]``: opt-in lossy INT8 QK / FP8 PV kernel with K smoothing + (``ends``: keep the first and last ICL fit call in PyTorch), gated + ``opt-in``; +- ``sm120+uncapped``: diagnostic, the BF16 kernel above ``MAX_S``; +- ``off``: PyTorch. ``kernels.enable(model, exact=True)`` also keeps it. +""" + +import functools +import operator +import os +import sys +from pathlib import Path + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch.fx import Graph, Node + +from benchmark.inference.kernels import fx_pass, registry +from benchmark.inference.kernels.fx_pass import is_call, val +from benchmark.inference.kernels.registry import Kernel +from benchmark.inference.kernels.sm120 import load_module + +aten = torch.ops.aten +MODULE = "icl_attention_sm120" +INT8_MODULE = "icl_attention_int8_sm120" +ENV = registry.ATTENTION_ENV +HEAD = 64 +MIN_S = 1024 +# Largest context the BF16 kernel's error and determinism were checked at; +# Medium at 100k misses the tolerance gate. +MAX_S = 65_535 +FA2_SYMBOL = "void flash::flash_fwd_kernel<" +INT8_SYMBOLS = ( + "icl_attention_int8::quant_kernel", + "icl_attention_int8::attend_kernel", +) +BSHD = [0, 2, 1, 3] + + +def _reference(q: Tensor, k: Tensor, v: Tensor, scale: float) -> Tensor: + return F.scaled_dot_product_attention( + q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), scale=scale + ).transpose(1, 2) + + +registry.define_op( + "icl_attention", + _reference, + "softmax(q k^T * scale) v on [B, S, H, 64], Sq == Skv, no mask", +) + + +@torch.library.custom_op( + "sdm_inference::icl_attention", + mutates_args=(), + tags=registry.EXACT_STRIDES, +) +def icl_attention( + q: Tensor, k: Tensor, v: Tensor, scale: float, kernel: str +) -> Tensor: + return registry.launch(registry.kernel(kernel), q, k, v, scale) + + +@icl_attention.register_fake +def _(q, k, v, scale, kernel): + return q.new_empty(q.shape) + + +def _admits( + q: Tensor, k: Tensor, v: Tensor, scale: float, dtypes=(torch.bfloat16,) +) -> str | None: + if any(t.device.type != "cuda" for t in (q, k, v)): + return "needs CUDA tensors" + if torch.cuda.get_device_capability(q.device) != (12, 0): + return "needs SM 12.0" + if q.dtype not in dtypes or any(t.dtype != q.dtype for t in (k, v)): + return f"needs q, k, v all one of {[str(d) for d in dtypes]}" + if torch.is_grad_enabled() and any(t.requires_grad for t in (q, k, v)): + return "no backward" + if q.dim() != 4 or q.shape != k.shape or q.shape != v.shape: + return "needs q, k, v [B, S, H, 64] of one shape (self-attention)" + s, d = q.shape[1], q.shape[3] + if d != HEAD: + return f"needs head dim {HEAD}" + if s < MIN_S: + return f"needs S >= {MIN_S} (fit); got {s}" + for t in (q, k, v): + if t.stride(3) != 1 or any(x % 8 for x in t.stride()[:3]): + return "needs last-dim stride 1 and 16-byte aligned rows" + if t.storage_offset() % 8: + return "needs a 16-byte aligned storage offset" + return None + + +def _bf16_limits(q: Tensor) -> str | None: + if q.size(1) > MAX_S: + return f"needs S <= {MAX_S} (checked); got {q.size(1)}" + return None + + +def _int8_limits(q: Tensor, k: Tensor, v: Tensor) -> str | None: + b, s, h, _ = q.shape + if b != 1: + return f"needs batch 1 (the ICL fit site); got {b}" + rows = max(t.stride(1) for t in (q, k, v)) + if s * rows >= 2**31 or b * h * s * HEAD >= 2**31: + return "needs 32-bit element offsets" + return None + + +def _setting() -> tuple[str, frozenset[str]]: + return registry.parse_attention(registry.attention()) + + +def _policy(want: str): + def supports(q: Tensor, k: Tensor, v: Tensor, scale: float): + try: + backend, chosen = _setting() + except ValueError as error: + return str(error) + if backend != want or chosen - {"ends", "uncapped"}: + return f"{ENV}={registry.attention()}" + if want == "int8": + return _admits( + q, k, v, scale, (torch.bfloat16, torch.float16) + ) or _int8_limits(q, k, v) + return _admits(q, k, v, scale) or ( + None if "uncapped" in chosen else _bf16_limits(q) + ) + + return supports + + +@functools.cache +def _flash_attn_func(): + sys.setdlopenflags(os.RTLD_LAZY | os.RTLD_GLOBAL) + try: + from flash_attn import flash_attn_func # noqa: PLC0415 + except ImportError: + return None + return flash_attn_func + + +def _fa2_supports(q: Tensor, k: Tensor, v: Tensor, scale: float): + if registry.attention() == "fa2" and _flash_attn_func() is None: + return "flash_attn is not importable" + return _policy("fa2")(q, k, v, scale) + + +def _fa2_impl(q: Tensor, k: Tensor, v: Tensor, scale: float) -> Tensor: + return _flash_attn_func()(q, k, v, softmax_scale=scale) + + +def _flash_node(node: Node) -> tuple[tuple[Node, ...], Node] | None: + """(q, k, v) and the sole getitem 0 of a plain flash SDPA node.""" + if not is_call(node, aten._scaled_dot_product_flash_attention.default): + return None + if len(node.args) != 3 or not set(node.kwargs) <= { + "dropout_p", + "is_causal", + "return_debug_mask", + "scale", + }: + return None + kwargs = node.kwargs + if ( + kwargs.get("dropout_p", 0.0) != 0.0 + or kwargs.get("is_causal", False) + or kwargs.get("return_debug_mask", False) + or kwargs.get("scale") is None + ): + return None + if not ( + len(node.users) == 1 + and all(is_call(u, operator.getitem, 0) for u in node.users) + ): + return None + return node.args, next(iter(node.users)) + + +def _fit_call(node: Node) -> bool: + """A batch-1 self-attention flash call: an ICL fit call.""" + found = _flash_node(node) + if found is None: + return False + q, k, _ = (val(t) for t in found[0]) + return q.shape == k.shape and q.size(0) == 1 and q.size(3) == HEAD + + +def _rule(graph: Graph, node: Node) -> bool: + found = _flash_node(node) + if found is None: + return False + try: + _, options = _setting() + except ValueError: + options = frozenset() + if "ends" in options and _fit_call(node): + calls = [n for n in graph.nodes if _fit_call(n)] + if node in (calls[0], calls[-1]): + return False + (q, k, v), out = found + scale = float(node.kwargs["scale"]) + kernel = registry.select( + "icl_attention", *(val(t).permute(BSHD) for t in (q, k, v)), scale + ) + if kernel is None: + return False + with graph.inserting_before(out): + views = [] + for t in (q, k, v): + views.append(graph.call_function(aten.permute.default, (t, BSHD))) + views[-1].meta["val"] = val(t).permute(BSHD) + fused = graph.call_function( + torch.ops.sdm_inference.icl_attention.default, + (*views, scale, kernel.name), + ) + fused.meta["val"] = val(views[0]).new_empty(val(views[0]).shape) + result = graph.call_function(aten.permute.default, (fused, BSHD)) + result.meta["val"] = val(fused).permute(BSHD) + fx_pass.replace(out, result) + return True + + +fx_pass.add_rule( + aten._scaled_dot_product_flash_attention.default, "icl_attention", _rule +) + + +def register(directory: Path) -> None: + module = load_module(directory, MODULE) + + def impl(q: Tensor, k: Tensor, v: Tensor, scale: float) -> Tensor: + return module.run(q, k, v, q.new_empty(q.shape), scale) + + registry.register( + Kernel( + op="icl_attention", + name="sm120_icl_attention_d64", + supports=_policy("sm120"), + impl=impl, + cuda_symbols=("icl_attention_kernel",), + bitwise=lambda *args: False, + ) + ) + registry.register( + Kernel( + op="icl_attention", + name="fa2_icl_attention_d64", + supports=_fa2_supports, + impl=_fa2_impl, + cuda_symbols=(FA2_SYMBOL,), + bitwise=lambda *args: False, + ) + ) + int8 = load_module(directory, INT8_MODULE) + + def int8_impl(q: Tensor, k: Tensor, v: Tensor, scale: float) -> Tensor: + out = q.new_empty(q.shape) + int8.run(*(t.permute(BSHD) for t in (q, k, v, out)), scale) + return out + + registry.register( + Kernel( + op="icl_attention", + name="sm120_int8_icl", + supports=_policy("int8"), + impl=int8_impl, + cuda_symbols=INT8_SYMBOLS, + bitwise=lambda *args: False, + exact=False, + ) + ) diff --git a/benchmark/inference/kernels/fusions/icl_qk.py b/benchmark/inference/kernels/fusions/icl_qk.py new file mode 100644 index 0000000000..aefddf4e7b --- /dev/null +++ b/benchmark/inference/kernels/fusions/icl_qk.py @@ -0,0 +1,417 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Per-head RMS of the ICL attention query and key, fused with their copies. + +``icl_query_rms_scale(q, head_scale, log_n)``: SDM's Triton +``_rmsnorm_cast_kernel`` on the raw query heads, then the LogScale multiply. +The compiled graph stores the RMS in BF16, then computes +``q * log_n * head_scale`` in FP32 (head_scale read as FP32, Inductor drops +its BF16 cast) with one rounding; the kernel does both in one pass. + +``icl_key_rms(kv, heads, key, cache)``: the key RMS read in place from the +packed KV addmm output (SDM's ``rmsnorm_cast`` copies the strided K chunk +first), plus the cache head slices ``k[:, :cache]`` and ``v[:, :cache]`` +written contiguous. With ``key=False`` (the last ICL fit layer, where only +the cache reads K and V) the rule also narrows the KV addmm to the weight +rows of those cache heads. +""" + +import operator +from pathlib import Path + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch.fx import Graph, Node + +from benchmark.inference.kernels import fx_pass, registry, sm120 +from benchmark.inference.kernels.fusions.kv_rms import ( + is_view, + unweighted_rms_launch, +) +from benchmark.inference.kernels.fx_pass import is_call, only_user, val + +aten, prims = torch.ops.aten, torch.ops.prims +MODULE = "icl_qk_sm120" +QUERY, KEY = "icl_query_rms_scale", "icl_key_rms" +HEAD = 64 +EPS = 1e-6 +# ICL widths: Small/Medium 512, Large 1024. +WIDTHS = (512, 1024) +# M range where cuBLAS sums the narrowed KV addmm in the same order as the +# full one (every M to 8192, every 7th to 65,535); it differs for M <= 512. +GATHER_M = (513, 65_535) + + +def _rms(x: Tensor) -> Tensor: + return F.rms_norm(x.float(), (HEAD,), eps=EPS).to(torch.bfloat16) + + +def _query_reference(q: Tensor, head_scale: Tensor, log_n: float) -> Tensor: + scaled = _rms(q).float() * log_n * head_scale.float()[:, None] + return scaled.to(torch.bfloat16) + + +def _key_reference( + kv: Tensor, heads: int, key: bool, cache: int +) -> tuple[Tensor, Tensor, Tensor]: + k, v = kv.unflatten(-1, (2, heads, HEAD)).unbind(1) + k = _rms(k) + return ( + k if key else k[:, :0].contiguous(), + k[:, :cache].contiguous(), + v[:, :cache].contiguous(), + ) + + +registry.define_op( + QUERY, + _query_reference, + "bf16(rms(q) * log_n * head_scale) per head of 64, eps 1e-6", +) +registry.define_op( + KEY, + _key_reference, + "rms of K read from packed [M, 2 * heads * 64] KV, plus K/V cache heads", +) + + +@torch.library.custom_op( + f"sdm_inference::{QUERY}", mutates_args=(), tags=registry.EXACT_STRIDES +) +def icl_query_rms_scale( + q: Tensor, head_scale: Tensor, log_n: float, kernel: str +) -> Tensor: + return registry.launch(registry.kernel(kernel), q, head_scale, log_n) + + +@icl_query_rms_scale.register_fake +def _(q: Tensor, head_scale: Tensor, log_n: float, kernel: str) -> Tensor: + return q.new_empty(q.shape) + + +@torch.library.custom_op( + f"sdm_inference::{KEY}", mutates_args=(), tags=registry.EXACT_STRIDES +) +def icl_key_rms( + kv: Tensor, heads: int, key: bool, cache: int, kernel: str +) -> tuple[Tensor, Tensor, Tensor]: + return registry.launch(registry.kernel(kernel), kv, heads, key, cache) + + +@icl_key_rms.register_fake +def _( + kv: Tensor, heads: int, key: bool, cache: int, kernel: str +) -> tuple[Tensor, Tensor, Tensor]: + m = kv.size(0) + return ( + kv.new_empty(m, heads if key else 0, HEAD), + kv.new_empty(m, cache, HEAD), + kv.new_empty(m, cache, HEAD), + ) + + +def _other(node: Node, first: object) -> object | None: + """The other operand of binary ``node`` if one operand is ``first``.""" + a, b = node.args[:2] + return b if a is first else a if b is first else None + + +def _query_rule(graph: Graph, norm: Node) -> bool: + """rmsnorm(q) * full(log n) * view(bf16(head_scale), [1, H, 1]).""" + launch = unweighted_rms_launch(norm) + if launch is None: + return False + q, out, channels, eps = launch + if channels != HEAD or eps != EPS or val(q).dim() != 3: + return False + if len(out.users) != 1: + return False + (scaled,) = out.users + if not is_call(scaled, aten.mul.Tensor) or len(scaled.users) != 1: + return False + full = _other(scaled, out) + if not ( + is_call(full, aten.full.default) + and full.kwargs.get("dtype") == torch.bfloat16 + and val(full).numel() == 1 + and only_user(full, scaled) + ): + return False + log_n = float(full.args[1]) + if torch.tensor(log_n, dtype=torch.bfloat16).item() != log_n: + return False + (product,) = scaled.users + if not is_call(product, aten.mul.Tensor): + return False + shaped = _other(product, scaled) + heads = val(q).shape[1] + if not ( + is_view(shaped) + and only_user(shaped, product) + and tuple(val(shaped).shape) == (1, heads, 1) + ): + return False + cast = shaped.args[0] + if not ( + is_call(cast, prims.convert_element_type.default, torch.bfloat16) + and only_user(cast, shaped) + and val(cast.args[0]).dtype == torch.float32 + ): + return False + head_scale = cast.args[0] + if heads * HEAD not in WIDTHS or val(product).shape != val(q).shape: + return False + kernel = registry.select(QUERY, val(q), val(head_scale), log_n) + if kernel is None: + return False + with graph.inserting_before(product): + fused = graph.call_function( + torch.ops.sdm_inference.icl_query_rms_scale.default, + (q, head_scale, log_n, kernel.name), + ) + fused.meta["val"] = val(q).new_empty(val(q).shape) + fx_pass.replace(product, fused) + return True + + +def _cache_copy(heads: Node) -> tuple[Node, Node, int] | None: + """The user ``clone(heads[:, :c])`` of ``heads``: (slice, clone, c).""" + for user in heads.users: + if not ( + is_call(user, aten.slice.Tensor, 1, 0) + and len(user.args) == 4 + and isinstance(user.args[3], int) + and 0 < user.args[3] <= val(heads).shape[1] + and len(user.users) == 1 + ): + continue + (copy,) = user.users + if is_call(copy, aten.clone.default) and val(copy).is_contiguous(): + return user, copy, user.args[3] + return None + + +def _through_clone(node: Node) -> Node: + """The full ``clone`` of ``node`` if any (SDM caches V contiguous).""" + clones = [ + u + for u in node.users + if is_call(u, aten.clone.default) and val(u).shape == val(node).shape + ] + return clones[0] if len(clones) == 1 else node + + +def _gather_rows(graph: Graph, weight: Node, width: int, rows: int) -> Node: + """``cat(weight[:rows], weight[width:width + rows])`` before any cast.""" + cast = None + if is_call(weight, prims.convert_element_type.default): + cast, weight = weight.args[1], weight.args[0] + parts = [] + for start in (0, width): + part = graph.call_function( + aten.slice.Tensor, (weight, 0, start, start + rows) + ) + part.meta["val"] = val(weight)[start : start + rows] + parts.append(part) + out = graph.call_function(aten.cat.default, (parts, 0)) + out.meta["val"] = torch.cat([val(p) for p in parts]) + if cast is None: + return out + converted = graph.call_function( + prims.convert_element_type.default, (out, cast) + ) + converted.meta["val"] = val(out).to(cast) + return converted + + +def _key_rule(graph: Graph, norm: Node) -> bool: + """Addmm -> split -> K view -> clone -> rmsnorm; cache head copies.""" + launch = unweighted_rms_launch(norm) + if launch is None: + return False + clone, out, channels, eps = launch + if channels != HEAD or eps != EPS: + return False + if not (is_call(clone, aten.clone.default) and only_user(clone, norm)): + return False + shaped = clone.args[0] + if not (is_view(shaped) and only_user(shaped, clone)): + return False + raw_k = shaped.args[0] + if not ( + is_call(raw_k, operator.getitem) + and raw_k.args[1] == 0 + and only_user(raw_k, shaped) + ): + return False + split = raw_k.args[0] + width = val(raw_k).shape[-1] + heads = width // HEAD + if not ( + is_call(split, aten.split.Tensor) + and split.args[1] == width + and split.args[2:] in ((-1,), (1,)) + and width in WIDTHS + and tuple(val(shaped).shape[1:]) == (heads, HEAD) + ): + return False + halves = {u.args[1]: u for u in split.users} + kv = split.args[0] + if ( + len(split.users) != 2 + or set(halves) != {0, 1} + or val(kv).dim() != 2 + or val(kv).shape[1] != 2 * width + ): + return False + k_copy = _cache_copy(out) + v_views = [ + u + for u in halves[1].users + if is_view(u) and tuple(val(u).shape[1:]) == (heads, HEAD) + ] + v_copy = next( + ( + c + for c in (_cache_copy(_through_clone(v)) for v in v_views) + if c is not None + ), + None, + ) + if k_copy is None or v_copy is None or k_copy[2] != v_copy[2]: + k_copy = v_copy = None + cache = 0 if k_copy is None else k_copy[2] + key = k_copy is None or list(out.users) != [k_copy[0]] + if not key and cache == 0: + return False + gather = ( + not key + and fx_pass.plain_addmm(kv) + and only_user(kv, split) + and list(halves[1].users) == v_views[:1] + and list(_through_clone(v_views[0]).users) == [v_copy[0]] + and len(v_views[0].users) == 1 + and is_call(kv.args[2], aten.permute.default, [1, 0]) + and GATHER_M[0] <= val(kv).shape[0] <= GATHER_M[1] + ) + order = {n: i for i, n in enumerate(graph.nodes)} + first = min( + (n for n in (out, v_copy[1] if v_copy else None) if n is not None), + key=order.__getitem__, + ) + heads_in = cache if gather else heads + m = val(kv).shape[0] + like = val(kv).new_empty(m, 2 * heads_in * HEAD) + kernel = registry.select(KEY, like, heads_in, key, cache) + if kernel is None: + return False + source = kv + if gather: + bias, x, transposed = kv.args + rows = cache * HEAD + with graph.inserting_before(first): + weight = _gather_rows(graph, transposed.args[0], width, rows) + bias = _gather_rows(graph, bias, width, rows) + permuted = graph.call_function( + aten.permute.default, (weight, [1, 0]) + ) + permuted.meta["val"] = val(weight).permute(1, 0) + source = graph.call_function( + aten.addmm.default, (bias, x, permuted) + ) + source.meta["val"] = like + with graph.inserting_before(first): + fused = graph.call_function( + torch.ops.sdm_inference.icl_key_rms.default, + (source, heads_in, key, cache, kernel.name), + ) + fused.meta["val"] = ( + val(source).new_empty(m, heads_in if key else 0, HEAD), + val(source).new_empty(m, cache, HEAD), + val(source).new_empty(m, cache, HEAD), + ) + parts = [] + for i in range(3): + parts.append(graph.call_function(operator.getitem, (fused, i))) + parts[-1].meta["val"] = fused.meta["val"][i] + if key: + fx_pass.replace(out, parts[0]) + if cache: + fx_pass.replace(k_copy[1], parts[1]) + fx_pass.replace(v_copy[1], parts[2]) + return True + + +fx_pass.add_rule(fx_pass.TRITON_KERNEL, QUERY, _query_rule) +fx_pass.add_rule(fx_pass.TRITON_KERNEL, KEY, _key_rule) + + +def _device(t: Tensor) -> str | None: + if t.device.type != "cuda": + return "needs CUDA tensors" + if torch.cuda.get_device_capability(t.device) != (12, 0): + return "needs SM 12.0" + if t.dtype != torch.bfloat16: + return "needs bfloat16" + return None + + +def _rows(t: Tensor, width: int, what: str) -> str | None: + if t.stride(-1) != 1 or t.stride(0) < width: + return f"needs {what} rows of {width} with unit stride" + if t.stride(0) % 8 or t.storage_offset() % 8: + return f"needs 16-byte aligned {what} rows" + return None + + +def _query_supports(q: Tensor, head_scale: Tensor, log_n: float) -> str | None: + if q.dim() != 3 or q.shape[2] != HEAD or q.stride(1) != HEAD: + return "needs q [M, H, 64] with head stride 64" + if q.numel() == 0: + return "needs a nonempty q" + if head_scale.dtype != torch.float32 or head_scale.shape != q.shape[1:2]: + return "needs FP32 head_scale [H]" + if not head_scale.is_contiguous() or head_scale.device != q.device: + return "needs contiguous head_scale on q's device" + return _device(q) or _rows(q, q.shape[1] * HEAD, "q") + + +def _key_supports(kv: Tensor, heads: int, key: bool, cache: int) -> str | None: + if kv.dim() != 2 or kv.shape[1] != 2 * heads * HEAD or kv.numel() == 0: + return "needs nonempty kv [M, 2 * heads * 64]" + if not 0 <= cache <= heads or not (key or cache): + return "needs 0 <= cache <= heads and some output" + return _device(kv) or _rows(kv, kv.shape[1], "kv") + + +def register(directory: Path) -> None: + module = sm120.load_module(directory, MODULE) + + def query(q: Tensor, head_scale: Tensor, log_n: float) -> Tensor: + return module.query(q, head_scale, log_n) + + def key( + kv: Tensor, heads: int, key: bool, cache: int + ) -> tuple[Tensor, Tensor, Tensor]: + return tuple(module.key(kv, heads, key, cache)) + + registry.register( + registry.Kernel( + op=QUERY, + name="sm120_icl_query_rms_scale", + supports=_query_supports, + impl=query, + cuda_symbols=("icl_qk::query_kernel",), + ) + ) + registry.register( + registry.Kernel( + op=KEY, + name="sm120_icl_key_rms", + supports=_key_supports, + impl=key, + cuda_symbols=("icl_qk::key_kernel",), + ) + ) diff --git a/benchmark/inference/kernels/fusions/kv_rms.py b/benchmark/inference/kernels/fusions/kv_rms.py new file mode 100644 index 0000000000..e6ce7681cb --- /dev/null +++ b/benchmark/inference/kernels/fusions/kv_rms.py @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Column KV projection with per-head key RMS fused (``linear_kv_key_rms``). + +Compiled SDM runs the column cross-attention KV as addmm into packed +[M, 2D], a split into K | V, a contiguous copy of K and SDM's Triton +``_rmsnorm_cast_kernel`` on it (no weight, eps 1e-6, channels = head). When +that copy is raw K's only reader, the kernel never stores raw K. V stays in +the packed buffer, so its readers are unchanged. +""" + +import operator +from pathlib import Path + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch._higher_order_ops.triton_kernel_wrap import kernel_side_table +from torch.fx import Graph, Node + +from benchmark.inference.kernels import fx_pass, registry, sm120 + +aten = torch.ops.aten +OP = "linear_kv_key_rms" +MODULE = "kv_rms_sm120" +EPS = 1e-6 +# Largest M per D checked bitwise against the compiled addmm + norm. +MAX_M = {128: 19_942_176, 256: 12_040_400} +# (D, head) -> smallest M whose bytes match cuBLAS addmm; at D=256 cuBLAS +# sums in another order for 17 <= M <= 128. +MIN_M = {(256, 64): 129, (128, 32): 1} + + +def _reference( + x: Tensor, weight: Tensor, bias: Tensor, head: int +) -> tuple[Tensor, Tensor]: + k, v = F.linear(x, weight, bias).chunk(2, -1) + k = F.rms_norm(k.float().unflatten(-1, (-1, head)), (head,), eps=EPS) + return k.to(v.dtype).flatten(-2), v + + +registry.define_op( + OP, + _reference, + "k, v = linear(x, w, b).chunk(2); per-head unweighted RMS (eps 1e-6) of k", +) + + +@torch.library.custom_op( + f"sdm_inference::{OP}", mutates_args=(), tags=registry.EXACT_STRIDES +) +def linear_kv_key_rms( + x: Tensor, weight: Tensor, bias: Tensor, head: int, kernel: str +) -> tuple[Tensor, Tensor]: + return registry.launch(registry.kernel(kernel), x, weight, bias, head) + + +@linear_kv_key_rms.register_fake +def _( + x: Tensor, weight: Tensor, bias: Tensor, head: int, kernel: str +) -> tuple[Tensor, Tensor]: + m, d = x.shape + return x.new_empty(m, d), x.new_empty(m, 2 * d)[:, d:] + + +def is_view(node: object) -> bool: + return fx_pass.is_call(node, aten.view.default) or fx_pass.is_call( + node, aten.reshape.default + ) + + +def unweighted_rms_launch(norm: Node) -> tuple[Node, Node, int, float] | None: + """SDM ``rmsnorm_cast`` without weight or RoPE: (x, out, channels, eps).""" + if fx_pass.triton_kernel_name(norm) != "_rmsnorm_cast_kernel": + return None + tensors = norm.kwargs["kwargs"] + const = kernel_side_table.get_constant_args( + norm.kwargs["constant_args_idx"] + ) + outs = list(norm.users) + if ( + set(tensors) != {"x_ptr", "out_ptr"} + or any( + const.get(k, 0) is not None + for k in ("weight_ptr", "cos_ptr", "sin_ptr") + ) + or not fx_pass.only_user(tensors["out_ptr"], norm) + or len(outs) != 1 + or outs[0].args[1:] != ("out_ptr",) + or fx_pass.val(outs[0]).dtype != fx_pass.val(tensors["x_ptr"]).dtype + ): + return None + return tensors["x_ptr"], outs[0], const.get("channels"), const.get("eps") + + +def _rule(graph: Graph, norm: Node) -> bool: + """Addmm -> [reshape] -> split(D); K -> reshape -> clone -> rmsnorm; V any. + + Raw K (split output 0) must have no reader other than that copy. + """ + launch = unweighted_rms_launch(norm) + if launch is None: + return False + clone, out, head, eps = launch + if not ( + fx_pass.is_call(clone, aten.clone.default) + and fx_pass.only_user(clone, norm) + ): + return False + shaped = clone.args[0] + if not (is_view(shaped) and fx_pass.only_user(shaped, clone)): + return False + raw_k = shaped.args[0] + if not ( + fx_pass.is_call(raw_k, operator.getitem) + and raw_k.args[1] == 0 + and fx_pass.only_user(raw_k, shaped) + ): + return False + split = raw_k.args[0] + if not fx_pass.is_call( + split, aten.split.Tensor, fx_pass.val(raw_k).shape[-1], -1 + ): + return False + halves = {u.args[1]: u for u in split.users} + if len(split.users) != 2 or set(halves) != {0, 1}: + return False + source = split.args[0] + addmm = source.args[0] if is_view(source) else source + if not ( + fx_pass.plain_addmm(addmm) + and fx_pass.only_user(source, split) + and (addmm is source or fx_pass.only_user(addmm, source)) + ): + return False + bias, x, transposed = addmm.args + if not ( + fx_pass.is_call(transposed, aten.permute.default, [1, 0]) + and fx_pass.val(raw_k).shape[-1] == fx_pass.val(x).shape[1] + and eps == EPS + and head == fx_pass.val(shaped).shape[-1] + ): + return False + weight = transposed.args[0] + kernel = registry.select( + OP, fx_pass.val(x), fx_pass.val(weight), fx_pass.val(bias), head + ) + if kernel is None: + return False + m, d = fx_pass.val(x).shape + with graph.inserting_before(out): + fused = graph.call_function( + torch.ops.sdm_inference.linear_kv_key_rms.default, + (x, weight, bias, head, kernel.name), + ) + packed = fx_pass.val(addmm) + fused.meta["val"] = (packed.new_empty(m, d), packed[:, d:]) + replacements = [] + for i, old in ((0, out), (1, halves[1])): + part = graph.call_function(operator.getitem, (fused, i)) + part.meta["val"] = fused.meta["val"][i] + view = graph.call_function( + aten.view.default, (part, list(fx_pass.val(old).shape)) + ) + view.meta["val"] = part.meta["val"].view(fx_pass.val(old).shape) + replacements.append((old, view)) + for old, new in replacements: + fx_pass.replace(old, new) + return True + + +fx_pass.add_rule(fx_pass.TRITON_KERNEL, OP, _rule) + + +def _supports( + x: Tensor, weight: Tensor, bias: Tensor, head: int +) -> str | None: + if x.dim() != 2 or (x.size(1), head) not in MIN_M: + return f"needs x [M, D] with (D, head) in {sorted(MIN_M)}" + m, d = x.shape + if not MIN_M[d, head] <= m <= MAX_M[d]: + return f"needs {MIN_M[d, head]} <= M <= {MAX_M[d]}" + if weight.shape != (2 * d, d) or bias.shape != (2 * d,): + return f"needs a {2 * d}x{d} weight and bias" + return sm120.policy(x, weight, bias) + + +def register(directory: Path) -> None: + module = sm120.load_module(directory, MODULE) + + def impl( + x: Tensor, weight: Tensor, bias: Tensor, head: int + ) -> tuple[Tensor, Tensor]: + m, d = x.shape + packed, key = x.new_empty(m, 2 * d), x.new_empty(m, d) + module.run(x, weight, bias, packed, key, head) + return key, packed[:, d:] + + # kv_rms::Key mangles to 6kv_rms3Key in profiler symbols. + registry.register( + registry.Kernel( + op=OP, + name="sm120_linear_kv_key_rms", + supports=_supports, + impl=impl, + cuda_symbols=("6kv_rms3Key",), + ) + ) diff --git a/benchmark/inference/kernels/fusions/linear_gelu.py b/benchmark/inference/kernels/fusions/linear_gelu.py new file mode 100644 index 0000000000..98679b3dbf --- /dev/null +++ b/benchmark/inference/kernels/fusions/linear_gelu.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``linear_gelu`` for the embedding MLPs and the dataset FFN. + +The first gating layer is also a ``linear_gelu``; its kernel is in +``gating.py``. +""" + +from pathlib import Path + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch.fx import Graph, Node + +from benchmark.inference.kernels import fx_pass, registry, sm120 +from benchmark.inference.kernels.fx_pass import is_call, only_user, val + +aten, prims = torch.ops.aten, torch.ops.prims + +MODULE = "linear_gelu_sm120" +# (K, N) -> largest M checked bitwise against compiled native. +MAX_M = {(128, 256): 33_029_640, (256, 512): 8_388_608, (1024, 2048): 500_000} +# Mangled names: linear_gelu::BiasGelu, linear_gelu::GeluEpilogue (K1024). +SYMBOLS = ("11linear_gelu8BiasGelu", "11linear_gelu12GeluEpilogue") + + +def _reference(x: Tensor, weight: Tensor, bias: Tensor | None) -> Tensor: + return F.gelu(F.linear(x, weight, bias)) + + +registry.define_op( + name="linear_gelu", + reference=_reference, + doc="gelu(linear(x, weight, bias)) with exact (erf) GELU", +) + + +@torch.library.custom_op( + "sdm_inference::linear_gelu", mutates_args=(), tags=registry.EXACT_STRIDES +) +def linear_gelu( + x: Tensor, weight: Tensor, bias: Tensor, kernel: str +) -> Tensor: + return registry.launch(registry.kernel(kernel), x, weight, bias) + + +@linear_gelu.register_fake +def _(x: Tensor, weight: Tensor, bias: Tensor, kernel: str) -> Tensor: + return x.new_empty(x.size(0), weight.size(0)) + + +def _rule(graph: Graph, erf: Node) -> bool: + """Addmm -> [view] -> f32 -> 0.5x(1 + erf(x/sqrt2)) -> bf16.""" + scaled = erf.args[0] + if not is_call(scaled, aten.mul.Tensor, 0.7071067811865476): + return False + upcast = scaled.args[0] + if not is_call(upcast, prims.convert_element_type.default, torch.float32): + return False + source = upcast.args[0] + view = ( + source + if is_call(source, aten.view.default) + or is_call(source, aten.reshape.default) + else None + ) + addmm = view.args[0] if view is not None else source + if not fx_pass.plain_addmm(addmm): + return False + bias, x, transposed = addmm.args + if not is_call(transposed, aten.permute.default, [1, 0]): + return False + half = [u for u in upcast.users if is_call(u, aten.mul.Tensor, 0.5)] + shifted = [u for u in erf.users if is_call(u, aten.add.Tensor, 1)] + if ( + len(half) != 1 + or len(shifted) != 1 + or set(upcast.users) + != { + scaled, + half[0], + } + ): + return False + product = [u for u in half[0].users if u.target == aten.mul.Tensor] + if len(product) != 1 or set(product[0].args) != {half[0], shifted[0]}: + return False + downcast = list(product[0].users) + if len(downcast) != 1 or not is_call( + downcast[0], prims.convert_element_type.default, torch.bfloat16 + ): + return False + if not ( + only_user(addmm, view or upcast) + and (view is None or only_user(view, upcast)) + and only_user(scaled, erf) + and only_user(erf, shifted[0]) + and only_user(shifted[0], product[0]) + and only_user(half[0], product[0]) + ): + return False + weight = transposed.args[0] + kernel = registry.select("linear_gelu", val(x), val(weight), val(bias)) + if kernel is None: + return False + with graph.inserting_before(downcast[0]): + fused = graph.call_function( + torch.ops.sdm_inference.linear_gelu.default, + (x, weight, bias, kernel.name), + ) + fused.meta["val"] = val(addmm) + if view is not None: + fused = graph.call_function(view.target, (fused, view.args[1])) + fused.meta["val"] = val(view) + fx_pass.replace(downcast[0], fused) + return True + + +fx_pass.add_rule(aten.erf.default, "linear_gelu", _rule) + + +def _kernel(module, k: int, n: int, max_m: int) -> registry.Kernel: + def supports(x: Tensor, weight: Tensor, bias: Tensor) -> str | None: + if ( + x.dim() != 2 + or x.size(1) != k + or not sm120.MIN_M <= x.size(0) <= max_m + ): + return f"needs x of shape [{sm120.MIN_M}<=M<={max_m}, {k}]" + if weight.shape != (n, k) or bias.shape != (n,): + return f"needs a {n}x{k} weight and bias" + return sm120.policy(x, weight, bias) + + def impl(x: Tensor, weight: Tensor, bias: Tensor) -> Tensor: + out = x.new_empty(x.size(0), n) + module.run(x, weight, bias, out) + return out + + return registry.Kernel( + op="linear_gelu", + name=f"sm120_linear_gelu_k{k}_n{n}", + supports=supports, + impl=impl, + cuda_symbols=(SYMBOLS[k == 1024],), + ) + + +def register(directory: Path) -> None: + module = sm120.load_module(directory, MODULE) + for (k, n), max_m in MAX_M.items(): + registry.register(_kernel(module, k, n, max_m)) diff --git a/benchmark/inference/kernels/fusions/projection_rms.py b/benchmark/inference/kernels/fusions/projection_rms.py new file mode 100644 index 0000000000..5a504446b7 --- /dev/null +++ b/benchmark/inference/kernels/fusions/projection_rms.py @@ -0,0 +1,315 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Column attention out-projection + residual + weighted RMSNorm (D 128/256). + +``(raw, rms_norm(RN(raw + table[:, 4:])))`` as ``[F, C, D]``. +""" + +import operator +from pathlib import Path + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch._higher_order_ops.triton_kernel_wrap import kernel_side_table +from torch.fx import Graph, Node + +from benchmark.inference.kernels import fx_pass, registry, sm120 + +aten = torch.ops.aten +MODULE = "projection_rms_sm120" +READOUTS = 4 +WIDTHS = (128, 256) +# Largest M = C * F per D checked bitwise against the compiled addmm + norm. +MAX_M = {128: 19_815_464, 256: 11_982_968} + + +def _norm(x: Tensor, norm_weight: Tensor, eps: float) -> Tensor: + return F.rms_norm(x.float(), (x.size(-1),), norm_weight, eps).bfloat16() + + +def _column_reference(x, weight, bias, table, norm_weight, eps): + c, t, d = table.shape + f = x.size(0) // c + raw = torch.addmm(bias, x, weight.t()) + residual = ( + raw.view(f, c, d).float() + table[:, t - f :].transpose(0, 1).float() + ).bfloat16() + return raw, _norm(residual, norm_weight, eps) + + +registry.define_op( + "column_projection_rms", + _column_reference, + "(raw, rms_norm(raw + table[:, 4:] as [F, C, D])), raw = addmm", +) + + +@torch.library.custom_op( + "sdm_inference::column_projection_rms", + mutates_args=(), + tags=registry.EXACT_STRIDES, +) +def column_projection_rms( + x: Tensor, + weight: Tensor, + bias: Tensor, + table: Tensor, + norm_weight: Tensor, + eps: float, + kernel: str, +) -> tuple[Tensor, Tensor]: + return registry.launch( + registry.kernel(kernel), x, weight, bias, table, norm_weight, eps + ) + + +@column_projection_rms.register_fake +def _(x, weight, bias, table, norm_weight, eps, kernel): + c, _, d = table.shape + return x.new_empty(x.shape), x.new_empty(x.size(0) // c, c, d) + + +def _supports(x, weight, bias, table, norm_weight, eps) -> str | None: + if ( + table.dim() != 3 + or table.size(0) < 1 + or x.dim() != 2 + or table.size(2) != x.size(1) + ): + return "needs x [M, D] and a [C, P + F, D] table" + c, t, _ = table.shape + if x.size(0) % c or t - x.size(0) // c != READOUTS: + return ( + f"needs M = C * F and a table of {READOUTS} + F rows per context" + ) + if ( + x.size(1) not in WIDTHS + or not sm120.MIN_M <= x.size(0) <= MAX_M[x.size(1)] + ): + return f"needs x of shape [{sm120.MIN_M}<=M<={MAX_M}, D in {WIDTHS}]" + d = x.size(1) + if weight.shape != (d, d) or bias.shape != (d,): + return f"needs a {d}x{d} weight and [{d}] bias" + if ( + norm_weight.dtype != torch.float32 + or norm_weight.shape != (d,) + or norm_weight.stride() != (1,) + or norm_weight.storage_offset() + or norm_weight.device != x.device + ): + return f"needs a contiguous FP32 [{d}] norm weight" + if eps != sm120.EPS: + return f"needs eps {sm120.EPS}" + return sm120.policy(x, weight, bias, table) + + +def _rmsnorm_launches(x: Node) -> list[tuple[Node, Node, float]]: + """SDM ``rmsnorm_cast`` launches reading ``x``: (output, weight, eps).""" + found = [] + for launch in x.users: + if fx_pass.triton_kernel_name(launch) != "_rmsnorm_cast_kernel": + continue + tensors = launch.kwargs["kwargs"] + constants = kernel_side_table.get_constant_args( + launch.kwargs["constant_args_idx"] + ) + outputs = list(launch.users) + if ( + tensors.get("x_ptr") is not x + or "weight_ptr" not in tensors + or "cos_ptr" in tensors + or constants.get("cos_ptr", 0) is not None + or len(outputs) != 1 + or outputs[0].args[1] != "out_ptr" + ): + continue + if fx_pass.val(outputs[0]).dtype != fx_pass.val(x).dtype: + continue + found.append((outputs[0], tensors["weight_ptr"], constants["eps"])) + return found + + +def _table_value(node: object) -> bool: + """``node`` is a value of the in-graph ``[C, P + F, D]`` row table. + + SDM allocates the table with ``torch.empty`` and updates it in place; + after functionalization each update (``slice_scatter`` into ``t[:, P:]``, + or the residual ``t + x``) is a new value. A + ``[C, P + F, D]`` graph input updated in place (copied back at the end) + counts too. + """ + pending, seen = [node], set() + while pending: + node = pending.pop() + if node in seen or not isinstance(node, Node): + continue + seen.add(node) + value = node.meta.get("val") + if not ( + isinstance(value, torch.Tensor) + and value.dim() == 3 + and value.is_contiguous() + ): + continue + if fx_pass.is_call(node, aten.empty.memory_format) or ( + node.op == "placeholder" + and any( + fx_pass.is_call(u, aten.copy_.default) and u.args[0] is node + for u in node.users + ) + ): + return True + if fx_pass.is_call(node, aten.slice_scatter.default): + pending.append(node.args[0]) + elif fx_pass.is_call(node, aten.add.Tensor): + pending.extend( + a + for a in node.args + if isinstance(a, Node) and fx_pass.val(a).shape == value.shape + ) + return False + + +def _table_rows(node: object) -> Node | None: + """The table whose rows ``P:`` ``node`` reads as ``[C, F, D]``. + + Either the flat graph input of ``as_strided(flat, [C, F, D], [(F + P)D, + D, 1], P * D)`` or the in-graph table ``t`` of ``t[:, P:]``, P = 4. + """ + skip = READOUTS + if fx_pass.is_call(node, aten.slice.Tensor): + t = node.args[0] + if ( + tuple(node.args[1:3]) == (1, skip) + and node.args[3:4] != () + and node.args[3] >= fx_pass.val(t).size(1) + and node.args[4:] in ((), (1,)) + and _table_value(t) + ): + return t + return None + if ( + not fx_pass.is_call(node, aten.as_strided.default) + or len(node.args) != 4 + or len(node.args[1]) != 3 + ): + return None + (_, f, d), stride, start = node.args[1:] + if list(stride) != [(f + skip) * d, d, 1] or start != skip * d: + return None + return node.args[0] + + +def _full_table(graph: Graph, flat: Node, like: Node) -> Node: + if fx_pass.val(flat).dim() == 3: + return flat + c, f, d = fx_pass.val(like).shape + t = f + READOUTS + table = graph.call_function( + aten.as_strided.default, (flat, [c, t, d], [t * d, d, 1], 0) + ) + table.meta["val"] = torch.as_strided( + fx_pass.val(flat), (c, t, d), (t * d, d, 1), 0 + ) + return table + + +def _other(add: Node, operand: Node) -> object: + """The other operand of a binary add that reads ``operand``.""" + a, b = add.args + return b if a is operand else a if b is operand else None + + +def _values(args): + return [fx_pass.val(a) if isinstance(a, Node) else a for a in args] + + +def _column( + graph: Graph, addmm: Node, view: Node, add: Node, order: dict +) -> bool: + """add(view(raw), permute(table[:, 4:])) -> rmsnorm.""" + prior = _other(add, view) + if not fx_pass.is_call(prior, aten.permute.default, [1, 0, 2]): + return False + rows = prior.args[0] + flat = _table_rows(rows) + norms = _rmsnorm_launches(add) + if ( + flat is None + or len(norms) != 1 + or fx_pass.val(view).shape != fx_pass.val(prior).shape + ): + return False + out, norm_weight, eps = norms[0] + # raw replaces addmm everywhere, so it goes before addmm's first user. + first = min(addmm.users, key=order.__getitem__) + if max(order[flat], order[norm_weight]) > order[first]: + return False + bias, x, transposed = addmm.args + with graph.inserting_before(first): + table = _full_table(graph, flat, rows) + args = (x, transposed.args[0], bias, table, norm_weight, eps) + kernel = registry.select("column_projection_rms", *_values(args)) + if kernel is None: + if table is not flat: + graph.erase_node(table) + return False + fused = graph.call_function( + torch.ops.sdm_inference.column_projection_rms.default, + (*args, kernel.name), + ) + fused.meta["val"] = (fx_pass.val(addmm), fx_pass.val(out)) + raw, norm = ( + graph.call_function(operator.getitem, (fused, i)) for i in range(2) + ) + raw.meta["val"], norm.meta["val"] = fx_pass.val(addmm), fx_pass.val(out) + fx_pass.replace(addmm, raw) + fx_pass.replace(out, norm) + return True + + +def _rule(graph: Graph, addmm: Node) -> bool: + """Match addmm -> reshape -> residual add -> rmsnorm.""" + if not fx_pass.plain_addmm(addmm) or not fx_pass.is_call( + addmm.args[2], aten.permute.default, [1, 0] + ): + return False + order = {n: i for i, n in enumerate(graph.nodes)} + for view in list(addmm.users): + if fx_pass.is_call(view, aten.reshape.default) or fx_pass.is_call( + view, aten.view.default + ): + for add in list(view.users): + if ( + fx_pass.is_call(add, aten.add.Tensor) + and len(add.args) == 2 + and not add.kwargs + and _column(graph, addmm, view, add, order) + ): + return True + return False + + +fx_pass.add_rule(aten.addmm.default, "column_projection_rms", _rule) + + +def register(directory: Path) -> None: + module = sm120.load_module(directory, MODULE) + + def impl(x, weight, bias, table, norm_weight, eps): + c, _, d = table.shape + raw, norm = torch.empty_like(x), x.new_empty(x.size(0) // c, c, d) + module.run(x, weight, bias, table, norm_weight, raw, norm, eps) + return raw, norm + + registry.register( + registry.Kernel( + "column_projection_rms", + "sm120_column_projection_rms", + _supports, + impl, + ("19ColumnProjectionRms",), + ) + ) diff --git a/benchmark/inference/kernels/fusions/query_rms.py b/benchmark/inference/kernels/fusions/query_rms.py new file mode 100644 index 0000000000..0530c90698 --- /dev/null +++ b/benchmark/inference/kernels/fusions/query_rms.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Column query projection with query RMS fused (``linear_query_rms``). + +Compiled SDM runs the column cross-attention query as addmm into a raw +[M, D] temporary, viewed per head, then SDM's Triton ``_rmsnorm_cast_kernel`` +(no weight, eps 1e-6, channels = head). The raw query has no other reader, so +the kernel never stores it. +""" + +from pathlib import Path + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch.fx import Graph, Node + +from benchmark.inference.kernels import fx_pass, registry, sm120 +from benchmark.inference.kernels.fusions.kv_rms import ( + is_view, + unweighted_rms_launch, +) + +aten = torch.ops.aten +OP = "linear_query_rms" +MODULE = "query_rms_sm120" +EPS = 1e-6 +# Largest M per D checked bitwise against the compiled addmm + norm. +MAX_M = {128: 33_029_640, 256: 20_600_904} +# (D, head) -> M band where cuBLAS uses another kernel (WMMA/split-K, other +# summation order). Bytes match cuBLAS elsewhere in 1..4096 and on a 7% log +# grid to 3.1M. Small's M = 128 is admitted; Medium's M = 256 is not. +M_GAP = {(256, 64): (1, 256), (128, 32): (934, 1024)} + + +def _reference(x: Tensor, weight: Tensor, bias: Tensor, head: int) -> Tensor: + q = F.linear(x, weight, bias) + return ( + F.rms_norm(q.float().unflatten(-1, (-1, head)), (head,), eps=EPS) + .to(q.dtype) + .flatten(-2) + ) + + +registry.define_op( + OP, + _reference, + "per-head unweighted RMS (eps 1e-6) of linear(x, weight, bias)", +) + + +@torch.library.custom_op( + f"sdm_inference::{OP}", mutates_args=(), tags=registry.EXACT_STRIDES +) +def linear_query_rms( + x: Tensor, weight: Tensor, bias: Tensor, head: int, kernel: str +) -> Tensor: + return registry.launch(registry.kernel(kernel), x, weight, bias, head) + + +@linear_query_rms.register_fake +def _( + x: Tensor, weight: Tensor, bias: Tensor, head: int, kernel: str +) -> Tensor: + return x.new_empty(x.shape) + + +def _rule(graph: Graph, norm: Node) -> bool: + """Addmm(bias, x, permute(w)) -> views -> per-head rmsnorm; sole reader.""" + launch = unweighted_rms_launch(norm) + if launch is None: + return False + view, out, head, eps = launch + if not is_view(view) or eps != EPS or head != fx_pass.val(view).shape[-1]: + return False + user = norm + while is_view(view) and fx_pass.only_user(view, user): + view, user = view.args[0], view + addmm = view + if not (fx_pass.plain_addmm(addmm) and fx_pass.only_user(addmm, user)): + return False + bias, x, transposed = addmm.args + if not fx_pass.is_call(transposed, aten.permute.default, [1, 0]): + return False + weight = transposed.args[0] + kernel = registry.select( + OP, fx_pass.val(x), fx_pass.val(weight), fx_pass.val(bias), head + ) + if kernel is None: + return False + with graph.inserting_before(out): + fused = graph.call_function( + torch.ops.sdm_inference.linear_query_rms.default, + (x, weight, bias, head, kernel.name), + ) + fused.meta["val"] = fx_pass.val(addmm).new_empty( + fx_pass.val(addmm).shape + ) + shaped = graph.call_function( + aten.view.default, (fused, list(fx_pass.val(out).shape)) + ) + shaped.meta["val"] = fx_pass.val(out) + fx_pass.replace(out, shaped) + return True + + +fx_pass.add_rule(fx_pass.TRITON_KERNEL, OP, _rule) + + +def _supports( + x: Tensor, weight: Tensor, bias: Tensor, head: int +) -> str | None: + if x.dim() != 2 or (x.size(1), head) not in M_GAP: + return f"needs x [M, D] with (D, head) in {sorted(M_GAP)}" + m, d = x.shape + lo, hi = M_GAP[d, head] + if not 1 <= m <= MAX_M[d]: + return f"needs M in [1, {MAX_M[d]}]; got M={m}" + if lo <= m <= hi: + return f"M in [{lo}, {hi}] differs from cuBLAS; got M={m}" + if weight.shape != (d, d) or bias.shape != (d,): + return f"needs a {d}x{d} weight and bias" + return sm120.policy(x, weight, bias) + + +def register(directory: Path) -> None: + module = sm120.load_module(directory, MODULE) + + def impl(x: Tensor, weight: Tensor, bias: Tensor, head: int) -> Tensor: + out = torch.empty_like(x) + module.run(x, weight, bias, out, head) + return out + + registry.register( + registry.Kernel( + op=OP, + name="sm120_linear_query_rms", + supports=_supports, + impl=impl, + cuda_symbols=("9query_rms5Query",), + ) + ) diff --git a/benchmark/inference/kernels/fusions/rope_rms.py b/benchmark/inference/kernels/fusions/rope_rms.py new file mode 100644 index 0000000000..2eb0fe4254 --- /dev/null +++ b/benchmark/inference/kernels/fusions/rope_rms.py @@ -0,0 +1,367 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Row-block Q/K transform: split-half RoPE then per-head RMS, one launch. + +``rope_rms(q, k, inv_freq) = (rms(rope(q)), rms(rope(k)))`` for ``q, k +[C, S, 4, head]`` (head 32 or 64), FP32 ``inv_freq [head / 2]``, positions +``0..S-1`` along dim 1, eps 1e-6, no weight. + +Compiled SDM runs Q and K each as Inductor's rope pointwise feeding SDM's +Triton ``_rmsnorm_cast_kernel``. The pass pairs two such regions sharing +``inv_freq`` and shape (a layer's Q and K) into one call. In the last row +layer the queries are only the 4 readout tokens, so Q and K differ in length +there; ``rope_rms_uneven`` covers that case. +""" + +import operator +from collections.abc import Callable +from pathlib import Path + +import torch +import torch.nn.functional as F +from torch import Size, Tensor +from torch.fx import Graph, Node + +from benchmark.inference.kernels import fx_pass, registry +from benchmark.inference.kernels.fusions.kv_rms import unweighted_rms_launch +from benchmark.inference.kernels.fx_pass import is_call, val +from benchmark.inference.kernels.registry import Kernel +from benchmark.inference.kernels.sm120 import load_module + +aten, prims = torch.ops.aten, torch.ops.prims +MODULE = "rope_rms_sm120" +EPS = 1e-6 +HEADS = 4 +WIDTHS = (32, 64) +MAX_S = 1024 # every position below it is checked against compiled native +# Head width -> largest C * S checked bitwise against compiled native. +MAX_TOKENS = {32: 20_234_480, 64: 8_388_737} + + +def _reference( + q: Tensor, k: Tensor, inv_freq: Tensor +) -> tuple[Tensor, Tensor]: + def one(x: Tensor) -> Tensor: + angle = ( + torch.arange(x.size(1), device=x.device, dtype=torch.float32)[ + :, None + ] + * inv_freq + ) + cos, sin = angle.cos()[:, None], angle.sin()[:, None] + x1, x2 = x.float().chunk(2, -1) + rotated = torch.cat((x1 * cos - x2 * sin, x2 * cos + x1 * sin), -1).to( + x.dtype + ) + return F.rms_norm(rotated.float(), (x.size(-1),), eps=EPS).to(x.dtype) + + return one(q), one(k) + + +registry.define_op( + "rope_rms", + _reference, + "(rms(rope(q)), rms(rope(k))): split-half RoPE, per-head RMS, eps 1e-6", +) + + +@torch.library.custom_op( + "sdm_inference::rope_rms", mutates_args=(), tags=registry.EXACT_STRIDES +) +def rope_rms( + q: Tensor, k: Tensor, inv_freq: Tensor, kernel: str +) -> tuple[Tensor, Tensor]: + return registry.launch(registry.kernel(kernel), q, k, inv_freq) + + +@rope_rms.register_fake +def _(q, k, inv_freq, kernel): + return q.new_empty(q.shape), q.new_empty(q.shape) + + +registry.define_op( + "rope_rms_uneven", + _reference, + "rope_rms with q [C, Sq, 4, head] and k [C, Sk, 4, head]", +) + + +@torch.library.custom_op( + "sdm_inference::rope_rms_uneven", + mutates_args=(), + tags=registry.EXACT_STRIDES, +) +def rope_rms_uneven( + q: Tensor, k: Tensor, inv_freq: Tensor, kernel: str +) -> tuple[Tensor, Tensor]: + return registry.launch(registry.kernel(kernel), q, k, inv_freq) + + +@rope_rms_uneven.register_fake +def _(q, k, inv_freq, kernel): + return q.new_empty(q.shape), k.new_empty(k.shape) + + +def _bf16(node: object, trig: object) -> Node | None: + """convert(unsqueeze(trig(angle), 1 or -2), bf16) -> angle.""" + if not ( + is_call(node, prims.convert_element_type.default) + and node.args[1] == torch.bfloat16 + ): + return None + unsqueeze = node.args[0] + if not ( + is_call(unsqueeze, aten.unsqueeze.default) + and unsqueeze.args[1] in (1, -2) + and is_call(unsqueeze.args[0], trig) + and val(unsqueeze.args[0]).dim() == 2 + ): + return None + return unsqueeze.args[0].args[0] + + +def _strip(node: object, target: object, value: int) -> object: + return ( + node.args[0] + if is_call(node, target) and node.args[1] == value + else node + ) + + +def _positions(node: object, seq: int) -> bool: + """Node = convert(iota(seq) [* 1] [+ 0], f32).""" + if not ( + is_call(node, prims.convert_element_type.default) + and node.args[1] == torch.float32 + ): + return False + iota = _strip(_strip(node.args[0], aten.add.Tensor, 0), aten.mul.Tensor, 1) + return ( + is_call(iota, prims.iota.default) + and iota.args[0] == seq + and iota.kwargs.get("start", 0) == 0 + and iota.kwargs.get("step", 1) == 1 + ) + + +def _inv_freq(angle: object, seq: int) -> Node | None: + """Angle = positions as [S, 1] * inv_freq (or inv_freq as [1, -1]).""" + if not is_call(angle, aten.mul.Tensor): + return None + pos, freq = angle.args + views = (aten.reshape.default, aten.view.default) + if any(is_call(pos, v) for v in views): + if list(pos.args[1]) != [-1, 1]: + return None + elif not (is_call(pos, aten.unsqueeze.default) and pos.args[1] in (1, -1)): + return None + if any(is_call(freq, v) for v in views) and list(freq.args[1]) == [1, -1]: + freq = freq.args[0] + if not isinstance(freq, Node) or val(freq).dim() != 1: + return None + return freq if _positions(pos.args[0], seq) else None + + +def _region(norm: Node) -> tuple[Node, Node, Node] | None: + """rmsnorm(cat([x1 cos - x2 sin, x2 cos + x1 sin], -1)): (x, inv_freq).""" + launch = unweighted_rms_launch(norm) + if launch is None: + return None + cat, out, channels, eps = launch + if eps != EPS or not out.users: + return None + if not ( + is_call(cat, aten.cat.default) + and len(cat.args) == 2 + and cat.args[1] in (-1, 3) + ): + return None + if len(cat.args[0]) != 2 or list(cat.users) != [norm]: + return None + first, second = cat.args[0] + if not ( + is_call(first, aten.sub.Tensor) and is_call(second, aten.add.Tensor) + ): + return None + products = (*first.args, *second.args) + if not all( + is_call(p, aten.mul.Tensor) and len(p.args) == 2 for p in products + ): + return None + (x1, cos), (x2, sin), (x2b, cos_b), (x1b, sin_b) = ( + p.args for p in products + ) + if x1 is not x1b or x2 is not x2b or cos is not cos_b or sin is not sin_b: + return None + if not ( + is_call(x1, operator.getitem) + and is_call(x2, operator.getitem) + and x1.args[0] is x2.args[0] + ): + return None + split = x1.args[0] + if not ( + is_call(split, aten.split.Tensor) + and x1.args[1] == 0 + and x2.args[1] == 1 + ): + return None + x = split.args[0] + if not isinstance(x, Node) or val(x).dim() != 4: + return None + seq, head = val(x).shape[1], val(x).shape[3] + if ( + split.args[1] != head // 2 + or split.args[2:] not in ((-1,), (3,)) + or channels != head + ): + return None + angle = _bf16(cos, aten.cos.default) + if angle is None or _bf16(sin, aten.sin.default) is not angle: + return None + inv_freq = _inv_freq(angle, seq) + return None if inv_freq is None else (x, inv_freq, out) + + +def _pair_rule(op: str, pairs: Callable[[Size, Size], bool]) -> fx_pass.Rule: + """Fuse a region with the next one sharing ``inv_freq`` that ``pairs``.""" + + def rule(graph: Graph, norm: Node) -> bool: + first = _region(norm) + if first is None: + return False + order = {n: i for i, n in enumerate(graph.nodes)} + later = ( + (n, r) + for n in graph.nodes + if order[n] > order[norm] + and fx_pass.triton_kernel_name(n) == "_rmsnorm_cast_kernel" + for r in [_region(n)] + if r is not None + and r[1] is first[1] + and pairs(val(first[0]).shape, val(r[0]).shape) + ) + pair = next(later, None) + if pair is None: + return False + second_norm, second = pair + if min(order[u] for u in first[2].users) < order[second_norm]: + return False + (q, inv_freq, q_out), (k, _, k_out) = first, second + kernel = registry.select(op, val(q), val(k), val(inv_freq)) + if kernel is None: + return False + with graph.inserting_before(second_norm): + fused = graph.call_function( + getattr(torch.ops.sdm_inference, op).default, + (q, k, inv_freq, kernel.name), + ) + fused.meta["val"] = (val(q_out), val(k_out)) + outputs = [] + for i, like in enumerate((q_out, k_out)): + outputs.append( + graph.call_function(operator.getitem, (fused, i)) + ) + outputs[-1].meta["val"] = val(like) + for old, new in zip((q_out, k_out), outputs): + fx_pass.replace(old, new) + return True + + return rule + + +def _uneven(q: Size, k: Size) -> bool: + return q[1] != k[1] and (q[0], *q[2:]) == (k[0], *k[2:]) + + +fx_pass.add_rule( + fx_pass.TRITON_KERNEL, "rope_rms", _pair_rule("rope_rms", operator.eq) +) +fx_pass.add_rule( + fx_pass.TRITON_KERNEL, + "rope_rms_uneven", + _pair_rule("rope_rms_uneven", _uneven), +) + + +def supports(q: Tensor, k: Tensor, inv_freq: Tensor) -> str | None: + if ( + any(t.device.type != "cuda" for t in (q, k, inv_freq)) + or len({q.device, k.device, inv_freq.device}) != 1 + ): + return "needs CUDA tensors on one device" + if torch.cuda.get_device_capability(q.device) != (12, 0): + return "needs SM 12.0" + if ( + q.dtype != torch.bfloat16 + or k.dtype != torch.bfloat16 + or inv_freq.dtype != torch.float32 + ): + return "needs bfloat16 q, k and float32 inv_freq" + if torch.is_grad_enabled() and any( + t.requires_grad for t in (q, k, inv_freq) + ): + return "no backward" + if q.dim() != 4 or q.shape != k.shape: + return "needs q, k [C, S, 4, head] of one shape" + c, s, h, head = q.shape + if h != HEADS or head not in WIDTHS: + return f"needs {HEADS} heads of width in {WIDTHS}" + if not 1 <= s <= MAX_S: + return f"needs 1 <= S <= {MAX_S} (qualified positions)" + if not 1 <= c * s <= MAX_TOKENS[head]: + return f"needs 1 <= C * S <= {MAX_TOKENS[head]}" + for t in (q, k): + ld = t.stride(1) + if t.stride() != (s * ld, ld, head, 1) or ld < HEADS * head: + return "needs q, k strides (S * ld, ld, head, 1), ld >= 4 * head" + if ld % 4 or t.storage_offset() % 4: + return "needs 8-byte aligned rows" + if inv_freq.shape != (head // 2,) or inv_freq.stride() != (1,): + return "needs contiguous inv_freq [head / 2]" + return None + + +def _uneven_supports(q: Tensor, k: Tensor, inv_freq: Tensor) -> str | None: + if q.dim() != 4 or k.dim() != 4 or q.shape[1] == k.shape[1]: + return "needs q, k [C, S, 4, head] of different S (else rope_rms)" + if (q.shape[0], *q.shape[2:]) != (k.shape[0], *k.shape[2:]): + return "needs q, k of one C, heads and head width" + longer = q if q.shape[1] > k.shape[1] else k + return ( + supports(longer, longer, inv_freq) + or supports(q, q, inv_freq) + or supports(k, k, inv_freq) + ) + + +def register(directory: Path) -> None: + module = load_module(directory, MODULE) + + def impl(q: Tensor, k: Tensor, inv_freq: Tensor) -> tuple[Tensor, Tensor]: + return tuple(module.run(q, k, inv_freq)) + + def uneven_impl( + q: Tensor, k: Tensor, inv_freq: Tensor + ) -> tuple[Tensor, Tensor]: + return tuple(module.run_uneven(q, k, inv_freq)) + + registry.register( + Kernel( + op="rope_rms", + name="sm120_rope_rms", + supports=supports, + impl=impl, + cuda_symbols=("rope_rms_kernel",), + ) + ) + registry.register( + Kernel( + op="rope_rms_uneven", + name="sm120_rope_rms_uneven", + supports=_uneven_supports, + impl=uneven_impl, + cuda_symbols=("rope_rms_uneven_kernel",), + ) + ) diff --git a/benchmark/inference/kernels/fx_pass.py b/benchmark/inference/kernels/fx_pass.py new file mode 100644 index 0000000000..a3796f5689 --- /dev/null +++ b/benchmark/inference/kernels/fx_pass.py @@ -0,0 +1,160 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Inductor post-grad pass that routes matched regions to selected kernels. + +Matches the pre-fusion ATen form of each named op. A region is rewritten only +if :func:`registry.select` admits its compile-time tensor metadata; otherwise +the graph is left untouched for Inductor. +""" + +from collections.abc import Callable + +import torch +from torch import Tensor +from torch._higher_order_ops.triton_kernel_wrap import ( + kernel_side_table, + triton_kernel_wrapper_functional, +) +from torch._inductor.custom_graph_pass import CustomGraphPass +from torch.fx import Graph, Node + +from benchmark.inference.kernels import registry + +aten = torch.ops.aten + + +def is_call(node: object, target: object, *args: object) -> bool: + return ( + isinstance(node, Node) + and node.op == "call_function" + and node.target == target + and all(a is None or a == b for a, b in zip(args, node.args[1:])) + ) + + +def plain_addmm(node: object) -> bool: + """``addmm(bias, x, w)`` with the default alpha and beta.""" + return ( + is_call(node, aten.addmm.default) + and len(node.args) == 3 + and not node.kwargs + ) + + +def only_user(node: Node, user: Node) -> bool: + return list(node.users) == [user] + + +def val(node: Node) -> torch.Tensor: + return node.meta["val"] + + +TRITON_KERNEL = triton_kernel_wrapper_functional + + +def triton_kernel_name(node: Node) -> str | None: + """Name of the user-defined Triton kernel ``node`` launches, if any.""" + if not is_call(node, TRITON_KERNEL): + return None + kernel = kernel_side_table.get_kernel(node.kwargs["kernel_idx"]) + while not hasattr(kernel, "__name__") and hasattr(kernel, "fn"): + kernel = kernel.fn + return getattr(kernel, "__name__", None) + + +Rule = Callable[[Graph, Node], bool] +RULES: list[tuple[object, str, Rule]] = [] + + +def add_rule(target: object, op: str, rule: Rule) -> None: + """Try ``rule`` at nodes calling ``target``, after earlier rules.""" + RULES.append((target, op, rule)) + + +_shadow: list[str] = [] + + +@torch.library.custom_op( + "sdm_inference::shadow_compare", + mutates_args=(), + tags=registry.EXACT_STRIDES, +) +def shadow_compare(native: Tensor, kernel: Tensor, op: str) -> Tensor: + registry.record_shadow(op, native, kernel) + return _like(native).copy_(native) + + +@shadow_compare.register_fake +def _(native: Tensor, kernel: Tensor, op: str) -> Tensor: + return _like(native) + + +def _like(t: Tensor) -> Tensor: + return t.new_empty_strided(t.shape, t.stride()) + + +def replace(old: Node, new: Node) -> None: + """Route the users of native ``old`` to the kernel's ``new``. + + In shadow mode ``old`` stays on the data path and every later user reads + ``shadow_compare(old, new)``, which records whether the two are bitwise + equal, so each kernel is checked on the exact inputs native sees. + """ + if not _shadow: + old.replace_all_uses_with(new) + return + graph = old.graph + order = {n: i for i, n in enumerate(graph.nodes)} + last = max(old, new, key=order.__getitem__) + with graph.inserting_after(last): + compare = graph.call_function( + torch.ops.sdm_inference.shadow_compare.default, + (old, new, _shadow[-1]), + ) + compare.meta["val"] = val(old) + old.replace_all_uses_with( + compare, + delete_user_cb=lambda u: ( + u is not compare and order.get(u, len(order)) > order[last] + ), + ) + + +class KernelPass(CustomGraphPass): + def __init__( + self, ops: frozenset[str] | None = None, shadow: bool = False + ) -> None: + self.rules: dict = {} + self.shadow = shadow + for target, op, rule in RULES: + if ops is None or op in ops: + self.rules.setdefault(target, []).append((op, rule)) + + def __call__(self, graph: Graph) -> None: + for node in list(graph.nodes): + if node.op == "call_function": + for op, rule in self.rules.get(node.target, ()): + if self.shadow: + _shadow.append(op) + try: + if rule(graph, node): + break + finally: + if self.shadow: + _shadow.pop() + graph.eliminate_dead_code() + + def uuid(self) -> None: + return None # never cache graphs compiled with this pass + + +def compile_options( + ops: frozenset[str] | None = None, shadow: bool = False +) -> dict: + """Inductor options adding the pass; ``ops`` limits it to those ops. + + ``shadow`` keeps native on the data path and compares each kernel output + with the native value it replaces (see :func:`replace`). + """ + return {"post_grad_custom_post_pass": KernelPass(ops, shadow)} diff --git a/benchmark/inference/kernels/integration.py b/benchmark/inference/kernels/integration.py new file mode 100644 index 0000000000..6ac59ca0e6 --- /dev/null +++ b/benchmark/inference/kernels/integration.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import functools +import os +from collections.abc import Callable +from pathlib import Path + +import torch + +from benchmark.inference.kernels import build, fusions, fx_pass, registry + + +def default_cache() -> Path: + root = os.environ.get("SDM_KERNEL_CACHE") + if root: + return Path(root) + version = f"{torch.__version__}-{torch.version.cuda}" + return Path.home() / ".cache" / "sdm-inference-kernels" / version + + +class Unsupported(RuntimeError): + """The kernels cannot run on this machine; the native arms still can.""" + + +NOT_BITWISE = frozenset({"icl_attention"}) + + +def install( + kernels: Path | None, + ops: tuple[str, ...] | None = None, + exact: bool = False, + shadow: bool = False, +) -> dict: + """Register the kernels, building missing ones; return Inductor options. + + ``kernels`` defaults to :func:`default_cache`. ``exact`` leaves the ops in + :data:`NOT_BITWISE` native. + """ + if torch.cuda.get_device_capability() != (12, 0): + raise Unsupported( + "the kernels need an SM 12.0 (Blackwell) GPU, found " + f"{torch.cuda.get_device_name()}" + ) + if not registry.registered(): + fusions.load(build.build(kernels or default_cache(), verbose=True)) + unknown = set(ops or ()) - {op for _, op, _ in fx_pass.RULES} + if unknown: + raise RuntimeError(f"no graph rule for ops {sorted(unknown)}") + if exact: + every = {op for _, op, _ in fx_pass.RULES} + ops = tuple(set(ops or every) - NOT_BITWISE) + return fx_pass.compile_options( + None if ops is None else frozenset(ops), shadow + ) + + +COMPILE = { + "fullgraph": False, + "dynamic": False, + "options": {"triton.cudagraphs": False}, +} + + +def isolate_compiles() -> None: + """Keep every ``torch.compile`` of a process on its own fresh graph.""" + torch._dynamo.config.cache_size_limit = 256 + torch._dynamo.config.accumulated_cache_size_limit = 4096 + torch._inductor.config.fx_graph_cache = False + torch._functorch.config.enable_autograd_cache = False + + +def compile_with_kernels( + fn: Callable, + kernels: Path | None = None, + ops: tuple[str, ...] | None = None, + exact: bool = False, + attention: str | None = None, + shadow: bool = False, +) -> Callable: + """``torch.compile(fn)`` with the kernels routed in and enabled. + + ``attention`` overrides ``$SDM_ICL_ATTENTION`` (see + :func:`registry.parse_attention`). ``shadow`` runs every kernel beside + native and records whether it reproduced native bit for bit + (:func:`fx_pass.replace`); the outputs stay native. + """ + if attention is not None: + registry.parse_attention(attention) + options = install(kernels, ops, exact, shadow) + compiled = torch.compile( + fn, **{**COMPILE, "options": {**COMPILE["options"], **options}} + ) + + @functools.wraps(fn) + def run(*args, **kwargs): + with registry.enabled(attention): + return compiled(*args, **kwargs) + + return run + + +def enable( + model: torch.nn.Module, + kernels: Path | None = None, + ops: tuple[str, ...] | None = None, + exact: bool = False, + attention: str | None = None, +) -> None: + """Compile every task model of an SDM ``model`` with the kernels. + + ``exact=True`` keeps ICL attention native, so the outputs stay bitwise + equal to compiled SDM. ``attention`` (default ``$SDM_ICL_ATTENTION``, + else ``sm120``) picks the ICL attention kernel, e.g. ``int8``. + Raises :class:`Unsupported` on a GPU the kernels do not support. Undo with + :func:`disable`. + """ + for inner in model.models.values(): + inner.forward = compile_with_kernels( + inner.forward, kernels, ops, exact, attention + ) + + +def disable(model: torch.nn.Module) -> None: + for inner in model.models.values(): + vars(inner).pop("forward", None) diff --git a/benchmark/inference/kernels/registry.py b/benchmark/inference/kernels/registry.py new file mode 100644 index 0000000000..2046926fe2 --- /dev/null +++ b/benchmark/inference/kernels/registry.py @@ -0,0 +1,239 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Named inference operations with optional kernels and native fallbacks. + +An :class:`Op` names an operation and its pure-PyTorch reference semantics. +A :class:`Kernel` implements one op for the inputs its ``supports`` admits. +Kernels are only selectable inside :func:`enabled`; everywhere else, and +whenever no kernel admits the inputs, the region stays native. +""" + +import contextlib +import os +from collections import Counter +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from typing import Any + +import torch +from torch import Tensor + +Output = Tensor | tuple[Tensor, ...] | None +# Ops read tables and views of the in-graph row table; Inductor must hand +# them the strides their fake implementations declare. +EXACT_STRIDES = (torch.Tag.needs_exact_strides,) + + +@dataclass(frozen=True) +class Op: + """A named operation. + + Args: + name: Unique op name. + reference: Pure-PyTorch implementation defining the semantics. + doc: One-line description. + """ + + name: str + reference: Callable[..., Output] + doc: str + + +@dataclass(frozen=True) +class Kernel: + """An implementation of an :class:`Op`. + + Args: + op: Name of the implemented op. + name: Unique kernel name. + supports: Takes the op arguments and returns ``None`` if the kernel + admits them, else a short reason. Inspects only metadata (shape, + dtype, stride, device, compute capability); never launches work + or synchronizes. + impl: Takes the op arguments and returns new tensors. It mutates + only the arguments its op declares in ``mutates_args``. + cuda_symbols: Substrings of the CUDA kernel names ``impl`` launches, + used to confirm in a profile that it ran. + bitwise: Takes the op arguments and returns whether they lie where + the kernel was checked bitwise equal to native; above that, + ``supports`` may still admit them within a measured tolerance. + exact: ``False`` for kernels that are lossy by design (opt-in 8-bit + attention); the benchmark then records the distance instead of + gating on it. + """ + + op: str + name: str + supports: Callable[..., str | None] + impl: Callable[..., Output] + cuda_symbols: tuple[str, ...] + bitwise: Callable[..., bool] = lambda *args, **kwargs: True + exact: bool = True + + +OPS: dict[str, Op] = {} +KERNELS: dict[str, list[Kernel]] = {} + +ATTENTION_ENV = "SDM_ICL_ATTENTION" +ATTENTION_BACKENDS = ("sm120", "fa2", "int8", "off") +ATTENTION_OPTIONS = ("ends",) +# Diagnostic only: lifts the BF16 kernel's checked row cap. +BF16_OPTIONS = ("uncapped",) + +_enabled = False +_attention: str | None = None +_decisions: list[dict] = [] +_launches: Counter[str] = Counter() +_shadow: dict[str, dict] = {} + + +def define_op(name: str, reference: Callable[..., Output], doc: str) -> Op: + if name in OPS: + raise ValueError(f"op {name!r} is already defined") + OPS[name] = op = Op(name=name, reference=reference, doc=doc) + return op + + +def register(kernel: Kernel) -> Kernel: + """Add ``kernel``; earlier registrations take precedence.""" + if kernel.op not in OPS: + raise ValueError( + f"kernel {kernel.name!r} implements unknown op {kernel.op!r}" + ) + if any(k.name == kernel.name for k in registered()): + raise ValueError(f"kernel {kernel.name!r} is already registered") + KERNELS.setdefault(kernel.op, []).append(kernel) + return kernel + + +def registered() -> list[Kernel]: + return [kernel for kernels in KERNELS.values() for kernel in kernels] + + +def parse_attention(setting: str) -> tuple[str, frozenset[str]]: + """Split ``[+option...]`` into the backend and its options.""" + backend, *options = setting.split("+") + if backend not in ATTENTION_BACKENDS: + raise ValueError( + f"ICL attention must be one of {ATTENTION_BACKENDS}, " + f"got {backend!r}" + ) + allowed = {"int8": ATTENTION_OPTIONS, "sm120": BF16_OPTIONS}.get( + backend, () + ) + if options and not allowed: + raise ValueError(f"ICL attention {backend!r} takes no options") + if set(options) - set(allowed) or len(set(options)) != len(options): + raise ValueError( + f"ICL attention {backend!r} options must be distinct and from " + f"{allowed}, got {options}" + ) + return backend, frozenset(options) + + +def attention() -> str: + """The ICL attention setting of :func:`enabled`, else the environment.""" + if _attention is not None: + return _attention + return os.environ.get(ATTENTION_ENV, "sm120") + + +@contextlib.contextmanager +def enabled(attention: str | None = None) -> Iterator[None]: + """Make registered kernels selectable within this context. + + ``attention`` overrides ``$SDM_ICL_ATTENTION`` (``[+option...]``, + see :func:`parse_attention`) for the ICL attention kernels. + """ + global _enabled, _attention + if attention is not None: + parse_attention(attention) + previous = _enabled, _attention + _enabled, _attention = True, attention + try: + yield + finally: + _enabled, _attention = previous + + +def _describe(value: Any) -> object: + if isinstance(value, Tensor): + return { + "shape": list(value.shape), + "dtype": str(value.dtype), + "stride": list(value.stride()), + "device": str(value.device), + } + return repr(value) + + +def select(op: str, *args: Any, **kwargs: Any) -> Kernel | None: + """The first registered kernel admitting the arguments, else ``None``. + + Inside :func:`enabled`, every decision is recorded in :func:`census`. + """ + if not _enabled: + return None + chosen, rejected = None, {} + for kernel in KERNELS.get(op, ()): + reason = kernel.supports(*args, **kwargs) + if reason is None: + chosen = kernel + break + rejected[kernel.name] = reason + _decisions.append( + { + "op": op, + "kernel": "native" if chosen is None else chosen.name, + "bitwise": chosen is None or chosen.bitwise(*args, **kwargs), + "exact": chosen is None or chosen.exact, + "rejected": rejected, + "args": [_describe(arg) for arg in args], + "kwargs": {key: _describe(value) for key, value in kwargs.items()}, + } + ) + return chosen + + +def kernel(name: str) -> Kernel: + return next(k for k in registered() if k.name == name) + + +def launch(kernel: Kernel, *args: Any, **kwargs: Any) -> Output: + _launches[kernel.name] += 1 + return kernel.impl(*args, **kwargs) + + +def record_shadow(op: str, native: Tensor, kernel: Tensor) -> None: + """Shadow mode: whether ``kernel`` reproduced ``native`` bit for bit.""" + entry = _shadow.setdefault( + op, {"outputs": 0, "bitwise": 0, "max_abs_diff": 0.0} + ) + entry["outputs"] += 1 + if native.shape != kernel.shape or native.dtype != kernel.dtype: + entry["max_abs_diff"] = float("inf") + return + same = torch.equal( + native.contiguous().view(torch.uint8).flatten(), + kernel.contiguous().view(torch.uint8).flatten(), + ) + entry["bitwise"] += same + if not same: + diff = (native.float() - kernel.float()).abs().nan_to_num(1e30) + entry["max_abs_diff"] = max(entry["max_abs_diff"], diff.max().item()) + + +def census() -> dict[str, Any]: + """Recorded selection decisions and per-kernel launch counts.""" + return { + "decisions": list(_decisions), + "launches": dict(_launches), + "shadow": {op: dict(e) for op, e in _shadow.items()}, + } + + +def reset_census() -> None: + _decisions.clear() + _launches.clear() + _shadow.clear() diff --git a/benchmark/inference/kernels/sm120.py b/benchmark/inference/kernels/sm120.py new file mode 100644 index 0000000000..c04d47c33d --- /dev/null +++ b/benchmark/inference/kernels/sm120.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Helpers shared by the SM120 (Blackwell) kernel registrations.""" + +import importlib.machinery +import importlib.util +from pathlib import Path +from types import ModuleType + +import torch +from torch import Tensor + +# Below this many rows cuBLAS can pick another algorithm, whose summation +# order the GEMM kernels do not reproduce, so small (query-side) calls stay +# native. +MIN_M = 1024 +EPS = 2.0**-23 # the RMS kernels hard-code finfo(float32).eps + + +def load_module(directory: Path, name: str) -> ModuleType: + path = directory / f"{name}.so" + if not path.is_file(): + raise FileNotFoundError(f"missing built kernel module {path}") + loader = importlib.machinery.ExtensionFileLoader(name, str(path)) + spec = importlib.util.spec_from_loader(name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +def policy(*tensors: Tensor) -> str | None: + if any(t.device.type != "cuda" for t in tensors): + return "needs CUDA tensors" + if torch.cuda.get_device_capability(tensors[0].device) != (12, 0): + return "needs SM 12.0" + if any(t.dtype != torch.bfloat16 for t in tensors): + return "needs bfloat16" + if any(not t.is_contiguous() or t.storage_offset() for t in tensors): + return "needs contiguous, offset-zero operands" + if torch.backends.cuda.matmul.fp32_precision == "tf32": + return "needs TF32 disabled" + if torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction: + return "needs reduced-precision BF16 reductions disabled" + return None diff --git a/benchmark/inference/report.py b/benchmark/inference/report.py new file mode 100644 index 0000000000..a5ee534866 --- /dev/null +++ b/benchmark/inference/report.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import csv +import hashlib +import importlib.metadata +import json +import os +import platform +import subprocess +from pathlib import Path + +import torch + +import sdm +from benchmark.inference.config import arm_attention + + +def _run(*command: str) -> str | None: + try: + return subprocess.run( + command, capture_output=True, text=True, check=True + ).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return None + + +def revision(path: Path) -> dict: + """Git revision of ``path``'s checkout, else a hash of its Python files.""" + head = _run("git", "-C", str(path), "rev-parse", "HEAD") + if head is None: + digest = hashlib.sha256() + for file in sorted(path.rglob("*.py")): + digest.update(file.relative_to(path).as_posix().encode()) + digest.update(file.read_bytes()) + return {"content_sha256": digest.hexdigest()} + status = _run("git", "-C", str(path), "status", "--porcelain", "--", ".") + return {"revision": head, "dirty": bool(status)} + + +def _version(package: str) -> str | None: + try: + return importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError: + return None + + +def environment(device: torch.device) -> dict: + properties = torch.cuda.get_device_properties(device) + driver = _run( + "nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader" + ) + return { + "sdm": revision(Path(sdm.__file__).parent), + "benchmark": revision(Path(__file__).parent), + "python": platform.python_version(), + "torch": torch.__version__, + "cuda": torch.version.cuda, + "triton": _version("triton"), + "gpu": { + "name": properties.name, + "capability": f"{properties.major}.{properties.minor}", + "memory_gib": round(properties.total_memory / 2**30, 1), + "driver": driver.splitlines()[0] if driver else "unknown", + }, + "cpu_count": os.cpu_count(), + "platform": platform.platform(), + } + + +def _ms(seconds: float) -> str: + return f"{1000 * seconds:.2f}" + + +def _gib(value: int) -> str: + return f"{value / 2**30:.2f}" + + +def _qualification( + arm: str, setting: str, entry: dict, scale: dict | None +) -> list[str]: + distance = entry["optimized_vs_compiled_distance"] + gate = entry["optimized_vs_compiled"] + if entry["gate"] == "opt-in": + lines = [ + f"**The {arm} arm is lossy: opt-in 8-bit ICL attention " + f"(`--attention {setting}`).** It is not bitwise equal to " + "compiled and is timed without a tolerance gate. Distance from " + f"compiled: max |dp| {distance['probability_max_abs_diff']:.3g}, " + f"mean |dp| {distance['probability_mean_abs_diff']:.3g}, argmax " + f"agreement {distance['argmax_agreement_pct']:.2f}%." + ] + if scale is not None: + lines.append( + "For scale, compiled vs eager: max |dp| " + f"{scale['probability_max_abs_diff']:.3g}, mean |dp| " + f"{scale['probability_mean_abs_diff']:.3g}, argmax agreement " + f"{scale['argmax_agreement_pct']:.2f}%." + ) + return lines + if entry["gate"] == "tolerance": + verdict = "passed" if entry["passed"] else "failed" + return [ + f"Tolerance gate {verdict} ({arm}): {arm} vs compiled max |dp| " + f"{distance['probability_max_abs_diff']:.3g} (allowed " + f"{scale['probability_max_abs_diff']:.3g}, the compiled vs eager " + f"distance), argmax agreement " + f"{distance['argmax_agreement_pct']:.2f}%." + ] + if not gate["bitwise_equal"]: + return [ + f"**WARNING: {arm} output is NOT bitwise equal to compiled** " + f"({gate['mismatched_leaves']} of {gate['leaves']} leaves " + f"differ; first: `{json.dumps(gate['first_mismatch'])}`)." + ] + return [f"{arm} vs compiled bitwise equal: True."] + + +def summary(report: dict) -> str: + config, env = report["config"], report.get("environment", {}) + gpu = env.get("gpu", {}) + lines = [ + "# KumoTabular fit + predict latency", + "", + f"Setup: {config['model_size']} model, {config['context_rows']} " + f"context rows, {config['query_rows']} query rows, " + f"{config['dataset']}, {gpu.get('name', '?')} " + f"(sm {gpu.get('capability', '?')}), torch {env.get('torch', '?')}, " + f"precision {config['precision']}, {config['blocks']} blocks.", + "", + f"Status: **{report['status']}**", + "", + ] + if "optimized_unavailable" in report: + lines += [ + "**Optimized arm skipped** (native arms only): " + f"{report['optimized_unavailable']}.", + "", + ] + if report["status"] in ("shadow_bitwise", "shadow_mismatch"): + lines += [ + "Shadow check (kernel vs native output on native's inputs):", + "", + "| arm | op | outputs | bitwise | max abs diff |", + "|---|---|---|---|---|", + ] + for arm, entry in report["kernels"]["arms"].items(): + for op, e in sorted(entry["shadow"].items()): + lines.append( + f"| {arm} | {op} | {e['outputs']} | {e['bitwise']} | " + f"{e['max_abs_diff']:.3g} |" + ) + return "\n".join(lines) + "\n" + qualification = report["qualification"] + scale = qualification.get("compiled_vs_eager") + for arm, entry in qualification.get("arms", {}).items(): + setting = arm_attention(arm, config["attention"]) + lines += [*_qualification(arm, setting, entry, scale), ""] + if scale is not None: + timed = "" if qualification.get("eager_timed", True) else " (untimed)" + lines += [ + f"Compiled vs eager{timed}: max |dp| " + f"{scale['probability_max_abs_diff']:.3g}, argmax agreement " + f"{scale['argmax_agreement_pct']:.2f}%.", + "", + ] + if "statistics" not in report: + lines.append( + "Not timed: --qualify-only." + if report["status"] == "qualified" + else "Not timed: qualification failed." + ) + return "\n".join(lines) + "\n" + + stats, peak = report["statistics"], report.get("peak_memory", {}) + lines += [ + "| arm | mean ms | median ms | min ms | stdev ms | samples | " + "peak alloc GiB | peak reserved GiB |", + "|---|---|---|---|---|---|---|---|", + ] + for arm, s in stats["arms"].items(): + memory = peak.get(arm) + lines.append( + f"| {arm} | {_ms(s['mean'])} | {_ms(s['median'])} | " + f"{_ms(s['min'])} | {_ms(s['stdev'])} | {s['samples']} | " + + ( + f"{_gib(memory['allocated_bytes'])} | " + f"{_gib(memory['reserved_bytes'])} |" + if memory + else "- | - |" + ) + ) + lines.append("") + for name, pair in stats["pairs"].items(): + lines.append( + f"- {name.replace('_to_', ' -> ')}: {pair['reduction_pct']:.2f}% " + f"lower mean latency; {pair['positive_blocks']}/{pair['blocks']} " + f"blocks faster; median block effect " + f"{_ms(pair['median_block_effect'])} ms" + ) + lines.append("") + for arm, entry in report["kernels"].get("arms", {}).items(): + if entry["verification"]: + lines.append( + f"Kernels verified in the {arm} profile: " + f"{entry['kernels_verified']}." + ) + for name, check in entry["verification"].items(): + verdict = "ran" if check["verified"] else "NOT verified" + lines.append(f"- {name}: {verdict}") + return "\n".join(lines) + "\n" + + +def write(output: Path, report: dict) -> None: + """Write report.json, samples.csv and summary.md to a new directory.""" + output.mkdir(parents=True) + (output / "report.json").write_text(json.dumps(report, indent=2)) + with (output / "samples.csv").open("w", newline="") as stream: + writer = csv.DictWriter( + stream, fieldnames=("block", "position", "arm", "seconds") + ) + writer.writeheader() + writer.writerows(report.get("samples", [])) + (output / "summary.md").write_text(summary(report)) diff --git a/benchmark/inference/runner.py b/benchmark/inference/runner.py new file mode 100644 index 0000000000..eabd065374 --- /dev/null +++ b/benchmark/inference/runner.py @@ -0,0 +1,503 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import gc +import sys +import time +from contextlib import ExitStack +from pathlib import Path +from typing import Protocol + +import torch +from torch import Tensor +from torch.autograd import DeviceType +from torch.profiler import ProfilerActivity, profile + +import sdm +from benchmark.inference import data +from benchmark.inference.config import Config +from benchmark.inference.kernels import integration, registry +from benchmark.inference.schedule import is_optimized, schedule, summarize +from benchmark.inference.snapshot import characterize, compare, leaves +from sdm.models._huggingface import download_checkpoint + +BOUNDARY = ( + "model.clear(), gc.collect() and a fresh seeded CUDA generator outside " + "the clock, Python GC disabled inside it; " + "clock = CUDA sync, fit(context, labels) + predict(query) under BF16 " + "autocast and inference mode on CUDA-resident tables, CUDA sync" +) + + +def set_precision(policy: str) -> None: + if policy == "ieee": + torch.backends.fp32_precision = "ieee" + torch.backends.cuda.matmul.fp32_precision = "ieee" + torch.backends.cudnn.fp32_precision = "ieee" + torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = ( + False + ) + + +def precision_flags() -> dict: + matmul = torch.backends.cuda.matmul + return { + "fp32_precision": torch.backends.fp32_precision, + "cuda.matmul.fp32_precision": matmul.fp32_precision, + "cudnn.fp32_precision": torch.backends.cudnn.fp32_precision, + "cuda.matmul.allow_bf16_reduced_precision_reduction": ( + matmul.allow_bf16_reduced_precision_reduction + ), + } + + +def checkpoint_path(config: Config) -> Path: + """The checkpoint file to load: ``--checkpoint`` or the pretrained one.""" + if config.checkpoint is not None: + return config.checkpoint + return Path( + download_checkpoint( + repo_id="nvidia/Kumo-Tabular", + filename=f"{config.model_size}/classifier.pt", + revision="v1.0.1", + config_filename="config.json", + ) + ) + + +def load_model(config: Config) -> sdm.models.KumoTabular: + kwargs = {"task": "classification", "size": config.model_size} + model = sdm.models.KumoTabular(**kwargs, pretrained=False, device="cpu") + path = checkpoint_path(config) + state = torch.load(path, map_location="cpu", weights_only=True) + try: + model.models[sdm.Task.classification].load_state_dict(state) + except RuntimeError as error: + raise SystemExit( + f"checkpoint {path} is not a {config.model_size} " + f"KumoTabular classifier:\n{error}" + ) from error + return model.to(config.device) + + +def kernel_counts(prof: profile) -> dict[str, dict]: + """CUDA kernel name -> launch count and total milliseconds.""" + kernels: dict[str, dict] = {} + for event in prof.events(): + if event.device_type == DeviceType.CUDA: + entry = kernels.setdefault(event.name, {"count": 0, "ms": 0.0}) + entry["count"] += 1 + entry["ms"] += event.time_range.elapsed_us() / 1000 + return kernels + + +def icl_attention_launches( + census: dict[str, int], kernels: dict[str, dict], expected: str | None +) -> dict[str, dict[str, int | None]]: + """ICL attention kernel -> registry and profiled CUDA launch counts. + + A kernel made of several CUDA kernels counts its least launched one. A + kernel sharing the expected kernel's CUDA kernels cannot be told apart + from it in a profile, so its profiled count is ``None``. + """ + attention = [k for k in registry.registered() if k.op == "icl_attention"] + expected_symbols = next( + (k.cuda_symbols for k in attention if k.name == expected), None + ) + launches = {} + for kernel in attention: + profiled = None + if kernel.name == expected or kernel.cuda_symbols != expected_symbols: + profiled = min( + sum( + e["count"] for name, e in kernels.items() if symbol in name + ) + for symbol in kernel.cuda_symbols + ) + launches[kernel.name] = { + "census": census.get(kernel.name, 0), + "profiled": profiled, + } + return launches + + +def verify_icl_attention( + launches: dict[str, dict[str, int]], expected: str | None, calls: int +) -> None: + """Raise unless only ``expected`` launched, ``calls`` times in both.""" + for name, counts in launches.items(): + want = calls if name == expected else 0 + if counts["census"] != want or counts["profiled"] not in (want, None): + raise RuntimeError( + f"ICL attention kernel {name} launched {counts}, " + f"expected {want} (kernel {expected or 'none'})" + ) + + +class Bench(Protocol): + unavailable: str | None + + def seconds(self, arm: str) -> float: ... + def snapshot(self, arm: str) -> dict[str, Tensor]: ... + def profile(self, arm: str) -> dict[str, dict]: ... + + +class CudaBench: + """One model and one set of resident tables shared by all arms.""" + + def __init__(self, config: Config) -> None: + if not torch.cuda.is_available(): + raise SystemExit( + "CUDA is not available; this benchmark times CUDA work" + ) + self.device = torch.device(config.device) + if self.device.type != "cuda": + raise SystemExit( + f"--device must be a CUDA device, got {config.device}" + ) + self.seed = config.seed + integration.isolate_compiles() + set_precision(config.precision) + self.model = load_model(config) + self.unavailable = None + self.x, self.y, self.query = data.tables(config) + self.inner = self.model.models[sdm.Task.classification] + self.icl_attention_calls = len(self.inner.icl_block.layers) - 1 + self.forwards = {} + if "compiled" in config.arms: + self.forwards["compiled"] = torch.compile( + self.inner.forward, **integration.COMPILE + ) + self.peak: dict[str, dict] = {} + for arm in filter(is_optimized, config.arms): + try: + self.forwards[arm] = integration.compile_with_kernels( + self.inner.forward, + config.kernels, + config.ops, + attention=config.arm_attention(arm), + shadow=config.shadow, + ) + except integration.Unsupported as error: + self.unavailable = str(error) + break + + def _run(self, arm: str) -> tuple[float, sdm.TableTensor]: + self.model.clear() + generator = torch.Generator(self.device).manual_seed(self.seed) + with ExitStack() as stack: + if arm != "eager": + self.inner.forward = self.forwards[arm] + stack.callback(vars(self.inner).pop, "forward") + stack.enter_context(torch.inference_mode()) + stack.enter_context(torch.autocast("cuda", dtype=torch.bfloat16)) + # Collect outside the clock and keep the collector out of it. + gc.collect() + gc.disable() + stack.callback(gc.enable) + torch.cuda.synchronize(self.device) + torch.cuda.reset_peak_memory_stats(self.device) + base = torch.cuda.memory_allocated(self.device) + start = time.perf_counter() + self.model.fit( + x=self.x, + y=self.y, + num_estimators=1, + generator=generator, + ) + result = self.model.predict(self.query) + torch.cuda.synchronize(self.device) + seconds = time.perf_counter() - start + memory = { + "base_bytes": base, + "allocated_bytes": torch.cuda.max_memory_allocated( + self.device + ), + "reserved_bytes": torch.cuda.max_memory_reserved(self.device), + } + peak = self.peak.setdefault(arm, dict.fromkeys(memory, 0)) + for key, value in memory.items(): + peak[key] = max(peak[key], value) + return seconds, result + + def seconds(self, arm: str) -> float: + return self._run(arm)[0] + + def snapshot(self, arm: str) -> dict[str, Tensor]: + _, result = self._run(arm) + out = leaves(self.model._cache, "cache") + out["probabilities"] = result.numerical.detach().cpu() + self.model.clear() + return out + + def profile(self, arm: str) -> dict[str, dict]: + activities = [ProfilerActivity.CPU, ProfilerActivity.CUDA] + with profile(activities=activities) as prof: + self._run(arm) + return kernel_counts(prof) + + +def _no_kernel_admitted(decisions: list[dict]) -> str: + if not decisions: + return ( + "no graph region matched any kernel rule; the compiled graph may " + "differ from the one the rules were written for" + ) + reasons = sorted( + { + f"{name}: {reason}" + for decision in decisions + for name, reason in decision["rejected"].items() + } + ) + return "no kernel admitted any region on this GPU and configuration" + ( + f" (rejections: {'; '.join(reasons)})" if reasons else "" + ) + + +def _drop( + arms: tuple[str, ...], dropped: list[str], reason: str +) -> tuple[str, ...]: + print( + f"WARNING: {', '.join(dropped)} skipped, {reason}. " + "Measuring the other arms only.", + file=sys.stderr, + ) + return tuple(arm for arm in arms if arm not in dropped) + + +def _matching(profile: dict[str, dict], symbols: tuple[str, ...]) -> list[str]: + return sorted(k for k in profile if any(s in k for s in symbols)) + + +def _kernel_report( + decisions: list[dict], + launches: dict[str, int], + profiles: dict[str, dict], + arm: str, +) -> dict: + kernels = {kernel.name: kernel for kernel in registry.registered()} + verification = {} + for name in sorted({d["kernel"] for d in decisions} - {"native"}): + symbols = kernels[name].cuda_symbols + optimized = _matching(profiles.get(arm, {}), symbols) + compiled = _matching(profiles.get("compiled", {}), symbols) + verification[name] = { + "optimized": optimized, + "compiled": compiled, + "verified": not compiled + and all(any(s in k for k in optimized) for s in symbols), + } + return { + "kernels_verified": bool(verification) + and all(v["verified"] for v in verification.values()), + "verification": verification, + "decisions": decisions, + "launches_per_optimized_call": launches, + } + + +def _gate(decisions: list[dict]) -> str: + """Gate an optimized arm must pass against compiled before timing. + + ``bitwise`` if every selected kernel is bitwise, ``tolerance`` (distance + to compiled within compiled vs eager) if some is not, ``opt-in`` (only + recorded) if some is lossy. + """ + if not all(d["exact"] for d in decisions): + return "opt-in" + if all(d["bitwise"] for d in decisions): + return "bitwise" + return "tolerance" + + +def _passes(entry: dict, scale: dict | None) -> bool: + if entry["gate"] == "opt-in": + return True + if entry["gate"] == "bitwise": + return entry["optimized_vs_compiled"]["bitwise_equal"] + distance = entry["optimized_vs_compiled_distance"] + return ( + distance["probability_max_abs_diff"] + <= scale["probability_max_abs_diff"] + and distance["argmax_agreement_pct"] == 100 + ) + + +def _verify_arms( + report: dict, + bench: Bench, + config: Config, + arms: tuple[str, ...], + decisions: dict[str, list[dict]], +) -> None: + calls = getattr(bench, "icl_attention_calls", None) + if calls is None or not any( + k.op == "icl_attention" for k in registry.registered() + ): + return + launches = {} + try: + for arm in arms: + chosen = { + d["kernel"] + for d in decisions.get(arm, ()) + if d["op"] == "icl_attention" + } - {"native"} + expected = None + if chosen: + (expected,) = chosen + expected_calls = calls + if is_optimized(arm): + _, options = registry.parse_attention( + config.arm_attention(arm) + ) + if "ends" in options: + expected_calls -= 2 + registry.reset_census() + profiled = bench.profile(arm) + launches[arm] = icl_attention_launches( + registry.census()["launches"], profiled, expected + ) + verify_icl_attention(launches[arm], expected, expected_calls) + except (RuntimeError, ValueError) as error: + report["status"] = "launch_verification_failed" + report["launch_verification_error"] = str(error) + report["kernels"]["icl_attention_launches"] = launches + + +def _single(per_arm: dict[str, dict], into: dict) -> dict: + """With one plain ``optimized`` arm, also keep its entries at the top.""" + if list(per_arm) == ["optimized"]: + into.update(per_arm["optimized"]) + return into + + +def measure(config: Config, bench: Bench) -> dict: + """Warm up, qualify, time and profile every arm of ``bench``.""" + report: dict = {"config": config.record(), "boundary": BOUNDARY} + arms = config.arms + if any(map(is_optimized, arms)) and bench.unavailable: + arms = _drop( + arms, [a for a in arms if is_optimized(a)], bench.unavailable + ) + report["optimized_unavailable"] = bench.unavailable + report["warmup"] = {} + decisions, shadow = {}, {} + for arm in arms: + registry.reset_census() + times = [bench.seconds(arm) for _ in range(config.warmup)] + report["warmup"][arm] = { + "first_call_seconds": times[0], + "seconds": times, + } + if is_optimized(arm): + decisions[arm] = registry.census()["decisions"] + shadow[arm] = registry.census()["shadow"] + native = [ + arm + for arm, made in decisions.items() + if all(d["kernel"] == "native" for d in made) + ] + if native: + reason = _no_kernel_admitted( + [d for arm in native for d in decisions[arm]] + ) + report["optimized_unavailable"] = reason + arms = _drop(arms, native, reason) + for arm in native: + del decisions[arm], shadow[arm] + optimized = [arm for arm in arms if is_optimized(arm)] + report["arms"] = list(arms) + if config.shadow: + exact = all( + e["bitwise"] == e["outputs"] + for entries in shadow.values() + for e in entries.values() + ) + report["status"] = "shadow_bitwise" if exact else "shadow_mismatch" + per = { + arm: {"decisions": decisions[arm], "shadow": shadow[arm]} + for arm in optimized + } + report["kernels"] = _single(per, {"arms": per}) + return report + + # Each snapshot is a fresh call; at most two are held at once. Eager is + # snapshotted for scale whenever a gate is not bitwise, timed or not. + gates = {arm: _gate(decisions[arm]) for arm in optimized} + qualification: dict = { + "gate": _gate([d for made in decisions.values() for d in made]), + "eager_timed": "eager" in arms, + "arms": {}, + } + if "compiled" in arms: + compiled = bench.snapshot("compiled") + if "eager" in arms or set(gates.values()) - {"bitwise"}: + qualification["compiled_vs_eager"] = characterize( + bench.snapshot("eager"), compiled + ) + for arm in optimized: + candidate = bench.snapshot(arm) + entry = { + "gate": gates[arm], + "optimized_vs_compiled": compare(compiled, candidate), + "optimized_vs_compiled_distance": characterize( + compiled, candidate + ), + } + del candidate + entry["passed"] = _passes( + entry, qualification.get("compiled_vs_eager") + ) + qualification["arms"][arm] = entry + del compiled + passed = all(e["passed"] for e in qualification["arms"].values()) + _single(qualification["arms"], qualification) + qualification["passed"] = passed + report["qualification"] = qualification + if config.qualify_only or (not passed and not config.allow_mismatch): + report["status"] = "qualified" if passed else "qualification_failed" + per = {arm: {"decisions": decisions[arm]} for arm in optimized} + report["kernels"] = _single(per, {"arms": per}) + if config.qualify_only: + _verify_arms(report, bench, config, arms, decisions) + return report + + # Warmup compiled every graph; a recompile inside the clock is an error. + peak = getattr(bench, "peak", {}) + peak.clear() + start = time.time() + with torch._dynamo.config.patch(error_on_recompile=True): + samples = [ + { + "block": block, + "position": position, + "arm": arm, + "seconds": bench.seconds(arm), + } + for block, order in enumerate(schedule(arms, config.blocks)) + for position, arm in enumerate(order) + ] + report["timing_window_unix"] = [start, time.time()] + report["peak_memory"] = {arm: dict(v) for arm, v in peak.items()} + report["samples"] = samples + report["statistics"] = summarize(samples, arms) + + profiles, launches = {}, {} + for arm in arms: + registry.reset_census() + profiles[arm] = bench.profile(arm) + launches[arm] = registry.census()["launches"] + per = { + arm: _kernel_report(decisions[arm], launches[arm], profiles, arm) + for arm in optimized + } + report["kernels"] = _single(per, {"arms": per, "profiles": profiles}) + if not passed: + report["status"] = "completed_with_mismatch" + elif "optimized_unavailable" in report: + report["status"] = "completed_without_optimized" + else: + report["status"] = "completed" + return report diff --git a/benchmark/inference/schedule.py b/benchmark/inference/schedule.py new file mode 100644 index 0000000000..def838c070 --- /dev/null +++ b/benchmark/inference/schedule.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import itertools +import random +import statistics + +SHUFFLE_SEED = 2026101507 + + +def is_optimized(arm: str) -> bool: + return arm == "optimized" or arm.startswith("optimized:") + + +def pairs(arms: tuple[str, ...]) -> list[tuple[str, str]]: + """(base, candidate) arm pairs compared in the statistics.""" + optimized = [arm for arm in arms if is_optimized(arm)] + bases = [base for base in ("compiled", "eager") if base in arms] + result = [(base, arm) for base in bases for arm in optimized] + if len(bases) == 2: + result.append(("eager", "compiled")) + return result + + +def _williams_square(n: int) -> list[tuple[int, ...]]: + """Latin square; for even ``n`` each index follows every other once.""" + first, low, high = [0], 1, n - 1 + while len(first) < n: + first.append(low) + low += 1 + if len(first) < n: + first.append(high) + high -= 1 + return [tuple((x + i) % n for x in first) for i in range(n)] + + +def orders(arms: tuple[str, ...], blocks: int) -> list[tuple[str, ...]]: + """Every permutation if ``blocks`` allows, else a Williams square.""" + every = list(itertools.permutations(arms)) + if blocks % len(every) == 0: + return every + if blocks % len(arms) == 0: + rows = _williams_square(len(arms)) + return [tuple(arms[i] for i in row) for row in rows] + raise ValueError( + f"blocks must be a multiple of {len(arms)} (or {len(every)} for " + "every order)" + ) + + +def schedule(arms: tuple[str, ...], blocks: int) -> list[tuple[str, ...]]: + """Call order per block: one of :func:`orders`, then its reverse. + + Every order is used equally often, shuffled with a fixed seed. + """ + rows = orders(arms, blocks) + rows = rows * (blocks // len(rows)) + random.Random(SHUFFLE_SEED).shuffle(rows) + return [row + row[::-1] for row in rows] + + +def summarize(samples: list[dict], arms: tuple[str, ...]) -> dict: + """Per-arm and paired statistics recomputed from every raw sample.""" + seconds = { + arm: [s["seconds"] for s in samples if s["arm"] == arm] for arm in arms + } + block_means: dict[tuple[int, str], float] = {} + for block in sorted({s["block"] for s in samples}): + for arm in arms: + block_means[block, arm] = statistics.mean( + s["seconds"] + for s in samples + if s["block"] == block and s["arm"] == arm + ) + blocks = sorted({block for block, _ in block_means}) + + result: dict = {"arms": {}, "pairs": {}} + for arm, values in seconds.items(): + result["arms"][arm] = { + "samples": len(values), + "mean": statistics.mean(values), + "median": statistics.median(values), + "min": min(values), + "stdev": statistics.stdev(values) if len(values) > 1 else 0.0, + } + for base, candidate in pairs(arms): + base_mean = result["arms"][base]["mean"] + effects = [ + block_means[block, base] - block_means[block, candidate] + for block in blocks + ] + result["pairs"][f"{base}_to_{candidate}"] = { + "reduction_pct": 100 + * (base_mean - result["arms"][candidate]["mean"]) + / base_mean, + "block_effects": effects, + "median_block_effect": statistics.median(effects), + "positive_blocks": sum(effect > 0 for effect in effects), + "blocks": len(effects), + } + return result diff --git a/benchmark/inference/snapshot.py b/benchmark/inference/snapshot.py new file mode 100644 index 0000000000..bec4cba8d1 --- /dev/null +++ b/benchmark/inference/snapshot.py @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import dataclasses +from collections.abc import Mapping + +import torch +from torch import Tensor + +_INT_OF_SIZE = {1: torch.uint8, 2: torch.int16, 4: torch.int32, 8: torch.int64} + + +def leaves(value: object, path: str = "") -> dict[str, Tensor]: + """CPU copies of tensors nested in mappings, sequences, dataclasses.""" + if isinstance(value, Tensor) and hasattr(value, "__tensor_flatten__"): + # Wrapper subclasses (e.g. SDM's StringTensor): compare their parts. + names, _ = value.__tensor_flatten__() + items = ((name, getattr(value, name)) for name in names) + return { + k: v + for name, item in items + for k, v in leaves(item, f"{path}/{name}").items() + } + if isinstance(value, Tensor): + return {path: value.detach().cpu()} + if isinstance(value, Mapping): + items = value.items() + elif isinstance(value, list | tuple): + items = enumerate(value) + elif dataclasses.is_dataclass(value) and not isinstance(value, type): + items = ( + (f.name, getattr(value, f.name)) for f in dataclasses.fields(value) + ) + else: + return {} + out: dict[str, Tensor] = {} + for key, item in items: + out.update(leaves(item, f"{path}/{key}")) + return out + + +def _max_abs_diff(a: Tensor, b: Tensor) -> float: + if a.numel() == 0: + return 0.0 + return (a.double() - b.double()).abs().max().item() + + +def _differing(a: Tensor, b: Tensor) -> int: + bits = _INT_OF_SIZE[a.element_size()] + return int((a.reshape(-1).view(bits) != b.reshape(-1).view(bits)).sum()) + + +def compare( + reference: dict[str, Tensor], candidate: dict[str, Tensor] +) -> dict: + """Bitwise comparison of two snapshots, leaf by leaf.""" + mismatches = [] + for path in [*reference, *(candidate.keys() - reference.keys())]: + a, b = reference.get(path), candidate.get(path) + if a is None or b is None: + side = "reference" if a is None else "candidate" + mismatches.append({"path": path, "reason": f"missing in {side}"}) + elif a.dtype != b.dtype or a.shape != b.shape: + mismatches.append( + { + "path": path, + "reason": f"{a.dtype}{list(a.shape)} vs " + f"{b.dtype}{list(b.shape)}", + } + ) + elif count := _differing(a, b): + mismatches.append( + { + "path": path, + "differing_elements": count, + "elements": a.numel(), + "max_abs_diff": _max_abs_diff(a, b), + } + ) + return { + "bitwise_equal": not mismatches, + "leaves": len(reference), + "mismatched_leaves": len(mismatches), + "first_mismatch": mismatches[0] if mismatches else None, + } + + +def characterize( + reference: dict[str, Tensor], + candidate: dict[str, Tensor], +) -> dict: + """Numerical distance of ``candidate`` from ``reference``.""" + p = reference["probabilities"].double() + q = candidate["probabilities"].double() + diff = (p - q).abs() + return { + "probability_max_abs_diff": diff.max().item(), + "probability_mean_abs_diff": diff.mean().item(), + "argmax_agreement_pct": 100 + * (p.argmax(-1) == q.argmax(-1)).double().mean().item(), + "leaf_max_abs_diff": { + path: _max_abs_diff(a, candidate[path]) + for path, a in reference.items() + if path in candidate and a.shape == candidate[path].shape + }, + } diff --git a/benchmark/inference/sweep.py b/benchmark/inference/sweep.py new file mode 100644 index 0000000000..273f66bcb5 --- /dev/null +++ b/benchmark/inference/sweep.py @@ -0,0 +1,358 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Speed sweep over model sizes and contexts, one benchmark run per cell. + + python -m benchmark.inference.sweep run OUT [--sizes ...] [--contexts ...] + python -m benchmark.inference.sweep table OUT + +``run`` skips cells that already have ``result.json``, so it resumes. A cell +that runs out of memory is retried once with expandable segments, then +recorded as OOM. ``table`` writes ``OUT/tables.md`` and ``OUT/table.csv``. +""" + +import argparse +import csv +import json +import math +import os +import statistics +import subprocess +import sys +import time +from datetime import datetime +from pathlib import Path + +BF16_MAX_ROWS = 65_535 +# Arm -> table column; a to d are the main comparison. +ARMS = { + "compiled": "a compiled", + "optimized:off": "b exact", + "optimized:sm120": "c +bf16 attn", + "optimized:int8": "d +int8 attn", + "optimized:int8+ends": "int8+ends", + "optimized:sm120+uncapped": "bf16 uncapped", + "eager": "eager", +} +SMI_FIELDS = ( + "timestamp,power.draw,power.limit,clocks.sm,clocks.mem," + "temperature.gpu,utilization.gpu,memory.used" +) +OOM_MARKERS = ("OutOfMemoryError", "CUDA out of memory") +EXPANDABLE = "expandable_segments:True" + + +def _arms(size: str, rows: int) -> list[str]: + arms = ["compiled", "optimized:off"] + if rows <= BF16_MAX_ROWS: + arms.append("optimized:sm120") + arms.append("optimized:int8") + if size == "large" and rows in (50_000, 100_000): + arms.append("optimized:int8+ends") + if size == "large" and rows == 100_000: + arms.append("optimized:sm120+uncapped") + if rows in (5_000, 50_000): + arms.append("eager") + return arms + + +def cells(sizes: list[str], contexts: list[int], repeat: bool) -> list[dict]: + out = [] + for size in sizes: + for rows in contexts: + name = f"{size}-{rows}" + arms = _arms(size, rows) + out.append( + {"name": name, "size": size, "rows": rows, "arms": arms} + ) + if size == "large" and repeat: + out += [ + {**c, "name": c["name"] + "-repeat"} + for c in out + if c["size"] == "large" and c["rows"] in (50_000, 100_000) + ] + return out + + +def blocks_for(arms: list[str], blocks: int) -> int: + """The smallest multiple of the arm count not below ``blocks``.""" + return math.ceil(blocks / len(arms)) * len(arms) + + +def _attempt( + cell: dict, args: argparse.Namespace, extra: list[str], expandable: bool +) -> dict: + root = args.out / cell["name"] + index = len(list(root.glob("attempt-*"))) + allocator = "expandable" if expandable else "default" + run = root / f"attempt-{index:02d}-{allocator}" + run.mkdir(parents=True) + env = dict(os.environ) + env.pop("PYTORCH_CUDA_ALLOC_CONF", None) + if expandable: + env["PYTORCH_CUDA_ALLOC_CONF"] = EXPANDABLE + command = [ + *(sys.executable, "-m", "benchmark.inference"), + *("--model-size", cell["size"]), + *("--context-rows", str(cell["rows"])), + *("--arms", *cell["arms"]), + *("--blocks", str(blocks_for(cell["arms"], args.blocks))), + *("--warmup", str(args.warmup)), + "--allow-mismatch", + *("--output", str(run / "run")), + *extra, + ] + smi_command = [ + *("nvidia-smi", f"--query-gpu={SMI_FIELDS}", "--format=csv,nounits"), + *("-lms", "500", "-i", args.gpu), + ] + start = time.time() + with ( + (run / "smi.csv").open("w") as smi_out, + (run / "log.txt").open("w") as log, + ): + smi = subprocess.Popen( + smi_command, stdout=smi_out, stderr=subprocess.DEVNULL + ) + try: + code = subprocess.run( + command, stdout=log, stderr=subprocess.STDOUT, env=env + ).returncode + finally: + smi.terminate() + smi.wait() + text = (run / "log.txt").read_text(errors="replace") + report = run / "run" / "report.json" + status = ( + json.loads(report.read_text())["status"] if report.exists() else None + ) + return { + "dir": run.name, + "allocator": env.get("PYTORCH_CUDA_ALLOC_CONF", "default"), + "returncode": code, + "seconds": time.time() - start, + "status": status, + "oom": any(marker in text for marker in OOM_MARKERS), + "command": command[1:], + } + + +def run(args: argparse.Namespace, extra: list[str]) -> None: + plan = cells(args.sizes, args.contexts, not args.no_repeat) + if args.only: + plan = [c for c in plan if c["name"] in args.only] + for cell in plan: + arms = cell["arms"] + blocks = blocks_for(arms, args.blocks) + listed = " ".join(arms) + print(f"{cell['name']}: {len(arms)} arms, {blocks} blocks: {listed}") + if args.dry_run: + return + for cell in plan: + result = args.out / cell["name"] / "result.json" + if result.exists(): + print(f"skip {cell['name']} (done)", flush=True) + continue + print(f"run {cell['name']} ...", flush=True) + attempts = [_attempt(cell, args, extra, expandable=False)] + if attempts[-1]["status"] is None and attempts[-1]["oom"]: + attempts.append(_attempt(cell, args, extra, expandable=True)) + last = attempts[-1] + if last["status"]: + outcome = "ok" + else: + outcome = "oom" if last["oom"] else "failed" + record = {**cell, "outcome": outcome, "attempts": attempts} + result.write_text(json.dumps(record, indent=2)) + line = ( + f"{cell['name']}: {outcome} ({last['status']}, " + f"exit {last['returncode']}, {last['seconds']:.0f} s)" + ) + print(line, flush=True) + if outcome != "ok" or last["status"] != "completed": + now = datetime.now().isoformat(timespec="seconds") + with (args.out / "failures.log").open("a") as log: + log.write(f"{now} {line} [{last['dir']}]\n") + + +def _smi(path: Path, window: list[float]) -> dict: + """Median power, power limit and SM clock inside the timing window.""" + rows = [] + with path.open() as stream: + for row in csv.reader(stream): + try: + at = datetime.strptime(row[0].strip(), "%Y/%m/%d %H:%M:%S.%f") + power, limit, clock = (float(value) for value in row[1:4]) + except (ValueError, IndexError): + continue + if window[0] <= at.timestamp() <= window[1]: + rows.append((power, limit, clock)) + if not rows: + return {} + power, limit, clock = zip(*rows) + return { + "power_w": statistics.median(power), + "power_limit_w": statistics.median(limit), + "sm_mhz": statistics.median(clock), + } + + +def _load(out: Path) -> list[dict]: + loaded = [] + for result in sorted(out.glob("*/result.json")): + cell = json.loads(result.read_text()) + run = result.parent / cell["attempts"][-1]["dir"] + report = run / "run" / "report.json" + cell["report"] = ( + json.loads(report.read_text()) if report.exists() else None + ) + window = (cell["report"] or {}).get("timing_window_unix") + cell["smi"] = _smi(run / "smi.csv", window) if window else {} + loaded.append(cell) + return loaded + + +def _row(cell: dict, arm: str) -> dict: + report = cell["report"] + entry = { + "cell": cell["name"], + "size": cell["size"], + "rows": cell["rows"], + "arm": arm, + "allocator": cell["attempts"][-1]["allocator"], + } + if report is None: + return {**entry, "outcome": cell["outcome"]} + if "statistics" not in report: + return {**entry, "outcome": report["status"]} + stats = report["statistics"]["arms"] + if arm not in stats: + return {**entry, "outcome": "not timed"} + median = stats[arm]["median"] + base = stats.get("compiled", {}).get("median") + peak = report.get("peak_memory", {}).get(arm, {}) + gate = report["qualification"].get("arms", {}).get(arm, {}) + distance = gate.get("optimized_vs_compiled_distance", {}) + return { + **entry, + "outcome": "ok" if gate.get("passed", True) else "gate failed", + "gate": gate.get("gate", ""), + "median_ms": 1000 * median, + "pct_vs_compiled": None + if base is None + else 100 * (median - base) / base, + "speedup_vs_compiled": None if base is None else base / median, + "peak_alloc_gib": peak.get("allocated_bytes", 0) / 2**30, + "peak_reserved_gib": peak.get("reserved_bytes", 0) / 2**30, + "max_dp_vs_compiled": distance.get("probability_max_abs_diff"), + **cell["smi"], + } + + +def _table_cell(cell: dict, arm: str) -> str: + if arm not in cell["arms"]: + capped = arm == "optimized:sm120" and cell["rows"] > BF16_MAX_ROWS + return "= b (cap)" if capped else "" + row = _row(cell, arm) + if "median_ms" not in row: + return row["outcome"].upper() + pct = "" + if row["pct_vs_compiled"] is not None and arm != "compiled": + pct = f" ({row['pct_vs_compiled']:+.1f}%)" + flag = " GATE FAIL" if row["outcome"] == "gate failed" else "" + return f"{row['median_ms']:.0f}{pct} {row['peak_alloc_gib']:.1f}G{flag}" + + +def table(out: Path) -> None: + loaded = _load(out) + rows = [_row(c, arm) for c in loaded for arm in ARMS if arm in c["arms"]] + keys = list(dict.fromkeys(key for row in rows for key in row)) + with (out / "table.csv").open("w", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=keys) + writer.writeheader() + writer.writerows(rows) + lines = [ + "Cells: median ms, % vs compiled (negative is faster), " + "peak allocated GiB.", + "", + ] + for size in ("large", "medium", "small"): + sized = [c for c in loaded if c["size"] == size] + if not sized: + continue + header = ["context", *ARMS.values(), "power W", "SM MHz"] + lines += [ + f"## {size}", + "", + "| " + " | ".join(header) + " |", + "|---" * len(header) + "|", + ] + for cell in sorted(sized, key=lambda c: (c["rows"], c["name"])): + label = f"{cell['rows'] // 1000}k" + if cell["name"].endswith("-repeat"): + label += " (repeat)" + if cell["attempts"][-1]["allocator"] != "default": + label += " (expandable)" + smi = cell["smi"] + parts = [ + label, + *(_table_cell(cell, arm) for arm in ARMS), + f"{smi.get('power_w', 0):.0f}", + f"{smi.get('sm_mhz', 0):.0f}", + ] + lines.append("| " + " | ".join(parts) + " |") + lines.append("") + (out / "tables.md").write_text("\n".join(lines)) + print("\n".join(lines)) + + +def main(argv: list[str] | None = None) -> None: + argv = sys.argv[1:] if argv is None else argv + extra = [] + if "--" in argv: + split = argv.index("--") + argv, extra = argv[:split], argv[split + 1 :] + parser = argparse.ArgumentParser( + prog="python -m benchmark.inference.sweep" + ) + commands = parser.add_subparsers(dest="command", required=True) + run_parser = commands.add_parser( + "run", + help="run missing cells; arguments after -- go to every benchmark run", + ) + run_parser.add_argument("out", type=Path) + run_parser.add_argument( + "--sizes", nargs="+", default=["large", "medium", "small"] + ) + run_parser.add_argument( + "--contexts", + nargs="+", + type=int, + default=[5_000, 25_000, 50_000, 75_000, 100_000], + ) + run_parser.add_argument( + "--only", nargs="+", help="cell names, e.g. large-50000" + ) + run_parser.add_argument( + "--blocks", + type=int, + default=12, + help="rounded up to a multiple of each cell's arm count", + ) + run_parser.add_argument("--warmup", type=int, default=3) + run_parser.add_argument("--no-repeat", action="store_true") + run_parser.add_argument( + "--gpu", + default=os.environ.get("CUDA_VISIBLE_DEVICES", "0").split(",")[0], + ) + run_parser.add_argument("--dry-run", action="store_true") + commands.add_parser("table").add_argument("out", type=Path) + args = parser.parse_args(argv) + if args.command == "run": + args.out.mkdir(parents=True, exist_ok=True) + run(args, extra) + table(args.out) + + +if __name__ == "__main__": + main() diff --git a/docs/inference-kernel-interface.md b/docs/inference-kernel-interface.md new file mode 100644 index 0000000000..6cbb7d86b6 --- /dev/null +++ b/docs/inference-kernel-interface.md @@ -0,0 +1,109 @@ +# Inference kernel interface + +How a kernel plugs into the KumoTabular fit + predict benchmark (`benchmark/inference/`). What each kernel does is in [`inference-kernels.md`](inference-kernels.md). + +## Three pieces + +1. **Op**: an operation with pure-PyTorch reference semantics, declared with `registry.define_op`, for example `linear_gelu(x, weight, bias) = gelu(linear(x, weight, bias))` with exact (erf) GELU. Each op is also a `torch.library` custom op `sdm_inference::` whose last argument is the name of the selected kernel. +2. **Kernel** (`registry.Kernel`): one implementation of one op. + - `op`, `name`: the op it implements and a unique name, e.g. `sm120_linear_gelu_k256_n512`. + - `supports(*args) -> None | str`: inspects tensor metadata only (shape, dtype, stride, storage offset, device, compute capability) and global policy (precision flags). Returns `None` to admit, else a short reason. Never launches work or synchronizes. It cannot check inference mode, because compilation runs outside it; the benchmark runs every call under `torch.inference_mode()`. + - `impl(*args) -> Tensor | tuple[Tensor, ...] | None`: allocates and returns new outputs. It mutates no argument. + - `cuda_symbols`: substrings of the CUDA kernel names `impl` launches, so a profile can show it ran. + - `bitwise(*args) -> bool` (optional, default always true): whether the args lie where the kernel was checked bitwise against native. `supports` may admit sizes above that, up to a hard limit, where the kernel stays within a measured tolerance; such runs use the benchmark's tolerance gate. + - `exact` (default `True`): `False` marks a lossy kernel that is selectable only through an opt-in setting (the 8-bit ICL attention); the benchmark then records its distance instead of gating on it. +3. **Graph rule**, run by the Inductor post-grad pass in `benchmark/inference/kernels/fx_pass.py`, which is installed only in the optimized arm's `torch.compile` options. A rule matches its op's pre-fusion ATen form, calls `registry.select(op, *fake_args)` on the compile-time metadata, and rewrites the region to `sdm_inference::(..., kernel=name)` only if a kernel admits it. Unmatched or rejected regions stay native and Inductor compiles them as usual. SDM source is not modified. The fallback is chosen at compile time, before any launch, and is recorded in the census with the rejection reasons. + +All three pieces of a kernel family live in one file, `benchmark/inference/kernels/fusions/.py`: the op's reference and `define_op`, its custom op and fake implementation, its rule (`fx_pass.add_rule`) and `register(directory)`, which loads the family's built CUDA modules from `directory` and registers its kernels. + +## The row table + +SDM's row embedding keeps a table `[C, F+4, D]` (4 readout tokens plus `F` cell features per context row) and updates it in place. The whole forward is one compiled graph, so after functionalization the table is an in-graph value: `torch.empty`, then one new value per update (`slice_scatter` into `t[:, 4:]` for the column blocks, `t + x` for the row residuals). `column_projection_rms` finds the table this way (`_table_value` and `_table_rows` in `fusions/projection_rms.py`); it also accepts the table as a flat graph input, which is how the unit tests build it. + +No op writes the table. Inductor decides per graph which values it stores: native SDM never stores the table after a column block, its norms read Inductor's private copies, and the row residual recomputes the table in FP32. A custom op that takes the table makes Inductor store it as BF16, and later readers load the rounded value, so the model is no longer bitwise end to end even though each kernel is (see [No row-table ops](inference-kernels.md#how-the-kernels-get-into-the-model)). A new kernel that reads an in-graph value with other native readers must keep native's storage decisions; the bitwise end-to-end gate checks this. + +## Contract a kernel must meet + +- **Exact against compiled native.** (The exceptions are `icl_attention`, below, and the lossy opt-in kernels.) On every input `supports` admits, the output must be bitwise equal to what clean SDM compiled with Inductor produces for that region, under the benchmark settings: BF16 autocast, inference mode, TF32 off, reduced-precision BF16 reductions off. Compiled native keeps FP32 between fused pointwise ops and rounds once at each stored buffer, so match that, not eager SDM (which rounds after every op). `--shadow` checks this on the real model: every kernel runs beside native on native's inputs and each output is compared bit for bit. +- **Honest `supports`.** Reject anything the kernel was not qualified for (shape, layout, M range, GPU arch). The benchmark gate checks whole-model outputs, so an over-broad `supports` shows up as a mismatch, not a crash. +- **No hidden state.** No caching across calls, no CUDA graphs, no stream changes; launch on the current stream. +- **Exact strides.** Custom ops are declared with `tags=registry.EXACT_STRIDES` (`torch.Tag.needs_exact_strides`). Otherwise Inductor picks its own layout for in-graph buffers, for example a feature-major table, and hands it to the kernel. + +## Adding a kernel + +```python +from benchmark.inference.kernels import registry, sm120 + +def register(directory): + module = sm120.load_module(directory, "my_family_sm120") + registry.register(registry.Kernel(op="linear_gelu", name="sm120_my_kernel", + supports=supports, impl=impl, cuda_symbols=("my_namespace",))) +``` + +`integration.install` builds the CUDA modules with `build.build` and passes the output directory to `fusions.load`, which imports every module in `fusions/` in name order and calls its `register`. Earlier registrations win. `sm120.py` holds the shared helpers: `load_module`, `policy` (CUDA, SM 12.0, BF16, contiguous, TF32 and reduced-precision reductions off), `MIN_M` and `EPS`. + +A new op needs a reference in `registry.define_op`, a custom op with a fake implementation, and a rule registered with `fx_pass.add_rule(target, op, rule)`; rules for one target are tried in order until one returns `True`. `fx_pass.triton_kernel_name(node)` names the user Triton kernel behind a `triton_kernel_wrapper_functional` node, e.g. SDM's `_rmsnorm_cast_kernel`; `fusions/kv_rms.py` has `unweighted_rms_launch` for SDM's unweighted per-head norm. + +CUDA sources live in `benchmark/inference/kernels/csrc//`, one directory per built module, listed in `build.MODULES` with their flags; headers shared by several modules are in `csrc/common/`, and `csrc/.clang-format` sets the style. The optimized arm builds missing or stale modules on first use (`--kernels DIR`, default `$SDM_KERNEL_CACHE` or `~/.cache/sdm-inference-kernels/-`); `python -m benchmark.inference.kernels.build --out DIR` builds ahead of time. `--ops a b ...` restricts the optimized arm to some ops, for cumulative measurements. + +## What the benchmark reports + +- `decisions`: every compile-time selection (op, kernel or `native`, rejection reasons, shapes). +- `launches_per_optimized_call`: runtime launch count per kernel. +- `verification`: each selected kernel's `cuda_symbols` present in the optimized profile and absent from the compiled one. +- Qualification: optimized vs compiled bitwise on the probabilities and every tensor reachable through the fitted cache's dicts, lists and dataclasses, before any timing. The preprocessing recipe object in the cache runs outside the compiled model and is not compared. + +## Ops + +Every op is a `sdm_inference::` custom op with a reference in `registry.OPS`; the docstring of its file in `benchmark/inference/kernels/fusions/` gives its exact semantics. CUDA modules are under `benchmark/inference/kernels/csrc/`. + +| Op | Python | CUDA | What it fuses | +| ------------------------------------ | --------------------------------------------------- | ---------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `linear_gelu` | `linear_gelu.py`; first gating layer in `gating.py` | `linear_gelu_sm120/linear_gelu.cu`, `gating_sm120/gating.cu` | linear + exact GELU (embedding MLPs, first gating layer, dataset FFN) | +| `gated_log_scale` | `gating.py` | `gating_sm120/gating.cu` | second gating linear + tanh + log-scale query multiply | +| `rope_rms`, `rope_rms_uneven` | `rope_rms.py` | `rope_rms_sm120/rope_rms.cu` | split-half RoPE + per-head Q/K RMSNorm; `rope_rms_uneven` is the last row layer (4 readout queries), with Q and K of different lengths | +| `linear_kv_key_rms` | `kv_rms.py` | `kv_rms_sm120/kv_rms.cu` | KV projection + per-head key RMSNorm | +| `linear_query_rms` | `query_rms.py` | `query_rms_sm120/query_rms.cu` | query projection + per-head RMSNorm | +| `column_projection_rms` | `projection_rms.py` | `projection_rms_sm120/projection_rms.cu` | column output projection + residual + RMSNorm | +| `icl_query_rms_scale`, `icl_key_rms` | `icl_qk.py` | `icl_qk_sm120/icl_qk.cu` | ICL query per-head RMSNorm + LogScale multiply; ICL key RMSNorm read in place from the packed KV projection, plus contiguous K/V cache heads (the last fit layer's KV projection narrowed to the cached heads) | +| `icl_attention` | `icl_attention.py` | `icl_attention_sm120/icl_attention.cu`, `icl_attention_int8_sm120/icl_attention_int8.cu` | not a fusion and **not bitwise**: ICL fit self-attention in a custom SM120 BF16 kernel, FlashAttention 2, or the lossy opt-in INT8 QK / FP8 PV kernel; see the setting below | + +## ICL attention setting + +`$SDM_ICL_ATTENTION`, `--attention`, or `kernels.enable(model, attention=...)` (which overrides the environment through `registry.enabled(attention=...)`) picks the `icl_attention` kernel: + +| Setting | Kernel | Gate | +| ----------------- | ----------------------------------------------------------- | ----------------------------------- | +| `sm120` (default) | `sm120_icl_attention_d64`, BF16, at most 65,535 rows | tolerance | +| `sm120+uncapped` | the same kernel without the row cap (diagnostic) | tolerance | +| `fa2` | `fa2_icl_attention_d64` (needs `flash_attn` for SM 12.0) | tolerance | +| `int8` | `sm120_int8_icl`: INT8 Q·Kᵀ with K smoothing, FP8 P·V | `opt-in` (distance recorded, timed) | +| `int8+ends` | as `int8`, first and last ICL fit call of each graph native | `opt-in` | +| `off` | none | bitwise (with the exact ops) | + +The rule matches the flash node of a self-attention with head dim 64 and at least 1,024 rows (fit; predict has 16 queries against the cache). The BF16 kernels admit at most 65,535 rows, the largest context at which their error and determinism were checked; the INT8 kernel has no row cap but needs batch 1 and 32-bit element offsets. `exact=True` keeps attention native whatever the setting. + +## Shape bounds and scope + +Exactness against cuBLAS depends on the algorithm cuBLAS picks per shape. Each kernel's `supports` admits rows only up to the largest size that was checked bitwise. `sm120.MIN_M = 1024` is the lower bound of the GEMM kernels; smaller (query-side) calls stay native, and some kernels have per-shape lower bounds. + +SDM chunks attention and cell embedding under `torch.compile` (a memory budget of `SDM_CHUNK_MEMORY_FRACTION`, default 5% of the GPU, per chunk). The row blocks run on chunks of about 7,200 contexts (Large and Medium; about 14,400 for Small; 57 table rows) and the column blocks on chunks of a few features once the context is above about 7,500 rows, so no upper bound of an exact kernel binds at 100k contexts. The ICL block is not chunked: its ops see M = C. `icl_key_rms` narrows the last fit layer's KV projection only for M in 513..65,535 (checked bitwise); above that it runs the full projection. Rules match the row-chunked row blocks (`rope_rms`, gating, the GEMM epilogues). Feature-chunked column blocks read a feature range of the table, which `column_projection_rms` does not take, so those regions stay native at large contexts. Re-check exactness before widening any bound; `--shadow` does that on the real model. + +## Testing a kernel + +The kernel tests are in `test/benchmark/inference/` (`test_fusions.py`, `test_icl_attention.py`, `test_icl_qk.py`). `conftest.py` provides the fixture `matches_native(fn, make_args, kernel)`: on an SM 12.0 GPU it builds and registers every kernel, compiles a small function reproducing the SDM pattern twice (plain, and with the kernel pass), and asserts that `kernel` was selected and launched and that every output and every input is bitwise equal. Off SM 12.0 the tests are skipped. For a kernel that is not bitwise, `close_to_native(fn, make_args, kernel, atol)` asserts the same within a maximum absolute difference. `stays_native(fn, make_args, op)` asserts that the rule ran but chose no kernel, `skips_rule` that the rule left the function alone, and `rejects(kernel, *args)` that `supports` refuses the arguments. + +On the real model, `python -m benchmark.inference --shadow ...` (also `fx_pass.compile_options(shadow=True)`) keeps native on the data path and routes each rewritten output through `sdm_inference::shadow_compare` (in `fx_pass.py`), which records per op how many outputs were bitwise equal (`report.json`, `kernels.shadow`). + +## Using the kernels in SDM code + +```python +from benchmark.inference import kernels + +kernels.enable(model) # bitwise ops + BF16 ICL attention +kernels.enable(model, exact=True) # only ops bitwise equal to compiled SDM +kernels.enable(model, attention="int8") # opt-in 8-bit ICL attention +kernels.disable(model) +``` + +`kernels.compile_with_kernels(fn)` does the same for one function; the benchmark's optimized arm uses it. diff --git a/docs/inference-kernel-results.md b/docs/inference-kernel-results.md new file mode 100644 index 0000000000..456a0c9481 --- /dev/null +++ b/docs/inference-kernel-results.md @@ -0,0 +1,235 @@ +# Inference kernel results + +Speed and accuracy of the kernels described in [`inference-kernels.md`](inference-kernels.md), measured on SDM `main` at 890521bfa with torch 2.14.1+cu130. Three kernel sets are compared: + +- the **exact fused kernels**, bitwise equal to compiled SDM end to end; +- the **BF16 SM120 ICL attention** kernel (default `--attention sm120`; close to compiled but not bitwise); +- the **opt-in 8-bit ICL attention** (`--attention int8`; lossy). + +Setup for every speed number: one RTX PRO 4500 Blackwell (SM 12.0, 165 W power cap), BF16 autocast, fresh-context `fit` + `predict` with 16 query rows on Covertype, `torch.compile(dynamic=False)`. The benchmark is described in [`benchmark/inference/README.md`](../benchmark/inference/README.md). + +| Arm | What runs | Flag | +| ------------- | ------------------------------------------------------------------ | ------------------------------------------------------------------- | +| a | compiled SDM | `compiled` | +| b | a + exact kernels, attention native | `optimized:off` | +| c | b + BF16 ICL attention | `optimized:sm120` (capped at 65,535 rows, so c = b at 75k and 100k) | +| d | b + 8-bit ICL attention | `optimized:int8` | +| int8+ends | b + 8-bit attention, first and last ICL call of each graph in BF16 | `optimized:int8+ends` (Large 50k and 100k) | +| bf16 uncapped | c without the row cap (diagnostic) | `optimized:sm120+uncapped` (Large 100k) | +| eager | eager SDM | `eager` (5k and 50k) | + +All arms of a cell run interleaved in one process: 12 timing blocks, or 15 for cells with 5 arms, in a balanced (Williams square) order, after 3 untimed warmup calls per arm. Large 50k and 100k were run a second time unchanged. + +Large, against compiled: exact kernels -11.1% / -8.4% / -6.0% / -4.6% / -3.7% at 5k / 25k / 50k / 75k / 100k; with BF16 attention -11.9% / -12.4% / -11.3% up to 50k; with 8-bit attention -14.4% / -23.6% / -28.4% / -31.2% / -33.1%. The exact kernels also lower peak allocated memory on Large from 25k up and on Medium from 75k up (Large 100k: 15.5 to 9.6 GiB). + +## Speed + +Each cell: median ms, change vs compiled (negative is faster), peak allocated GiB. Power and SM clock are medians over the timed window. + +### Large + +| Context | a compiled | b exact | c +bf16 attn | d +int8 attn | int8+ends | bf16 uncapped | eager | Power W | SM MHz | +| ----------- | ------------ | ------------------- | ------------------- | ------------------- | ------------------- | ------------------- | ------------------ | ------- | ------ | +| 5k | 207, 2.1 | 184 (-11.1%), 2.2 | 183 (-11.9%), 2.2 | 177 (-14.4%), 2.2 | | | 230 (+10.8%), 2.2 | 70 | 2,400 | +| 25k | 1,264, 4.5 | 1,157 (-8.4%), 4.1 | 1,108 (-12.4%), 4.1 | 965 (-23.6%), 4.1 | | | | 113 | 2,126 | +| 50k | 3,521, 8.1 | 3,310 (-6.0%), 5.5 | 3,122 (-11.3%), 5.5 | 2,520 (-28.4%), 5.5 | 2,588 (-26.5%), 5.5 | | 3,572 (+1.5%), 4.1 | 165 | 1,792 | +| 50k repeat | 3,536, 8.1 | 3,325 (-6.0%), 5.5 | 3,134 (-11.4%), 5.5 | 2,530 (-28.5%), 5.5 | 2,597 (-26.5%), 5.5 | | 3,587 (+1.4%), 4.1 | 165 | 1,785 | +| 75k | 6,824, 11.7 | 6,508 (-4.6%), 7.5 | = b (cap) | 4,695 (-31.2%), 7.5 | | | | 165 | 1,770 | +| 100k | 11,131, 15.5 | 10,721 (-3.7%), 9.6 | = b (cap) | 7,445 (-33.1%), 9.6 | 7,725 (-30.6%), 9.6 | 9,935 (-10.7%), 9.6 | | 165 | 1,762 | +| 100k repeat | 11,102, 15.5 | 10,688 (-3.7%), 9.6 | = b (cap) | 7,425 (-33.1%), 9.6 | 7,703 (-30.6%), 9.6 | 9,910 (-10.7%), 9.6 | | 165 | 1,770 | + +### Medium + +| Context | a compiled | b exact | c +bf16 attn | d +int8 attn | eager | Power W | SM MHz | +| ------- | ----------- | ------------------ | ------------------- | ------------------- | ------------------ | ------- | ------ | +| 5k | 184, 1.5 | 162 (-12.1%), 1.7 | 161 (-12.7%), 1.7 | 158 (-14.1%), 1.7 | 201 (+9.4%), 1.7 | 60 | 2,407 | +| 25k | 955, 3.5 | 856 (-10.4%), 3.5 | 830 (-13.1%), 3.5 | 756 (-20.8%), 3.5 | | 114 | 2,070 | +| 50k | 2,443, 5.0 | 2,243 (-8.2%), 5.0 | 2,145 (-12.2%), 5.0 | 1,850 (-24.3%), 5.0 | 2,398 (-1.9%), 3.5 | 147 | 1,815 | +| 75k | 4,477, 7.8 | 4,175 (-6.7%), 6.9 | = b (cap) | 3,275 (-26.8%), 6.9 | | 165 | 1,792 | +| 100k | 7,067, 12.1 | 6,670 (-5.6%), 9.2 | = b (cap) | 5,046 (-28.6%), 9.2 | | 165 | 1,770 | + +### Small + +| Context | a compiled | b exact | c +bf16 attn | d +int8 attn | eager | Power W | SM MHz | +| ------- | ---------- | ------------------ | ----------------- | ------------------- | ----------------- | ------- | ------ | +| 5k | 89, 0.8 | 82 (-8.0%), 0.9 | 82 (-8.5%), 0.9 | 80 (-9.7%), 0.9 | 100 (+11.8%), 1.0 | 59 | 2,377 | +| 25k | 370, 1.9 | 332 (-10.1%), 1.9 | 320 (-13.4%), 1.9 | 286 (-22.8%), 1.9 | | 103 | 2,257 | +| 50k | 933, 3.2 | 861 (-7.7%), 3.2 | 814 (-12.8%), 3.2 | 675 (-27.7%), 3.2 | 963 (+3.2%), 3.0 | 126 | 2,014 | +| 75k | 1,739, 3.6 | 1,630 (-6.3%), 3.6 | = b (cap) | 1,204 (-30.8%), 3.6 | | 134 | 1,856 | +| 100k | 2,767, 5.0 | 2,620 (-5.3%), 4.9 | = b (cap) | 1,863 (-32.7%), 4.9 | | 162 | 1,815 | + +No cell ran out of memory. Every b run was bitwise equal to compiled (probabilities and every fitted-cache tensor), and every c run passed the tolerance gate with 100% class agreement. The profiler found every kernel of each arm in each cell. + +The card is at its 165 W cap for Large from 50k up, Medium from 75k up and Small at 100k (about 1.8 GHz). Smaller cells run below the cap at higher clocks (up to 2.4 GHz), so their percentages are not power-bound. At large contexts the fixed-cost savings of the exact kernels shrink relative to ICL attention, which grows with the square of the context; the 8-bit attention gains more as the context grows. + +### Speed vs accuracy + +Max |Δprob| against compiled on the 16 timed query rows of each cell, with compiled vs eager on the same rows as the reference gap (eager is run untimed for the comparison where it is not timed). Repeats give identical distances. + +| Model | Context | compiled vs eager | c BF16 | d int8 | int8+ends | BF16 uncapped | +| ------ | ------- | ----------------- | ------ | ------ | --------- | ------------- | +| Large | 5k | 0.0333 | 0.0058 | 0.0135 | | | +| Large | 25k | 0.0224 | 0.0032 | 0.0119 | | | +| Large | 50k | 0.1063 | 0.0110 | 0.0338 | 0.0057 | | +| Large | 75k | 0.0117 | | 0.0059 | | | +| Large | 100k | 0.0075 | | 0.0430 | 0.0148 | 0.0132 | +| Medium | 5k | 0.0198 | 0.0090 | 0.0125 | | | +| Medium | 25k | 0.0368 | 0.0056 | 0.0088 | | | +| Medium | 50k | 0.0445 | 0.0017 | 0.0322 | | | +| Medium | 75k | 0.0047 | | 0.0044 | | | +| Medium | 100k | 0.0095 | | 0.0039 | | | +| Small | 5k | 0.0430 | 0.0074 | 0.0124 | | | +| Small | 25k | 0.0732 | 0.0019 | 0.0071 | | | +| Small | 50k | 0.0277 | 0.0092 | 0.0126 | | | +| Small | 75k | 0.0134 | | 0.0031 | | | +| Small | 100k | 0.0929 | | 0.0497 | | | + +Class agreement was 100% for every arm in every cell. The 8-bit attention stays inside the gap except at Large 100k, where int8 is 5.7x and int8+ends 2x over a small gap; the 16-row gap varies a lot between cells, so the accuracy study below is the better guide. + +## Effect of SDM #1055 on the compiled baseline + +SDM #1055 makes attention and cell embedding chunk under `torch.compile` with a memory budget per chunk. Compiled Large 50k, compiled arm only, 2 processes x 6 blocks each: + +| Build | Median ms | Peak allocated / reserved GiB | +| --------------------------------------------------------------- | --------- | ----------------------------- | +| (i) 890521bfa, default | 3,468 | 8.13 / 11.26 | +| (ii) 890521bfa, chunking off (`SDM_CHUNK_MEMORY_FRACTION=1000`) | 3,358 | 13.00 / 20.97 | +| (iii) 842c408fe on torch 2.14.1 | 3,315 | 10.47 / 17.37 | +| (iv) 890521bfa with only #1055 reverted | 3,353 | 13.00 / 20.97 | + +(ii) was checked to run as one chunk: its graph has 41 SDPA nodes against 149 with chunking. 842c408fe measured 3,313 ms on torch 2.13, so the torch version accounts for almost nothing. (iv) matches (ii), so #1055 alone costs about 110 ms (+3.2%) and saves 4.9 GiB allocated (-37%) and 9.7 GiB reserved. A fresh repeat of (i) after the sweep gave 3,460 ms. The remaining ~40 ms (1.1%) between (iv) and (iii) is other SDM changes. The first compiled call takes 165 s against 55 s on 842c408fe, with an FX graph of about 18,000 nodes against 5,000. Compiled Large 100k runs (11.1 s, 15.5 GiB allocated); on 842c408fe it fails, because flash attention rejects more than 65,535 batch entries. + +## Correctness + +### Exact kernels + +- Kernel tests: 240 passed, 2 skipped (FlashAttention 2 not installed); 274 passed, 2 skipped with the row-table ops included. + +- Bitwise end to end, default exact set (`--attention off`), every fitted-cache tensor and the probabilities (64 tensors; 36 for Small), Small, Medium and Large at 5k, 50k and 100k: all equal. Nine ops match at 5k and eight from 50k up (`column_projection_rms` drops out because columns are feature-chunked there). + + | Model | Sites at 5k | 50k | 100k | + | ------ | ----------- | --- | ---- | + | Small | 93 | 205 | 301 | + | Medium | 153 | 513 | 921 | + | Large | 176 | 536 | 944 | + +- `--shadow` (native stays on the data path, each kernel output compared bit for bit with the value it replaces): every op exact. + +- SDM tests: `test/nn/test_attention.py` 16 passed; `FULL_TEST=1 test/nn/test_compile.py` 31 passed. Nothing under `sdm/` is changed. + +### ICL attention accuracy + +Large, 50,000 context rows (47,818 for adult, the whole table), 16 and 1,024 query rows, seeds 0 to 2. Each setting is compiled SDM plus only the attention op. Per cell: worst max |Δprob| over seeds vs compiled, class flips summed over the 3 seeds (48 or 3,072 rows), change in log-loss (mean over seeds). + +| Dataset | Query rows | compiled vs eager | c BF16 | d int8 | int8+ends | +| --------- | ---------- | ------------------- | ------------------ | ------------------ | ------------------ | +| adult | 16 | 0.0186, 0, +0.0017 | 0.0098, 0, -0.0001 | 0.0122, 0, -0.0002 | 0.0054, 0, -0.0001 | +| adult | 1,024 | 0.1372, 19, -0.0002 | 0.0045, 0, +0.0000 | 0.0094, 0, -0.0000 | 0.0103, 0, -0.0000 | +| covertype | 16 | 0.1125, 1, -0.0090 | 0.0137, 1, +0.0007 | 0.0338, 1, +0.0006 | 0.0234, 1, -0.0022 | +| covertype | 1,024 | 0.1761, 19, -0.0001 | 0.0155, 1, -0.0001 | 0.0915, 9, +0.0002 | 0.0407, 2, +0.0002 | +| higgs | 16 | 0.0203, 1, +0.0005 | 0.0029, 0, -0.0001 | 0.0071, 0, +0.0004 | 0.0050, 0, -0.0006 | +| higgs | 1,024 | 0.0814, 37, +0.0008 | 0.0063, 2, +0.0000 | 0.0100, 7, +0.0002 | 0.0092, 4, +0.0001 | +| volkert | 16 | 0.0885, 0, +0.0108 | 0.0091, 0, +0.0013 | 0.0203, 0, +0.0062 | 0.0098, 0, +0.0052 | +| volkert | 1,024 | 0.4468, 51, +0.0009 | 0.0199, 3, -0.0000 | 0.0566, 7, +0.0001 | 0.0500, 6, +0.0003 | + +- **BF16 (c)** is inside the compiled-vs-eager gap in every run. At 16 query rows it flips one row: a covertype near-tie that eager flips too. At 1,024 rows it flips 0 to 3 of 3,072 rows (eager: 19 to 51). Log-loss moves by at most ±0.0013. +- **int8 (d)** is over the per-seed gap in 1 of 3 covertype 16-row seeds (0.0275 against 0.0098). At 1,024 rows it flips 0 to 0.29% of rows (eager 0.6 to 1.7%). Log-loss moves by -0.0002 to +0.0062. +- **int8+ends** is over the gap in 1 of 3 covertype 16-row seeds, flips fewer rows than int8, and moves log-loss by -0.0022 to +0.0052. + +Two more classification tables, same layout. bank_marketing uses 44,187 context rows and letter 18,976, the whole table in each case. On letter the 8-bit settings stay native (see below). + +| Dataset | Query rows | compiled vs eager | c BF16 | d int8 | int8+ends | +| -------------- | ---------- | ------------------ | ------------------ | ------------------ | ------------------ | +| bank_marketing | 16 | 0.0248, 0, -0.0023 | 0.0037, 0, +0.0000 | 0.0062, 0, -0.0006 | 0.0035, 0, -0.0001 | +| bank_marketing | 1,024 | 0.0397, 6, -0.0001 | 0.0064, 0, +0.0000 | 0.0063, 2, -0.0000 | 0.0060, 0, -0.0000 | +| letter | 16 | 0.0154, 0, +0.0010 | 0.0014, 0, -0.0001 | native | native | +| letter | 1,024 | 0.0675, 0, +0.0001 | 0.0104, 0, +0.0000 | native | native | + +Small and Medium, 50,000 context rows, seed 0 only, so the flips are out of 16 or 1,024 rows: + +| Model | Dataset | Query rows | compiled vs eager | c BF16 | d int8 | int8+ends | +| ------ | --------- | ---------- | ------------------- | ------------------ | ------------------ | ------------------ | +| Medium | covertype | 16 | 0.0445, 0, -0.0009 | 0.0017, 0, -0.0001 | 0.0322, 0, -0.0017 | 0.0139, 0, -0.0022 | +| Medium | covertype | 1,024 | 0.1317, 3, -0.0014 | 0.0216, 2, +0.0002 | 0.0534, 1, +0.0005 | 0.0337, 3, +0.0004 | +| Medium | volkert | 16 | 0.0644, 0, -0.0694 | 0.0079, 0, +0.0074 | 0.0137, 0, +0.0048 | 0.0115, 0, -0.0007 | +| Medium | volkert | 1,024 | 0.1054, 13, -0.0017 | 0.0328, 1, +0.0003 | 0.0505, 6, +0.0005 | 0.0290, 4, +0.0009 | +| Small | covertype | 16 | 0.0277, 0, -0.0004 | 0.0092, 0, -0.0017 | 0.0126, 0, -0.0004 | 0.0123, 0, -0.0004 | +| Small | covertype | 1,024 | 0.2246, 11, -0.0002 | 0.0234, 2, +0.0004 | 0.0546, 4, +0.0002 | 0.0267, 1, +0.0001 | +| Small | volkert | 16 | 0.0521, 0, +0.0554 | 0.0061, 0, +0.0019 | 0.0256, 0, -0.0061 | 0.0114, 0, -0.0007 | +| Small | volkert | 1,024 | 0.1952, 23, +0.0025 | 0.0225, 1, +0.0000 | 0.0396, 5, +0.0003 | 0.0292, 2, -0.0004 | + +- Every setting is inside the gap in every classification run, except the two covertype 16-row seeds noted above. +- Flips at 1,024 rows: BF16 flips 0 to 3 per cell, always fewer than eager. int8 flips 0 to 9 and int8+ends 0 to 6; the one cell where int8+ends equals eager is Medium covertype (3 against 3). At 16 rows no setting flips any row except the covertype near-tie. +- Log-loss changes by at most ±0.001 at 1,024 rows for every setting. At 16 rows the change is noisier, up to ±0.0074 (BF16 on Medium volkert, one seed, where eager moves -0.069). + +Regression, Large, 6 TabArena tables (context: the table minus 1,024, at most 50,000), seeds 0 to 2. The model predicts 999 quantiles. Distances are |Δ| / the context target's standard deviation, against compiled. Per cell: worst over seeds of the max over all quantiles, worst point-prediction (quantile mean) |Δ|, and the change in RMSE (mean over seeds). + +| Table | Context | Query rows | compiled vs eager | c BF16 | d int8 | int8+ends | +| ---------------------- | ------- | ---------- | --------------------- | --------------------- | --------------------- | --------------------- | +| diamonds | 50,000 | 16 | 0.125, 0.0129, -1.13% | 0.063, 0.0032, +0.05% | 0.047, 0.0040, +0.39% | 0.063, 0.0043, +0.86% | +| diamonds | 50,000 | 1,024 | 0.445, 0.0582, -0.55% | 0.305, 0.0171, -0.07% | 0.535, 0.0189, +0.04% | 0.367, 0.0181, +0.03% | +| food_delivery_time | 44,427 | 16 | 0.814, 0.1828, +0.85% | 0.125, 0.0085, +0.02% | 0.178, 0.0105, -0.01% | 0.172, 0.0051, -0.01% | +| food_delivery_time | 44,427 | 1,024 | 1.352, 0.5197, +0.06% | 0.266, 0.0291, +0.00% | 0.453, 0.0552, +0.00% | 0.328, 0.0466, -0.00% | +| houses | 19,616 | 16 | 0.375, 0.0837, -0.62% | 0.141, 0.0116, -0.40% | 0.219, 0.0300, -0.07% | 0.281, 0.0187, -0.35% | +| houses | 19,616 | 1,024 | 2.555, 0.2360, +0.25% | 0.637, 0.0270, -0.03% | 1.008, 0.0572, -0.01% | 0.773, 0.0491, -0.05% | +| physiochemical_protein | 44,706 | 16 | 1.248, 0.1921, +2.18% | 0.319, 0.0128, -0.13% | 0.383, 0.0442, +0.73% | 0.474, 0.0617, +0.37% | +| physiochemical_protein | 44,706 | 1,024 | 3.594, 0.4222, +0.42% | 1.098, 0.0465, -0.02% | 1.973, 0.1214, -0.05% | 1.701, 0.0979, -0.05% | +| qsar_tid_11 | 4,718 | 16 | 0.324, 0.0667, +0.24% | 0.141, 0.0076, -0.08% | 0.281, 0.0352, +0.03% | 0.195, 0.0164, +0.68% | +| qsar_tid_11 | 4,718 | 1,024 | 1.250, 0.1792, -0.20% | 0.531, 0.0243, +0.01% | 1.969, 0.0845, -0.01% | 0.965, 0.0408, -0.00% | +| superconductivity | 20,239 | 16 | 0.281, 0.0502, +0.91% | 0.148, 0.0079, -0.16% | 0.234, 0.0230, +0.50% | 0.367, 0.0143, +0.12% | +| superconductivity | 20,239 | 1,024 | 1.445, 0.2830, +0.67% | 0.703, 0.0563, -0.08% | 1.313, 0.1271, -0.23% | 0.875, 0.1293, -0.03% | + +- **BF16 (c):** the max over all quantiles is inside the same seed's eager gap in 36 of 36 runs, and so is the point prediction. Within each band of quantiles (5% tails, 45 to 55% median) it passes eager's band max in 2 of 108 cases, both in the high tail (diamonds 16 rows: 0.063 against 0.031; qsar_tid_11 1,024 rows: 0.453 against 0.441). RMSE moves at most 1.1% in any seed (eager: up to 7.0%). +- **int8 (d):** the point prediction is inside the gap in 36 of 36 runs; the max over all quantiles in 31 of 36. The misses are on diamonds, qsar_tid_11 and superconductivity, in the tail quantiles (12 of 108 band cases). RMSE moves at most 2.4% in a seed. +- **int8+ends:** point prediction 36 of 36; all quantiles 33 of 36; 10 of 108 band cases over; RMSE at most 3.4% in a seed (diamonds, 16 rows). +- **letter:** with 26 classes ECOC runs the ICL attention as a batch of 8. The 8-bit kernel takes only batch 1 and refuses it ("needs batch 1 (the ICL fit site); got 8"), so attention stays native and d and int8+ends equal compiled. BF16 takes batch 8 and is inside the gap. + +### Determinism + +Large 50k, 10 runs each: compiled, BF16, int8 and int8+ends are each bitwise identical from run to run. + +### Why the BF16 cap stays at 65,535 rows + +Uncapped BF16 at Large 100k is 10.7% faster than compiled, but its max |Δprob| is 0.0132 against a gap of 0.0075. Medium 100k uncapped missed the gate narrowly (0.00952 against 0.00948). The kernel's error and determinism were checked up to 65,535 rows; raising the cap needs the accuracy study at those sizes. + +## Coverage at these sizes + +- `column_projection_rms` stays native from 50k up, because a feature chunk reads a range of the table it does not take (passing the range would recover about 0.4%). `linear_query_rms` stays native at Small 100k, whose M falls in the band where cuBLAS reduces differently. `icl_key_rms` narrows the last layer's KV projection only up to 65,535 rows. +- The eight row-table ops (see [`inference-kernels.md`](inference-kernels.md#how-the-kernels-get-into-the-model)) are excluded. Each matched native bit for bit on native's inputs, but with them on top of b, Large 25k ran 1,158 ms (b: 1,157) and Large 100k the same as b, while max |Δprob| at Large 100k was 0.144 against a gap of 0.0075. + +## Reproduce + +From the repository root on an SM 12.0 GPU, with a CUDA 13.0 toolkit for the kernel build: + +```bash +# environment as in .github/workflows/test-gpu.yml (resolves torch 2.14.1+cu130) +uv pip install --torch-backend=cu130 --extra-index-url=https://pypi.nvidia.com \ + --find-links=https://data.pyg.org/whl/torch-2.14.0+cu130.html \ + --index-strategy=unsafe-best-match --group=test . scikit-learn pandas ninja + +# speed sweep (resumable; --dry-run lists cells, arms and blocks) +python -m benchmark.inference.sweep run OUT --dry-run +python -m benchmark.inference.sweep run OUT +python -m benchmark.inference.sweep table OUT # OUT/tables.md, OUT/table.csv + +# one cell +python -m benchmark.inference --model-size large --context-rows 50000 \ + --arms compiled optimized:off optimized:sm120 optimized:int8 eager --blocks 15 --output runs/large-50k + +# bitwise and shadow checks of the exact set +python -m benchmark.inference --model-size large --context-rows 50000 --attention off \ + --arms compiled optimized --qualify-only --output runs/bit-large-50k +python -m benchmark.inference --model-size large --context-rows 50000 --attention off \ + --arms compiled optimized --shadow --output runs/shadow-large-50k + +# attention accuracy and determinism +python -m benchmark.attention.compare_data --out DATA +python -m benchmark.attention.compare run --data-dir DATA --out RES --settings sm120 int8 int8+ends \ + --contexts 50000 --queries 16 1024 --seeds 0 1 2 +python -m benchmark.attention.compare tabulate --results RES --output RES/compare.md +python -m benchmark.attention.determinism --out det-int8.json --size large --rows 50000 --attention int8 + +# #1055 attribution: chunking off +SDM_CHUNK_MEMORY_FRACTION=1000 python -m benchmark.inference --model-size large --context-rows 50000 \ + --arms compiled --output runs/nochunk +``` diff --git a/docs/inference-kernels.md b/docs/inference-kernels.md new file mode 100644 index 0000000000..5a0ba103a5 --- /dev/null +++ b/docs/inference-kernels.md @@ -0,0 +1,195 @@ +# Inference kernels for KumoTabular + +SM120 (Blackwell) CUDA kernels that speed up fresh-context `fit` + `predict` of the KumoTabular classifier. Kernel authors should read [`inference-kernel-interface.md`](inference-kernel-interface.md). The benchmark is described in [`benchmark/inference/README.md`](../benchmark/inference/README.md), and the latest measurements are in [`inference-kernel-results.md`](inference-kernel-results.md). + +## How the kernels get into the model + +SDM source is not changed. The model's `forward` is compiled with `torch.compile`, and one extra Inductor pass (`benchmark/inference/kernels/fx_pass.py`) runs on the post-grad graph, the ATen graph before Inductor fuses anything. + +1. Each kernel family has a rule that matches the ATen pattern compiled SDM produces for one computation, for example an `addmm` followed by the decomposed erf-GELU. +2. The rule asks the registry whether a kernel admits the operands' shapes, dtypes, strides and device. Only compile-time metadata is used, so the choice is made before anything runs. +3. If a kernel admits them, the region is replaced by one custom op (`sdm_inference::`) that calls that kernel. Otherwise the graph is left alone and Inductor compiles the native code. + +Every kernel except ICL attention reproduces compiled native SDM bit for bit on its inputs: the same rounding points, the same GEMM reduction order as the cuBLAS kernel it replaces, and the same RMS arithmetic as SDM's Triton `rmsnorm_cast`. `--shadow` checks each kernel on the real model against the native value it replaces. + +Whether the whole model then stays bitwise equal to compiled SDM depends on the op: + +- **Bitwise end to end:** `linear_gelu`, `gated_log_scale`, `rope_rms`, `rope_rms_uneven`, `linear_kv_key_rms`, `linear_query_rms`, `column_projection_rms`, `icl_query_rms_scale`, `icl_key_rms`. They are on by default and the benchmark gates them bitwise. +- **ICL attention** (`icl_attention`): the BF16 kernel is close to compiled SDM but not bitwise; see [Default on or opt-in](#default-on-or-opt-in). The 8-bit kernel is lossy and opt-in. `SDM_ICL_ATTENTION=off` or `kernels.enable(model, exact=True)` keeps attention native. + +**No row-table ops.** SDM's row embedding updates a table `[C, F+4, D]` in place. Eight ops that fused the residual adds, norms and down-projections reading or writing this table (`strided_rmsnorm`, `column_residual_rms`, `dual_rmsnorm`, `row_residual_rms`, `linear_residual_dual_rmsnorm`, `row_down_residual`, `row_projection_rms`, `linear_residual_kv_query_rms`) are not included. Each matched native bit for bit on native's inputs, but on SDM 890521bfa, where the row embedding is one compiled graph, the model was no longer bitwise end to end, and Large at 25k and 100k ran no faster than with the exact set. They moved the probabilities by a max |Δprob| of 0.144 at Large 100k, against 0.0075 for compiled vs eager. The cause is which values Inductor stores. Native code never stores the table after a column block (`(a + t) + c`): the norms read Inductor's private copies, and the row residual recomputes the table in FP32. A custom op that takes the table makes Inductor store it as BF16, so the residual adds the rounded table (in predict, the reverse). + +The time saved comes from two sources: + +- **Fewer round trips to memory.** A GEMM followed by a pointwise op (GELU, residual add, RMSNorm) normally writes a large intermediate tensor and reads it back. A fused epilogue never stores it. +- **Fewer temporaries.** Some native regions copy a strided view into a contiguous buffer before a norm. The fused kernel reads the strided view directly. + +Notation: + +- `C` is context rows and `F` is cell features after preprocessing (Covertype: `F = 53`). Each table row also has 4 readout tokens, so it has `F + 4` entries. +- `D` is the embedding width: 256 for Large and Medium, 128 for Small. +- `M` is the row count a GEMM or norm sees, for example `F * C` in the column stage. + +## Families + +Each family has one Python file in `benchmark/inference/kernels/fusions/` and its CUDA code in `benchmark/inference/kernels/csrc/`; the [op table](inference-kernel-interface.md#ops) lists them. The contribution figures were measured on SDM 842c408fe with torch 2.13.0, together with the row-table ops (see [Measured on SDM 842c408fe](#measured-on-sdm-842c408fe)); they have not been re-measured on 890521bfa. + +### `linear_gelu`: linear + exact GELU + +- **Replaces.** Every `Linear` followed by `GELU`: the embedding MLP up-projections (column, row and inducing blocks), the first layer of the row block's gating MLP, and the dataset (ICL) feed-forward. Native compiled code is a cuBLAS `addmm` producing a BF16 `[M, N]` tensor, then a Triton kernel that reads it, computes `0.5·x·(1 + erf(x/√2))` in FP32 and writes BF16. + +- **Fuses.** GELU becomes the GEMM's epilogue, so the BF16 pre-activation is never stored or re-read. + +- **Kernels.** `sm120_linear_gelu_k_n`. The embedding MLP and dataset FFN kernels are in `linear_gelu_sm120/linear_gelu.cu`; the first gating layer runs the first-layer kernel of `gating_sm120/gating.cu`. + + | Shape (K→N) | Use | Models | + | ----------- | ------------------ | ------------- | + | 256→512 | embedding MLPs | Large, Medium | + | 64→64 | first gating layer | Large, Medium | + | 1024→2048 | dataset FFN | Large only | + | 128→256 | embedding MLPs | Small | + | 32→64 | first gating layer | Small | + +- **Exactness.** CUTLASS GEMMs with split-K 1 reproduce cuBLAS's reduction order, the bias is rounded once, and the GELU arithmetic is transcribed from Triton's libdevice. Below M = 1024 cuBLAS can choose a different reduction (it does for 256→512 at M = 64), so smaller query-side calls stay native. + +- **Limits.** 1024 ≤ M ≤ the largest M checked bitwise: 8.39M for 256→512, 40M for 64→64, 500k for 1024→2048 (the dataset FFN sees M = C, at most 65,535). The Small kernels admit up to 33.0M (128→256) and 132M (32→64). + +- **Contribution (Large 50k).** 2.1% (71 ms) on its own, the first row of the cumulative table below. + +### `gated_log_scale`: second gating layer + query scaling + +- **Replaces.** `GatedLogScale` in the row block's attention: a 64→head linear, then `q · log(key_len) · head_scale · (1 + tanh(z))`. Native code is an `addmm` plus one Inductor pointwise kernel. +- **Fuses.** The tanh gate and the query multiply become the GEMM epilogue (`sm120_gated_log_scale`). +- **Shapes.** Head 64 (Large, Medium) or head 32 (Small). +- **Exactness.** Bitwise equal to cuBLAS's WMMA kernel plus Inductor's pointwise kernel. +- **Limits.** 1024 ≤ M ≤ 80.9M (head 32) or 49.6M (head 64). +- **Contribution (Large 50k).** +0.7% (20 ms). + +### `rope_rms`: RoPE + per-head RMSNorm for Q and K + +- **Replaces.** The row block's `_RoPERMSNorm` on queries and keys. Compiled SDM applies rotary embedding as an Inductor pointwise kernel, then runs `rmsnorm_cast` per head: two passes for Q and two for K. +- **Fuses.** One launch rotates and normalises both Q and K; the rule pairs a layer's Q and K regions. +- **Shapes.** 4 heads of 64 (Large, Medium) or 32 (Small). +- **Exactness.** FP32 sin/cos with a single rounding, as compiled native. +- **Limits.** At most 1024 positions per row, and C·S ≤ 20.2M tokens (head 32) or 8.39M (head 64). +- **Contribution (Large 50k).** +1.3% (42 ms). + +### `rope_rms_uneven`: RoPE + RMSNorm in the last row layer + +- **Replaces.** RoPE and the per-head norms of the last row layer, whose queries are only the 4 readout tokens. Q has 4 positions and K has F + 4, so the `rope_rms` pairing does not apply. +- **Fuses.** Rotates and normalises Q and K of different lengths in one launch. Same module as `rope_rms`. +- **Shapes and limits.** Those of `rope_rms`, on the longer of Q and K. + +### `linear_kv_key_rms`: KV projection + key RMSNorm + +- **Replaces.** The column cross-attention's KV projection. Native code is an `addmm` into packed `[M, 2D]`, a split, a contiguous copy of K, and `rmsnorm_cast` on it. +- **Fuses.** The key norm runs in the GEMM epilogue, and raw K is never stored. V keeps its native packed layout. +- **Shapes.** D = 256 with head 64, or D = 128 with head 32. +- **Limits.** M ≤ 19.9M (D=128) or 12.0M (D=256). At D = 256, M must be at least 129, because cuBLAS sums smaller M in another order. +- **Contribution (Large 50k).** +1.2% (41 ms). + +### `linear_query_rms`: query projection + query RMSNorm + +- **Replaces.** The column cross-attention's query projection plus its per-head norm. +- **Fuses.** The norm runs in the epilogue; the raw query is never stored. +- **Limits.** M ≤ 33.0M (D=128) or 20.6M (D=256). A band of small M where cuBLAS reduces differently stays native. +- **Contribution (Large 50k).** +0.6% (20 ms). + +### `column_projection_rms`: attention output projection + residual + RMSNorm + +- **Replaces.** The column block's attention output projection, the residual add of `table[:, 4:]`, and the following norm. +- **Fuses.** All three run in one GEMM epilogue. The raw projection is still returned where later code reads it. +- **Limits.** M ≤ 19.8M (D=128) or 12.0M (D=256). + +### `icl_query_rms_scale`, `icl_key_rms`: ICL query and key norms without the copies + +- **Replaces.** Around each dataset (ICL) attention: the per-head query RMSNorm followed by a separate LogScale multiply `q · log(n) · head_scale`; the key RMSNorm, which `rmsnorm_cast` runs on a contiguous copy of the K view of the packed KV projection; and the copies of the K/V heads kept in the fitted cache. +- **Fuses.** The query norm and LogScale run in one pass. The key norm reads K in place from the packed projection and writes the cached K/V heads contiguous in the same launch. In the last fit layer, where only the cache reads K and V, the rule narrows the KV projection to the weight rows of the cached heads (2 of 16 on Large). +- **Shapes.** Head 64; ICL width 1024 (Large) or 512 (Medium, Small). +- **Exactness.** Native rounds the RMS to BF16, then computes the LogScale product in FP32 with one rounding; the kernel does the same. The narrowed projection sums in the same order as the full one for M from 513 to 65,535 (every M checked up to 8,192, then every 7th). +- **Limits.** The norms have no GEMM, so they do not depend on cuBLAS, and they index in 64 bits. The narrowed last-layer projection is used only for M in 513..65,535; outside that range the full projection runs and the kernel still does the norms and cache heads. +- **Contribution (Large 50k).** Profiler, fit: query side 12.5 to 6.3 ms, key side and cache slices 13.7 to 7.2 ms, last-layer projection 2.35 to 0.27 ms. + +### `icl_attention`: ICL fit attention (not bitwise) + +- **Replaces.** The dataset (ICL) attention in `fit`: context self-attention, R x R, head dim 64, 16 heads on Large and 8 on Medium and Small, 23 calls per fit. Compiled SDM runs PyTorch's flash attention, which is about 56% of fit + predict at Large 50k. +- **Does.** Exact softmax attention in a custom kernel (`sm120_icl_attention_d64`): 256 query rows per CTA with no register spills (flash spills), TMA loads into a 2-stage ring with per-stage barriers, online softmax that rescales only when a row maximum grew, and the launch's last wave split along keys and merged in a fixed order. The card runs at its power cap, so the gain comes from doing less non-MMA work per FLOP. +- **Exactness.** Not bitwise: it sums in a different order than flash. Outputs are deterministic from run to run. The benchmark qualifies it with the tolerance gate (below). +- **Shapes.** BF16, head dim 64, self-attention (same Q and K length) with 1,024 to 65,535 rows, no mask or dropout. Predict calls (16 query rows against the cache) and the row-block attention stay native. 65,535 is the largest context at which its error and determinism were checked; above it ICL attention stays native. Uncapped, Medium at 100k missed the tolerance gate narrowly (max |Δprob| 0.00952 against compiled-vs-eager 0.00948, 100% class agreement), so raising the cap needs the accuracy study at those sizes. +- **Switch.** `SDM_ICL_ATTENTION=sm120` (default), `fa2` (FlashAttention 2 at the same site, if installed), `int8` or `int8+ends` (below), or `off`; `kernels.enable(model, exact=True)` is the same as `off`. `sm120+uncapped` lifts the row cap, for diagnosis only. +- **Contribution.** Attention op alone, Large 50k: 5.7% (3,405 to 3,209 ms); Medium 4.4%, Small 5.3%. On top of every exact kernel, Large goes from 16.0% / 14.0% / 10.1% to 16.9% / 18.0% / 15.9% faster than compiled at 5k / 25k / 50k. +- **Evidence.** The study on 842c408fe covered 5 classification and 6 regression tables. Harnesses: `benchmark/attention/{accuracy,regression,determinism,compare}.py`. Results on 890521bfa are in [`inference-kernel-results.md`](inference-kernel-results.md#icl-attention-accuracy). + +### Opt-in 8-bit ICL attention (`SDM_ICL_ATTENTION=int8`) + +- **Does.** Quantizes K to INT8 per 64-key block after subtracting the per-head K mean (smoothing), V to FP8 E4M3 per channel and Q to INT8 per row; Q·Kᵀ runs on the INT8 `mma.sync` path, softmax in FP32, P·V in FP8 with FP32 accumulation. Source: `benchmark/inference/kernels/csrc/icl_attention_int8_sm120/icl_attention_int8.cu`, kernel `sm120_int8_icl`. The K mean is summed in a fixed order, so outputs are deterministic. +- **Settings.** `int8`, or `int8+ends`, which keeps the first and last ICL fit call of each graph native. +- **Gate.** Lossy by design (`exact=False`), never selected by default. The benchmark records the distance to compiled (and compiled vs eager when the eager arm runs) as gate `opt-in` and times the run. +- **Shapes.** BF16 or FP16, batch 1, head dim 64, at least 1,024 rows, 32-bit element offsets. Classification with more than 10 classes runs its ECOC codes as an ICL batch above 1 (8 for 26 classes), so attention stays native there; `benchmark.attention.compare` skips the 8-bit settings on those tables. +- **Measured on SDM 842c408fe** (torch 2.13): 1.6 to 1.7x flash standalone; Large 50k fit + predict 23.6% faster with the 8-bit attention alone and 33.8% with every exact kernel; class flips 0.16% at 1,024 query rows (compiled vs eager 0.92%); on TabArena, 1 of 74 per-task tests significant after correction (+0.014% log-loss). + +## Default on or opt-in + +The BF16 ICL attention kernel is on whenever the kernels are on. Whether it should stay on by default is for the SDM maintainers to decide. The rule used here for a default-on kernel that is not bitwise: the maximum probability difference against compiled SDM must stay within compiled SDM's own difference from eager SDM, with 100% class agreement, at 16 and at 1,024 query rows, on several datasets and seeds. The kernel meets that rule at 16 query rows. It does not meet it strictly at 1,024 query rows, or on the most extreme regression quantiles. The evidence below was measured on SDM 842c408fe; the 890521bfa study is in [`inference-kernel-results.md`](inference-kernel-results.md#icl-attention-accuracy). + +| Evidence | Result | +| ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Bitwise equal to compiled | No | +| 16 query rows, 4 classification datasets x 3 model/seed settings | 100% class agreement in 12 of 12 | +| 1,024 query rows, same cases | 4 of 12 flip 1 to 3 of 1,024 classes; compiled vs eager flips more rows in each (4 to 17); FlashAttention 2 flips the same rows in 3 of the 4 | +| Regression point prediction, 6 tables x 2 query counts x 3 seeds | inside the compiled-vs-eager gap in 36 of 36; RMSE shift at most 1.0% (eager vs compiled: 7.1%) | +| Regression per-quantile max | over the gap in 3 of 36 runs, on one or two cells of the 0.001 or 0.002 quantile; the 99.9th percentile of \|Δ\| is 1.7 to 15x below the gap in every run | +| Run-to-run determinism | bitwise identical | +| Large, all kernels vs compiled, 5k / 25k / 50k | 16.9% / 18.0% / 15.9% with it, 16.0% / 14.0% / 10.1% without | + +To make the kernels bitwise: `SDM_ICL_ATTENTION=off`, or `kernels.enable(model, exact=True)`. + +## Measured on SDM 842c408fe + +On SDM 842c408fe the compiled forward was split into several graphs around the row embedding and nothing was chunked under compile. Fresh-context `fit` + `predict`, clean compiled SDM vs the same plus all exact kernels without `icl_attention` (its numbers are above). Settings: one RTX PRO 4500 Blackwell, torch 2.13.0+cu130, BF16, Covertype, Q16, 12 mirrored blocks per run plus an unchanged repeat. Every run was bitwise equal to clean compiled on the probabilities and the fitted cache. These kernel sets include the row-table ops, which were bitwise end to end on that revision. The table shows the initial runs; the repeats differ by at most 0.11 percentage points. + +| Model | 5k | 25k | 50k | +| ------ | ----- | ----- | ----- | +| Large | 15.2% | 13.1% | 9.5% | +| Medium | 16.9% | 17.4% | 13.9% | +| Small | 9.0% | 12.9% | 10.6% | + +Large 50k, adding one family at a time (short 4-block runs, attribution only): + +| Kernels | Faster than compiled | +| ----------------------------------------------- | -------------------- | +| `linear_gelu` | 2.10% | +| + `strided_rmsnorm` | 3.20% | +| + `gated_log_scale` | 3.91% | +| + `rope_rms` | 5.17% | +| + `linear_kv_key_rms` | 6.38% | +| + `column_residual_rms`, `dual_rmsnorm` | 7.37% | +| + `linear_query_rms` | 7.98% | +| + in-place column-down and row residual | 8.40% | +| + `row_down_residual` | 8.90% | +| + `column_projection_rms`, `row_projection_rms` | 9.67% | + +Adding the attention-adjacent ops (`icl_query_rms_scale`, `icl_key_rms`, `rope_rms_uneven` and the row-table op `linear_residual_kv_query_rms`) makes Large 50k about 10.2% faster than compiled, against about 9.5% with the set above in the same process. This is a same-process A/B of compiled, the set above and every op, 12 interleaved blocks plus an unchanged repeat, all bitwise equal. The attention-adjacent ops add 0.3 to 1.1% across Large, Medium and Small at 5k to 50k. + +## Coverage and limits + +Compiled SDM runs any context: SDPA splits flash calls above 65,535 batch entries (SDM #1181), and attention and cell embedding chunk under `torch.compile` with a memory budget per chunk (SDM #1055, `SDM_CHUNK_MEMORY_FRACTION`, default 5% of the GPU). No flash call in the graph has more than 65,535 batch entries. + +Chunking changes what the rules see: + +- **Row blocks** run on chunks of a few thousand contexts (about 7,200 for Large at 53 features on a 32 GB GPU) from about 7,000 contexts up. The row-local ops (`rope_rms`, `rope_rms_uneven`, gating, the GEMM epilogues) match every chunk, and no upper bound binds at 100k contexts. +- **Column blocks** run on chunks of a few features once the context is above about 7,500 rows (Large and Medium; about 15,000 for Small). A chunk reads a feature range of the table, which `column_projection_rms` does not take, so that region stays native at large contexts. `linear_kv_key_rms` and `linear_query_rms` work on the chunk's own GEMM and still match. +- **ICL block** is not chunked: its ops see M = C. `icl_key_rms` narrows the last fit layer's KV projection only for M in 513..65,535, where it was checked bitwise; above that it runs the full projection and still does the norms and cache heads. The BF16 `icl_attention` kernel admits at most 65,535 rows (checked); the 8-bit kernel has no row cap. + +| Family | Models | Rows admitted (M) | Outside the limits | +| ------------------------------------ | ------------------------- | ------------------------------------------------------------------------------------- | ------------------------------------------------------------ | +| `linear_gelu` | all; 1024→2048 Large only | 1024 to 8.39M (256→512), 40M (64→64), 500k (1024→2048), 33.0M (128→256), 132M (32→64) | native | +| `gated_log_scale` | all | 1024 to 80.9M (head 32), 49.6M (head 64) | native | +| `rope_rms` | all | ≤ 1024 positions, ≤ 20.2M tokens (head 32), 8.39M (head 64) | native | +| `linear_kv_key_rms` | all | ≤ 19.9M (D=128), 12.0M (D=256) | native | +| `linear_query_rms` | all | ≤ 33.0M (D=128), 20.6M (D=256), minus a small-M band | native | +| `column_projection_rms` | all | 19.8M (D=128), 12.0M (D=256) | native | +| `icl_query_rms_scale`, `icl_key_rms` | all | no row bound; narrowed last-layer KV projection for M in 513..65,535 | full projection (outside the M range), native (other shapes) | +| `rope_rms_uneven` | all | as `rope_rms`, on the longer of Q and K | native | +| `icl_attention` BF16 (not bitwise) | all | self-attention of 1,024 to 65,535 rows, head dim 64 | native | +| `icl_attention` int8 (opt-in, lossy) | all | self-attention of at least 1,024 rows, head dim 64, batch 1 | native | diff --git a/test/benchmark/inference/conftest.py b/test/benchmark/inference/conftest.py new file mode 100644 index 0000000000..783cecbe97 --- /dev/null +++ b/test/benchmark/inference/conftest.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fixtures checking real SM120 kernels against native Inductor, bitwise. + +Each check compiles a small function reproducing an SDM pattern twice: plain +(the native result) and with the kernel pass enabled, so the rule, the op and +the kernel are all exercised. +""" + +from collections.abc import Callable +from contextlib import nullcontext + +import pytest +import torch +from torch import Tensor +from torch._inductor.utils import run_and_get_code + +# Defines the linear_gelu op and rule that the unit tests give fake kernels. +import benchmark.inference.kernels.fusions.linear_gelu # noqa: F401 +from benchmark.inference.kernels import fx_pass, integration, registry +from benchmark.inference.runner import set_precision + + +@pytest.fixture(scope="session") +def kernels() -> None: + """Build and register every kernel; skip without an SM 12.0 GPU.""" + if not torch.cuda.is_available() or torch.cuda.get_device_capability() != ( + 12, + 0, + ): + pytest.skip("needs an SM 12.0 (Blackwell) GPU") + set_precision("ieee") + integration.install(None) + + +def _bits(value: Tensor) -> Tensor: + return value.detach().contiguous().reshape(-1).view(torch.uint8) + + +def _tensors(value: object) -> list[Tensor]: + if isinstance(value, Tensor): + return [value] + if isinstance(value, (list, tuple)): + return [t for v in value for t in _tensors(v)] + return [] + + +def _run( + fn: Callable, make_args: Callable[[], tuple], options: dict, enabled: bool +) -> tuple[list[Tensor], str]: + args = make_args() + torch._dynamo.reset() + compiled = torch.compile( + fn, + **{ + **integration.COMPILE, + "options": {**integration.COMPILE["options"], **options}, + }, + ) + with ( + torch.inference_mode(), + torch.autocast("cuda", dtype=torch.bfloat16), + registry.enabled() if enabled else nullcontext(), + ): + out, codes = run_and_get_code(compiled, *args) + torch.cuda.synchronize() + return _tensors(out) + _tensors(args), codes[-1] + + +def _with_kernels( + fn: Callable, make_args: Callable[[], tuple], atol: float | None = None +) -> tuple[dict, str]: + """Census and fused code of ``fn``, whose results match native bitwise. + + With ``atol`` they match within that max absolute difference instead. + + ``make_args`` must return identical fresh inputs on every call, so a + mutated input (an in-place table) starts from the same values on both + paths; outputs and all inputs are compared after the call. + """ + registry.reset_census() + native, _ = _run(fn, make_args, {}, enabled=False) + fused, code = _run(fn, make_args, fx_pass.compile_options(), enabled=True) + census = registry.census() + assert len(native) == len(fused) + for i, (a, b) in enumerate(zip(native, fused)): + assert a.shape == b.shape, i + assert a.dtype == b.dtype, i + if atol is None: + assert torch.equal(_bits(a), _bits(b)), f"tensor {i} differs" + else: + diff = (a.float() - b.float()).abs().max().item() + assert diff <= atol, f"tensor {i} differs by {diff}" + return census, code + + +def assert_kernel_matches_native( + fn: Callable, make_args: Callable[[], tuple], kernel: str +) -> str: + """``kernel`` runs inside ``fn`` and changes no bit of its results. + + Returns the generated code of the run with kernels. + """ + census, code = _with_kernels(fn, make_args) + assert kernel in {d["kernel"] for d in census["decisions"]}, census + assert census["launches"].get(kernel, 0) > 0, census["launches"] + return code + + +def assert_kernel_close_to_native( + fn: Callable, make_args: Callable[[], tuple], kernel: str, atol: float +) -> str: + """:func:`assert_kernel_matches_native`, within ``atol``, not bitwise.""" + census, code = _with_kernels(fn, make_args, atol) + assert kernel in {d["kernel"] for d in census["decisions"]}, census + assert census["launches"].get(kernel, 0) > 0, census["launches"] + return code + + +def assert_stays_native( + fn: Callable, make_args: Callable[[], tuple], op: str +) -> None: + """``op``'s rule is tried but no kernel is chosen; results stay bitwise.""" + census, _ = _with_kernels(fn, make_args) + chosen = [d for d in census["decisions"] if d["op"] == op] + assert chosen, census + assert all(d["kernel"] == "native" for d in chosen), census + + +def assert_rule_skips( + fn: Callable, make_args: Callable[[], tuple], op: str +) -> None: + """``op``'s rule leaves ``fn`` alone; any other kernel stays exact.""" + census, _ = _with_kernels(fn, make_args) + assert all(d["op"] != op for d in census["decisions"]), census + + +def assert_kernel_rejects( + kernel: str, *args: object, reason: str | None = None +) -> None: + """``kernel``'s ``supports`` refuses ``args``, naming ``reason``.""" + refused = registry.kernel(kernel).supports(*args) + assert refused is not None, f"{kernel} admits the arguments" + assert reason is None or reason in refused, refused + + +@pytest.fixture +def matches_native(kernels: None) -> Callable: + """:func:`assert_kernel_matches_native`, on an SM 12.0 GPU only.""" + return assert_kernel_matches_native + + +@pytest.fixture +def close_to_native(kernels: None) -> Callable: + """:func:`assert_kernel_close_to_native`, on an SM 12.0 GPU only.""" + return assert_kernel_close_to_native + + +@pytest.fixture +def stays_native(kernels: None) -> Callable: + """:func:`assert_stays_native`, on an SM 12.0 GPU only.""" + return assert_stays_native + + +@pytest.fixture +def skips_rule(kernels: None) -> Callable: + """:func:`assert_rule_skips`, on an SM 12.0 GPU only.""" + return assert_rule_skips + + +@pytest.fixture +def rejects(kernels: None) -> Callable: + """:func:`assert_kernel_rejects`, on an SM 12.0 GPU only.""" + return assert_kernel_rejects diff --git a/test/benchmark/inference/test_build.py b/test/benchmark/inference/test_build.py new file mode 100644 index 0000000000..ccab2636f7 --- /dev/null +++ b/test/benchmark/inference/test_build.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import shutil +from pathlib import Path + +import pytest + +from benchmark.inference.kernels import build + +MODULE = "linear_gelu_sm120" + + +def test_stamp_tracks_module_sources( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + shutil.copytree(build.CSRC / MODULE, tmp_path / MODULE) + monkeypatch.setattr(build, "CSRC", tmp_path) + first = build.stamp(MODULE) + assert first == build.stamp(MODULE) + source = next((tmp_path / MODULE).iterdir()) + source.write_text(source.read_text() + "\n") + assert build.stamp(MODULE) != first + + +def test_up_to_date_module_skips_toolchain( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / f"{MODULE}.so").write_bytes(b"") + (tmp_path / f"{MODULE}.sha256").write_text(build.stamp(MODULE)) + monkeypatch.setattr( + build, "check_toolchain", lambda: pytest.fail("rebuilt") + ) + assert build.build(tmp_path, [MODULE]) == tmp_path + + +def test_stale_module_needs_toolchain( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / f"{MODULE}.so").write_bytes(b"") + (tmp_path / f"{MODULE}.sha256").write_text("stale") + monkeypatch.setattr(build.cpp_extension, "CUDA_HOME", None) + with pytest.raises(RuntimeError, match="CUDA toolkit"): + build.build(tmp_path, [MODULE]) diff --git a/test/benchmark/inference/test_config.py b/test/benchmark/inference/test_config.py new file mode 100644 index 0000000000..efd2152a98 --- /dev/null +++ b/test/benchmark/inference/test_config.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest + +from benchmark.inference.config import ARMS, parse + +_ARGS = ["--model-size", "large", "--context-rows", "50000"] + + +def test_defaults(tmp_path: Path) -> None: + config = parse([*_ARGS, "--output", str(tmp_path / "out")]) + assert config.arms == ARMS + assert config.query_rows == 16 + assert config.precision == "ieee" + assert config.checkpoint is None + assert config.record()["dataset"] == "covertype" + + +@pytest.mark.parametrize( + ("args", "message"), + [ + (["--model-size", "huge"], "invalid choice"), + (["--context-rows", "0"], "positive integer"), + (["--blocks", "4"], "multiple of 3"), + (["--arms", "compiled", "optimized:fp8"], "ICL attention"), + (["--arms", "compiled", "optimized:off:x"], "optimized:"), + (["--arms", "eager", "optimized:off"], "requires compiled"), + (["--data", "table.csv"], "--data and --target"), + (["--data", "table.txt", "--target", "y"], ".parquet or .csv"), + (["--arms", "eager", "optimized"], "requires compiled"), + (["--arms", "eager", "eager"], "must not repeat"), + (["--checkpoint", "missing.pt"], "does not exist"), + ], +) +def test_invalid_configuration_exits_with_message( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + args: list[str], + message: str, +) -> None: + with pytest.raises(SystemExit): + parse([*_ARGS, "--output", str(tmp_path / "out"), *args]) + assert message in capsys.readouterr().err + + +def test_existing_output_is_refused( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + with pytest.raises(SystemExit): + parse([*_ARGS, "--output", str(tmp_path)]) + assert "already exists" in capsys.readouterr().err + + +def test_attention_comes_from_the_flag_or_the_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("SDM_ICL_ATTENTION", raising=False) + assert ( + parse([*_ARGS, "--output", str(tmp_path / "a")]).attention == "sm120" + ) + monkeypatch.setenv("SDM_ICL_ATTENTION", "off") + assert parse([*_ARGS, "--output", str(tmp_path / "b")]).attention == "off" + config = parse( + [*_ARGS, "--output", str(tmp_path / "c"), "--attention", "int8+ends"] + ) + assert config.record()["attention"] == "int8+ends" + + +def test_attention_is_validated(tmp_path: Path) -> None: + with pytest.raises(SystemExit): + parse([*_ARGS, "--output", str(tmp_path / "o"), "--attention", "fp8"]) + + +def test_optimized_variants(tmp_path: Path) -> None: + config = parse( + [ + *(*_ARGS, "--output", str(tmp_path / "out"), "--blocks", "4"), + *("--arms", "compiled", "optimized", "optimized:int8+ends"), + "optimized:off", + ] + ) + assert config.arm_attention("optimized") == "sm120" + assert config.arm_attention("optimized:int8+ends") == "int8+ends" + assert config.arm_attention("optimized:off") == "off" diff --git a/test/benchmark/inference/test_fusions.py b/test/benchmark/inference/test_fusions.py new file mode 100644 index 0000000000..b2fc06e142 --- /dev/null +++ b/test/benchmark/inference/test_fusions.py @@ -0,0 +1,438 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Each fusion against native Inductor on small SDM-shaped inputs.""" + +import gc +from collections.abc import Callable + +import pytest +import torch +import torch.nn.functional as F +from torch import Tensor + +from benchmark.inference.kernels import registry, sm120 +from benchmark.inference.kernels.fusions import ( + gating, + kv_rms, + linear_gelu, + projection_rms, + query_rms, + rope_rms, +) +from sdm._kernels import rmsnorm_cast +from sdm.nn import GatedLogScale, RMSNorm, RotaryEmbedding + +EPS = torch.finfo(torch.float32).eps +CONTEXTS, FEATURES = 64, 24 +WIDTHS = pytest.mark.parametrize(("d", "head"), [(128, 32), (256, 64)]) +# (K, N) of every linear_gelu kernel. +LINEAR_GELU = [*linear_gelu.MAX_M, *((h, gating.HIDDEN) for h in gating.HEADS)] + + +def _seeded(build: Callable[[Callable[..., Tensor]], tuple]) -> Callable: + """Identical fresh inputs on every call, drawn by ``build``.""" + + def make() -> tuple: + g = torch.Generator("cuda").manual_seed(0) + return build( + lambda *shape, dtype=torch.bfloat16: torch.randn( + *shape, device="cuda", generator=g, dtype=dtype + ) + ) + + return make + + +def _add(a: Tensor, b: Tensor, swapped: bool) -> Tensor: + return b + a if swapped else a + b + + +def _table( + r: Callable[..., Tensor], d: int, readouts: int = 4 +) -> tuple[Tensor, Tensor]: + """The table and its feature rows, views of one hidden flat buffer. + + AOT then passes that buffer as one flat graph input, as for SDM's table. + """ + table = r(CONTEXTS * (FEATURES + 4) * d).view(CONTEXTS, FEATURES + 4, d) + return table, table[:, readouts:] + + +@pytest.mark.parametrize(("k", "n"), LINEAR_GELU) +def test_linear_gelu(matches_native: Callable, k: int, n: int) -> None: + def fn(x: Tensor, w: Tensor, b: Tensor) -> Tensor: + return F.gelu(F.linear(x, w, b)) + + m = 2048 if k == 1024 else 4096 + matches_native( + fn, + _seeded(lambda r: (r(m, k), r(n, k), r(n))), + f"sm120_linear_gelu_k{k}_n{n}", + ) + + +@pytest.mark.parametrize("head", [32, 64]) +def test_gated_log_scale(matches_native: Callable, head: int) -> None: + scaling = GatedLogScale( + channels=head, num_heads=4, hidden_channels=64, device="cuda" + ) + torch.nn.init.normal_(scaling.gate[-1].weight, std=0.125) + + def fn(q: Tensor) -> Tensor: + return scaling(q, key_len=q.size(-3)) + + matches_native( + fn, _seeded(lambda r: (r(16, 32, 4, head),)), "sm120_gated_log_scale" + ) + + +@WIDTHS +def test_rope_rms(matches_native: Callable, d: int, head: int) -> None: + rope = RotaryEmbedding( + head, "split_half", requires_grad=False, device="cuda" + ) + norm = RMSNorm(head, eps=1e-6, elementwise_affine=False, device="cuda") + + def fn(q: Tensor, kv: Tensor) -> tuple[Tensor, Tensor]: + k = kv[..., :d].unflatten(-1, [4, head]) + return norm(q.unflatten(-1, [4, head]), rope=rope), norm(k, rope=rope) + + matches_native( + fn, + _seeded(lambda r: (r(50, 57, d), r(50, 57, 2 * d))), + "sm120_rope_rms", + ) + + +@WIDTHS +def test_rope_rms_uneven(matches_native: Callable, d: int, head: int) -> None: + rope = RotaryEmbedding( + head, "split_half", requires_grad=False, device="cuda" + ) + norm = RMSNorm(head, eps=1e-6, elementwise_affine=False, device="cuda") + + def fn(q: Tensor, kv: Tensor) -> tuple[Tensor, Tensor]: + k = kv[..., :d].unflatten(-1, [4, head]) + return norm(q.unflatten(-1, [4, head]), rope=rope), norm(k, rope=rope) + + matches_native( + fn, + _seeded(lambda r: (r(50, 4, d), r(50, 57, 2 * d))), + "sm120_rope_rms_uneven", + ) + + +@WIDTHS +def test_linear_kv_key_rms( + matches_native: Callable, d: int, head: int +) -> None: + norm = RMSNorm(head, eps=1e-6, elementwise_affine=False, device="cuda") + + def fn(x: Tensor, w: Tensor, b: Tensor) -> tuple[Tensor, Tensor]: + k, v = F.linear(x, w, b).chunk(2, -1) + return norm(k.unflatten(-1, [4, head])), v + + matches_native( + fn, + _seeded(lambda r: (r(FEATURES, CONTEXTS, d), r(2 * d, d), r(2 * d))), + "sm120_linear_kv_key_rms", + ) + + +@WIDTHS +def test_linear_query_rms(matches_native: Callable, d: int, head: int) -> None: + norm = RMSNorm(head, eps=1e-6, elementwise_affine=False, device="cuda") + + def fn(x: Tensor, w: Tensor, b: Tensor) -> Tensor: + return norm(F.linear(x, w, b).unflatten(-1, [4, head])) + + matches_native( + fn, + _seeded(lambda r: (r(FEATURES, CONTEXTS, d), r(d, d), r(d))), + "sm120_linear_query_rms", + ) + + +def _column_projection(d: int, swapped: bool) -> tuple[Callable, Callable]: + def fn(t: Tensor, rows: Tensor, x: Tensor, *params: Tensor) -> tuple: + w, b, norm_weight = params + raw = F.linear(x, w, b) + prior = rows.transpose(0, 1) + residual = _add(raw.view(prior.shape), prior, swapped) + out = rmsnorm_cast(residual, norm_weight, EPS) + t[:, :4] = 0 + return raw, out + + make = _seeded( + lambda r: ( + *_table(r, d), + r(FEATURES * CONTEXTS, d), + r(d, d), + r(d), + r(d, dtype=torch.float32), + ) + ) + return fn, make + + +@WIDTHS +def test_column_projection_rms( + matches_native: Callable, d: int, head: int +) -> None: + matches_native( + *_column_projection(d, swapped=False), "sm120_column_projection_rms" + ) + + +@WIDTHS +def test_column_projection_rms_skips_a_transposed_sum( + skips_rule: Callable, d: int, head: int +) -> None: + """A transposed prior + raw gets cloned before the norm; stay native.""" + skips_rule(*_column_projection(d, swapped=True), "column_projection_rms") + + +D = 128 + + +def test_gated_log_scale_needs_four_heads(skips_rule: Callable) -> None: + scaling = GatedLogScale( + channels=32, num_heads=2, hidden_channels=64, device="cuda" + ) + torch.nn.init.normal_(scaling.gate[-1].weight, std=0.125) + + def fn(q: Tensor) -> Tensor: + return scaling(q, key_len=q.size(-3)) + + skips_rule(fn, _seeded(lambda r: (r(32, 32, 2, 32),)), "gated_log_scale") + + +def test_linear_kv_key_rms_keeps_a_read_raw_key(skips_rule: Callable) -> None: + norm = RMSNorm(32, eps=1e-6, elementwise_affine=False, device="cuda") + + def fn(x: Tensor, w: Tensor, b: Tensor) -> tuple[Tensor, ...]: + k, v = F.linear(x, w, b).chunk(2, -1) + return norm(k.unflatten(-1, [4, 32])), v, k * 2 + + skips_rule( + fn, + _seeded(lambda r: (r(FEATURES, CONTEXTS, D), r(2 * D, D), r(2 * D))), + "linear_kv_key_rms", + ) + + +def test_linear_query_rms_keeps_a_read_raw_query(skips_rule: Callable) -> None: + norm = RMSNorm(32, eps=1e-6, elementwise_affine=False, device="cuda") + + def fn(x: Tensor, w: Tensor, b: Tensor) -> tuple[Tensor, Tensor]: + q = F.linear(x, w, b) + return norm(q.unflatten(-1, [4, 32])), q * 2 + + skips_rule( + fn, + _seeded(lambda r: (r(FEATURES, CONTEXTS, D), r(D, D), r(D))), + "linear_query_rms", + ) + + +def test_column_projection_rms_needs_four_readouts( + skips_rule: Callable, +) -> None: + def fn(t: Tensor, rows: Tensor, x: Tensor, *params: Tensor) -> tuple: + w, b, norm_weight = params + raw = F.linear(x, w, b) + prior = rows.transpose(0, 1) + out = rmsnorm_cast(raw.view(prior.shape) + prior, norm_weight, EPS) + t[:, :3] = 0 + return raw, out + + skips_rule( + fn, + _seeded( + lambda r: ( + *_table(r, D, readouts=3), + r((FEATURES + 1) * CONTEXTS, D), + r(D, D), + r(D), + r(D, dtype=torch.float32), + ) + ), + "column_projection_rms", + ) + + +def _empty(*shape: int, dtype: torch.dtype = torch.bfloat16) -> Tensor: + return torch.empty(*shape, device="cuda", dtype=dtype) + + +def _f32(*shape: int) -> Tensor: + return _empty(*shape, dtype=torch.float32) + + +REJECTED = { + "sm120_linear_gelu_k32_n64": ( + lambda: (_empty(512, 32), _empty(64, 32), _empty(64)), + "needs x of shape [1024<=M", + ), + "sm120_linear_gelu_k128_n256": ( + lambda: (_empty(4096, 128), _empty(128, 128), _empty(128)), + "needs a 256x128 weight", + ), + "sm120_gated_log_scale": ( + lambda: ( + _empty(2048, 64), + _empty(48, 64), + _empty(48), + _empty(2048, 48), + _f32(4), + 1.0, + ), + "with head in (32, 64)", + ), + "sm120_rope_rms": ( + lambda: (_empty(1, 2048, 4, 32), _empty(1, 2048, 4, 32), _f32(16)), + "needs 1 <= S <= 1024", + ), + "sm120_linear_kv_key_rms": ( + lambda: (_empty(128, 256), _empty(512, 256), _empty(512), 64), + "needs 129 <= M", + ), + "sm120_linear_query_rms": ( + lambda: (_empty(1000, 128), _empty(128, 128), _empty(128), 32), + "differs from cuBLAS", + ), + "sm120_column_projection_rms": ( + lambda: ( + _empty(CONTEXTS * 25, 128), + _empty(128, 128), + _empty(128), + _empty(CONTEXTS, 28, 128), + _f32(128), + EPS, + ), + "a table of 4 + F rows", + ), + "sm120_rope_rms_uneven": ( + lambda: (_empty(8, 57, 4, 64), _empty(8, 57, 4, 64), _f32(32)), + "different S", + ), +} + + +@pytest.mark.parametrize("kernel", sorted(REJECTED)) +def test_supports_rejects(rejects: Callable, kernel: str) -> None: + make, reason = REJECTED[kernel] + rejects(kernel, *make(), reason=reason) + + +def _rows(m: int, *widths: int) -> list[Tensor]: + """``[m, w]`` views of one buffer; ``supports`` reads only metadata.""" + flat = _empty(m * max(widths)) + return [flat[: m * w].view(m, w) for w in widths] + + +def _column(m: int, d: int) -> tuple[Tensor, Tensor]: + """Raw ``[C * F, D]`` and table ``[C, 4 + F, D]`` at C = 1.""" + (rows,) = _rows(m + 4, d) + return rows[:m], rows.view(1, m + 4, d) + + +def _linear_gelu_args(m: int, k: int, n: int) -> tuple: + return *_rows(m, k), _empty(n, k), _empty(n) + + +def _gls_args(m: int, head: int) -> tuple: + x, query = _rows(m, 64, head) + return x, _empty(head, 64), _empty(head), query, _f32(4), 1.0 + + +def _rope_args(m: int, head: int) -> tuple: + (q,) = _rows(m, 4 * head) + q = q.view(m, 1, 4, head) + return q, q, _f32(head // 2) + + +def _column_projection_args(m: int, d: int) -> tuple: + x, table = _column(m, d) + return x, _empty(d, d), _empty(d), table, _f32(d), EPS + + +D_HEAD = ((128, 32), (256, 64)) +# kernel -> [(largest checked M of one instance, M -> supports() args)] +ROW_LIMITS = { + **{ + f"sm120_linear_gelu_k{k}_n{n}": [ + (limit, lambda m, k=k, n=n: _linear_gelu_args(m, k, n)) + ] + for (k, n), limit in ( + *linear_gelu.MAX_M.items(), + *(((h, gating.HIDDEN), m) for h, m in gating.FIRST_MAX_M.items()), + ) + }, + "sm120_gated_log_scale": [ + (gating.SECOND_MAX_M[h], lambda m, h=h: _gls_args(m, h)) + for h in (32, 64) + ], + "sm120_linear_kv_key_rms": [ + ( + kv_rms.MAX_M[d], + lambda m, d=d, h=h: ( + *_rows(m, d), + _empty(2 * d, d), + _empty(2 * d), + h, + ), + ) + for d, h in D_HEAD + ], + "sm120_linear_query_rms": [ + ( + query_rms.MAX_M[d], + lambda m, d=d, h=h: (*_rows(m, d), _empty(d, d), _empty(d), h), + ) + for d, h in D_HEAD + ], + "sm120_rope_rms": [ + (rope_rms.MAX_TOKENS[h], lambda m, h=h: _rope_args(m, h)) + for _, h in D_HEAD + ], + "sm120_column_projection_rms": [ + ( + projection_rms.MAX_M[d], + lambda m, d=d: _column_projection_args(m, d), + ) + for d, _ in D_HEAD + ], +} + + +@pytest.mark.parametrize( + ("kernel", "instance"), + [(k, i) for k in sorted(ROW_LIMITS) for i in range(len(ROW_LIMITS[k]))], +) +def test_row_limits(rejects: Callable, kernel: str, instance: int) -> None: + """The largest checked row count is admitted; one more is refused.""" + limit, make = ROW_LIMITS[kernel][instance] + for m in (limit, limit + 1): + args = make(m) + if m == limit: + assert registry.kernel(kernel).supports(*args) is None + else: + rejects(kernel, *args, reason=str(limit)) + del args + gc.collect() + torch.cuda.empty_cache() + + +@pytest.mark.parametrize(("k", "n"), LINEAR_GELU) +def test_linear_gelu_leaves_query_side_calls_native( + rejects: Callable, k: int, n: int +) -> None: + kernel = f"sm120_linear_gelu_k{k}_n{n}" + for m in (sm120.MIN_M, sm120.MIN_M - 1): + args = _linear_gelu_args(m, k, n) + if m == sm120.MIN_M: + assert registry.kernel(kernel).supports(*args) is None + else: + rejects(kernel, *args, reason=f"{sm120.MIN_M}<=M") diff --git a/test/benchmark/inference/test_fx_pass.py b/test/benchmark/inference/test_fx_pass.py new file mode 100644 index 0000000000..f9aafaf01a --- /dev/null +++ b/test/benchmark/inference/test_fx_pass.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Callable + +import pytest +import torch +import torch.nn.functional as F +from torch import Tensor + +from benchmark.inference.kernels import fx_pass, registry +from benchmark.inference.kernels.registry import Kernel + +needs_cuda = pytest.mark.skipif( + not torch.cuda.is_available(), reason="needs CUDA" +) + + +@pytest.fixture(autouse=True) +def _empty_registry(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(registry, "KERNELS", {}) + registry.reset_census() + + +def _mlp(x: Tensor, w: Tensor, b: Tensor) -> Tensor: + return F.gelu(F.linear(x, w, b)).reshape(2, -1, w.size(0)).sum(0) + + +def _tanh_mlp(x: Tensor, w: Tensor, b: Tensor) -> Tensor: + return F.gelu(F.linear(x, w, b), approximate="tanh") + + +def _run( + reason: str | None, fn: Callable[..., Tensor] = _mlp +) -> tuple[Tensor, Tensor]: + registry.register( + Kernel( + op="linear_gelu", + name="test_kernel", + supports=lambda x, w, b: reason, + impl=lambda x, w, b: F.gelu(F.linear(x, w, b)), + cuda_symbols=("test",), + ) + ) + torch._dynamo.reset() + compiled = torch.compile(fn, options=fx_pass.compile_options()) + args = tuple( + torch.randn(*shape, dtype=torch.bfloat16) + for shape in ((8, 16), (32, 16), (32,)) + ) + with registry.enabled(): + return compiled(*args), fn(*args) + + +def test_admitted_region_runs_the_selected_kernel() -> None: + out, expected = _run(None) + torch.testing.assert_close(out, expected) + census = registry.census() + assert [d["kernel"] for d in census["decisions"]] == ["test_kernel"] + assert census["launches"] == {"test_kernel": 1} + + +def test_rejected_region_stays_native() -> None: + _run("unsupported shape") + census = registry.census() + assert census["decisions"][0]["kernel"] == "native" + assert census["decisions"][0]["rejected"] == { + "test_kernel": "unsupported shape" + } + assert census["launches"] == {} + + +def test_tanh_gelu_is_not_rewritten() -> None: + out, expected = _run(None, _tanh_mlp) + torch.testing.assert_close(out, expected) + assert _selected() == [] + + +def test_rules_for_one_target_run_in_order_until_one_matches( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: list[str] = [] + monkeypatch.setattr(fx_pass, "RULES", []) + erf = torch.ops.aten.erf.default + fx_pass.add_rule(erf, "a", lambda g, n: seen.append("a")) + fx_pass.add_rule(erf, "b", lambda g, n: seen.append("b") or True) + fx_pass.add_rule(erf, "c", lambda g, n: seen.append("c")) + torch._dynamo.reset() + fn = torch.compile(torch.erf, options=fx_pass.compile_options()) + fn(torch.randn(4)) + assert seen == ["a", "b"] + + +@needs_cuda +def test_triton_kernel_node_is_matchable_by_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + triton = pytest.importorskip("triton") + tl = pytest.importorskip("triton.language") + + @triton.jit + def _double_kernel(x_ptr, out_ptr, n, BLOCK: tl.constexpr): + i = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + tl.store(out_ptr + i, tl.load(x_ptr + i, mask=i < n) * 2, mask=i < n) + + def double(x: Tensor) -> Tensor: + out = torch.empty_like(x) + _double_kernel[(1,)](x, out, x.numel(), BLOCK=64) + return out + + names: list[str | None] = [] + monkeypatch.setattr(fx_pass, "RULES", []) + fx_pass.add_rule( + fx_pass.TRITON_KERNEL, + "double", + lambda g, n: names.append(fx_pass.triton_kernel_name(n)), + ) + torch._dynamo.reset() + torch.compile(double, options=fx_pass.compile_options())( + torch.randn(64, device="cuda") + ) + assert names == ["_double_kernel"] + + +def _selected() -> list[str]: + return [ + d["kernel"] + for d in registry.census()["decisions"] + if d["kernel"] != "native" + ] + + +def test_shadow_keeps_native_outputs_and_records_the_kernel() -> None: + registry.register( + Kernel( + op="linear_gelu", + name="test_kernel", + supports=lambda x, w, b: None, + impl=lambda x, w, b: F.gelu(F.linear(x, w, b)) + 1, + cuda_symbols=("test",), + ) + ) + args = tuple( + torch.randn(*shape, dtype=torch.bfloat16) + for shape in ((8, 16), (32, 16), (32,)) + ) + torch._dynamo.reset() + native = torch.compile(_mlp)(*args) + torch._dynamo.reset() + shadow = torch.compile(_mlp, options=fx_pass.compile_options(shadow=True)) + with registry.enabled(): + out = shadow(*args) + # Native stays on the data path (the kernel adds 1); Inductor may still + # fuse it differently around the comparison. + assert (out.float() - native.float()).abs().max() < 0.5 + entry = registry.census()["shadow"]["linear_gelu"] + assert (entry["outputs"], entry["bitwise"]) == (1, 0) + assert entry["max_abs_diff"] > 0.5 diff --git a/test/benchmark/inference/test_icl_attention.py b/test/benchmark/inference/test_icl_attention.py new file mode 100644 index 0000000000..5dc68ee206 --- /dev/null +++ b/test/benchmark/inference/test_icl_attention.py @@ -0,0 +1,208 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import operator +from collections.abc import Callable + +import pytest +import torch +import torch.nn.functional as F +from torch import Tensor +from torch._inductor.utils import run_and_get_code +from torch.fx import Graph + +from benchmark.inference.kernels import fx_pass, integration, registry +from benchmark.inference.kernels.fusions import icl_attention +from benchmark.inference.kernels.registry import Kernel + +HEADS, HEAD = 8, 64 +KERNELS = { + "sm120": "sm120_icl_attention_d64", + "fa2": "fa2_icl_attention_d64", + "int8": "sm120_int8_icl", +} +ATOL = {"sm120": 2e-2, "fa2": 2e-2, "int8": 8e-2} + + +def _seeded(queries: int, keys: int) -> Callable[[], tuple]: + def make() -> tuple: + g = torch.Generator("cuda").manual_seed(0) + return tuple( + torch.randn( + 1, + n, + HEADS, + HEAD, + device="cuda", + generator=g, + dtype=torch.bfloat16, + ) + for n in (queries, keys, keys) + ) + + return make + + +def _attend(q: Tensor, k: Tensor, v: Tensor) -> Tensor: + return F.scaled_dot_product_attention( + q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2) + ).transpose(1, 2) + + +@pytest.fixture(params=sorted(KERNELS)) +def mode(request, monkeypatch, kernels: None) -> str: + if request.param == "fa2" and icl_attention._flash_attn_func() is None: + pytest.skip("flash_attn is not importable") + monkeypatch.setenv(icl_attention.ENV, request.param) + return request.param + + +def test_fit_attention_is_replaced( + close_to_native: Callable, mode: str +) -> None: + make = _seeded(2048, 2048) + code = close_to_native(_attend, make, KERNELS[mode], atol=ATOL[mode]) + assert "sdm_inference.icl_attention.default(" in code + q, k, v = make() + reference = _attend(q.float(), k.float(), v.float()) + kernel = registry.kernel(KERNELS[mode]) + with torch.inference_mode(): + out = registry.launch(kernel, q, k, v, HEAD**-0.5) + assert out.shape == q.shape + assert (out.float() - reference).abs().max().item() <= ATOL[mode] + + +def test_predict_shape_stays_native(stays_native: Callable, mode: str) -> None: + stays_native(_attend, _seeded(16, 2048), "icl_attention") + + +def test_off_stays_native( + stays_native: Callable, monkeypatch, kernels: None +) -> None: + monkeypatch.setenv(icl_attention.ENV, "off") + stays_native(_attend, _seeded(2048, 2048), "icl_attention") + + +@pytest.mark.parametrize( + ("rows", "admitted"), [(1023, False), (65_535, True), (65_536, False)] +) +def test_bf16_row_bounds( + monkeypatch, kernels: None, rows: int, admitted: bool +) -> None: + monkeypatch.setenv(icl_attention.ENV, "sm120") + t = torch.empty(1, rows, HEADS, HEAD, device="cuda", dtype=torch.bfloat16) + kernel = registry.kernel(KERNELS["sm120"]) + assert (kernel.supports(t, t, t, HEAD**-0.5) is None) == admitted + + +def test_uncapped_bf16_admits_contexts_above_65535( + monkeypatch, kernels: None +) -> None: + monkeypatch.setenv(icl_attention.ENV, "sm120+uncapped") + t = torch.empty( + 1, 65_536, HEADS, HEAD, device="cuda", dtype=torch.bfloat16 + ) + kernel = registry.kernel(KERNELS["sm120"]) + assert kernel.supports(t, t, t, HEAD**-0.5) is None + + +def test_int8_admits_contexts_above_65535(monkeypatch, kernels: None) -> None: + monkeypatch.setenv(icl_attention.ENV, "int8") + t = torch.empty( + 1, 100_000, HEADS, HEAD, device="cuda", dtype=torch.bfloat16 + ) + assert registry.kernel(KERNELS["int8"]).supports(t, t, t, 0.125) is None + + +def test_int8_needs_batch_1(monkeypatch, kernels: None) -> None: + monkeypatch.setenv(icl_attention.ENV, "int8") + t = torch.empty(2, 2048, HEADS, HEAD, device="cuda", dtype=torch.bfloat16) + reason = registry.kernel(KERNELS["int8"]).supports(t, t, t, 0.125) + assert "batch 1" in reason + + +@pytest.mark.parametrize("exact", [False, True]) +def test_exact_keeps_attention_native( + monkeypatch, kernels: None, exact: bool +) -> None: + monkeypatch.setenv(icl_attention.ENV, "sm120") + torch._dynamo.reset() + fn = integration.compile_with_kernels(_attend, exact=exact) + with torch.inference_mode(): + _, codes = run_and_get_code(fn, *_seeded(2048, 2048)()) + assert ("sdm_inference.icl_attention.default(" in codes[-1]) != exact + + +def test_split_merge_is_deterministic(monkeypatch, kernels: None) -> None: + monkeypatch.setenv(icl_attention.ENV, "sm120") + q, k, v = _seeded(20000, 20000)() + kernel = registry.kernel(KERNELS["sm120"]) + with torch.inference_mode(): + first = registry.launch(kernel, q, k, v, HEAD**-0.5) + for _ in range(9): + out = registry.launch(kernel, q, k, v, HEAD**-0.5) + assert torch.equal(out, first) + + +class _Graph: + """``count`` flash calls with the shapes of the ICL fit site.""" + + def __init__(self, count: int) -> None: + self.graph = Graph() + like = torch.empty(1, 2, 8, 64, dtype=torch.bfloat16) + inputs = [] + for name in "qkv": + inputs.append(self.graph.placeholder(name)) + inputs[-1].meta["val"] = like + items = [] + for _ in range(count): + call = self.graph.call_function( + torch.ops.aten._scaled_dot_product_flash_attention.default, + tuple(inputs), + {"scale": 0.125}, + ) + items.append(self.graph.call_function(operator.getitem, (call, 0))) + items[-1].meta["val"] = like + self.graph.output(tuple(items)) + + def rewritten(self, setting: str) -> int: + with registry.enabled(attention=setting): + fx_pass.KernelPass(frozenset({"icl_attention"}))(self.graph) + target = torch.ops.sdm_inference.icl_attention.default + return sum(n.target == target for n in self.graph.nodes) + + +@pytest.mark.parametrize( + ("setting", "rewritten"), + [("off", 0), ("int8", 4), ("int8+ends", 2)], +) +def test_rule_follows_the_setting( + monkeypatch, setting: str, rewritten: int +) -> None: + monkeypatch.setattr(registry, "KERNELS", {}) + registry.register( + Kernel( + op="icl_attention", + name="int8", + supports=lambda q, k, v, scale: ( + None + if registry.attention().replace("+ends", "") == "int8" + else "other setting" + ), + impl=lambda q, k, v, scale: q, + cuda_symbols=("int8",), + exact=False, + ) + ) + assert _Graph(4).rewritten(setting) == rewritten + + +def _ops(options: dict) -> set[str]: + rules = options["post_grad_custom_post_pass"].rules.values() + return {op for pairs in rules for op, _ in pairs} + + +def test_exact_leaves_attention_native(kernels: None) -> None: + exact = _ops(integration.install(None, exact=True)) + assert exact + assert not exact & integration.NOT_BITWISE diff --git a/test/benchmark/inference/test_icl_qk.py b/test/benchmark/inference/test_icl_qk.py new file mode 100644 index 0000000000..39bddd8126 --- /dev/null +++ b/test/benchmark/inference/test_icl_qk.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Callable + +import pytest +import torch +import torch.nn.functional as F +from torch import Tensor + +from benchmark.inference.kernels import registry +from sdm._kernels import rmsnorm_cast +from sdm.cache import Cache +from sdm.models.kumo.tabular.icl import ICLBlock +from sdm.nn import LogScale + +SIZES = pytest.mark.parametrize( + ("d", "heads", "kv"), [(1024, 16, 2), (512, 8, 2), (512, 8, None)] +) +TRAIN = 512 + + +def _fit( + d: int, heads: int, kv: int | None, rows: int, train: int = TRAIN +) -> tuple: + torch.manual_seed(0) + block = ICLBlock(3, 3, d, 2, heads, kv, device="cuda") + + def fn(x: Tensor, y: Tensor) -> tuple: + cache = Cache() + out = block(x, y, cache=cache) + return out, [t for entry in cache.values() for t in entry] + + def make() -> tuple[Tensor, Tensor]: + g = torch.Generator("cuda").manual_seed(0) + x = torch.randn( + rows, d, device="cuda", generator=g, dtype=torch.bfloat16 + ) + return x, torch.randint(0, 3, (train,), device="cuda", generator=g) + + return fn, make + + +@SIZES +@pytest.mark.parametrize("rows", [TRAIN, TRAIN + 96]) +@pytest.mark.parametrize( + "kernel", ["sm120_icl_query_rms_scale", "sm120_icl_key_rms"] +) +def test_icl_fit( + matches_native: Callable, + d: int, + heads: int, + kv: int | None, + rows: int, + kernel: str, +) -> None: + matches_native(*_fit(d, heads, kv, rows), kernel) + + +@pytest.mark.parametrize(("d", "heads"), [(1024, 16), (512, 8)]) +def test_icl_last_layer_narrows_the_kv_projection( + matches_native: Callable, d: int, heads: int +) -> None: + code = matches_native(*_fit(d, heads, 2, 1024, 1024), "sm120_icl_key_rms") + shapes = [ + c["args"][0]["shape"] + for c in registry.census()["decisions"] + if c["kernel"] == "sm120_icl_key_rms" + ] + assert [1024, 4 * 64] in shapes, shapes + body = code[code.index("def call(") :] + assert "clone_split" not in body + assert "clone_slice" not in body + + +@pytest.mark.parametrize("m", [16, 1000]) +@pytest.mark.parametrize(("d", "heads"), [(1024, 16), (512, 8)]) +def test_icl_predict_query( + matches_native: Callable, m: int, d: int, heads: int +) -> None: + scaling = LogScale(heads, device="cuda") + + def fn(x: Tensor, w: Tensor, b: Tensor) -> Tensor: + q = F.linear(x, w, b).unflatten(-1, [heads, 64]) + return scaling(rmsnorm_cast(q, None, 1e-6), key_len=5000) + + def make() -> tuple[Tensor, ...]: + g = torch.Generator("cuda").manual_seed(0) + return tuple( + torch.randn(*s, device="cuda", generator=g) + for s in ((m, d), (d, d), (d,)) + ) + + matches_native(fn, make, "sm120_icl_query_rms_scale") diff --git a/test/benchmark/inference/test_measure.py b/test/benchmark/inference/test_measure.py new file mode 100644 index 0000000000..7915de6070 --- /dev/null +++ b/test/benchmark/inference/test_measure.py @@ -0,0 +1,319 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import csv +import json +from pathlib import Path + +import pytest +import torch + +from benchmark.inference.config import Config +from benchmark.inference.kernels import registry +from benchmark.inference.kernels.registry import Kernel +from benchmark.inference.report import summary, write +from benchmark.inference.runner import measure, verify_icl_attention +from benchmark.inference.schedule import is_optimized + +SECONDS = {"eager": 3.0, "compiled": 2.0, "optimized": 1.5} + + +class FakeBench: + """Fixed time per arm; the optimized arm selects and launches a kernel.""" + + def __init__( + self, + optimized_probability: float = 0.25, + eager_probability: float = 0.25, + unavailable: str | None = None, + compiled_kernels: tuple[str, ...] = ("triton_mm",), + optimized_kernels: tuple[str, ...] = ("fake_gemm_gelu",), + ) -> None: + self.optimized_probability = optimized_probability + self.eager_probability = eager_probability + self.unavailable = unavailable + self.kernels = { + "eager": ("triton_mm",), + "compiled": compiled_kernels, + "optimized": optimized_kernels, + } + + def _call(self, arm: str) -> None: + if is_optimized(arm): + args = torch.randn(2, 3), torch.randn(4, 3), None + with registry.enabled(): + kernel = registry.select("linear_gelu", *args) + if kernel is not None: + registry.launch(kernel, *args) + + def seconds(self, arm: str) -> float: + self._call(arm) + return SECONDS[arm.split(":")[0]] + + def snapshot(self, arm: str) -> dict[str, torch.Tensor]: + self._call(arm) + p = { + "optimized": self.optimized_probability, + "eager": self.eager_probability, + }.get(arm.split(":")[0], 0.25) + return {"probabilities": torch.tensor([[p, 1 - p]])} + + def profile(self, arm: str) -> dict[str, dict]: + self._call(arm) + base = arm.split(":")[0] + return { + name: {"count": 1, "ms": SECONDS[base]} + for name in self.kernels[base] + } + + +@pytest.fixture(autouse=True) +def _fake_kernel(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(registry, "KERNELS", {}) + registry.register( + Kernel( + op="linear_gelu", + name="fake", + supports=lambda x, weight, bias: None, + impl=lambda x, weight, bias: x @ weight.T, + cuda_symbols=("fake_gemm",), + ) + ) + + +def _config(tmp_path: Path, **kwargs: object) -> Config: + kwargs = {"blocks": 6, "warmup": 2, **kwargs} + return Config( + model_size="small", + context_rows=5000, + output=tmp_path / "out", + **kwargs, + ) + + +def test_completed_run_writes_all_outputs(tmp_path: Path) -> None: + config = _config(tmp_path) + report = measure(config, FakeBench()) + write(config.output, report) + + assert report["status"] == "completed" + assert report["qualification"]["optimized_vs_compiled"]["bitwise_equal"] + pairs = report["statistics"]["pairs"] + assert pairs["compiled_to_optimized"]["reduction_pct"] == 25.0 + assert pairs["eager_to_optimized"]["positive_blocks"] == 6 + assert report["kernels"]["kernels_verified"] + assert report["kernels"]["launches_per_optimized_call"] == {"fake": 1} + + with (config.output / "samples.csv").open() as stream: + rows = list(csv.DictReader(stream)) + assert len(rows) == 6 * 2 * 3 + assert {float(row["seconds"]) for row in rows} == set(SECONDS.values()) + saved = json.loads((config.output / "report.json").read_text()) + assert saved["config"]["context_rows"] == 5000 + summary = (config.output / "summary.md").read_text() + assert "25.00%" in summary + assert "WARNING" not in summary + with pytest.raises(FileExistsError): + write(config.output, report) + + +def test_mismatch_blocks_timing(tmp_path: Path) -> None: + config = _config(tmp_path) + report = measure(config, FakeBench(optimized_probability=0.5)) + write(config.output, report) + + assert report["status"] == "qualification_failed" + gate = report["qualification"]["optimized_vs_compiled"] + assert gate["first_mismatch"]["path"] == "probabilities" + assert "statistics" not in report + summary = (config.output / "summary.md").read_text() + assert "NOT bitwise equal" in summary + assert "Not timed" in summary + + +def test_mismatch_is_timed_when_allowed(tmp_path: Path) -> None: + config = _config(tmp_path, allow_mismatch=True) + report = measure(config, FakeBench(optimized_probability=0.5)) + assert report["status"] == "completed_with_mismatch" + assert len(report["samples"]) == 6 * 2 * 3 + + +def test_optimized_without_selected_kernel_falls_back_to_native( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(registry, "KERNELS", {}) + registry.register( + Kernel( + op="linear_gelu", + name="sm100_only", + supports=lambda x, weight, bias: "requires sm_100", + impl=lambda x, weight, bias: x, + cuda_symbols=("sm100",), + ) + ) + report = measure(_config(tmp_path), FakeBench()) + assert "requires sm_100" in report["optimized_unavailable"] + assert report["status"] == "completed_without_optimized" + assert set(report["statistics"]["arms"]) == {"eager", "compiled"} + + +def test_unavailable_kernels_fall_back_to_native(tmp_path: Path) -> None: + report = measure(_config(tmp_path), FakeBench(unavailable="no nvcc")) + assert report["optimized_unavailable"] == "no nvcc" + assert report["status"] == "completed_without_optimized" + assert set(report["statistics"]["arms"]) == {"eager", "compiled"} + + +def test_eager_and_compiled_only(tmp_path: Path) -> None: + config = _config(tmp_path, arms=("eager", "compiled"), blocks=2) + report = measure(config, FakeBench()) + assert report["status"] == "completed" + assert set(report["statistics"]["pairs"]) == {"eager_to_compiled"} + assert ( + report["qualification"]["compiled_vs_eager"]["argmax_agreement_pct"] + == 100.0 + ) + + +def test_kernel_also_in_compiled_profile_is_not_verified( + tmp_path: Path, +) -> None: + bench = FakeBench(compiled_kernels=("triton_mm", "fake_gemm_gelu")) + report = measure(_config(tmp_path), bench) + assert report["status"] == "completed" + assert not report["kernels"]["verification"]["fake"]["verified"] + assert not report["kernels"]["kernels_verified"] + + +def test_kernel_with_a_missing_symbol_is_not_verified( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(registry, "KERNELS", {}) + registry.register( + Kernel( + op="linear_gelu", + name="two_launches", + supports=lambda x, weight, bias: None, + impl=lambda x, weight, bias: x @ weight.T, + cuda_symbols=("fake_gemm", "fake_epilogue"), + ) + ) + report = measure(_config(tmp_path), FakeBench()) + verification = report["kernels"]["verification"]["two_launches"] + assert verification["optimized"] == ["fake_gemm_gelu"] + assert not verification["verified"] + assert not report["kernels"]["kernels_verified"] + + +def _register_tolerance_kernel(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(registry, "KERNELS", {}) + registry.register( + Kernel( + op="linear_gelu", + name="fake", + supports=lambda x, weight, bias: None, + impl=lambda x, weight, bias: x @ weight.T, + cuda_symbols=("fake_gemm",), + bitwise=lambda x, weight, bias: False, + ) + ) + + +def test_tolerance_gate_admits_a_difference_within_eager( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _register_tolerance_kernel(monkeypatch) + bench = FakeBench(optimized_probability=0.26, eager_probability=0.3) + report = measure(_config(tmp_path), bench) + assert report["qualification"]["gate"] == "tolerance" + assert report["status"] == "completed" + + +def test_tolerance_gate_rejects_a_difference_beyond_eager( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _register_tolerance_kernel(monkeypatch) + bench = FakeBench(optimized_probability=0.4, eager_probability=0.3) + report = measure(_config(tmp_path), bench) + assert report["status"] == "qualification_failed" + assert "Tolerance gate failed" in summary(report) + + +def test_tolerance_gate_snapshots_eager_without_timing_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _register_tolerance_kernel(monkeypatch) + config = _config(tmp_path, arms=("compiled", "optimized"), blocks=2) + bench = FakeBench(optimized_probability=0.26, eager_probability=0.3) + report = measure(config, bench) + assert report["status"] == "completed" + assert not report["qualification"]["eager_timed"] + assert "compiled_vs_eager" in report["qualification"] + assert set(report["statistics"]["arms"]) == {"compiled", "optimized"} + assert set(report["peak_memory"]) <= {"compiled", "optimized"} + assert "(untimed)" in summary(report) + + +def test_optimized_variants_are_qualified_and_timed_per_arm( + tmp_path: Path, +) -> None: + arms = ("compiled", "optimized:off", "optimized:int8", "eager") + config = _config(tmp_path, arms=arms, blocks=4) + report = measure(config, FakeBench()) + write(config.output, report) + + assert report["status"] == "completed" + assert set(report["qualification"]["arms"]) == set(arms[1:3]) + pairs = report["statistics"]["pairs"] + assert pairs["compiled_to_optimized:int8"]["reduction_pct"] == 25.0 + assert "eager_to_compiled" in pairs + for arm in arms[1:3]: + assert report["kernels"]["arms"][arm]["kernels_verified"] + assert "launches_per_optimized_call" not in report["kernels"] + assert len(report["samples"]) == 4 * 2 * 4 + assert "optimized:int8 vs compiled bitwise equal" in summary(report) + + +def test_icl_attention_launch_verification() -> None: + ok = { + "a": {"census": 23, "profiled": 23}, + "b": {"census": 0, "profiled": 0}, + } + verify_icl_attention(ok, "a", 23) + verify_icl_attention( + {k: {"census": 0, "profiled": 0} for k in ok}, None, 23 + ) + with pytest.raises(RuntimeError): + verify_icl_attention(ok, None, 23) + with pytest.raises(RuntimeError): + verify_icl_attention({"a": {"census": 23, "profiled": 22}}, "a", 23) + shared = {**ok, "b": {"census": 0, "profiled": None}} + verify_icl_attention(shared, "a", 23) + + +def test_lossy_kernel_is_opt_in_and_reported( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(registry, "KERNELS", {}) + registry.register( + Kernel( + op="linear_gelu", + name="fake", + supports=lambda x, weight, bias: None, + impl=lambda x, weight, bias: x @ weight.T, + cuda_symbols=("fake_gemm",), + exact=False, + ) + ) + config = _config(tmp_path, attention="int8") + report = measure(config, FakeBench(optimized_probability=0.4)) + write(config.output, report) + + qualification = report["qualification"] + assert report["status"] == "completed" + assert qualification["gate"] == "opt-in" + assert qualification["passed"] + summary = (config.output / "summary.md").read_text() + assert "optimized arm is lossy" in summary + assert "`--attention int8`" in summary + assert "compiled vs eager" in summary diff --git a/test/benchmark/inference/test_registry.py b/test/benchmark/inference/test_registry.py new file mode 100644 index 0000000000..b5abc40d1c --- /dev/null +++ b/test/benchmark/inference/test_registry.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import torch + +from benchmark.inference.kernels import registry +from benchmark.inference.kernels.registry import Kernel + + +@pytest.fixture(autouse=True) +def _empty_registry(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(registry, "KERNELS", {}) + registry.reset_census() + + +def _kernel(name: str, reason: str | None = None) -> Kernel: + return Kernel( + op="linear_gelu", + name=name, + supports=lambda x, weight, bias: reason, + impl=lambda x, weight, bias: torch.full( + (x.size(0), weight.size(0)), 7.0 + ), + cuda_symbols=(name,), + ) + + +def _args() -> tuple[torch.Tensor, ...]: + return torch.randn(2, 3), torch.randn(4, 3), torch.randn(4) + + +def test_nothing_is_selected_outside_enabled() -> None: + registry.register(_kernel("fast")) + assert registry.select("linear_gelu", *_args()) is None + assert registry.census() == { + "decisions": [], + "launches": {}, + "shadow": {}, + } + + +def test_first_admitting_kernel_is_selected_and_recorded() -> None: + registry.register(_kernel("strict", reason="needs bfloat16")) + registry.register(_kernel("fast")) + with registry.enabled(): + for _ in range(2): + kernel = registry.select("linear_gelu", *_args()) + out = registry.launch(kernel, *_args()) + assert torch.equal(out, torch.full((2, 4), 7.0)) + + census = registry.census() + assert census["launches"] == {"fast": 2} + decision = census["decisions"][0] + assert decision["kernel"] == "fast" + assert decision["rejected"] == {"strict": "needs bfloat16"} + assert decision["args"][0]["shape"] == [2, 3] + assert decision["args"][0]["dtype"] == "torch.float32" + + registry.reset_census() + assert registry.census() == { + "decisions": [], + "launches": {}, + "shadow": {}, + } + + +def test_native_when_no_kernel_admits() -> None: + registry.register(_kernel("strict", reason="needs sm_120")) + with registry.enabled(): + assert registry.select("linear_gelu", *_args()) is None + census = registry.census() + assert census["decisions"][0]["kernel"] == "native" + assert census["decisions"][0]["rejected"] == {"strict": "needs sm_120"} + assert census["launches"] == {} + + +def test_register_rejects_unknown_op_and_duplicate_name() -> None: + registry.register(_kernel("fast")) + with pytest.raises(ValueError, match="already registered"): + registry.register(_kernel("fast")) + with pytest.raises(ValueError, match="unknown op"): + registry.register( + Kernel( + op="missing", + name="other", + supports=lambda: None, + impl=lambda: torch.zeros(1), + cuda_symbols=(), + ) + ) + + +@pytest.mark.parametrize( + ("setting", "backend", "options"), + [ + ("sm120", "sm120", set()), + ("off", "off", set()), + ("sm120+uncapped", "sm120", {"uncapped"}), + ("int8+ends", "int8", {"ends"}), + ], +) +def test_parse_attention(setting: str, backend: str, options: set) -> None: + assert registry.parse_attention(setting) == (backend, options) + + +@pytest.mark.parametrize( + "setting", + ["", "fp8", "sm120+ends", "int8+fast", "int8+ends+ends", "int8+uncapped"], +) +def test_parse_attention_rejects(setting: str) -> None: + with pytest.raises(ValueError, match="ICL attention"): + registry.parse_attention(setting) + + +def test_attention_setting_is_scoped_to_enabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv(registry.ATTENTION_ENV, raising=False) + assert registry.attention() == "sm120" + monkeypatch.setenv(registry.ATTENTION_ENV, "off") + with registry.enabled(attention="int8+ends"): + assert registry.attention() == "int8+ends" + with registry.enabled(): + assert registry.attention() == "off" + assert registry.attention() == "off" + + +def test_decisions_record_lossy_kernels() -> None: + registry.register(Kernel(**{**_kernel("lossy").__dict__, "exact": False})) + with registry.enabled(): + registry.select("linear_gelu", *_args()) + assert not registry.census()["decisions"][0]["exact"] diff --git a/test/benchmark/inference/test_schedule.py b/test/benchmark/inference/test_schedule.py new file mode 100644 index 0000000000..ea8e240f89 --- /dev/null +++ b/test/benchmark/inference/test_schedule.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from collections import Counter +from itertools import pairwise, permutations + +import pytest + +from benchmark.inference.schedule import pairs, schedule, summarize + +ARMS = ("eager", "compiled", "optimized") + + +def test_schedule_is_mirrored_and_balanced() -> None: + blocks = schedule(ARMS, 12) + assert len(blocks) == 12 + for block in blocks: + assert block[3:] == block[:3][::-1] + assert Counter(block) == dict.fromkeys(ARMS, 2) + assert Counter(block[:3] for block in blocks) == dict.fromkeys( + permutations(ARMS), 2 + ) + assert schedule(ARMS, 12) == blocks + + +def test_schedule_rejects_unbalanced_block_count() -> None: + with pytest.raises(ValueError, match="multiple of 3"): + schedule(ARMS, 4) + + +@pytest.mark.parametrize("n", [4, 5, 6]) +def test_schedule_uses_a_williams_square_for_many_arms(n: int) -> None: + arms = tuple(f"arm{i}" for i in range(n)) + blocks = schedule(arms, 2 * n) + firsts = [block[:n] for block in blocks] + assert len(set(firsts)) == n + for position in range(n): + assert Counter(row[position] for row in firsts) == dict.fromkeys( + arms, 2 + ) + if n % 2 == 0: + follows = Counter(p for row in set(firsts) for p in pairwise(row)) + assert set(follows.values()) == {1} + assert len(follows) == n * (n - 1) + + +def test_pairs_compare_every_optimized_arm() -> None: + arms = ("compiled", "optimized:off", "optimized:int8", "eager") + assert pairs(arms) == [ + ("compiled", "optimized:off"), + ("compiled", "optimized:int8"), + ("eager", "optimized:off"), + ("eager", "optimized:int8"), + ("eager", "compiled"), + ] + + +def test_summarize_from_raw_samples() -> None: + seconds = { + (0, "compiled"): [1.0, 3.0], + (0, "optimized"): [1.0, 1.0], + (1, "compiled"): [2.0, 2.0], + (1, "optimized"): [3.0, 3.0], + } + samples = [ + {"block": block, "arm": arm, "seconds": value} + for (block, arm), values in seconds.items() + for value in values + ] + stats = summarize(samples, ("compiled", "optimized")) + + assert stats["arms"]["compiled"]["mean"] == 2.0 + assert stats["arms"]["compiled"]["median"] == 2.0 + assert stats["arms"]["optimized"]["min"] == 1.0 + assert stats["arms"]["optimized"]["stdev"] == pytest.approx(1.1547005) + assert set(stats["pairs"]) == {"compiled_to_optimized"} + pair = stats["pairs"]["compiled_to_optimized"] + assert pair["reduction_pct"] == 0.0 + assert pair["block_effects"] == [1.0, -1.0] + assert pair["median_block_effect"] == 0.0 + assert pair["positive_blocks"] == 1 diff --git a/test/benchmark/inference/test_snapshot.py b/test/benchmark/inference/test_snapshot.py new file mode 100644 index 0000000000..40d2d42896 --- /dev/null +++ b/test/benchmark/inference/test_snapshot.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass + +import pyarrow as pa +import pytest +import torch + +from benchmark.inference.snapshot import characterize, compare, leaves +from sdm import StringTensor + + +@dataclass +class _State: + weight: torch.Tensor + name: str + + +def test_leaves_walks_nested_containers() -> None: + tree = { + "a": (torch.ones(2), [torch.zeros(1)]), + 3: _State(torch.ones(1), "x"), + } + assert set(leaves(tree, "cache")) == { + "cache/a/0", + "cache/a/1/0", + "cache/3/weight", + } + + +def test_compare_is_bitwise() -> None: + a = {"p": torch.tensor([0.0, float("nan"), 1.0], dtype=torch.bfloat16)} + assert compare(a, {"p": a["p"].clone()})["bitwise_equal"] + signed_zero = {"p": torch.tensor([-0.0, float("nan"), 1.0]).bfloat16()} + assert not compare(a, signed_zero)["bitwise_equal"] + + +def test_compare_reports_first_mismatch() -> None: + reference = {"x": torch.ones(4), "y": torch.arange(4), "z": torch.ones(2)} + candidate = { + "x": torch.ones(4), + "y": torch.tensor([0, 1, 5, 7]), + "z": torch.ones(3), + } + result = compare(reference, candidate) + assert not result["bitwise_equal"] + assert result["mismatched_leaves"] == 2 + assert result["first_mismatch"] == { + "path": "y", + "differing_elements": 2, + "elements": 4, + "max_abs_diff": 4.0, + } + missing = compare(reference, {"x": torch.ones(4)}) + assert missing["first_mismatch"]["reason"] == "missing in candidate" + + +def test_characterize_probability_distance() -> None: + p = torch.tensor([[0.9, 0.1], [0.4, 0.6]]) + q = torch.tensor([[0.8, 0.2], [0.6, 0.4]]) + result = characterize({"probabilities": p}, {"probabilities": q}) + assert result["probability_max_abs_diff"] == pytest.approx(0.2) + assert result["probability_mean_abs_diff"] == pytest.approx(0.15) + assert result["argmax_agreement_pct"] == 50.0 + + +def test_string_tensor_leaves_compare_bitwise() -> None: + strings = StringTensor.from_arrow(pa.array(["a", "bc", "def"])) + other = StringTensor.from_arrow(pa.array(["a", "bc", "deg"])) + ref = leaves({"classes": strings}, "cache") + assert ref + assert compare(ref, leaves({"classes": strings}, "cache"))["bitwise_equal"] + assert not compare(ref, leaves({"classes": other}, "cache"))[ + "bitwise_equal" + ] From 8acec331abcf3f6118e1f825b74823346e88cd19 Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Mon, 26 Oct 2026 23:09:11 +0000 Subject: [PATCH 2/2] Let the 8-bit ICL attention take ECOC batches Classification with more than 10 classes runs its ECOC codes as an ICL batch above 1. The 8-bit kernel already keeps per (batch, head) statistics, so admit any batch and head count up to 65,535 with batch * heads * padded rows below 2^31, and match +ends fit calls by row count instead of batch 1. Add the ECOC speed and accuracy results. Signed-off-by: Vedaant Jain --- benchmark/attention/compare.py | 13 +---- benchmark/attention/compare_data.py | 4 ++ .../icl_attention_int8.cu | 6 +- .../kernels/fusions/icl_attention.py | 18 +++--- docs/inference-kernel-interface.md | 2 +- docs/inference-kernel-results.md | 22 ++++++-- docs/inference-kernels.md | 4 +- .../benchmark/inference/test_icl_attention.py | 56 +++++++++++++++---- 8 files changed, 83 insertions(+), 42 deletions(-) diff --git a/benchmark/attention/compare.py b/benchmark/attention/compare.py index 19ff0a100b..b7b8e895e2 100644 --- a/benchmark/attention/compare.py +++ b/benchmark/attention/compare.py @@ -38,9 +38,6 @@ "int8": "sm120_int8_icl", } CLIP = 1e-7 -# More classes than this run ECOC codes as an ICL batch above 1, which the -# 8-bit kernel does not take (batch 1 only). -MAX_INT8_CLASSES = 10 # LEVELS[i] is the (i + 1) / 1000 quantile. LEVELS = (np.arange(999) + 1) / 1000 MEDIAN = 499 @@ -193,15 +190,9 @@ def distance(reference, candidate, regression, scale): def run_group(args, name, info, df, size, context, model, task): regression = info["task"] == "regression" - settings = [ - s - for s in args.settings - if (info["classes"] or 0) <= MAX_INT8_CLASSES - or registry.parse_attention(s)[0] != "int8" - ] - arms = ["eager", "compiled", *settings] + arms = ["eager", "compiled", *args.settings] torch._dynamo.reset() - forwards = Arms(model, task, settings, args.device) + forwards = Arms(model, task, args.settings, args.device) for query_rows in args.queries: for seed in args.seeds: path = ( diff --git a/benchmark/attention/compare_data.py b/benchmark/attention/compare_data.py index fea6641ec7..8577ae2c16 100644 --- a/benchmark/attention/compare_data.py +++ b/benchmark/attention/compare_data.py @@ -31,6 +31,10 @@ "volkert": 41166, "letter": 6, "bank_marketing": 1461, + "kropt": 184, + "walking_activity": 1509, + "helena": 41169, + "dionis": 41167, } diff --git a/benchmark/inference/kernels/csrc/icl_attention_int8_sm120/icl_attention_int8.cu b/benchmark/inference/kernels/csrc/icl_attention_int8_sm120/icl_attention_int8.cu index e0803c8593..ad48ac3b5a 100644 --- a/benchmark/inference/kernels/csrc/icl_attention_int8_sm120/icl_attention_int8.cu +++ b/benchmark/inference/kernels/csrc/icl_attention_int8_sm120/icl_attention_int8.cu @@ -489,10 +489,10 @@ void run(const at::Tensor& q, const at::Tensor& k, const at::Tensor& v, at::Tens out.stride(0) % 2 == 0 && out.stride(1) % 2 == 0 && out.stride(2) % 2 == 0, "icl_attention_int8: out must have the dtype and shape of q, a unit last stride and even strides"); TORCH_CHECK(k.sizes() == q.sizes() && v.sizes() == q.sizes(), "icl_attention_int8: k and v must have the shape of q"); + TORCH_CHECK(q.size(2) > 0 && q.size(0) <= 65535 && q.size(1) <= 65535 && + q.size(0) * q.size(1) * (q.size(2) + BN) < (1ll << 31), + "icl_attention_int8: S must be positive, B and H at most 65535 and B * H * (S + 64) below 2^31"); int B = q.size(0), H = q.size(1), S = q.size(2); - TORCH_CHECK(S > 0 && (int64_t)S * std::max(q.stride(2), std::max(k.stride(2), v.stride(2))) < (1ll << 31) && - (int64_t)B * H * S * D < (1ll << 31), - "icl_attention_int8: S must be positive and element offsets must fit in int32"); c10::cuda::CUDAGuard guard(q.device()); auto stream = at::cuda::getCurrentCUDAStream(); int S_pad = (S + BN - 1) / BN * BN; diff --git a/benchmark/inference/kernels/fusions/icl_attention.py b/benchmark/inference/kernels/fusions/icl_attention.py index f1bafc77fc..1ad2b1851d 100644 --- a/benchmark/inference/kernels/fusions/icl_attention.py +++ b/benchmark/inference/kernels/fusions/icl_attention.py @@ -45,6 +45,7 @@ # Largest context the BF16 kernel's error and determinism were checked at; # Medium at 100k misses the tolerance gate. MAX_S = 65_535 +GRID = 65_535 FA2_SYMBOL = "void flash::flash_fwd_kernel<" INT8_SYMBOLS = ( "icl_attention_int8::quant_kernel", @@ -114,13 +115,12 @@ def _bf16_limits(q: Tensor) -> str | None: return None -def _int8_limits(q: Tensor, k: Tensor, v: Tensor) -> str | None: +def _int8_limits(q: Tensor) -> str | None: b, s, h, _ = q.shape - if b != 1: - return f"needs batch 1 (the ICL fit site); got {b}" - rows = max(t.stride(1) for t in (q, k, v)) - if s * rows >= 2**31 or b * h * s * HEAD >= 2**31: - return "needs 32-bit element offsets" + if b > GRID or h > GRID: + return f"needs batch and heads <= {GRID}; got {b}, {h}" + if b * h * (s + 64) >= 2**31: + return "needs batch * heads * padded S < 2**31" return None @@ -139,7 +139,7 @@ def supports(q: Tensor, k: Tensor, v: Tensor, scale: float): if want == "int8": return _admits( q, k, v, scale, (torch.bfloat16, torch.float16) - ) or _int8_limits(q, k, v) + ) or _int8_limits(q) return _admits(q, k, v, scale) or ( None if "uncapped" in chosen else _bf16_limits(q) ) @@ -195,12 +195,12 @@ def _flash_node(node: Node) -> tuple[tuple[Node, ...], Node] | None: def _fit_call(node: Node) -> bool: - """A batch-1 self-attention flash call: an ICL fit call.""" + """A self-attention flash call over at least ``MIN_S`` rows: ICL fit.""" found = _flash_node(node) if found is None: return False q, k, _ = (val(t) for t in found[0]) - return q.shape == k.shape and q.size(0) == 1 and q.size(3) == HEAD + return q.shape == k.shape and q.size(2) >= MIN_S and q.size(3) == HEAD def _rule(graph: Graph, node: Node) -> bool: diff --git a/docs/inference-kernel-interface.md b/docs/inference-kernel-interface.md index 6cbb7d86b6..7212a9719d 100644 --- a/docs/inference-kernel-interface.md +++ b/docs/inference-kernel-interface.md @@ -81,7 +81,7 @@ Every op is a `sdm_inference::` custom op with a reference in `registry.OPS` | `int8+ends` | as `int8`, first and last ICL fit call of each graph native | `opt-in` | | `off` | none | bitwise (with the exact ops) | -The rule matches the flash node of a self-attention with head dim 64 and at least 1,024 rows (fit; predict has 16 queries against the cache). The BF16 kernels admit at most 65,535 rows, the largest context at which their error and determinism were checked; the INT8 kernel has no row cap but needs batch 1 and 32-bit element offsets. `exact=True` keeps attention native whatever the setting. +The rule matches the flash node of a self-attention with head dim 64 and at least 1,024 rows (fit; predict has 16 queries against the cache). The BF16 kernels admit at most 65,535 rows, the largest context at which their error and determinism were checked; the INT8 kernel has no row cap and takes any batch (ECOC runs more than 10 classes as an ICL batch of codes) up to 65,535, with batch × heads × padded rows below 2³¹. `+ends` counts as fit calls the self-attention calls of at least 1,024 rows. `exact=True` keeps attention native whatever the setting. ## Shape bounds and scope diff --git a/docs/inference-kernel-results.md b/docs/inference-kernel-results.md index 456a0c9481..fb650ae45a 100644 --- a/docs/inference-kernel-results.md +++ b/docs/inference-kernel-results.md @@ -86,6 +86,20 @@ Max |Δprob| against compiled on the 16 timed query rows of each cell, with comp Class agreement was 100% for every arm in every cell. The 8-bit attention stays inside the gap except at Large 100k, where int8 is 5.7x and int8+ends 2x over a small gap; the 16-row gap varies a lot between cells, so the accuracy study below is the better guide. +### More than 10 classes (ECOC) + +Classification with more than 10 classes runs its ECOC codes as an ICL batch of T entries, so each fit attention call is T times larger. The 8-bit kernel takes these batches (each entry keeps its own quantization statistics). Large, whole table up to the context shown, same cell layout: + +| Dataset | Classes | T | Context | a compiled | c +bf16 attn | d +int8 attn | +| ---------------- | ------- | -- | ------- | ----------- | ------------------- | -------------------- | +| letter | 26 | 8 | 18,976 | 4,677, 22.9 | 4,269 (-8.7%), 18.2 | 3,561 (-23.9%), 18.2 | +| kropt | 18 | 8 | 18,000 | 3,365, 21.6 | 3,097 (-8.0%), 16.8 | 2,476 (-26.4%), 16.8 | +| walking_activity | 22 | 8 | 18,000 | 3,186, 21.5 | 2,936 (-7.8%), 16.7 | 2,314 (-27.4%), 16.7 | +| helena | 100 | 12 | 12,000 | 4,864, 22.1 | 4,439 (-8.7%), 17.8 | 4,047 (-16.8%), 17.8 | +| dionis | 355 | 40 | 3,500 | 8,063, 26.0 | 7,468 (-7.4%), 25.6 | 7,380 (-8.5%), 25.6 | + +The 8-bit kernel launched at all 23 fit calls in every cell (21 with `+ends`). Memory limits these tasks: T x context rows of about 150,000 fills a 32 GB card. The 8-bit gain shrinks with the context per entry (dionis: 3,500 rows), as at small contexts above. + ## Effect of SDM #1055 on the compiled baseline SDM #1055 makes attention and cell embedding chunk under `torch.compile` with a memory budget per chunk. Compiled Large 50k, compiled arm only, 2 processes x 6 blocks each: @@ -136,14 +150,14 @@ Large, 50,000 context rows (47,818 for adult, the whole table), 16 and 1,024 que - **int8 (d)** is over the per-seed gap in 1 of 3 covertype 16-row seeds (0.0275 against 0.0098). At 1,024 rows it flips 0 to 0.29% of rows (eager 0.6 to 1.7%). Log-loss moves by -0.0002 to +0.0062. - **int8+ends** is over the gap in 1 of 3 covertype 16-row seeds, flips fewer rows than int8, and moves log-loss by -0.0022 to +0.0052. -Two more classification tables, same layout. bank_marketing uses 44,187 context rows and letter 18,976, the whole table in each case. On letter the 8-bit settings stay native (see below). +Two more classification tables, same layout. bank_marketing uses 44,187 context rows and letter 18,976, the whole table in each case. | Dataset | Query rows | compiled vs eager | c BF16 | d int8 | int8+ends | | -------------- | ---------- | ------------------ | ------------------ | ------------------ | ------------------ | | bank_marketing | 16 | 0.0248, 0, -0.0023 | 0.0037, 0, +0.0000 | 0.0062, 0, -0.0006 | 0.0035, 0, -0.0001 | | bank_marketing | 1,024 | 0.0397, 6, -0.0001 | 0.0064, 0, +0.0000 | 0.0063, 2, -0.0000 | 0.0060, 0, -0.0000 | -| letter | 16 | 0.0154, 0, +0.0010 | 0.0014, 0, -0.0001 | native | native | -| letter | 1,024 | 0.0675, 0, +0.0001 | 0.0104, 0, +0.0000 | native | native | +| letter | 16 | 0.0154, 0, +0.0010 | 0.0014, 0, -0.0001 | 0.0009, 0, -0.0001 | 0.0031, 0, -0.0001 | +| letter | 1,024 | 0.0675, 0, +0.0001 | 0.0104, 0, +0.0000 | 0.0274, 0, +0.0000 | 0.0280, 0, -0.0000 | Small and Medium, 50,000 context rows, seed 0 only, so the flips are out of 16 or 1,024 rows: @@ -182,7 +196,7 @@ Regression, Large, 6 TabArena tables (context: the table minus 1,024, at most 50 - **BF16 (c):** the max over all quantiles is inside the same seed's eager gap in 36 of 36 runs, and so is the point prediction. Within each band of quantiles (5% tails, 45 to 55% median) it passes eager's band max in 2 of 108 cases, both in the high tail (diamonds 16 rows: 0.063 against 0.031; qsar_tid_11 1,024 rows: 0.453 against 0.441). RMSE moves at most 1.1% in any seed (eager: up to 7.0%). - **int8 (d):** the point prediction is inside the gap in 36 of 36 runs; the max over all quantiles in 31 of 36. The misses are on diamonds, qsar_tid_11 and superconductivity, in the tail quantiles (12 of 108 band cases). RMSE moves at most 2.4% in a seed. - **int8+ends:** point prediction 36 of 36; all quantiles 33 of 36; 10 of 108 band cases over; RMSE at most 3.4% in a seed (diamonds, 16 rows). -- **letter:** with 26 classes ECOC runs the ICL attention as a batch of 8. The 8-bit kernel takes only batch 1 and refuses it ("needs batch 1 (the ICL fit site); got 8"), so attention stays native and d and int8+ends equal compiled. BF16 takes batch 8 and is inside the gap. +- **More than 10 classes (ECOC):** on Large, letter (26 classes, above), kropt and walking_activity (18,000 context rows), helena (12,000, 100 classes) and dionis (3,500, 355 classes, seed 0 only), the 8-bit settings behave as on the tables with 10 classes or fewer: inside the same seed's gap in every run, and at 1,024 query rows no more flips than eager (int8: 0, 5, 8, 24 and 3 rows over the seeds; eager: 0, 11, 24, 131 and 11) with log-loss moving by at most ±0.0008. ### Determinism diff --git a/docs/inference-kernels.md b/docs/inference-kernels.md index 5a0ba103a5..9e31b74dfe 100644 --- a/docs/inference-kernels.md +++ b/docs/inference-kernels.md @@ -125,7 +125,7 @@ Each family has one Python file in `benchmark/inference/kernels/fusions/` and it - **Does.** Quantizes K to INT8 per 64-key block after subtracting the per-head K mean (smoothing), V to FP8 E4M3 per channel and Q to INT8 per row; Q·Kᵀ runs on the INT8 `mma.sync` path, softmax in FP32, P·V in FP8 with FP32 accumulation. Source: `benchmark/inference/kernels/csrc/icl_attention_int8_sm120/icl_attention_int8.cu`, kernel `sm120_int8_icl`. The K mean is summed in a fixed order, so outputs are deterministic. - **Settings.** `int8`, or `int8+ends`, which keeps the first and last ICL fit call of each graph native. - **Gate.** Lossy by design (`exact=False`), never selected by default. The benchmark records the distance to compiled (and compiled vs eager when the eager arm runs) as gate `opt-in` and times the run. -- **Shapes.** BF16 or FP16, batch 1, head dim 64, at least 1,024 rows, 32-bit element offsets. Classification with more than 10 classes runs its ECOC codes as an ICL batch above 1 (8 for 26 classes), so attention stays native there; `benchmark.attention.compare` skips the 8-bit settings on those tables. +- **Shapes.** BF16 or FP16, head dim 64, at least 1,024 rows, batch and heads at most 65,535, batch × heads × padded rows below 2³¹. Classification with more than 10 classes runs its ECOC codes as an ICL batch above 1 (8 for 26 classes); each batch entry gets its own quantization statistics, so a batched call equals its entries run alone. - **Measured on SDM 842c408fe** (torch 2.13): 1.6 to 1.7x flash standalone; Large 50k fit + predict 23.6% faster with the 8-bit attention alone and 33.8% with every exact kernel; class flips 0.16% at 1,024 query rows (compiled vs eager 0.92%); on TabArena, 1 of 74 per-task tests significant after correction (+0.014% log-loss). ## Default on or opt-in @@ -192,4 +192,4 @@ Chunking changes what the rules see: | `icl_query_rms_scale`, `icl_key_rms` | all | no row bound; narrowed last-layer KV projection for M in 513..65,535 | full projection (outside the M range), native (other shapes) | | `rope_rms_uneven` | all | as `rope_rms`, on the longer of Q and K | native | | `icl_attention` BF16 (not bitwise) | all | self-attention of 1,024 to 65,535 rows, head dim 64 | native | -| `icl_attention` int8 (opt-in, lossy) | all | self-attention of at least 1,024 rows, head dim 64, batch 1 | native | +| `icl_attention` int8 (opt-in, lossy) | all | self-attention of at least 1,024 rows, head dim 64, batch at most 65,535 | native | diff --git a/test/benchmark/inference/test_icl_attention.py b/test/benchmark/inference/test_icl_attention.py index 5dc68ee206..f26b1970e8 100644 --- a/test/benchmark/inference/test_icl_attention.py +++ b/test/benchmark/inference/test_icl_attention.py @@ -24,12 +24,12 @@ ATOL = {"sm120": 2e-2, "fa2": 2e-2, "int8": 8e-2} -def _seeded(queries: int, keys: int) -> Callable[[], tuple]: +def _seeded(queries: int, keys: int, batch: int = 1) -> Callable[[], tuple]: def make() -> tuple: g = torch.Generator("cuda").manual_seed(0) return tuple( torch.randn( - 1, + batch, n, HEADS, HEAD, @@ -57,10 +57,11 @@ def mode(request, monkeypatch, kernels: None) -> str: return request.param +@pytest.mark.parametrize("batch", [1, 8]) def test_fit_attention_is_replaced( - close_to_native: Callable, mode: str + close_to_native: Callable, mode: str, batch: int ) -> None: - make = _seeded(2048, 2048) + make = _seeded(2048, 2048, batch) code = close_to_native(_attend, make, KERNELS[mode], atol=ATOL[mode]) assert "sdm_inference.icl_attention.default(" in code q, k, v = make() @@ -114,11 +115,39 @@ def test_int8_admits_contexts_above_65535(monkeypatch, kernels: None) -> None: assert registry.kernel(KERNELS["int8"]).supports(t, t, t, 0.125) is None -def test_int8_needs_batch_1(monkeypatch, kernels: None) -> None: +def test_int8_batch_bounds(monkeypatch, kernels: None) -> None: monkeypatch.setenv(icl_attention.ENV, "int8") - t = torch.empty(2, 2048, HEADS, HEAD, device="cuda", dtype=torch.bfloat16) - reason = registry.kernel(KERNELS["int8"]).supports(t, t, t, 0.125) - assert "batch 1" in reason + t = torch.empty(1, 2048, HEADS, HEAD, device="cuda", dtype=torch.bfloat16) + kernel = registry.kernel(KERNELS["int8"]) + assert kernel.supports(*(t.expand(65_535, -1, -1, -1),) * 3, 0.125) is None + wide = t.expand(65_536, -1, -1, -1) + assert "batch and heads" in kernel.supports(wide, wide, wide, 0.125) + long = t[:, :1].expand(8, 2**25, -1, -1) + assert "padded S" in kernel.supports(long, long, long, 0.125) + + +@pytest.mark.parametrize( + ("batch", "rows"), [(2, 1024), (8, 3001), (12, 20_000)] +) +def test_int8_batch_equals_each_entry_alone( + kernels: None, batch: int, rows: int +) -> None: + g = torch.Generator("cuda").manual_seed(batch) + qkv = torch.randn( + batch, + rows, + 3 * HEADS * HEAD, + device="cuda", + generator=g, + dtype=torch.bfloat16, + ) + q, k, v = (t.unflatten(-1, (HEADS, HEAD)) for t in qkv.chunk(3, -1)) + impl = registry.kernel(KERNELS["int8"]).impl + with torch.inference_mode(): + out = impl(q, k, v, HEAD**-0.5) + for i in range(batch): + alone = impl(q[i : i + 1], k[i : i + 1], v[i : i + 1], HEAD**-0.5) + assert torch.equal(out[i : i + 1], alone), i @pytest.mark.parametrize("exact", [False, True]) @@ -147,9 +176,11 @@ def test_split_merge_is_deterministic(monkeypatch, kernels: None) -> None: class _Graph: """``count`` flash calls with the shapes of the ICL fit site.""" - def __init__(self, count: int) -> None: + def __init__(self, count: int, batch: int = 1) -> None: self.graph = Graph() - like = torch.empty(1, 2, 8, 64, dtype=torch.bfloat16) + like = torch.empty( + batch, 8, 1024, 64, dtype=torch.bfloat16, device="meta" + ) inputs = [] for name in "qkv": inputs.append(self.graph.placeholder(name)) @@ -172,12 +203,13 @@ def rewritten(self, setting: str) -> int: return sum(n.target == target for n in self.graph.nodes) +@pytest.mark.parametrize("batch", [1, 8]) @pytest.mark.parametrize( ("setting", "rewritten"), [("off", 0), ("int8", 4), ("int8+ends", 2)], ) def test_rule_follows_the_setting( - monkeypatch, setting: str, rewritten: int + monkeypatch, setting: str, rewritten: int, batch: int ) -> None: monkeypatch.setattr(registry, "KERNELS", {}) registry.register( @@ -194,7 +226,7 @@ def test_rule_follows_the_setting( exact=False, ) ) - assert _Graph(4).rewritten(setting) == rewritten + assert _Graph(4, batch).rewritten(setting) == rewritten def _ops(options: dict) -> set[str]: