From 0f3e87718c20b78c2df41c51b47f6e1ef844d251 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:54:14 -0800 Subject: [PATCH 1/4] Slice attention batches above the kernel limit in an operator CUDA attention kernels accept at most 65,535 batch entries per call, so `SDPA` slices larger batches. A compiled graph held one kernel call per slice of the batch it was traced with: a dynamic batch compiled a new graph for every number of slices. The slicing now runs inside the `sdm::sdpa` operator whenever the batch does not provably fit a single call. The operator is not traced, so one graph serves any batch. It returns a contiguous tensor, which is what compiled code expects from it. Batches that provably fit and calls with gradients keep the plain call. Compiled code does not apply autocast inside the operator, so attention would silently run in full precision there. `SDPA` therefore converts the inputs of the operator to the autocast dtype, except float64, as autocast does. --- sdm/nn/attention.py | 122 +++++++++++++++++++++++++++++++--------- test/nn/test_compile.py | 35 ++++++++++++ 2 files changed, 130 insertions(+), 27 deletions(-) diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index 2a6dc0f3c..6670e993a 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -9,6 +9,7 @@ import torch import torch.nn.functional as F from torch import Tensor +from torch.fx.experimental.symbolic_shapes import statically_known_true from torch.nn import Linear from sdm._memory import _split_tensor, chunk_memory_limit @@ -16,6 +17,75 @@ from sdm.nn import QueryScaling +def _sdpa( + query: Tensor, # [B, Hq, Q, C] + key: Tensor, # [B, Hkv, KV, C] + value: Tensor, # [B, Hkv, KV, C] + attn_mask: Tensor | None, # [B, Hq, Q, KV] + dropout_p: float, + enable_gqa: bool, + is_causal: bool, + scale: float | None, +) -> Tensor: # [B, Q, Hq, C] + # CUDA attention kernels accept at most 65,535 batch entries per call. + queries = _split_tensor(query, 65_535, dim=0) + masks = ( + (None,) * len(queries) + if attn_mask is None + else _split_tensor(attn_mask, 65_535, dim=0) + ) + outs = [] + for chunk_query, chunk_key, chunk_value, chunk_mask in zip( + queries, + _split_tensor(key, 65_535, dim=0), + _split_tensor(value, 65_535, dim=0), + masks, + strict=True, + ): + out = F.scaled_dot_product_attention( + query=chunk_query, + key=chunk_key, + value=chunk_value, + attn_mask=chunk_mask, + dropout_p=dropout_p, + enable_gqa=enable_gqa, + is_causal=is_causal, + scale=scale, + ) + outs.append(out.transpose(-3, -2)) + return outs[0] if len(outs) == 1 else torch.cat(outs) + + +@torch.library.custom_op("sdm::sdpa", mutates_args=()) +def _sdpa_operator( + query: Tensor, + key: Tensor, + value: Tensor, + attn_mask: Tensor | None, + dropout_p: float, + enable_gqa: bool, + is_causal: bool, + scale: float | None, +) -> Tensor: + """Run `_sdpa` as one operator that `torch.compile` does not trace. + + Tracing its loop would recompile whenever the number of slices changes. + """ + # Compiled code expects a contiguous output, as the fake below declares. + return _sdpa( + query, key, value, attn_mask, dropout_p, enable_gqa, is_causal, scale + ).contiguous() + + +@_sdpa_operator.register_fake +def _sdpa_operator_fake( + query: Tensor, key: Tensor, value: Tensor, *args: Any, **kwargs: Any +) -> Tensor: + return query.new_empty( + (query.size(0), query.size(2), query.size(1), value.size(-1)) + ) + + class SDPA(torch.nn.Module): r"""Scaled Dot-Product Attention (SDPA). @@ -189,34 +259,32 @@ def forward( key = key.transpose(-3, -2) # [B, Hkv, KV, C] value = value.transpose(-3, -2) # [B, Hkv, KV, C] - # CUDA attention kernels accept at most 65,535 batch entries per call. - # Slicing a dynamic batch that provably fits would add a size guard. - queries = _split_tensor(query, 65_535, dim=0) - masks = ( - (None,) * len(queries) - if attn_mask is None - else _split_tensor(attn_mask, 65_535, dim=0) - ) - outs = [] - for chunk_query, chunk_key, chunk_value, chunk_mask in zip( - queries, - _split_tensor(key, 65_535, dim=0), - _split_tensor(value, 65_535, dim=0), - masks, - strict=True, + # The operator avoids a recompile when the number of slices changes. + # It has no gradients, and a batch known to need no slicing skips it. + sdpa = _sdpa + if not torch.is_grad_enabled() and not statically_known_true( + query.size(0) <= 65_535 ): - out = F.scaled_dot_product_attention( - query=chunk_query, - key=chunk_key, - value=chunk_value, - attn_mask=chunk_mask, # [B, Hq, Q, KV] - dropout_p=self.dropout if self.training else 0.0, - enable_gqa=enable_gqa, - is_causal=is_causal, - scale=self.scale, - ) - outs.append(out.transpose(-3, -2)) # [B, Q, Hq, C] - out = outs[0] if len(outs) == 1 else torch.cat(outs) + sdpa = _sdpa_operator + device_type = query.device.type + if torch.is_autocast_enabled(device_type): + # Compiled code skips autocast inside the operator, so + # convert the inputs here. Autocast never converts float64. + dtype = torch.get_autocast_dtype(device_type) + query, key, value = ( + x if x.dtype == torch.float64 else x.to(dtype) + for x in (query, key, value) + ) + out = sdpa( + query, + key, + value, + attn_mask, # [B, Hq, Q, KV] + self.dropout if self.training else 0.0, + enable_gqa, + is_causal, + self.scale, + ) # [B, Q, Hq, C] return out.view(batch_shape + out.size()[-3:]) # [..., Q, Hq, C] diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index 1612bacda..8f2f2b2ee 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -7,6 +7,7 @@ import pytest import torch from torch import Tensor +from torch._dynamo.testing import CompileCounter from sdm.nn import ( SDPA, @@ -184,6 +185,40 @@ def test_sdpa_compile_dynamic_batch() -> None: torch.testing.assert_close(out, expected) +def test_sdpa_compile_batch_above_kernel_limit() -> None: + module = SDPA(num_query_heads=2).eval() + counter = CompileCounter() + compiled = torch.compile(module, fullgraph=True, backend=counter) + with torch.inference_mode(): + for batch_size in [3, 65_536]: + query = torch.randn(batch_size, 4, 2, 8) + key = torch.randn(batch_size, 5, 2, 8) + value = torch.randn_like(key) + for tensor in (query, key, value): + torch._dynamo.mark_dynamic(tensor, 0) + expected = module(query=query, key=key, value=value) + out = compiled(query=query, key=key, value=value) + torch.testing.assert_close(out, expected) + assert counter.frame_count == 1 + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float64]) +def test_sdpa_compile_dynamic_batch_autocast(dtype: torch.dtype) -> None: + module = SDPA(num_query_heads=2).eval() + query = torch.randn(3, 4, 2, 8, dtype=dtype) + key = torch.randn(3, 5, 2, 8, dtype=dtype) + value = torch.randn_like(key) + for tensor in (query, key, value): + torch._dynamo.mark_dynamic(tensor, 0) + with torch.inference_mode(), torch.autocast("cpu", torch.bfloat16): + expected = module(query=query, key=key, value=value) + out = torch.compile(module, fullgraph=True)( + query=query, key=key, value=value + ) + assert out.dtype == expected.dtype + torch.testing.assert_close(out, expected) + + @withCUDA @pytest.mark.parametrize( ("num_key_value_heads", "qassmax", "self_attn"), From f40f85dbf6d9d69c81523820197a2a7d86c0a91c Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:52:48 -0800 Subject: [PATCH 2/4] Move the attention operator to sdm/_kernels --- sdm/_kernels/sdpa.py | 111 +++++++++++++++++++++++++++++++++++++++++++ sdm/nn/attention.py | 89 +--------------------------------- 2 files changed, 113 insertions(+), 87 deletions(-) create mode 100644 sdm/_kernels/sdpa.py diff --git a/sdm/_kernels/sdpa.py b/sdm/_kernels/sdpa.py new file mode 100644 index 000000000..d06cf9366 --- /dev/null +++ b/sdm/_kernels/sdpa.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch.fx.experimental.symbolic_shapes import statically_known_true + +from sdm._memory import _split_tensor + + +def _sdpa( + query: Tensor, # [B, Hq, Q, C] + key: Tensor, # [B, Hkv, KV, C] + value: Tensor, # [B, Hkv, KV, C] + attn_mask: Tensor | None, # [B, Hq, Q, KV] + dropout_p: float, + enable_gqa: bool, + is_causal: bool, + scale: float | None, +) -> Tensor: # [B, Q, Hq, C] + # CUDA attention kernels accept at most 65,535 batch entries per call. + queries = _split_tensor(query, 65_535, dim=0) + masks = ( + (None,) * len(queries) + if attn_mask is None + else _split_tensor(attn_mask, 65_535, dim=0) + ) + outs = [] + for chunk_query, chunk_key, chunk_value, chunk_mask in zip( + queries, + _split_tensor(key, 65_535, dim=0), + _split_tensor(value, 65_535, dim=0), + masks, + strict=True, + ): + out = F.scaled_dot_product_attention( + query=chunk_query, + key=chunk_key, + value=chunk_value, + attn_mask=chunk_mask, + dropout_p=dropout_p, + enable_gqa=enable_gqa, + is_causal=is_causal, + scale=scale, + ) + outs.append(out.transpose(-3, -2)) + return outs[0] if len(outs) == 1 else torch.cat(outs) + + +@torch.library.custom_op("sdm::sdpa", mutates_args=()) +def _sdpa_operator( + query: Tensor, + key: Tensor, + value: Tensor, + attn_mask: Tensor | None, + dropout_p: float, + enable_gqa: bool, + is_causal: bool, + scale: float | None, +) -> Tensor: + """Run `_sdpa` as one operator that `torch.compile` does not trace. + + Tracing its loop would recompile whenever the number of slices changes. + """ + # Compiled code expects a contiguous output, as the fake below declares. + return _sdpa( + query, key, value, attn_mask, dropout_p, enable_gqa, is_causal, scale + ).contiguous() + + +@_sdpa_operator.register_fake +def _sdpa_operator_fake( + query: Tensor, key: Tensor, value: Tensor, *args: Any, **kwargs: Any +) -> Tensor: + return query.new_empty( + (query.size(0), query.size(2), query.size(1), value.size(-1)) + ) + + +def sdpa( + query: Tensor, # [B, Hq, Q, C] + key: Tensor, # [B, Hkv, KV, C] + value: Tensor, # [B, Hkv, KV, C] + attn_mask: Tensor | None, # [B, Hq, Q, KV] + dropout_p: float, + enable_gqa: bool, + is_causal: bool, + scale: float | None, +) -> Tensor: # [B, Q, Hq, C] + # The operator avoids a recompile when the number of slices changes. + # It has no gradients, and a batch known to need no slicing skips it. + kernel = _sdpa + if not torch.is_grad_enabled() and not statically_known_true( + query.size(0) <= 65_535 + ): + kernel = _sdpa_operator + device_type = query.device.type + if torch.is_autocast_enabled(device_type): + # Compiled code skips autocast inside the operator, so + # convert the inputs here. Autocast never converts float64. + dtype = torch.get_autocast_dtype(device_type) + query, key, value = ( + x if x.dtype == torch.float64 else x.to(dtype) + for x in (query, key, value) + ) + return kernel( + query, key, value, attn_mask, dropout_p, enable_gqa, is_causal, scale + ) diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index aa42abe83..d3fc05065 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -9,84 +9,15 @@ import torch import torch.nn.functional as F from torch import Tensor -from torch.fx.experimental.symbolic_shapes import statically_known_true from torch.nn import Linear from sdm._kernels.flash_attention import flash_attention -from sdm._memory import _split_tensor, chunk_memory_limit +from sdm._kernels.sdpa import sdpa +from sdm._memory import chunk_memory_limit from sdm.cache import KVCacheEntry from sdm.nn import QueryScaling -def _sdpa( - query: Tensor, # [B, Hq, Q, C] - key: Tensor, # [B, Hkv, KV, C] - value: Tensor, # [B, Hkv, KV, C] - attn_mask: Tensor | None, # [B, Hq, Q, KV] - dropout_p: float, - enable_gqa: bool, - is_causal: bool, - scale: float | None, -) -> Tensor: # [B, Q, Hq, C] - # CUDA attention kernels accept at most 65,535 batch entries per call. - queries = _split_tensor(query, 65_535, dim=0) - masks = ( - (None,) * len(queries) - if attn_mask is None - else _split_tensor(attn_mask, 65_535, dim=0) - ) - outs = [] - for chunk_query, chunk_key, chunk_value, chunk_mask in zip( - queries, - _split_tensor(key, 65_535, dim=0), - _split_tensor(value, 65_535, dim=0), - masks, - strict=True, - ): - out = F.scaled_dot_product_attention( - query=chunk_query, - key=chunk_key, - value=chunk_value, - attn_mask=chunk_mask, - dropout_p=dropout_p, - enable_gqa=enable_gqa, - is_causal=is_causal, - scale=scale, - ) - outs.append(out.transpose(-3, -2)) - return outs[0] if len(outs) == 1 else torch.cat(outs) - - -@torch.library.custom_op("sdm::sdpa", mutates_args=()) -def _sdpa_operator( - query: Tensor, - key: Tensor, - value: Tensor, - attn_mask: Tensor | None, - dropout_p: float, - enable_gqa: bool, - is_causal: bool, - scale: float | None, -) -> Tensor: - """Run `_sdpa` as one operator that `torch.compile` does not trace. - - Tracing its loop would recompile whenever the number of slices changes. - """ - # Compiled code expects a contiguous output, as the fake below declares. - return _sdpa( - query, key, value, attn_mask, dropout_p, enable_gqa, is_causal, scale - ).contiguous() - - -@_sdpa_operator.register_fake -def _sdpa_operator_fake( - query: Tensor, key: Tensor, value: Tensor, *args: Any, **kwargs: Any -) -> Tensor: - return query.new_empty( - (query.size(0), query.size(2), query.size(1), value.size(-1)) - ) - - class SDPA(torch.nn.Module): r"""Scaled Dot-Product Attention (SDPA). @@ -281,22 +212,6 @@ def forward( key = key.transpose(-3, -2) # [B, Hkv, KV, C] value = value.transpose(-3, -2) # [B, Hkv, KV, C] - # The operator avoids a recompile when the number of slices changes. - # It has no gradients, and a batch known to need no slicing skips it. - sdpa = _sdpa - if not torch.is_grad_enabled() and not statically_known_true( - query.size(0) <= 65_535 - ): - sdpa = _sdpa_operator - device_type = query.device.type - if torch.is_autocast_enabled(device_type): - # Compiled code skips autocast inside the operator, so - # convert the inputs here. Autocast never converts float64. - dtype = torch.get_autocast_dtype(device_type) - query, key, value = ( - x if x.dtype == torch.float64 else x.to(dtype) - for x in (query, key, value) - ) out = sdpa( query, key, From cf22d7a741d86e7453997ce8eeeaef6de88eb251 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:04:31 -0800 Subject: [PATCH 3/4] Convert floating-point attention masks under autocast for the operator --- sdm/_kernels/sdpa.py | 12 +++++++----- test/nn/test_compile.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/sdm/_kernels/sdpa.py b/sdm/_kernels/sdpa.py index d06cf9366..361a6444f 100644 --- a/sdm/_kernels/sdpa.py +++ b/sdm/_kernels/sdpa.py @@ -99,12 +99,14 @@ def sdpa( kernel = _sdpa_operator device_type = query.device.type if torch.is_autocast_enabled(device_type): - # Compiled code skips autocast inside the operator, so - # convert the inputs here. Autocast never converts float64. + # Compiled code skips autocast inside the operator, so convert the + # inputs here. Autocast never converts float64 or boolean tensors. dtype = torch.get_autocast_dtype(device_type) - query, key, value = ( - x if x.dtype == torch.float64 else x.to(dtype) - for x in (query, key, value) + query, key, value, attn_mask = ( + x + if x is None or x.dtype in (torch.float64, torch.bool) + else x.to(dtype) + for x in (query, key, value, attn_mask) ) return kernel( query, key, value, attn_mask, dropout_p, enable_gqa, is_causal, scale diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index 8f2f2b2ee..3fa6dc004 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -202,18 +202,25 @@ def test_sdpa_compile_batch_above_kernel_limit() -> None: assert counter.frame_count == 1 -@pytest.mark.parametrize("dtype", [torch.float32, torch.float64]) -def test_sdpa_compile_dynamic_batch_autocast(dtype: torch.dtype) -> None: +@pytest.mark.parametrize( + ("dtype", "mask_dtype"), + [(torch.float32, torch.float16), (torch.float64, torch.float64)], +) +def test_sdpa_compile_dynamic_batch_autocast( + dtype: torch.dtype, + mask_dtype: torch.dtype, +) -> None: module = SDPA(num_query_heads=2).eval() query = torch.randn(3, 4, 2, 8, dtype=dtype) key = torch.randn(3, 5, 2, 8, dtype=dtype) value = torch.randn_like(key) - for tensor in (query, key, value): + attn_mask = torch.randn(3, 2, 4, 5, dtype=mask_dtype) + for tensor in (query, key, value, attn_mask): torch._dynamo.mark_dynamic(tensor, 0) with torch.inference_mode(), torch.autocast("cpu", torch.bfloat16): - expected = module(query=query, key=key, value=value) + expected = module(query, key, value, attn_mask=attn_mask) out = torch.compile(module, fullgraph=True)( - query=query, key=key, value=value + query, key, value, attn_mask=attn_mask ) assert out.dtype == expected.dtype torch.testing.assert_close(out, expected) From 581ccdba3646db5f7e4a029d264e61e8b5a28f75 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:12:05 -0800 Subject: [PATCH 4/4] Keep the tensor types exact in the autocast conversion --- sdm/_kernels/sdpa.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sdm/_kernels/sdpa.py b/sdm/_kernels/sdpa.py index 361a6444f..7a23af4ca 100644 --- a/sdm/_kernels/sdpa.py +++ b/sdm/_kernels/sdpa.py @@ -102,12 +102,13 @@ def sdpa( # Compiled code skips autocast inside the operator, so convert the # inputs here. Autocast never converts float64 or boolean tensors. dtype = torch.get_autocast_dtype(device_type) - query, key, value, attn_mask = ( - x - if x is None or x.dtype in (torch.float64, torch.bool) - else x.to(dtype) - for x in (query, key, value, attn_mask) + kept = (torch.float64, torch.bool) + query, key, value = ( + x if x.dtype in kept else x.to(dtype) + for x in (query, key, value) ) + if attn_mask is not None and attn_mask.dtype not in kept: + attn_mask = attn_mask.to(dtype) return kernel( query, key, value, attn_mask, dropout_p, enable_gqa, is_causal, scale )