diff --git a/sdm/_kernels/sdpa.py b/sdm/_kernels/sdpa.py new file mode 100644 index 000000000..7a23af4ca --- /dev/null +++ b/sdm/_kernels/sdpa.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch.fx.experimental.symbolic_shapes import statically_known_true + +from sdm._memory import _split_tensor + + +def _sdpa( + query: Tensor, # [B, Hq, Q, C] + key: Tensor, # [B, Hkv, KV, C] + value: Tensor, # [B, Hkv, KV, C] + attn_mask: Tensor | None, # [B, Hq, Q, KV] + dropout_p: float, + enable_gqa: bool, + is_causal: bool, + scale: float | None, +) -> Tensor: # [B, Q, Hq, C] + # CUDA attention kernels accept at most 65,535 batch entries per call. + queries = _split_tensor(query, 65_535, dim=0) + masks = ( + (None,) * len(queries) + if attn_mask is None + else _split_tensor(attn_mask, 65_535, dim=0) + ) + outs = [] + for chunk_query, chunk_key, chunk_value, chunk_mask in zip( + queries, + _split_tensor(key, 65_535, dim=0), + _split_tensor(value, 65_535, dim=0), + masks, + strict=True, + ): + out = F.scaled_dot_product_attention( + query=chunk_query, + key=chunk_key, + value=chunk_value, + attn_mask=chunk_mask, + dropout_p=dropout_p, + enable_gqa=enable_gqa, + is_causal=is_causal, + scale=scale, + ) + outs.append(out.transpose(-3, -2)) + return outs[0] if len(outs) == 1 else torch.cat(outs) + + +@torch.library.custom_op("sdm::sdpa", mutates_args=()) +def _sdpa_operator( + query: Tensor, + key: Tensor, + value: Tensor, + attn_mask: Tensor | None, + dropout_p: float, + enable_gqa: bool, + is_causal: bool, + scale: float | None, +) -> Tensor: + """Run `_sdpa` as one operator that `torch.compile` does not trace. + + Tracing its loop would recompile whenever the number of slices changes. + """ + # Compiled code expects a contiguous output, as the fake below declares. + return _sdpa( + query, key, value, attn_mask, dropout_p, enable_gqa, is_causal, scale + ).contiguous() + + +@_sdpa_operator.register_fake +def _sdpa_operator_fake( + query: Tensor, key: Tensor, value: Tensor, *args: Any, **kwargs: Any +) -> Tensor: + return query.new_empty( + (query.size(0), query.size(2), query.size(1), value.size(-1)) + ) + + +def sdpa( + query: Tensor, # [B, Hq, Q, C] + key: Tensor, # [B, Hkv, KV, C] + value: Tensor, # [B, Hkv, KV, C] + attn_mask: Tensor | None, # [B, Hq, Q, KV] + dropout_p: float, + enable_gqa: bool, + is_causal: bool, + scale: float | None, +) -> Tensor: # [B, Q, Hq, C] + # The operator avoids a recompile when the number of slices changes. + # It has no gradients, and a batch known to need no slicing skips it. + kernel = _sdpa + if not torch.is_grad_enabled() and not statically_known_true( + query.size(0) <= 65_535 + ): + kernel = _sdpa_operator + device_type = query.device.type + if torch.is_autocast_enabled(device_type): + # Compiled code skips autocast inside the operator, so convert the + # inputs here. Autocast never converts float64 or boolean tensors. + dtype = torch.get_autocast_dtype(device_type) + kept = (torch.float64, torch.bool) + query, key, value = ( + x if x.dtype in kept else x.to(dtype) + for x in (query, key, value) + ) + if attn_mask is not None and attn_mask.dtype not in kept: + attn_mask = attn_mask.to(dtype) + return kernel( + query, key, value, attn_mask, dropout_p, enable_gqa, is_causal, scale + ) diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index 1ee7ed482..d3fc05065 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -12,7 +12,8 @@ from torch.nn import Linear from sdm._kernels.flash_attention import flash_attention -from sdm._memory import _split_tensor, chunk_memory_limit +from sdm._kernels.sdpa import sdpa +from sdm._memory import chunk_memory_limit from sdm.cache import KVCacheEntry from sdm.nn import QueryScaling @@ -211,34 +212,16 @@ def forward( key = key.transpose(-3, -2) # [B, Hkv, KV, C] value = value.transpose(-3, -2) # [B, Hkv, KV, C] - # CUDA attention kernels accept at most 65,535 batch entries per call. - # Slicing a dynamic batch that provably fits would add a size guard. - queries = _split_tensor(query, 65_535, dim=0) - masks = ( - (None,) * len(queries) - if attn_mask is None - else _split_tensor(attn_mask, 65_535, dim=0) - ) - outs = [] - for chunk_query, chunk_key, chunk_value, chunk_mask in zip( - queries, - _split_tensor(key, 65_535, dim=0), - _split_tensor(value, 65_535, dim=0), - masks, - strict=True, - ): - out = F.scaled_dot_product_attention( - query=chunk_query, - key=chunk_key, - value=chunk_value, - attn_mask=chunk_mask, # [B, Hq, Q, KV] - dropout_p=self.dropout if self.training else 0.0, - enable_gqa=enable_gqa, - is_causal=is_causal, - scale=self.scale, - ) - outs.append(out.transpose(-3, -2)) # [B, Q, Hq, C] - out = outs[0] if len(outs) == 1 else torch.cat(outs) + out = sdpa( + query, + key, + value, + attn_mask, # [B, Hq, Q, KV] + self.dropout if self.training else 0.0, + enable_gqa, + is_causal, + self.scale, + ) # [B, Q, Hq, C] return out.view(batch_shape + out.size()[-3:]) # [..., Q, Hq, C] diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index 1612bacda..3fa6dc004 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -7,6 +7,7 @@ import pytest import torch from torch import Tensor +from torch._dynamo.testing import CompileCounter from sdm.nn import ( SDPA, @@ -184,6 +185,47 @@ def test_sdpa_compile_dynamic_batch() -> None: torch.testing.assert_close(out, expected) +def test_sdpa_compile_batch_above_kernel_limit() -> None: + module = SDPA(num_query_heads=2).eval() + counter = CompileCounter() + compiled = torch.compile(module, fullgraph=True, backend=counter) + with torch.inference_mode(): + for batch_size in [3, 65_536]: + query = torch.randn(batch_size, 4, 2, 8) + key = torch.randn(batch_size, 5, 2, 8) + value = torch.randn_like(key) + for tensor in (query, key, value): + torch._dynamo.mark_dynamic(tensor, 0) + expected = module(query=query, key=key, value=value) + out = compiled(query=query, key=key, value=value) + torch.testing.assert_close(out, expected) + assert counter.frame_count == 1 + + +@pytest.mark.parametrize( + ("dtype", "mask_dtype"), + [(torch.float32, torch.float16), (torch.float64, torch.float64)], +) +def test_sdpa_compile_dynamic_batch_autocast( + dtype: torch.dtype, + mask_dtype: torch.dtype, +) -> None: + module = SDPA(num_query_heads=2).eval() + query = torch.randn(3, 4, 2, 8, dtype=dtype) + key = torch.randn(3, 5, 2, 8, dtype=dtype) + value = torch.randn_like(key) + attn_mask = torch.randn(3, 2, 4, 5, dtype=mask_dtype) + for tensor in (query, key, value, attn_mask): + torch._dynamo.mark_dynamic(tensor, 0) + with torch.inference_mode(), torch.autocast("cpu", torch.bfloat16): + expected = module(query, key, value, attn_mask=attn_mask) + out = torch.compile(module, fullgraph=True)( + query, key, value, attn_mask=attn_mask + ) + assert out.dtype == expected.dtype + torch.testing.assert_close(out, expected) + + @withCUDA @pytest.mark.parametrize( ("num_key_value_heads", "qassmax", "self_attn"),