Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 114 additions & 0 deletions sdm/_kernels/sdpa.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from typing import Any

import torch
import torch.nn.functional as F
from torch import Tensor
from torch.fx.experimental.symbolic_shapes import statically_known_true

from sdm._memory import _split_tensor


def _sdpa(
query: Tensor, # [B, Hq, Q, C]
key: Tensor, # [B, Hkv, KV, C]
value: Tensor, # [B, Hkv, KV, C]
attn_mask: Tensor | None, # [B, Hq, Q, KV]
dropout_p: float,
enable_gqa: bool,
is_causal: bool,
scale: float | None,
) -> Tensor: # [B, Q, Hq, C]
# CUDA attention kernels accept at most 65,535 batch entries per call.
queries = _split_tensor(query, 65_535, dim=0)
masks = (
(None,) * len(queries)
if attn_mask is None
else _split_tensor(attn_mask, 65_535, dim=0)
)
outs = []
for chunk_query, chunk_key, chunk_value, chunk_mask in zip(
queries,
_split_tensor(key, 65_535, dim=0),
_split_tensor(value, 65_535, dim=0),
masks,
strict=True,
):
out = F.scaled_dot_product_attention(
query=chunk_query,
key=chunk_key,
value=chunk_value,
attn_mask=chunk_mask,
dropout_p=dropout_p,
enable_gqa=enable_gqa,
is_causal=is_causal,
scale=scale,
)
outs.append(out.transpose(-3, -2))
return outs[0] if len(outs) == 1 else torch.cat(outs)


@torch.library.custom_op("sdm::sdpa", mutates_args=())
def _sdpa_operator(
query: Tensor,
key: Tensor,
value: Tensor,
attn_mask: Tensor | None,
dropout_p: float,
enable_gqa: bool,
is_causal: bool,
scale: float | None,
) -> Tensor:
"""Run `_sdpa` as one operator that `torch.compile` does not trace.

Tracing its loop would recompile whenever the number of slices changes.
"""
# Compiled code expects a contiguous output, as the fake below declares.
return _sdpa(
query, key, value, attn_mask, dropout_p, enable_gqa, is_causal, scale
).contiguous()


@_sdpa_operator.register_fake
def _sdpa_operator_fake(
query: Tensor, key: Tensor, value: Tensor, *args: Any, **kwargs: Any
) -> Tensor:
return query.new_empty(
(query.size(0), query.size(2), query.size(1), value.size(-1))
)


def sdpa(
query: Tensor, # [B, Hq, Q, C]
key: Tensor, # [B, Hkv, KV, C]
value: Tensor, # [B, Hkv, KV, C]
attn_mask: Tensor | None, # [B, Hq, Q, KV]
dropout_p: float,
enable_gqa: bool,
is_causal: bool,
scale: float | None,
) -> Tensor: # [B, Q, Hq, C]
# The operator avoids a recompile when the number of slices changes.
# It has no gradients, and a batch known to need no slicing skips it.
kernel = _sdpa
if not torch.is_grad_enabled() and not statically_known_true(
query.size(0) <= 65_535
):
kernel = _sdpa_operator
device_type = query.device.type
if torch.is_autocast_enabled(device_type):
# Compiled code skips autocast inside the operator, so convert the
# inputs here. Autocast never converts float64 or boolean tensors.
dtype = torch.get_autocast_dtype(device_type)
kept = (torch.float64, torch.bool)
query, key, value = (
x if x.dtype in kept else x.to(dtype)
for x in (query, key, value)
)
if attn_mask is not None and attn_mask.dtype not in kept:
attn_mask = attn_mask.to(dtype)
return kernel(
query, key, value, attn_mask, dropout_p, enable_gqa, is_causal, scale
)
41 changes: 12 additions & 29 deletions sdm/nn/attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
from torch.nn import Linear

from sdm._kernels.flash_attention import flash_attention
from sdm._memory import _split_tensor, chunk_memory_limit
from sdm._kernels.sdpa import sdpa
from sdm._memory import chunk_memory_limit
from sdm.cache import KVCacheEntry
from sdm.nn import QueryScaling

Expand Down Expand Up @@ -211,34 +212,16 @@ def forward(
key = key.transpose(-3, -2) # [B, Hkv, KV, C]
value = value.transpose(-3, -2) # [B, Hkv, KV, C]

# CUDA attention kernels accept at most 65,535 batch entries per call.
# Slicing a dynamic batch that provably fits would add a size guard.
queries = _split_tensor(query, 65_535, dim=0)
masks = (
(None,) * len(queries)
if attn_mask is None
else _split_tensor(attn_mask, 65_535, dim=0)
)
outs = []
for chunk_query, chunk_key, chunk_value, chunk_mask in zip(
queries,
_split_tensor(key, 65_535, dim=0),
_split_tensor(value, 65_535, dim=0),
masks,
strict=True,
):
out = F.scaled_dot_product_attention(
query=chunk_query,
key=chunk_key,
value=chunk_value,
attn_mask=chunk_mask, # [B, Hq, Q, KV]
dropout_p=self.dropout if self.training else 0.0,
enable_gqa=enable_gqa,
is_causal=is_causal,
scale=self.scale,
)
outs.append(out.transpose(-3, -2)) # [B, Q, Hq, C]
out = outs[0] if len(outs) == 1 else torch.cat(outs)
out = sdpa(
query,
key,
value,
attn_mask, # [B, Hq, Q, KV]
self.dropout if self.training else 0.0,
enable_gqa,
is_causal,
self.scale,
) # [B, Q, Hq, C]

return out.view(batch_shape + out.size()[-3:]) # [..., Q, Hq, C]

Expand Down
42 changes: 42 additions & 0 deletions test/nn/test_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import pytest
import torch
from torch import Tensor
from torch._dynamo.testing import CompileCounter

from sdm.nn import (
SDPA,
Expand Down Expand Up @@ -184,6 +185,47 @@ def test_sdpa_compile_dynamic_batch() -> None:
torch.testing.assert_close(out, expected)


def test_sdpa_compile_batch_above_kernel_limit() -> None:
module = SDPA(num_query_heads=2).eval()
counter = CompileCounter()
compiled = torch.compile(module, fullgraph=True, backend=counter)
with torch.inference_mode():
for batch_size in [3, 65_536]:
query = torch.randn(batch_size, 4, 2, 8)
key = torch.randn(batch_size, 5, 2, 8)
value = torch.randn_like(key)
for tensor in (query, key, value):
torch._dynamo.mark_dynamic(tensor, 0)
expected = module(query=query, key=key, value=value)
out = compiled(query=query, key=key, value=value)
torch.testing.assert_close(out, expected)
assert counter.frame_count == 1


@pytest.mark.parametrize(
("dtype", "mask_dtype"),
[(torch.float32, torch.float16), (torch.float64, torch.float64)],
)
def test_sdpa_compile_dynamic_batch_autocast(
dtype: torch.dtype,
mask_dtype: torch.dtype,
) -> None:
module = SDPA(num_query_heads=2).eval()
query = torch.randn(3, 4, 2, 8, dtype=dtype)
key = torch.randn(3, 5, 2, 8, dtype=dtype)
value = torch.randn_like(key)
attn_mask = torch.randn(3, 2, 4, 5, dtype=mask_dtype)
for tensor in (query, key, value, attn_mask):
torch._dynamo.mark_dynamic(tensor, 0)
with torch.inference_mode(), torch.autocast("cpu", torch.bfloat16):
expected = module(query, key, value, attn_mask=attn_mask)
out = torch.compile(module, fullgraph=True)(
query, key, value, attn_mask=attn_mask
)
assert out.dtype == expected.dtype
torch.testing.assert_close(out, expected)


@withCUDA
@pytest.mark.parametrize(
("num_key_value_heads", "qassmax", "self_attn"),
Expand Down