Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions sdm/_kernels/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
# SPDX-License-Identifier: Apache-2.0

from sdm._kernels.segment_multi_reduce import segment_multi_reduce
from sdm._kernels.rmsnorm_cast import rmsnorm_cast
from sdm._kernels.rms_norm_cast import rms_norm_cast

__all__ = [
"segment_multi_reduce",
"rmsnorm_cast",
"rms_norm_cast",
]
14 changes: 7 additions & 7 deletions sdm/_kernels/rmsnorm_cast.py → sdm/_kernels/rms_norm_cast.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,18 @@
[Tensor, Tensor | None, float, tuple[Tensor, Tensor] | None], Tensor
]

_triton_rmsnorm_cast: _RMSNormCast | None = None
_triton_rms_norm_cast: _RMSNormCast | None = None
try:
from sdm._kernels.triton.rmsnorm_cast import (
rmsnorm_cast as _triton_rmsnorm_cast_impl,
from sdm._kernels.triton.rms_norm_cast import (
rms_norm_cast as _triton_rms_norm_cast_impl,
)
except ImportError:
pass
else:
_triton_rmsnorm_cast = _triton_rmsnorm_cast_impl
_triton_rms_norm_cast = _triton_rms_norm_cast_impl


def rmsnorm_cast(
def rms_norm_cast(
x: Tensor,
weight: Tensor | None,
eps: float,
Expand All @@ -45,7 +45,7 @@ def rmsnorm_cast(
is the number of heads. Rotation rounds in the input dtype.
"""
if (
_triton_rmsnorm_cast is not None
_triton_rms_norm_cast is not None
and x.is_cuda
and x.dtype in {torch.float16, torch.bfloat16, torch.float32}
and x.size(-1) in {32, 64, 128, 256, 512}
Expand Down Expand Up @@ -82,7 +82,7 @@ def rmsnorm_cast(
and torch.is_grad_enabled()
)
):
return _triton_rmsnorm_cast(x, weight, eps, rope)
return _triton_rms_norm_cast(x, weight, eps, rope)
if rope is not None:
cos, sin = (table.unsqueeze(-2) for table in rope)
x1, x2 = x.chunk(2, dim=-1)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ def _load_input(


@triton.jit(do_not_specialize=["rows", "seq_len", "num_heads"])
def _rmsnorm_cast_kernel(
def _rms_norm_cast_kernel(
x_ptr,
weight_ptr,
cos_ptr,
Expand Down Expand Up @@ -98,7 +98,7 @@ def _rmsnorm_cast_kernel(
tl.store(out_ptr + offsets + c, value, mask=mask)


def rmsnorm_cast(
def rms_norm_cast(
x: Tensor,
weight: Tensor | None,
eps: float,
Expand All @@ -118,7 +118,7 @@ def rmsnorm_cast(
grid = ((rows + block_rows - 1) // block_rows,)
out = torch.empty_like(x, dtype=torch.get_autocast_dtype("cuda"))
with torch.cuda.device(x.device):
cast(Any, _rmsnorm_cast_kernel)[grid](
cast(Any, _rms_norm_cast_kernel)[grid](
x,
weight,
cos,
Expand Down
4 changes: 2 additions & 2 deletions sdm/nn/rms_norm.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import torch
from torch import Tensor

from sdm._kernels import rmsnorm_cast
from sdm._kernels import rms_norm_cast
from sdm.nn.rope import RotaryEmbedding


Expand Down Expand Up @@ -55,5 +55,5 @@ def forward(
if self.eps is not None
else torch.finfo(torch.float32).eps
)
return rmsnorm_cast(x, self.weight, eps, rope=cos_sin)
return rms_norm_cast(x, self.weight, eps, rope=cos_sin)
return super().forward(x if rope is None else rope(x))
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import torch
import torch.nn.functional as F

from sdm._kernels import rmsnorm_cast
from sdm._kernels import rms_norm_cast
from sdm.nn import RotaryEmbedding
from sdm.testing import onlyCUDA, withCUDA

Expand All @@ -19,7 +19,7 @@
@pytest.mark.parametrize("affine", [False, True])
@pytest.mark.parametrize("strided", [False, True])
@pytest.mark.parametrize("channels", [32, 64])
def test_rmsnorm_cast(
def test_rms_norm_cast(
dtype: torch.dtype,
affine: bool,
strided: bool,
Expand Down Expand Up @@ -57,7 +57,7 @@ def test_rmsnorm_cast(
torch.autocast("cuda", dtype=output_dtype),
):
expected = F.rms_norm(x, (channels,), weight, eps).to(output_dtype)
actual = rmsnorm_cast(x, weight, eps)
actual = rms_norm_cast(x, weight, eps)

assert torch.cuda.current_device() == current_device
torch.testing.assert_close(
Expand All @@ -74,7 +74,7 @@ def test_rmsnorm_cast(
((0, 64), torch.bfloat16),
],
)
def test_rmsnorm_cast_fallback(
def test_rms_norm_cast_fallback(
device: torch.device,
shape: tuple[int, int],
dtype: torch.dtype,
Expand All @@ -85,33 +85,33 @@ def test_rmsnorm_cast_fallback(
torch.inference_mode(),
torch.autocast(device.type, dtype=torch.bfloat16, enabled=False),
):
actual = rmsnorm_cast(x, weight, 1e-6)
actual = rms_norm_cast(x, weight, 1e-6)
expected = F.rms_norm(x, (shape[-1],), weight, 1e-6).to(torch.bfloat16)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)


@onlyCUDA
def test_rmsnorm_cast_without_triton(
def test_rms_norm_cast_without_triton(
monkeypatch: pytest.MonkeyPatch,
) -> None:
module = importlib.import_module("sdm._kernels.rmsnorm_cast")
monkeypatch.setattr(module, "_triton_rmsnorm_cast", None)
module = importlib.import_module("sdm._kernels.rms_norm_cast")
monkeypatch.setattr(module, "_triton_rms_norm_cast", None)
x = torch.randn(2, 64, device="cuda", dtype=torch.bfloat16)
weight = torch.randn(64, device="cuda")
with torch.inference_mode(), torch.autocast("cuda", dtype=x.dtype):
actual = rmsnorm_cast(x, weight, 1e-6)
actual = rms_norm_cast(x, weight, 1e-6)
expected = F.rms_norm(x, (64,), weight, 1e-6).to(x.dtype)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)


@onlyCUDA
def test_rmsnorm_cast_grad_fallback() -> None:
def test_rms_norm_cast_grad_fallback() -> None:
x = torch.randn(
2, 64, device="cuda", dtype=torch.bfloat16, requires_grad=True
)
weight = torch.randn(64, device="cuda", requires_grad=True)
with torch.autocast("cuda", dtype=x.dtype):
actual = rmsnorm_cast(x, weight, 1e-6)
actual = rms_norm_cast(x, weight, 1e-6)
expected = F.rms_norm(x, (64,), weight, 1e-6).to(x.dtype)
actual_grads = torch.autograd.grad(actual.sum(), (x, weight))
expected_grads = torch.autograd.grad(expected.sum(), (x, weight))
Expand All @@ -124,7 +124,7 @@ def test_rmsnorm_cast_grad_fallback() -> None:
)
@pytest.mark.parametrize("channels", [32, 64, 128, 256, 512])
@pytest.mark.parametrize("strided", [False, True])
def test_rmsnorm_cast_rope(
def test_rms_norm_cast_rope(
dtype: torch.dtype,
channels: int,
strided: bool,
Expand Down Expand Up @@ -152,7 +152,7 @@ def test_rmsnorm_cast_rope(
tables = (freq.cos().to(dtype), freq.sin().to(dtype))

with torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16):
actual = rmsnorm_cast(x, weight, 1e-6, rope=tables)
actual = rms_norm_cast(x, weight, 1e-6, rope=tables)
expected = F.rms_norm(rope(x), (channels,), weight, 1e-6).to(
torch.float16
)
Expand All @@ -161,12 +161,12 @@ def test_rmsnorm_cast_rope(


@withCUDA
def test_rmsnorm_cast_rope_grad_fallback(device: torch.device) -> None:
def test_rms_norm_cast_rope_grad_fallback(device: torch.device) -> None:
x = torch.randn(2, 5, 3, 64, device=device)
freq = torch.randn(5, 32, device=device, requires_grad=True)
tables = (freq.cos(), freq.sin())
with torch.autocast(device.type, dtype=torch.bfloat16):
actual = rmsnorm_cast(x, None, 1e-6, rope=tables)
actual = rms_norm_cast(x, None, 1e-6, rope=tables)
cos, sin = (table.unsqueeze(-2) for table in tables)
x1, x2 = x.chunk(2, dim=-1)
rotated = torch.cat((x1 * cos - x2 * sin, x2 * cos + x1 * sin), -1)
Expand Down
Loading