From 8969712767a93edf3b7f2593adaddc03db746cf4 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:06:58 +0000 Subject: [PATCH 01/44] Refactor KV cache entries as dataclasses with quantized metadata --- sdm/cache.py | 61 +++++++++++++++++++++++++++++----- sdm/models/kumo/tabular/icl.py | 25 ++++++-------- test/test_cache.py | 52 ++++++++++++++++++++++++++++- 3 files changed, 114 insertions(+), 24 deletions(-) diff --git a/sdm/cache.py b/sdm/cache.py index 59c6c0db06..38bc333333 100644 --- a/sdm/cache.py +++ b/sdm/cache.py @@ -9,8 +9,9 @@ Mapping, MutableMapping, ) +from dataclasses import dataclass, replace from enum import StrEnum -from typing import NamedTuple, Self, cast +from typing import Self, cast import torch from torch import Tensor @@ -19,12 +20,8 @@ from sdm.tensor.mixin import DeviceMixin -class _KVCacheEntry(NamedTuple): - key: Tensor - value: Tensor - - -class KVCacheEntry(_KVCacheEntry, DeviceMixin): +@dataclass +class KVCacheEntry(DeviceMixin): r"""Cached key/value projections for a single transformer block. Args: @@ -32,12 +29,60 @@ class KVCacheEntry(_KVCacheEntry, DeviceMixin): value: Cached value projection tensor. """ + key: Tensor + value: Tensor + + def select_heads(self, num_heads: int) -> Self: + """Retain the leading key/value heads for grouped-query attention.""" + return replace( + self, + key=self.key[..., :num_heads, :].clone(), + value=self.value[..., :num_heads, :].clone(), + ) + def _tensors(self) -> Iterator[Tensor]: yield self.key yield self.value def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self: - return self.__class__(key=fn(self.key), value=fn(self.value)) + return replace(self, key=fn(self.key), value=fn(self.value)) + + +@dataclass +class QuantizedKVCacheEntry(KVCacheEntry): + """FP8 attention projections and their per-head dequantization scales. + + Keys and values have shape ``[..., rows, heads, channels]``. Scales have + shape ``[..., 1, heads, 1]``. The query scale is learned from context + queries and reused for prediction. ``dtype`` is the attention output dtype. + """ + + key_scale: Tensor + value_scale: Tensor + query_scale: Tensor + dtype: torch.dtype + + def select_heads(self, num_heads: int) -> Self: + """Retain leading K/V heads and scales, keeping all query scales.""" + return replace( + super().select_heads(num_heads), + key_scale=self.key_scale[..., :num_heads, :].clone(), + value_scale=self.value_scale[..., :num_heads, :].clone(), + ) + + def _tensors(self) -> Iterator[Tensor]: + yield from super()._tensors() + yield self.key_scale + yield self.value_scale + yield self.query_scale + + def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self: + return replace( + super()._apply_tensor(fn), + key_scale=fn(self.key_scale), + value_scale=fn(self.value_scale), + query_scale=fn(self.query_scale), + ) class Cache(MutableMapping[Hashable, object], DeviceMixin): diff --git a/sdm/models/kumo/tabular/icl.py b/sdm/models/kumo/tabular/icl.py index 7f76b6dd86..8027daf18d 100644 --- a/sdm/models/kumo/tabular/icl.py +++ b/sdm/models/kumo/tabular/icl.py @@ -109,21 +109,19 @@ def forward( ) if cache is not None and cache.is_recording: - x, (key, value) = result - kv_heads = self.kv_heads - if kv_heads is not None and kv_heads < key.size(-2): - key, value = _copy_key_value( - key=key[..., :kv_heads, :], - value=value[..., :kv_heads, :], - ) - cache[cache_key] = KVCacheEntry(key, value) - del key, value + x, entry = result + cache[cache_key] = ( + entry.select_heads(self.kv_heads) + if self.kv_heads is not None + else entry + ) + del entry else: x = result del result continue - x_context, (key, value) = layer( + x_context, entry = layer( query=x if train_size is not None else x[..., :0, :] @@ -138,11 +136,8 @@ def forward( if last_layer else x[..., :R_train, :], ) - key_value = KVCacheEntry( - key=key[..., : self.kv_heads, :].contiguous(), - value=value[..., : self.kv_heads, :].contiguous(), - ) - del key, value + key_value = entry.select_heads(self.kv_heads) + del entry x_query = layer( query=x if train_size is not None else x[..., R_train:, :], key_value=key_value, diff --git a/test/test_cache.py b/test/test_cache.py index d05102f090..d33a6bafb0 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -7,7 +7,7 @@ import pytest import torch -from sdm.cache import Cache, KVCacheEntry +from sdm.cache import Cache, KVCacheEntry, QuantizedKVCacheEntry from sdm.testing import onlyCUDA, withCUDA @@ -118,3 +118,53 @@ def test_cache_serialization_without_cuda( torch.testing.assert_close( restored["tensor"], cache["tensor"], rtol=0, atol=0 ) + + +@withCUDA +@pytest.mark.parametrize("quantized", [False, True]) +def test_cache_head_selection_and_transfer( + device: torch.device, quantized: bool +) -> None: + key = torch.arange(192).reshape(2, 3, 4, 8).float() + entry = ( + QuantizedKVCacheEntry( + key=key.to(torch.float8_e4m3fn), + value=(key / 2).to(torch.float8_e4m3fn), + key_scale=torch.arange(8).reshape(2, 1, 4, 1).float() + 1, + value_scale=torch.ones(2, 1, 4, 1), + query_scale=torch.ones(2, 1, 8, 1), + dtype=torch.float32, + ) + if quantized + else KVCacheEntry(key, key / 2) + ) + selected = entry.select_heads(2) + restored = selected.to(device).cpu() + assert isinstance(restored, KVCacheEntry) + assert entry.key.size(-2) == 4 + for name in ("key", "value"): + actual = getattr(restored, name) + expected = getattr(entry, name)[..., :2, :] + torch.testing.assert_close(actual.float(), expected.float()) + assert actual.dtype == expected.dtype + assert ( + actual.untyped_storage().nbytes() + == actual.numel() * actual.element_size() + ) + expected_bytes = restored.key.numel() * restored.key.element_size() * 2 + if isinstance(entry, QuantizedKVCacheEntry): + assert isinstance(restored, QuantizedKVCacheEntry) + assert restored.dtype == entry.dtype + torch.testing.assert_close( + restored.key_scale, entry.key_scale[..., :2, :] + ) + torch.testing.assert_close( + restored.value_scale, entry.value_scale[..., :2, :] + ) + torch.testing.assert_close(restored.query_scale, entry.query_scale) + expected_bytes += ( + restored.key_scale.numel() + + restored.value_scale.numel() + + restored.query_scale.numel() + ) * 4 + assert Cache(entry=restored).size() == expected_bytes From f8d2f3ea989c04345919116492803d67b15e7b2b Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:23:51 +0000 Subject: [PATCH 02/44] Describe quantized cache metadata independently of storage format --- sdm/cache.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdm/cache.py b/sdm/cache.py index 38bc333333..67a021ec05 100644 --- a/sdm/cache.py +++ b/sdm/cache.py @@ -50,11 +50,12 @@ def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self: @dataclass class QuantizedKVCacheEntry(KVCacheEntry): - """FP8 attention projections and their per-head dequantization scales. + """Quantized attention projections with per-head scaling metadata. Keys and values have shape ``[..., rows, heads, channels]``. Scales have shape ``[..., 1, heads, 1]``. The query scale is learned from context - queries and reused for prediction. ``dtype`` is the attention output dtype. + queries and reused for prediction. The key/value tensor dtypes identify + their storage formats; ``dtype`` is the attention output dtype. """ key_scale: Tensor From 9b7950dc2cd4792fb1fcd62f478d266f33bab3eb Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:38:49 +0000 Subject: [PATCH 03/44] Preserve contiguous head-selection behavior in cache entries --- sdm/cache.py | 8 ++++---- test/test_cache.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/sdm/cache.py b/sdm/cache.py index 67a021ec05..cca32cb0e8 100644 --- a/sdm/cache.py +++ b/sdm/cache.py @@ -36,8 +36,8 @@ def select_heads(self, num_heads: int) -> Self: """Retain the leading key/value heads for grouped-query attention.""" return replace( self, - key=self.key[..., :num_heads, :].clone(), - value=self.value[..., :num_heads, :].clone(), + key=self.key[..., :num_heads, :].contiguous(), + value=self.value[..., :num_heads, :].contiguous(), ) def _tensors(self) -> Iterator[Tensor]: @@ -67,8 +67,8 @@ def select_heads(self, num_heads: int) -> Self: """Retain leading K/V heads and scales, keeping all query scales.""" return replace( super().select_heads(num_heads), - key_scale=self.key_scale[..., :num_heads, :].clone(), - value_scale=self.value_scale[..., :num_heads, :].clone(), + key_scale=self.key_scale[..., :num_heads, :].contiguous(), + value_scale=self.value_scale[..., :num_heads, :].contiguous(), ) def _tensors(self) -> Iterator[Tensor]: diff --git a/test/test_cache.py b/test/test_cache.py index d33a6bafb0..34f809ea0a 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -147,10 +147,7 @@ def test_cache_head_selection_and_transfer( expected = getattr(entry, name)[..., :2, :] torch.testing.assert_close(actual.float(), expected.float()) assert actual.dtype == expected.dtype - assert ( - actual.untyped_storage().nbytes() - == actual.numel() * actual.element_size() - ) + assert actual.is_contiguous() expected_bytes = restored.key.numel() * restored.key.element_size() * 2 if isinstance(entry, QuantizedKVCacheEntry): assert isinstance(restored, QuantizedKVCacheEntry) @@ -168,3 +165,11 @@ def test_cache_head_selection_and_transfer( + restored.query_scale.numel() ) * 4 assert Cache(entry=restored).size() == expected_bytes + + +def test_cache_head_selection_preserves_contiguous_storage() -> None: + entry = KVCacheEntry(torch.ones(1, 1, 4, 8), torch.zeros(1, 1, 4, 8)) + selected = entry.select_heads(2) + assert selected.key.shape == selected.value.shape == (1, 1, 2, 8) + assert selected.key.data_ptr() == entry.key.data_ptr() + assert selected.value.data_ptr() == entry.value.data_ptr() From 4949725dd3caf997579b2013d588f1bcfc60e7d7 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:51:24 +0000 Subject: [PATCH 04/44] Simplify quantized cache tests --- test/test_cache.py | 62 +++++++++++++++++----------------------------- 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/test/test_cache.py b/test/test_cache.py index 34f809ea0a..8733ce37c9 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -121,50 +121,34 @@ def test_cache_serialization_without_cuda( @withCUDA -@pytest.mark.parametrize("quantized", [False, True]) -def test_cache_head_selection_and_transfer( - device: torch.device, quantized: bool +def test_quantized_cache_head_selection_and_transfer( + device: torch.device, ) -> None: - key = torch.arange(192).reshape(2, 3, 4, 8).float() - entry = ( - QuantizedKVCacheEntry( - key=key.to(torch.float8_e4m3fn), - value=(key / 2).to(torch.float8_e4m3fn), - key_scale=torch.arange(8).reshape(2, 1, 4, 1).float() + 1, - value_scale=torch.ones(2, 1, 4, 1), - query_scale=torch.ones(2, 1, 8, 1), - dtype=torch.float32, - ) - if quantized - else KVCacheEntry(key, key / 2) + key = torch.arange(192).reshape(2, 3, 4, 8).to(torch.float8_e4m3fn) + scale = torch.arange(8).reshape(2, 1, 4, 1).float() + 1 + entry = QuantizedKVCacheEntry( + key=key, + value=key, + key_scale=scale, + value_scale=scale, + query_scale=torch.ones(2, 1, 8, 1), + dtype=torch.float32, ) - selected = entry.select_heads(2) - restored = selected.to(device).cpu() - assert isinstance(restored, KVCacheEntry) + selected = entry.select_heads(2).to(device) + assert isinstance(selected, QuantizedKVCacheEntry) + assert selected.dtype == entry.dtype assert entry.key.size(-2) == 4 - for name in ("key", "value"): - actual = getattr(restored, name) - expected = getattr(entry, name)[..., :2, :] - torch.testing.assert_close(actual.float(), expected.float()) + for name in ("key", "value", "key_scale", "value_scale", "query_scale"): + expected = getattr(entry, name) + if name != "query_scale": + expected = expected[..., :2, :] + actual = getattr(selected, name) + assert actual.device == device assert actual.dtype == expected.dtype assert actual.is_contiguous() - expected_bytes = restored.key.numel() * restored.key.element_size() * 2 - if isinstance(entry, QuantizedKVCacheEntry): - assert isinstance(restored, QuantizedKVCacheEntry) - assert restored.dtype == entry.dtype - torch.testing.assert_close( - restored.key_scale, entry.key_scale[..., :2, :] - ) - torch.testing.assert_close( - restored.value_scale, entry.value_scale[..., :2, :] - ) - torch.testing.assert_close(restored.query_scale, entry.query_scale) - expected_bytes += ( - restored.key_scale.numel() - + restored.value_scale.numel() - + restored.query_scale.numel() - ) * 4 - assert Cache(entry=restored).size() == expected_bytes + torch.testing.assert_close(actual.cpu().float(), expected.float()) + # 192 K/V bytes + 32 K/V scale bytes + 64 query scale bytes. + assert Cache(entry=selected).size() == 288 def test_cache_head_selection_preserves_contiguous_storage() -> None: From 09e302614f668919ec7d808da4f29a2f4d64f6b6 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:41:44 +0000 Subject: [PATCH 05/44] Document cache entry metadata and head selection --- sdm/cache.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/sdm/cache.py b/sdm/cache.py index cca32cb0e8..90befd5a4c 100644 --- a/sdm/cache.py +++ b/sdm/cache.py @@ -33,7 +33,14 @@ class KVCacheEntry(DeviceMixin): value: Tensor def select_heads(self, num_heads: int) -> Self: - """Retain the leading key/value heads for grouped-query attention.""" + """Retain the leading key/value heads for grouped-query attention. + + Args: + num_heads: Number of leading K/V heads to retain. + + Returns: + An entry with contiguous K/V slices, reusing storage when possible. + """ return replace( self, key=self.key[..., :num_heads, :].contiguous(), @@ -56,6 +63,14 @@ class QuantizedKVCacheEntry(KVCacheEntry): shape ``[..., 1, heads, 1]``. The query scale is learned from context queries and reused for prediction. The key/value tensor dtypes identify their storage formats; ``dtype`` is the attention output dtype. + + Args: + key: Quantized key projections. + value: Quantized value projections. + key_scale: Per-head key dequantization scales. + value_scale: Per-head value dequantization scales. + query_scale: Per-query-head scales derived from the context. + dtype: Attention output dtype. """ key_scale: Tensor @@ -64,7 +79,14 @@ class QuantizedKVCacheEntry(KVCacheEntry): dtype: torch.dtype def select_heads(self, num_heads: int) -> Self: - """Retain leading K/V heads and scales, keeping all query scales.""" + """Retain leading K/V heads and scales, keeping all query scales. + + Args: + num_heads: Number of leading K/V heads to retain. + + Returns: + A quantized entry with selected K/V heads and matching scales. + """ return replace( super().select_heads(num_heads), key_scale=self.key_scale[..., :num_heads, :].contiguous(), From dc307146b7faa60c6f661c0affd3ac73b6ff8222 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:43:20 +0000 Subject: [PATCH 06/44] Preserve compiled cache-copy behavior after rebasing on main --- sdm/cache.py | 25 ++++++++++++++++++++----- sdm/models/kumo/tabular/icl.py | 13 ------------- test/test_cache.py | 9 ++++++--- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/sdm/cache.py b/sdm/cache.py index 90befd5a4c..a11e375716 100644 --- a/sdm/cache.py +++ b/sdm/cache.py @@ -39,13 +39,15 @@ def select_heads(self, num_heads: int) -> Self: num_heads: Number of leading K/V heads to retain. Returns: - An entry with contiguous K/V slices, reusing storage when possible. + An entry with contiguous K/V copies when reducing the head count. """ - return replace( - self, - key=self.key[..., :num_heads, :].contiguous(), - value=self.value[..., :num_heads, :].contiguous(), + if num_heads >= self.key.size(-2): + return self + key, value = _copy_key_value( + key=self.key[..., :num_heads, :], + value=self.value[..., :num_heads, :], ) + return replace(self, key=key, value=value) def _tensors(self) -> Iterator[Tensor]: yield self.key @@ -266,3 +268,16 @@ def _apply(value: object) -> object: ) out._mode = self._mode return out + + +def _copy_key_value(key: Tensor, value: Tensor) -> tuple[Tensor, Tensor]: + return ( + key.clone(memory_format=torch.contiguous_format), + value.clone(memory_format=torch.contiguous_format), + ) + + +# Keep the guard for PyTorch 2.7; 2.8+ can use the decorator directly. +if hasattr(torch.compiler, "nested_compile_region"): + # Compile cache copies separately to free each layer's full K/V tensors. + _copy_key_value = torch.compiler.nested_compile_region(_copy_key_value) diff --git a/sdm/models/kumo/tabular/icl.py b/sdm/models/kumo/tabular/icl.py index 8027daf18d..066001790f 100644 --- a/sdm/models/kumo/tabular/icl.py +++ b/sdm/models/kumo/tabular/icl.py @@ -154,16 +154,3 @@ def forward( x = torch.cat([x_context, x_query], -2) return self.head(self.norm(x)) - - -def _copy_key_value(key: Tensor, value: Tensor) -> tuple[Tensor, Tensor]: - return ( - key.clone(memory_format=torch.contiguous_format), - value.clone(memory_format=torch.contiguous_format), - ) - - -# Keep the guard for PyTorch 2.7; 2.8+ can use the decorator directly. -if hasattr(torch.compiler, "nested_compile_region"): - # Compile cache copies separately to free each layer's full K/V tensors. - _copy_key_value = torch.compiler.nested_compile_region(_copy_key_value) diff --git a/test/test_cache.py b/test/test_cache.py index 8733ce37c9..0edbcaff0b 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -151,9 +151,12 @@ def test_quantized_cache_head_selection_and_transfer( assert Cache(entry=selected).size() == 288 -def test_cache_head_selection_preserves_contiguous_storage() -> None: +def test_cache_head_selection_releases_unused_heads() -> None: entry = KVCacheEntry(torch.ones(1, 1, 4, 8), torch.zeros(1, 1, 4, 8)) selected = entry.select_heads(2) assert selected.key.shape == selected.value.shape == (1, 1, 2, 8) - assert selected.key.data_ptr() == entry.key.data_ptr() - assert selected.value.data_ptr() == entry.value.data_ptr() + assert selected.key.data_ptr() != entry.key.data_ptr() + assert selected.value.data_ptr() != entry.value.data_ptr() + assert selected.key.is_contiguous() + assert selected.value.is_contiguous() + assert entry.select_heads(4) is entry From 10c63aeecd918d8e878d4ded13e29056f4aa2dee Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:06:58 +0000 Subject: [PATCH 07/44] Implement FP8 attention and quantization kernels --- sdm/_kernels/triton/fp8_attention.py | 200 +++++++++++++++++++++++++++ test/nn/test_fp8_attention.py | 151 ++++++++++++++++++++ 2 files changed, 351 insertions(+) create mode 100644 sdm/_kernels/triton/fp8_attention.py create mode 100644 test/nn/test_fp8_attention.py diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py new file mode 100644 index 0000000000..30fea4e747 --- /dev/null +++ b/sdm/_kernels/triton/fp8_attention.py @@ -0,0 +1,200 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any, cast + +import torch +import triton +import triton.language as _tl + +tl: Any = _tl + + +@triton.jit +def attention_kernel( + Q, + K, + V, + QS, + KS, + VS, + Out, + HQ: tl.constexpr, + HK: tl.constexpr, + M: tl.constexpr, + N: tl.constexpr, + D: tl.constexpr, + SCALE: tl.constexpr, + BM: tl.constexpr, + BN: tl.constexpr, + ACC_CHUNK: tl.constexpr = 0, + LIFT_EXP: tl.constexpr = False, + FUSED_SOFTMAX: tl.constexpr = False, +): + block, head = tl.program_id(0), tl.program_id(1) + kv_head = (head // HQ) * HK + (head % HQ) // (HQ // HK) + rows = block * BM + tl.arange(0, BM) + cols = tl.arange(0, BN) + dims = tl.arange(0, D) + q = tl.load( + Q + head * M * D + rows[:, None] * D + dims[None, :], + rows[:, None] < M, + 0.0, + ) + scale = ( + tl.load(QS + head) * tl.load(KS + kv_head) * SCALE * 1.4426950408889634 + ) + vmax = tl.load(VS + kv_head) + maximum = tl.full((BM,), -float("inf"), tl.float32) + denominator = tl.full((BM,), 0, tl.float32) + acc = tl.full((BM, D), 0, tl.float32) + if ACC_CHUNK > 0: + outer_acc = tl.full((BM, D), 0, tl.float32) + outer_max = tl.full((BM,), -float("inf"), tl.float32) + for start in range(tl.cdiv(N, BN)): + indices = start * BN + cols + k = tl.load( + K + kv_head * N * D + indices[None, :] * D + dims[:, None], + indices[None, :] < N, + 0.0, + ) + score = tl.dot(q, k, max_num_imprecise_acc=32) + if FUSED_SOFTMAX and SCALE > 0: + score = tl.where(indices[None, :] < N, score, -float("inf")) + new_max = tl.maximum(maximum, tl.max(score, 1) * scale) + shift = new_max - 8.0 if LIFT_EXP else new_max + p = tl.exp2(score * scale - shift[:, None]) + else: + score = tl.where( + indices[None, :] < N, score * scale, -float("inf") + ) + new_max = tl.maximum(maximum, tl.max(score, 1)) + if LIFT_EXP: + p = tl.exp2(score - (new_max[:, None] - 8.0)) + else: + p = tl.exp2(score - new_max[:, None]) + correction = tl.exp2(maximum - new_max) + denominator = denominator * correction + tl.sum(p, 1) + acc *= correction[:, None] + v_offsets = indices[:, None] + dims[None, :] * N + v = tl.load( + V + kv_head * N * D + v_offsets, + indices[:, None] < N, + 0.0, + ) + # Lift softmax probabilities by 256 to retain precision in FP8. + acc = tl.dot( + (p if LIFT_EXP else p * 256.0).to(tl.float8e4nv), + v, + acc, + max_num_imprecise_acc=32, + ) + # Periodically flush the dot accumulator to FP32 for long L4 contexts. + if ACC_CHUNK > 0: # noqa: SIM102 - constexpr guard avoids modulo zero. + if ((start + 1) % ACC_CHUNK == 0) | (start + 1 == tl.cdiv(N, BN)): + outer_acc = ( + outer_acc * tl.exp2(outer_max - new_max)[:, None] + acc + ) + outer_max = new_max + acc = tl.full((BM, D), 0, tl.float32) + maximum = new_max + if ACC_CHUNK > 0: + acc = outer_acc + out = acc * vmax / denominator[:, None] + if not LIFT_EXP: + out *= 1.0 / 256.0 + tl.store( + Out + head * M * D + rows[:, None] * D + dims[None, :], + out, + rows[:, None] < M, + ) + + +@triton.jit +def quantize_kernel( + X, + S, + Y, + LENGTH: tl.constexpr, + BLOCK: tl.constexpr, + H: tl.constexpr, + D: tl.constexpr, + S0: tl.constexpr, + S1: tl.constexpr, + S2: tl.constexpr, + S3: tl.constexpr, +): + offset = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + head = tl.program_id(1) + source = ( + (head // H) * S0 + + (head % H) * S1 + + (offset // D) * S2 + + (offset % D) * S3 + ) + x = tl.load(X + source, offset < LENGTH, 0.0).to(tl.float32) + scale = tl.load(S + head) + y = tl.minimum(tl.maximum(x / scale, -448.0), 448.0) + tl.store(Y + head * LENGTH + offset, y, offset < LENGTH) + + +def quantize( + x: torch.Tensor, scale: torch.Tensor | None = None +) -> tuple[torch.Tensor, torch.Tensor]: + if scale is None: + scale = ( + x.abs().amax(dim=(-2, -1), keepdim=True).float().clamp_min(1e-12) + / 448.0 + ) + output = torch.empty(x.shape, device=x.device, dtype=torch.float8_e4m3fn) + length = x.size(-2) * x.size(-1) + cast(Any, quantize_kernel)[ + (((length + 1023) // 1024), x.size(0) * x.size(1)) + ](x, scale, output, length, 1024, x.size(1), x.size(-1), *x.stride()) + return output, scale + + +def quantized_attention( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + qs: torch.Tensor, + ks: torch.Tensor, + vs: torch.Tensor, + dtype: torch.dtype, + block: int, + scale: float | None = None, + tile: int = 64, + warps: int = 4, + stages: int = 2, + accumulation_chunk: int = 0, + lift_exp: bool = False, + fused_softmax: bool = False, +) -> torch.Tensor: + b, h, m, d = q.shape + n = k.size(-2) + # Q/K are [batch, heads, sequence, channels]; V is transposed. + output = torch.empty((b, h, m, d), device=q.device, dtype=dtype) + cast(Any, attention_kernel)[(((m + block - 1) // block), b * h)]( + q, + k, + v, + qs, + ks, + vs, + output, + h, + k.size(1), + m, + n, + d, + d**-0.5 if scale is None else scale, + block, + tile, + accumulation_chunk, + lift_exp, + fused_softmax, + num_warps=warps, + num_stages=stages, + ) + return output diff --git a/test/nn/test_fp8_attention.py b/test/nn/test_fp8_attention.py new file mode 100644 index 0000000000..5f396ea11a --- /dev/null +++ b/test/nn/test_fp8_attention.py @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any + +import pytest +import torch + +pytest.importorskip("triton") +from sdm._kernels.triton.fp8_attention import ( + quantize, + quantized_attention, +) + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available() + or torch.cuda.get_device_capability() < (8, 9), + reason="FP8 attention requires an Ada or newer CUDA GPU", +) + + +@pytest.mark.parametrize( + ("batch", "queries", "context"), [(2, 129, 513), (1, 129, 8192)] +) +def test_fp8_partial_tiles(batch: int, queries: int, context: int) -> None: + with torch.inference_mode(): + q = torch.randn( + batch, 8, queries, 64, device="cuda", dtype=torch.float16 + ) + k = torch.randn( + batch, 8, context, 64, device="cuda", dtype=torch.float16 + ) + v = torch.randn_like(k) + q8, qs = quantize(q) + k8, ks = quantize(k) + v8, vs = quantize(v) + actual = quantized_attention( + q8, + k8, + v8.transpose(-1, -2).contiguous(), + qs, + ks, + vs, + q.dtype, + 128, + ) + expected = torch.nn.functional.scaled_dot_product_attention(q, k, v) + assert actual.isfinite().all() + relative_rms = ( + actual.float() - expected.float() + ).square().mean().sqrt() / expected.float().square().mean().sqrt() + assert relative_rms < 0.08 + + +@pytest.mark.parametrize("scale", [None, 1.0]) +@pytest.mark.parametrize("tile", [64, 128]) +def test_grouped_heads_and_model_scale(scale: float | None, tile: int) -> None: + with torch.inference_mode(): + q = ( + torch.randn(2, 8, 129, 64, device="cuda", dtype=torch.float16) + * 0.3 + ) + k = ( + torch.randn(2, 2, 8192, 64, device="cuda", dtype=torch.float16) + * 0.3 + ) + v = torch.randn_like(k) + q8, qs = quantize(q) + k8, ks = quantize(k) + v8, vs = quantize(v) + actual = quantized_attention( + q8, + k8, + v8.transpose(-1, -2).contiguous(), + qs, + ks, + vs, + q.dtype, + 128, + scale, + tile=tile, + ) + expected = torch.nn.functional.scaled_dot_product_attention( + q, k, v, enable_gqa=True, scale=scale + ) + assert actual.isfinite().all() + relative_rms = ( + actual.float() - expected.float() + ).square().mean().sqrt() / expected.float().square().mean().sqrt() + assert relative_rms < 0.08 + + +@pytest.mark.parametrize("accumulation_chunk", [0, 16]) +@pytest.mark.parametrize("optimized", [False, True]) +@pytest.mark.parametrize("tile", [64, 128]) +@pytest.mark.parametrize("context", [8193, 65536]) +def test_periodic_attention_matches_cached_query_slice( + context: int, tile: int, optimized: bool, accumulation_chunk: int +) -> None: + with torch.inference_mode(): + q = torch.randn(1, 1, context, 64, device="cuda", dtype=torch.float16) + k, v = torch.randn_like(q), torch.randn_like(q) + q8, qs = quantize(q) + k8, ks = quantize(k) + v8, vs = quantize(v) + v8 = v8.transpose(-1, -2).contiguous() + opts: dict[str, Any] = { + "accumulation_chunk": accumulation_chunk, + "tile": tile, + "lift_exp": optimized, + "fused_softmax": optimized, + } + full = quantized_attention( + q8, k8, v8, qs, ks, vs, q.dtype, 128, **opts + ) + cached = quantized_attention( + q8[:, :, -129:].contiguous(), + k8, + v8, + qs, + ks, + vs, + q.dtype, + 128, + **opts, + ) + torch.testing.assert_close( + cached, full[:, :, -129:], rtol=1e-3, atol=2e-5 + ) + reference = torch.nn.functional.scaled_dot_product_attention( + q[:, :, -129:], k, v + ) + relative_rms = ( + (cached.float() - reference.float()).square().mean() + / reference.float().square().mean() + ).sqrt() + assert relative_rms < 0.08 + assert cached.isfinite().all() + + +def test_quantization_reads_strided_heads_without_fp16_copy() -> None: + x = torch.randn( + 2, 513, 8, 64, device="cuda", dtype=torch.float16 + ).transpose(1, 2) + actual, actual_scale = quantize(x) + expected, expected_scale = quantize(x.contiguous()) + assert actual.is_contiguous() + torch.testing.assert_close( + actual.float(), expected.float(), rtol=0, atol=0 + ) + torch.testing.assert_close(actual_scale, expected_scale, rtol=0, atol=0) From 10b39a554273c1d8c35a8b2af4a17c2858da0213 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:39:47 +0000 Subject: [PATCH 08/44] Clarify FP8 attention arithmetic and kernel stages --- sdm/_kernels/triton/fp8_attention.py | 68 +++++++++++++++++----------- test/nn/test_fp8_attention.py | 4 +- 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index 30fea4e747..0defb102d9 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -28,9 +28,10 @@ def attention_kernel( BM: tl.constexpr, BN: tl.constexpr, ACC_CHUNK: tl.constexpr = 0, - LIFT_EXP: tl.constexpr = False, - FUSED_SOFTMAX: tl.constexpr = False, + SCALE_WEIGHTS_IN_EXP: tl.constexpr = False, + FUSE_SCORE_SCALE: tl.constexpr = False, ): + # Each program owns BM query rows for one head and visits every K/V tile. block, head = tl.program_id(0), tl.program_id(1) kv_head = (head // HQ) * HK + (head % HQ) // (HQ // HK) rows = block * BM + tl.arange(0, BM) @@ -41,12 +42,13 @@ def attention_kernel( rows[:, None] < M, 0.0, ) - scale = ( + # Restore Q/K magnitudes and convert natural-exponential scores to base 2. + score_scale = ( tl.load(QS + head) * tl.load(KS + kv_head) * SCALE * 1.4426950408889634 ) - vmax = tl.load(VS + kv_head) - maximum = tl.full((BM,), -float("inf"), tl.float32) - denominator = tl.full((BM,), 0, tl.float32) + value_scale = tl.load(VS + kv_head) + row_max = tl.full((BM,), -float("inf"), tl.float32) + weight_sum = tl.full((BM,), 0, tl.float32) acc = tl.full((BM, D), 0, tl.float32) if ACC_CHUNK > 0: outer_acc = tl.full((BM, D), 0, tl.float32) @@ -58,23 +60,27 @@ def attention_kernel( indices[None, :] < N, 0.0, ) + # QK^T: [BM, D] @ [D, BN] -> [BM, BN]. score = tl.dot(q, k, max_num_imprecise_acc=32) - if FUSED_SOFTMAX and SCALE > 0: + # Scale after max() so score * scale - shift can fuse. + if FUSE_SCORE_SCALE and SCALE > 0: score = tl.where(indices[None, :] < N, score, -float("inf")) - new_max = tl.maximum(maximum, tl.max(score, 1) * scale) - shift = new_max - 8.0 if LIFT_EXP else new_max - p = tl.exp2(score * scale - shift[:, None]) + new_max = tl.maximum(row_max, tl.max(score, 1) * score_scale) + # exp2(x + 8) = 256 * exp2(x): scale weights before FP8 conversion. + shift = new_max - 8.0 if SCALE_WEIGHTS_IN_EXP else new_max + weights = tl.exp2(score * score_scale - shift[:, None]) else: score = tl.where( - indices[None, :] < N, score * scale, -float("inf") + indices[None, :] < N, score * score_scale, -float("inf") ) - new_max = tl.maximum(maximum, tl.max(score, 1)) - if LIFT_EXP: - p = tl.exp2(score - (new_max[:, None] - 8.0)) + new_max = tl.maximum(row_max, tl.max(score, 1)) + if SCALE_WEIGHTS_IN_EXP: + weights = tl.exp2(score - (new_max[:, None] - 8.0)) else: - p = tl.exp2(score - new_max[:, None]) - correction = tl.exp2(maximum - new_max) - denominator = denominator * correction + tl.sum(p, 1) + weights = tl.exp2(score - new_max[:, None]) + # Rescale previous tiles when the running maximum increases. + correction = tl.exp2(row_max - new_max) + weight_sum = weight_sum * correction + tl.sum(weights, 1) acc *= correction[:, None] v_offsets = indices[:, None] + dims[None, :] * N v = tl.load( @@ -82,9 +88,11 @@ def attention_kernel( indices[:, None] < N, 0.0, ) - # Lift softmax probabilities by 256 to retain precision in FP8. + # Weighted V: scale unnormalized weights by 256 unless exp2 did it. acc = tl.dot( - (p if LIFT_EXP else p * 256.0).to(tl.float8e4nv), + (weights if SCALE_WEIGHTS_IN_EXP else weights * 256.0).to( + tl.float8e4nv + ), v, acc, max_num_imprecise_acc=32, @@ -97,11 +105,12 @@ def attention_kernel( ) outer_max = new_max acc = tl.full((BM, D), 0, tl.float32) - maximum = new_max + row_max = new_max if ACC_CHUNK > 0: acc = outer_acc - out = acc * vmax / denominator[:, None] - if not LIFT_EXP: + # Normalize and restore V's scale; an exp2 factor of 256 cancels here. + out = acc * value_scale / weight_sum[:, None] + if not SCALE_WEIGHTS_IN_EXP: out *= 1.0 / 256.0 tl.store( Out + head * M * D + rows[:, None] * D + dims[None, :], @@ -168,9 +177,16 @@ def quantized_attention( warps: int = 4, stages: int = 2, accumulation_chunk: int = 0, - lift_exp: bool = False, - fused_softmax: bool = False, + scale_weights_in_exp: bool = False, + fuse_score_scale: bool = False, ) -> torch.Tensor: + """Run tiled attention on per-head scaled FP8 inputs. + + ``scale_weights_in_exp`` applies the weight factor 256 inside exp2; + otherwise it is applied before the weighted-value multiplication. + ``fuse_score_scale`` permits fused score scaling and max subtraction. + Both select arithmetic variants, not different attention operations. + """ b, h, m, d = q.shape n = k.size(-2) # Q/K are [batch, heads, sequence, channels]; V is transposed. @@ -192,8 +208,8 @@ def quantized_attention( block, tile, accumulation_chunk, - lift_exp, - fused_softmax, + scale_weights_in_exp, + fuse_score_scale, num_warps=warps, num_stages=stages, ) diff --git a/test/nn/test_fp8_attention.py b/test/nn/test_fp8_attention.py index 5f396ea11a..5d0998ec27 100644 --- a/test/nn/test_fp8_attention.py +++ b/test/nn/test_fp8_attention.py @@ -107,8 +107,8 @@ def test_periodic_attention_matches_cached_query_slice( opts: dict[str, Any] = { "accumulation_chunk": accumulation_chunk, "tile": tile, - "lift_exp": optimized, - "fused_softmax": optimized, + "scale_weights_in_exp": optimized, + "fuse_score_scale": optimized, } full = quantized_attention( q8, k8, v8, qs, ks, vs, q.dtype, 128, **opts From c7cab2bb0c6e9d7acdfbe091283dd3c3d81be1bc Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:41:44 +0000 Subject: [PATCH 09/44] Document quantization and attention kernel helpers --- sdm/_kernels/triton/fp8_attention.py | 34 ++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index 0defb102d9..a8aeeb7091 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -150,6 +150,16 @@ def quantize_kernel( def quantize( x: torch.Tensor, scale: torch.Tensor | None = None ) -> tuple[torch.Tensor, torch.Tensor]: + """Quantize a tensor to FP8 using per-head scales. + + Args: + x: Input with shape ``[batch, heads, rows, channels]``. + scale: Contiguous scales shaped ``[batch, heads, 1, 1]``. If omitted, + derive scales from each head's maximum absolute input value. + + Returns: + The contiguous FP8 tensor and its dequantization scales. + """ if scale is None: scale = ( x.abs().amax(dim=(-2, -1), keepdim=True).float().clamp_min(1e-12) @@ -182,10 +192,26 @@ def quantized_attention( ) -> torch.Tensor: """Run tiled attention on per-head scaled FP8 inputs. - ``scale_weights_in_exp`` applies the weight factor 256 inside exp2; - otherwise it is applied before the weighted-value multiplication. - ``fuse_score_scale`` permits fused score scaling and max subtraction. - Both select arithmetic variants, not different attention operations. + Args: + q: Contiguous FP8 queries shaped ``[batch, query_heads, queries, D]``. + k: Contiguous FP8 keys shaped ``[batch, kv_heads, context, D]``. + v: Contiguous FP8 values shaped ``[batch, kv_heads, D, context]``. + qs: Query scales shaped ``[batch, query_heads, 1, 1]``. + ks: Key scales shaped ``[batch, kv_heads, 1, 1]``. + vs: Value scales shaped ``[batch, kv_heads, 1, 1]``. + dtype: Output dtype. + block: Query rows handled by each program. + scale: Attention score multiplier. Defaults to ``1 / sqrt(D)``. + tile: Context rows processed per loop iteration. + warps: Cooperating warps per program. + stages: Software pipeline stages. + accumulation_chunk: Tiles per FP32 accumulator flush; zero disables it. + scale_weights_in_exp: Apply the weight factor 256 inside exp2 instead + of before the weighted-value multiplication. + fuse_score_scale: Permit fused score scaling and maximum subtraction. + + Returns: + Attention output with the query shape and requested dtype. """ b, h, m, d = q.shape n = k.size(-2) From d176c9411da99be90b63eb1e952bf2a9088d8e93 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:44:58 +0000 Subject: [PATCH 10/44] Document Triton kernel entry points in Google style --- sdm/_kernels/triton/fp8_attention.py | 38 +++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index a8aeeb7091..a9016b192b 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -31,7 +31,28 @@ def attention_kernel( SCALE_WEIGHTS_IN_EXP: tl.constexpr = False, FUSE_SCORE_SCALE: tl.constexpr = False, ): - # Each program owns BM query rows for one head and visits every K/V tile. + """Write attention outputs for one query tile and head per program. + + Args: + Q: FP8 queries shaped ``[batch, HQ, M, D]``. + K: FP8 keys shaped ``[batch, HK, N, D]``. + V: FP8 values shaped ``[batch, HK, D, N]``. + QS: Per-query-head dequantization scales. + KS: Per-key-head dequantization scales. + VS: Per-value-head dequantization scales. + Out: Output buffer shaped ``[batch, HQ, M, D]``. + HQ: Query heads per batch item. + HK: K/V heads per batch item; must divide HQ. + M: Number of query rows. + N: Number of context rows. + D: Channels per head. + SCALE: Attention score multiplier. + BM: Query rows handled by each program. + BN: Context rows processed per iteration. + ACC_CHUNK: Tiles per FP32 accumulator flush; zero disables it. + SCALE_WEIGHTS_IN_EXP: Include the weight factor 256 inside exp2. + FUSE_SCORE_SCALE: Permit fused score scaling and max subtraction. + """ block, head = tl.program_id(0), tl.program_id(1) kv_head = (head // HQ) * HK + (head % HQ) // (HQ // HK) rows = block * BM + tl.arange(0, BM) @@ -133,6 +154,21 @@ def quantize_kernel( S2: tl.constexpr, S3: tl.constexpr, ): + """Write scaled, clamped input values into a contiguous FP8 buffer. + + Args: + X: Input shaped ``[batch, heads, rows, channels]``. + S: Contiguous per-head dequantization scales. + Y: Contiguous FP8 output buffer with the input shape. + LENGTH: Number of elements per head. + BLOCK: Elements handled by each program. + H: Heads per batch item. + D: Channels per head. + S0: Input batch stride in elements. + S1: Input head stride in elements. + S2: Input row stride in elements. + S3: Input channel stride in elements. + """ offset = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) head = tl.program_id(1) source = ( From de1545ee353483d7d172e2e10f5491b1ecc39b12 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:49:34 +0000 Subject: [PATCH 11/44] Mark tiled matmul and online softmax stages in attention --- sdm/_kernels/triton/fp8_attention.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index a9016b192b..de428c4a72 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -33,6 +33,9 @@ def attention_kernel( ): """Write attention outputs for one query tile and head per program. + Each program keeps a ``[BM, D]`` query tile and loops over context tiles + of ``BN`` rows. Both matrix multiplications use the full head width D. + Args: Q: FP8 queries shaped ``[batch, HQ, M, D]``. K: FP8 keys shaped ``[batch, HK, N, D]``. @@ -50,8 +53,10 @@ def attention_kernel( BM: Query rows handled by each program. BN: Context rows processed per iteration. ACC_CHUNK: Tiles per FP32 accumulator flush; zero disables it. - SCALE_WEIGHTS_IN_EXP: Include the weight factor 256 inside exp2. - FUSE_SCORE_SCALE: Permit fused score scaling and max subtraction. + SCALE_WEIGHTS_IN_EXP: Scale weights via ``exp2(x + 8)`` instead of + multiplying ``exp2(x)`` by 256 afterward. + FUSE_SCORE_SCALE: Move positive score scaling after the maximum + reduction to permit fused scaling and maximum subtraction. """ block, head = tl.program_id(0), tl.program_id(1) kv_head = (head // HQ) * HK + (head % HQ) // (HQ // HK) @@ -75,19 +80,19 @@ def attention_kernel( outer_acc = tl.full((BM, D), 0, tl.float32) outer_max = tl.full((BM,), -float("inf"), tl.float32) for start in range(tl.cdiv(N, BN)): + # 1. Tiled QK^T multiplication: [BM, D] @ [D, BN] -> [BM, BN]. indices = start * BN + cols k = tl.load( K + kv_head * N * D + indices[None, :] * D + dims[:, None], indices[None, :] < N, 0.0, ) - # QK^T: [BM, D] @ [D, BN] -> [BM, BN]. score = tl.dot(q, k, max_num_imprecise_acc=32) - # Scale after max() so score * scale - shift can fuse. + + # 2. Online softmax: update row maxima and weight sums across tiles. if FUSE_SCORE_SCALE and SCALE > 0: score = tl.where(indices[None, :] < N, score, -float("inf")) new_max = tl.maximum(row_max, tl.max(score, 1) * score_scale) - # exp2(x + 8) = 256 * exp2(x): scale weights before FP8 conversion. shift = new_max - 8.0 if SCALE_WEIGHTS_IN_EXP else new_max weights = tl.exp2(score * score_scale - shift[:, None]) else: @@ -99,17 +104,17 @@ def attention_kernel( weights = tl.exp2(score - (new_max[:, None] - 8.0)) else: weights = tl.exp2(score - new_max[:, None]) - # Rescale previous tiles when the running maximum increases. correction = tl.exp2(row_max - new_max) weight_sum = weight_sum * correction + tl.sum(weights, 1) acc *= correction[:, None] + + # 3. Tiled weights-V multiplication: [BM, BN] @ [BN, D] -> [BM, D]. v_offsets = indices[:, None] + dims[None, :] * N v = tl.load( V + kv_head * N * D + v_offsets, indices[:, None] < N, 0.0, ) - # Weighted V: scale unnormalized weights by 256 unless exp2 did it. acc = tl.dot( (weights if SCALE_WEIGHTS_IN_EXP else weights * 256.0).to( tl.float8e4nv @@ -129,7 +134,7 @@ def attention_kernel( row_max = new_max if ACC_CHUNK > 0: acc = outer_acc - # Normalize and restore V's scale; an exp2 factor of 256 cancels here. + # After all tiles, divide accumulated weighted values by the weight sum. out = acc * value_scale / weight_sum[:, None] if not SCALE_WEIGHTS_IN_EXP: out *= 1.0 / 256.0 From d2fa03fcbcff97a9c4f86a30e1d9ba387df85faf Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 16 Sep 2026 03:54:15 +0000 Subject: [PATCH 12/44] Clarify batch head indexing and attention tile dimensions --- sdm/_kernels/triton/fp8_attention.py | 38 ++++++++++++++++------------ test/nn/test_fp8_attention.py | 14 +++++----- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index de428c4a72..fe0d00075e 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -58,21 +58,27 @@ def attention_kernel( FUSE_SCORE_SCALE: Move positive score scaling after the maximum reduction to permit fused scaling and maximum subtraction. """ - block, head = tl.program_id(0), tl.program_id(1) - kv_head = (head // HQ) * HK + (head % HQ) // (HQ // HK) - rows = block * BM + tl.arange(0, BM) + query_block, batch_query_head = tl.program_id(0), tl.program_id(1) + batch = batch_query_head // HQ + query_head = batch_query_head % HQ + # Query heads share K/V heads only when HQ > HK (grouped-query attention). + batch_kv_head = batch * HK + query_head // (HQ // HK) + rows = query_block * BM + tl.arange(0, BM) cols = tl.arange(0, BN) dims = tl.arange(0, D) q = tl.load( - Q + head * M * D + rows[:, None] * D + dims[None, :], + Q + batch_query_head * M * D + rows[:, None] * D + dims[None, :], rows[:, None] < M, 0.0, ) # Restore Q/K magnitudes and convert natural-exponential scores to base 2. score_scale = ( - tl.load(QS + head) * tl.load(KS + kv_head) * SCALE * 1.4426950408889634 + tl.load(QS + batch_query_head) + * tl.load(KS + batch_kv_head) + * SCALE + * 1.4426950408889634 ) - value_scale = tl.load(VS + kv_head) + value_scale = tl.load(VS + batch_kv_head) row_max = tl.full((BM,), -float("inf"), tl.float32) weight_sum = tl.full((BM,), 0, tl.float32) acc = tl.full((BM, D), 0, tl.float32) @@ -83,7 +89,7 @@ def attention_kernel( # 1. Tiled QK^T multiplication: [BM, D] @ [D, BN] -> [BM, BN]. indices = start * BN + cols k = tl.load( - K + kv_head * N * D + indices[None, :] * D + dims[:, None], + K + batch_kv_head * N * D + indices[None, :] * D + dims[:, None], indices[None, :] < N, 0.0, ) @@ -111,7 +117,7 @@ def attention_kernel( # 3. Tiled weights-V multiplication: [BM, BN] @ [BN, D] -> [BM, D]. v_offsets = indices[:, None] + dims[None, :] * N v = tl.load( - V + kv_head * N * D + v_offsets, + V + batch_kv_head * N * D + v_offsets, indices[:, None] < N, 0.0, ) @@ -139,7 +145,7 @@ def attention_kernel( if not SCALE_WEIGHTS_IN_EXP: out *= 1.0 / 256.0 tl.store( - Out + head * M * D + rows[:, None] * D + dims[None, :], + Out + batch_query_head * M * D + rows[:, None] * D + dims[None, :], out, rows[:, None] < M, ) @@ -222,9 +228,9 @@ def quantized_attention( ks: torch.Tensor, vs: torch.Tensor, dtype: torch.dtype, - block: int, + query_tile: int, scale: float | None = None, - tile: int = 64, + context_tile: int = 64, warps: int = 4, stages: int = 2, accumulation_chunk: int = 0, @@ -241,9 +247,9 @@ def quantized_attention( ks: Key scales shaped ``[batch, kv_heads, 1, 1]``. vs: Value scales shaped ``[batch, kv_heads, 1, 1]``. dtype: Output dtype. - block: Query rows handled by each program. + query_tile: Query rows handled by each program. scale: Attention score multiplier. Defaults to ``1 / sqrt(D)``. - tile: Context rows processed per loop iteration. + context_tile: Context rows processed per loop iteration. warps: Cooperating warps per program. stages: Software pipeline stages. accumulation_chunk: Tiles per FP32 accumulator flush; zero disables it. @@ -258,7 +264,7 @@ def quantized_attention( n = k.size(-2) # Q/K are [batch, heads, sequence, channels]; V is transposed. output = torch.empty((b, h, m, d), device=q.device, dtype=dtype) - cast(Any, attention_kernel)[(((m + block - 1) // block), b * h)]( + cast(Any, attention_kernel)[(((m + query_tile - 1) // query_tile), b * h)]( q, k, v, @@ -272,8 +278,8 @@ def quantized_attention( n, d, d**-0.5 if scale is None else scale, - block, - tile, + query_tile, + context_tile, accumulation_chunk, scale_weights_in_exp, fuse_score_scale, diff --git a/test/nn/test_fp8_attention.py b/test/nn/test_fp8_attention.py index 5d0998ec27..c89bf8bdcf 100644 --- a/test/nn/test_fp8_attention.py +++ b/test/nn/test_fp8_attention.py @@ -53,8 +53,10 @@ def test_fp8_partial_tiles(batch: int, queries: int, context: int) -> None: @pytest.mark.parametrize("scale", [None, 1.0]) -@pytest.mark.parametrize("tile", [64, 128]) -def test_grouped_heads_and_model_scale(scale: float | None, tile: int) -> None: +@pytest.mark.parametrize("context_tile", [64, 128]) +def test_grouped_query_attention_and_model_scale( + scale: float | None, context_tile: int +) -> None: with torch.inference_mode(): q = ( torch.randn(2, 8, 129, 64, device="cuda", dtype=torch.float16) @@ -78,7 +80,7 @@ def test_grouped_heads_and_model_scale(scale: float | None, tile: int) -> None: q.dtype, 128, scale, - tile=tile, + context_tile=context_tile, ) expected = torch.nn.functional.scaled_dot_product_attention( q, k, v, enable_gqa=True, scale=scale @@ -92,10 +94,10 @@ def test_grouped_heads_and_model_scale(scale: float | None, tile: int) -> None: @pytest.mark.parametrize("accumulation_chunk", [0, 16]) @pytest.mark.parametrize("optimized", [False, True]) -@pytest.mark.parametrize("tile", [64, 128]) +@pytest.mark.parametrize("context_tile", [64, 128]) @pytest.mark.parametrize("context", [8193, 65536]) def test_periodic_attention_matches_cached_query_slice( - context: int, tile: int, optimized: bool, accumulation_chunk: int + context: int, context_tile: int, optimized: bool, accumulation_chunk: int ) -> None: with torch.inference_mode(): q = torch.randn(1, 1, context, 64, device="cuda", dtype=torch.float16) @@ -106,7 +108,7 @@ def test_periodic_attention_matches_cached_query_slice( v8 = v8.transpose(-1, -2).contiguous() opts: dict[str, Any] = { "accumulation_chunk": accumulation_chunk, - "tile": tile, + "context_tile": context_tile, "scale_weights_in_exp": optimized, "fuse_score_scale": optimized, } From 428e074231a5ee481a94a92cafadebc0344775c2 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 16 Sep 2026 22:23:26 +0000 Subject: [PATCH 13/44] Name quantization load and store mask arguments --- sdm/_kernels/triton/fp8_attention.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index fe0d00075e..c64d9de64b 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -188,10 +188,10 @@ def quantize_kernel( + (offset // D) * S2 + (offset % D) * S3 ) - x = tl.load(X + source, offset < LENGTH, 0.0).to(tl.float32) + x = tl.load(X + source, mask=offset < LENGTH, other=0.0).to(tl.float32) scale = tl.load(S + head) y = tl.minimum(tl.maximum(x / scale, -448.0), 448.0) - tl.store(Y + head * LENGTH + offset, y, offset < LENGTH) + tl.store(Y + head * LENGTH + offset, y, mask=offset < LENGTH) def quantize( From fd956c972f10273c210c9bf685dc805a3dd8fb99 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:08:10 +0000 Subject: [PATCH 14/44] Clarify quantization channel and input stride names --- sdm/_kernels/triton/fp8_attention.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index c64d9de64b..9a93ea8d8f 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -159,11 +159,11 @@ def quantize_kernel( LENGTH: tl.constexpr, BLOCK: tl.constexpr, H: tl.constexpr, - D: tl.constexpr, - S0: tl.constexpr, - S1: tl.constexpr, - S2: tl.constexpr, - S3: tl.constexpr, + CHANNELS: tl.constexpr, + BATCH_STRIDE: tl.constexpr, + HEAD_STRIDE: tl.constexpr, + ROW_STRIDE: tl.constexpr, + CHANNEL_STRIDE: tl.constexpr, ): """Write scaled, clamped input values into a contiguous FP8 buffer. @@ -174,19 +174,19 @@ def quantize_kernel( LENGTH: Number of elements per head. BLOCK: Elements handled by each program. H: Heads per batch item. - D: Channels per head. - S0: Input batch stride in elements. - S1: Input head stride in elements. - S2: Input row stride in elements. - S3: Input channel stride in elements. + CHANNELS: Channels per head. + BATCH_STRIDE: Input batch stride in elements. + HEAD_STRIDE: Input head stride in elements. + ROW_STRIDE: Input row stride in elements. + CHANNEL_STRIDE: Input channel stride in elements. """ offset = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) head = tl.program_id(1) source = ( - (head // H) * S0 - + (head % H) * S1 - + (offset // D) * S2 - + (offset % D) * S3 + (head // H) * BATCH_STRIDE + + (head % H) * HEAD_STRIDE + + (offset // CHANNELS) * ROW_STRIDE + + (offset % CHANNELS) * CHANNEL_STRIDE ) x = tl.load(X + source, mask=offset < LENGTH, other=0.0).to(tl.float32) scale = tl.load(S + head) From d0c654598a4dab71910407ddaa8f4a4281fc3314 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:12:04 +0000 Subject: [PATCH 15/44] Use concise quantization stride names --- sdm/_kernels/triton/fp8_attention.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index 9a93ea8d8f..8fdee93474 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -159,11 +159,11 @@ def quantize_kernel( LENGTH: tl.constexpr, BLOCK: tl.constexpr, H: tl.constexpr, - CHANNELS: tl.constexpr, - BATCH_STRIDE: tl.constexpr, - HEAD_STRIDE: tl.constexpr, - ROW_STRIDE: tl.constexpr, - CHANNEL_STRIDE: tl.constexpr, + C: tl.constexpr, + stride_b: tl.constexpr, + stride_h: tl.constexpr, + stride_r: tl.constexpr, + stride_c: tl.constexpr, ): """Write scaled, clamped input values into a contiguous FP8 buffer. @@ -174,19 +174,19 @@ def quantize_kernel( LENGTH: Number of elements per head. BLOCK: Elements handled by each program. H: Heads per batch item. - CHANNELS: Channels per head. - BATCH_STRIDE: Input batch stride in elements. - HEAD_STRIDE: Input head stride in elements. - ROW_STRIDE: Input row stride in elements. - CHANNEL_STRIDE: Input channel stride in elements. + C: Channels per head. + stride_b: Input batch stride in elements. + stride_h: Input head stride in elements. + stride_r: Input row stride in elements. + stride_c: Input channel stride in elements. """ offset = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) head = tl.program_id(1) source = ( - (head // H) * BATCH_STRIDE - + (head % H) * HEAD_STRIDE - + (offset // CHANNELS) * ROW_STRIDE - + (offset % CHANNELS) * CHANNEL_STRIDE + (head // H) * stride_b + + (head % H) * stride_h + + (offset // C) * stride_r + + (offset % C) * stride_c ) x = tl.load(X + source, mask=offset < LENGTH, other=0.0).to(tl.float32) scale = tl.load(S + head) From b8115d9446eec6e7b4055f780e95b5b863cc7a81 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:28:32 +0000 Subject: [PATCH 16/44] Clarify attention kernel arguments and launch dimensions --- sdm/_kernels/triton/fp8_attention.py | 198 +++++++++++++++------------ 1 file changed, 108 insertions(+), 90 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index 8fdee93474..095f72b6a3 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -12,101 +12,114 @@ @triton.jit def attention_kernel( - Q, - K, - V, - QS, - KS, - VS, - Out, - HQ: tl.constexpr, - HK: tl.constexpr, - M: tl.constexpr, - N: tl.constexpr, - D: tl.constexpr, - SCALE: tl.constexpr, - BM: tl.constexpr, - BN: tl.constexpr, - ACC_CHUNK: tl.constexpr = 0, - SCALE_WEIGHTS_IN_EXP: tl.constexpr = False, - FUSE_SCORE_SCALE: tl.constexpr = False, + q_ptr, + k_ptr, + v_ptr, + q_scale_ptr, + k_scale_ptr, + v_scale_ptr, + out_ptr, + n_q_heads: tl.constexpr, + n_kv_heads: tl.constexpr, + n_queries: tl.constexpr, + n_context: tl.constexpr, + head_dim: tl.constexpr, + attn_scale: tl.constexpr, + block_q: tl.constexpr, + block_kv: tl.constexpr, + acc_flush_tiles: tl.constexpr = 0, + scale_weights_in_exp: tl.constexpr = False, + fuse_score_scale: tl.constexpr = False, ): """Write attention outputs for one query tile and head per program. - Each program keeps a ``[BM, D]`` query tile and loops over context tiles - of ``BN`` rows. Both matrix multiplications use the full head width D. + Each program keeps a ``[block_q, head_dim]`` query tile and loops over + context tiles of ``block_kv`` rows. Both matrix multiplications use + the full head width. Args: - Q: FP8 queries shaped ``[batch, HQ, M, D]``. - K: FP8 keys shaped ``[batch, HK, N, D]``. - V: FP8 values shaped ``[batch, HK, D, N]``. - QS: Per-query-head dequantization scales. - KS: Per-key-head dequantization scales. - VS: Per-value-head dequantization scales. - Out: Output buffer shaped ``[batch, HQ, M, D]``. - HQ: Query heads per batch item. - HK: K/V heads per batch item; must divide HQ. - M: Number of query rows. - N: Number of context rows. - D: Channels per head. - SCALE: Attention score multiplier. - BM: Query rows handled by each program. - BN: Context rows processed per iteration. - ACC_CHUNK: Tiles per FP32 accumulator flush; zero disables it. - SCALE_WEIGHTS_IN_EXP: Scale weights via ``exp2(x + 8)`` instead of + q_ptr: FP8 queries shaped ``[batch, n_q_heads, n_queries, head_dim]``. + k_ptr: FP8 keys shaped ``[batch, n_kv_heads, n_context, head_dim]``. + v_ptr: FP8 values shaped ``[batch, n_kv_heads, head_dim, n_context]``. + q_scale_ptr: Per-query-head dequantization scales. + k_scale_ptr: Per-key-head dequantization scales. + v_scale_ptr: Per-value-head dequantization scales. + out_ptr: Output buffer with the query shape. + n_q_heads: Query heads per batch item. + n_kv_heads: K/V heads per batch item; must divide n_q_heads. + n_queries: Number of query rows. + n_context: Number of context rows. + head_dim: Channels per head. + attn_scale: Attention score multiplier. + block_q: Query rows handled by each program. + block_kv: Context rows processed per iteration. + acc_flush_tiles: Tiles per FP32 accumulator flush; zero disables it. + scale_weights_in_exp: Scale weights via ``exp2(x + 8)`` instead of multiplying ``exp2(x)`` by 256 afterward. - FUSE_SCORE_SCALE: Move positive score scaling after the maximum + fuse_score_scale: Move positive score scaling after the maximum reduction to permit fused scaling and maximum subtraction. """ query_block, batch_query_head = tl.program_id(0), tl.program_id(1) - batch = batch_query_head // HQ - query_head = batch_query_head % HQ - # Query heads share K/V heads only when HQ > HK (grouped-query attention). - batch_kv_head = batch * HK + query_head // (HQ // HK) - rows = query_block * BM + tl.arange(0, BM) - cols = tl.arange(0, BN) - dims = tl.arange(0, D) + batch = batch_query_head // n_q_heads + query_head = batch_query_head % n_q_heads + # Grouped-query attention shares K/V heads when n_q_heads > n_kv_heads. + batch_kv_head = batch * n_kv_heads + query_head // ( + n_q_heads // n_kv_heads + ) + rows = query_block * block_q + tl.arange(0, block_q) + cols = tl.arange(0, block_kv) + dims = tl.arange(0, head_dim) q = tl.load( - Q + batch_query_head * M * D + rows[:, None] * D + dims[None, :], - rows[:, None] < M, + q_ptr + + batch_query_head * n_queries * head_dim + + rows[:, None] * head_dim + + dims[None, :], + rows[:, None] < n_queries, 0.0, ) # Restore Q/K magnitudes and convert natural-exponential scores to base 2. score_scale = ( - tl.load(QS + batch_query_head) - * tl.load(KS + batch_kv_head) - * SCALE + tl.load(q_scale_ptr + batch_query_head) + * tl.load(k_scale_ptr + batch_kv_head) + * attn_scale * 1.4426950408889634 ) - value_scale = tl.load(VS + batch_kv_head) - row_max = tl.full((BM,), -float("inf"), tl.float32) - weight_sum = tl.full((BM,), 0, tl.float32) - acc = tl.full((BM, D), 0, tl.float32) - if ACC_CHUNK > 0: - outer_acc = tl.full((BM, D), 0, tl.float32) - outer_max = tl.full((BM,), -float("inf"), tl.float32) - for start in range(tl.cdiv(N, BN)): - # 1. Tiled QK^T multiplication: [BM, D] @ [D, BN] -> [BM, BN]. - indices = start * BN + cols + value_scale = tl.load(v_scale_ptr + batch_kv_head) + row_max = tl.full((block_q,), -float("inf"), tl.float32) + weight_sum = tl.full((block_q,), 0, tl.float32) + acc = tl.full((block_q, head_dim), 0, tl.float32) + if acc_flush_tiles > 0: + outer_acc = tl.full((block_q, head_dim), 0, tl.float32) + outer_max = tl.full((block_q,), -float("inf"), tl.float32) + for start in range(tl.cdiv(n_context, block_kv)): + # 1. Tiled QK^T: reduce over channels to get [block_q, block_kv]. + indices = start * block_kv + cols k = tl.load( - K + batch_kv_head * N * D + indices[None, :] * D + dims[:, None], - indices[None, :] < N, + k_ptr + + batch_kv_head * n_context * head_dim + + indices[None, :] * head_dim + + dims[:, None], + indices[None, :] < n_context, 0.0, ) score = tl.dot(q, k, max_num_imprecise_acc=32) # 2. Online softmax: update row maxima and weight sums across tiles. - if FUSE_SCORE_SCALE and SCALE > 0: - score = tl.where(indices[None, :] < N, score, -float("inf")) + if fuse_score_scale and attn_scale > 0: + score = tl.where( + indices[None, :] < n_context, score, -float("inf") + ) new_max = tl.maximum(row_max, tl.max(score, 1) * score_scale) - shift = new_max - 8.0 if SCALE_WEIGHTS_IN_EXP else new_max + shift = new_max - 8.0 if scale_weights_in_exp else new_max weights = tl.exp2(score * score_scale - shift[:, None]) else: score = tl.where( - indices[None, :] < N, score * score_scale, -float("inf") + indices[None, :] < n_context, + score * score_scale, + -float("inf"), ) new_max = tl.maximum(row_max, tl.max(score, 1)) - if SCALE_WEIGHTS_IN_EXP: + if scale_weights_in_exp: weights = tl.exp2(score - (new_max[:, None] - 8.0)) else: weights = tl.exp2(score - new_max[:, None]) @@ -114,15 +127,15 @@ def attention_kernel( weight_sum = weight_sum * correction + tl.sum(weights, 1) acc *= correction[:, None] - # 3. Tiled weights-V multiplication: [BM, BN] @ [BN, D] -> [BM, D]. - v_offsets = indices[:, None] + dims[None, :] * N + # 3. Tiled weights-V: reduce over context to get [block_q, head_dim]. + v_offsets = indices[:, None] + dims[None, :] * n_context v = tl.load( - V + batch_kv_head * N * D + v_offsets, - indices[:, None] < N, + v_ptr + batch_kv_head * n_context * head_dim + v_offsets, + indices[:, None] < n_context, 0.0, ) acc = tl.dot( - (weights if SCALE_WEIGHTS_IN_EXP else weights * 256.0).to( + (weights if scale_weights_in_exp else weights * 256.0).to( tl.float8e4nv ), v, @@ -130,24 +143,29 @@ def attention_kernel( max_num_imprecise_acc=32, ) # Periodically flush the dot accumulator to FP32 for long L4 contexts. - if ACC_CHUNK > 0: # noqa: SIM102 - constexpr guard avoids modulo zero. - if ((start + 1) % ACC_CHUNK == 0) | (start + 1 == tl.cdiv(N, BN)): + if acc_flush_tiles > 0: # noqa: SIM102 - constexpr guard avoids modulo zero. + if ((start + 1) % acc_flush_tiles == 0) | ( + start + 1 == tl.cdiv(n_context, block_kv) + ): outer_acc = ( outer_acc * tl.exp2(outer_max - new_max)[:, None] + acc ) outer_max = new_max - acc = tl.full((BM, D), 0, tl.float32) + acc = tl.full((block_q, head_dim), 0, tl.float32) row_max = new_max - if ACC_CHUNK > 0: + if acc_flush_tiles > 0: acc = outer_acc # After all tiles, divide accumulated weighted values by the weight sum. out = acc * value_scale / weight_sum[:, None] - if not SCALE_WEIGHTS_IN_EXP: + if not scale_weights_in_exp: out *= 1.0 / 256.0 tl.store( - Out + batch_query_head * M * D + rows[:, None] * D + dims[None, :], + out_ptr + + batch_query_head * n_queries * head_dim + + rows[:, None] * head_dim + + dims[None, :], out, - rows[:, None] < M, + rows[:, None] < n_queries, ) @@ -272,17 +290,17 @@ def quantized_attention( ks, vs, output, - h, - k.size(1), - m, - n, - d, - d**-0.5 if scale is None else scale, - query_tile, - context_tile, - accumulation_chunk, - scale_weights_in_exp, - fuse_score_scale, + n_q_heads=h, + n_kv_heads=k.size(1), + n_queries=m, + n_context=n, + head_dim=d, + attn_scale=d**-0.5 if scale is None else scale, + block_q=query_tile, + block_kv=context_tile, + acc_flush_tiles=accumulation_chunk, + scale_weights_in_exp=scale_weights_in_exp, + fuse_score_scale=fuse_score_scale, num_warps=warps, num_stages=stages, ) From 5c49f0335779ed315843d504f63f8e53e6dd3a62 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 17 Sep 2026 22:05:57 +0000 Subject: [PATCH 17/44] Follow segment kernel conventions in FP8 Triton launches --- sdm/_kernels/triton/fp8_attention.py | 200 ++++++++++++--------------- 1 file changed, 91 insertions(+), 109 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index 095f72b6a3..e8d1d16cd7 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -1,17 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import sys from typing import Any, cast import torch -import triton -import triton.language as _tl +from torch import Tensor + +if sys.platform != "linux": + raise ImportError("Triton kernels are only available on Linux") -tl: Any = _tl +import triton +import triton.language as tl @triton.jit -def attention_kernel( +def _attention_kernel( q_ptr, k_ptr, v_ptr, @@ -27,37 +31,16 @@ def attention_kernel( attn_scale: tl.constexpr, block_q: tl.constexpr, block_kv: tl.constexpr, - acc_flush_tiles: tl.constexpr = 0, - scale_weights_in_exp: tl.constexpr = False, - fuse_score_scale: tl.constexpr = False, -): + acc_flush_tiles: tl.constexpr, + scale_weights_in_exp: tl.constexpr, + fuse_score_scale: tl.constexpr, +) -> None: """Write attention outputs for one query tile and head per program. Each program keeps a ``[block_q, head_dim]`` query tile and loops over context tiles of ``block_kv`` rows. Both matrix multiplications use the full head width. - Args: - q_ptr: FP8 queries shaped ``[batch, n_q_heads, n_queries, head_dim]``. - k_ptr: FP8 keys shaped ``[batch, n_kv_heads, n_context, head_dim]``. - v_ptr: FP8 values shaped ``[batch, n_kv_heads, head_dim, n_context]``. - q_scale_ptr: Per-query-head dequantization scales. - k_scale_ptr: Per-key-head dequantization scales. - v_scale_ptr: Per-value-head dequantization scales. - out_ptr: Output buffer with the query shape. - n_q_heads: Query heads per batch item. - n_kv_heads: K/V heads per batch item; must divide n_q_heads. - n_queries: Number of query rows. - n_context: Number of context rows. - head_dim: Channels per head. - attn_scale: Attention score multiplier. - block_q: Query rows handled by each program. - block_kv: Context rows processed per iteration. - acc_flush_tiles: Tiles per FP32 accumulator flush; zero disables it. - scale_weights_in_exp: Scale weights via ``exp2(x + 8)`` instead of - multiplying ``exp2(x)`` by 256 afterward. - fuse_score_scale: Move positive score scaling after the maximum - reduction to permit fused scaling and maximum subtraction. """ query_block, batch_query_head = tl.program_id(0), tl.program_id(1) batch = batch_query_head // n_q_heads @@ -74,8 +57,8 @@ def attention_kernel( + batch_query_head * n_queries * head_dim + rows[:, None] * head_dim + dims[None, :], - rows[:, None] < n_queries, - 0.0, + mask=rows[:, None] < n_queries, + other=0.0, ) # Restore Q/K magnitudes and convert natural-exponential scores to base 2. score_scale = ( @@ -91,7 +74,7 @@ def attention_kernel( if acc_flush_tiles > 0: outer_acc = tl.full((block_q, head_dim), 0, tl.float32) outer_max = tl.full((block_q,), -float("inf"), tl.float32) - for start in range(tl.cdiv(n_context, block_kv)): + for start in range(tl.cdiv(n_context, block_kv)): # ty: ignore[invalid-argument-type] # 1. Tiled QK^T: reduce over channels to get [block_q, block_kv]. indices = start * block_kv + cols k = tl.load( @@ -99,8 +82,8 @@ def attention_kernel( + batch_kv_head * n_context * head_dim + indices[None, :] * head_dim + dims[:, None], - indices[None, :] < n_context, - 0.0, + mask=indices[None, :] < n_context, + other=0.0, ) score = tl.dot(q, k, max_num_imprecise_acc=32) @@ -109,7 +92,7 @@ def attention_kernel( score = tl.where( indices[None, :] < n_context, score, -float("inf") ) - new_max = tl.maximum(row_max, tl.max(score, 1) * score_scale) + new_max = tl.maximum(row_max, tl.max(score, 1) * score_scale) # ty: ignore[invalid-argument-type] shift = new_max - 8.0 if scale_weights_in_exp else new_max weights = tl.exp2(score * score_scale - shift[:, None]) else: @@ -118,21 +101,21 @@ def attention_kernel( score * score_scale, -float("inf"), ) - new_max = tl.maximum(row_max, tl.max(score, 1)) + new_max = tl.maximum(row_max, tl.max(score, 1)) # ty: ignore[invalid-argument-type] if scale_weights_in_exp: weights = tl.exp2(score - (new_max[:, None] - 8.0)) else: weights = tl.exp2(score - new_max[:, None]) correction = tl.exp2(row_max - new_max) - weight_sum = weight_sum * correction + tl.sum(weights, 1) + weight_sum = weight_sum * correction + tl.sum(weights, 1) # ty: ignore[invalid-argument-type] acc *= correction[:, None] # 3. Tiled weights-V: reduce over context to get [block_q, head_dim]. v_offsets = indices[:, None] + dims[None, :] * n_context v = tl.load( v_ptr + batch_kv_head * n_context * head_dim + v_offsets, - indices[:, None] < n_context, - 0.0, + mask=indices[:, None] < n_context, + other=0.0, ) acc = tl.dot( (weights if scale_weights_in_exp else weights * 256.0).to( @@ -144,8 +127,8 @@ def attention_kernel( ) # Periodically flush the dot accumulator to FP32 for long L4 contexts. if acc_flush_tiles > 0: # noqa: SIM102 - constexpr guard avoids modulo zero. - if ((start + 1) % acc_flush_tiles == 0) | ( - start + 1 == tl.cdiv(n_context, block_kv) + if ((start + 1) % acc_flush_tiles == 0) | ( # ty: ignore[unsupported-operator] + start + 1 == tl.cdiv(n_context, block_kv) # ty: ignore[invalid-argument-type] ): outer_acc = ( outer_acc * tl.exp2(outer_max - new_max)[:, None] + acc @@ -165,56 +148,40 @@ def attention_kernel( + rows[:, None] * head_dim + dims[None, :], out, - rows[:, None] < n_queries, + mask=rows[:, None] < n_queries, ) @triton.jit -def quantize_kernel( - X, - S, - Y, - LENGTH: tl.constexpr, - BLOCK: tl.constexpr, - H: tl.constexpr, - C: tl.constexpr, +def _quantize_kernel( + x_ptr, + scale_ptr, + out_ptr, + numel: tl.constexpr, + block_size: tl.constexpr, + num_heads: tl.constexpr, + num_channels: tl.constexpr, stride_b: tl.constexpr, stride_h: tl.constexpr, stride_r: tl.constexpr, stride_c: tl.constexpr, -): - """Write scaled, clamped input values into a contiguous FP8 buffer. - - Args: - X: Input shaped ``[batch, heads, rows, channels]``. - S: Contiguous per-head dequantization scales. - Y: Contiguous FP8 output buffer with the input shape. - LENGTH: Number of elements per head. - BLOCK: Elements handled by each program. - H: Heads per batch item. - C: Channels per head. - stride_b: Input batch stride in elements. - stride_h: Input head stride in elements. - stride_r: Input row stride in elements. - stride_c: Input channel stride in elements. - """ - offset = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) +) -> None: + """Write scaled, clamped input values into a contiguous FP8 buffer.""" + offset = tl.program_id(0) * block_size + tl.arange(0, block_size) head = tl.program_id(1) source = ( - (head // H) * stride_b - + (head % H) * stride_h - + (offset // C) * stride_r - + (offset % C) * stride_c + (head // num_heads) * stride_b + + (head % num_heads) * stride_h + + (offset // num_channels) * stride_r + + (offset % num_channels) * stride_c ) - x = tl.load(X + source, mask=offset < LENGTH, other=0.0).to(tl.float32) - scale = tl.load(S + head) + x = tl.load(x_ptr + source, mask=offset < numel, other=0.0).to(tl.float32) + scale = tl.load(scale_ptr + head) y = tl.minimum(tl.maximum(x / scale, -448.0), 448.0) - tl.store(Y + head * LENGTH + offset, y, mask=offset < LENGTH) + tl.store(out_ptr + head * numel + offset, y, mask=offset < numel) -def quantize( - x: torch.Tensor, scale: torch.Tensor | None = None -) -> tuple[torch.Tensor, torch.Tensor]: +def quantize(x: Tensor, scale: Tensor | None = None) -> tuple[Tensor, Tensor]: """Quantize a tensor to FP8 using per-head scales. Args: @@ -232,19 +199,32 @@ def quantize( ) output = torch.empty(x.shape, device=x.device, dtype=torch.float8_e4m3fn) length = x.size(-2) * x.size(-1) - cast(Any, quantize_kernel)[ - (((length + 1023) // 1024), x.size(0) * x.size(1)) - ](x, scale, output, length, 1024, x.size(1), x.size(-1), *x.stride()) + block_size = 1024 + grid = ( + triton.cdiv(length, block_size), # ty: ignore[invalid-argument-type] + x.size(0) * x.size(1), + ) + with torch.cuda.device(x.device): + cast(Any, _quantize_kernel)[grid]( + x, + scale, + output, + length, + block_size, + x.size(1), + x.size(-1), + *x.stride(), + ) return output, scale def quantized_attention( - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - qs: torch.Tensor, - ks: torch.Tensor, - vs: torch.Tensor, + q: Tensor, + k: Tensor, + v: Tensor, + qs: Tensor, + ks: Tensor, + vs: Tensor, dtype: torch.dtype, query_tile: int, scale: float | None = None, @@ -254,7 +234,7 @@ def quantized_attention( accumulation_chunk: int = 0, scale_weights_in_exp: bool = False, fuse_score_scale: bool = False, -) -> torch.Tensor: +) -> Tensor: """Run tiled attention on per-head scaled FP8 inputs. Args: @@ -282,26 +262,28 @@ def quantized_attention( n = k.size(-2) # Q/K are [batch, heads, sequence, channels]; V is transposed. output = torch.empty((b, h, m, d), device=q.device, dtype=dtype) - cast(Any, attention_kernel)[(((m + query_tile - 1) // query_tile), b * h)]( - q, - k, - v, - qs, - ks, - vs, - output, - n_q_heads=h, - n_kv_heads=k.size(1), - n_queries=m, - n_context=n, - head_dim=d, - attn_scale=d**-0.5 if scale is None else scale, - block_q=query_tile, - block_kv=context_tile, - acc_flush_tiles=accumulation_chunk, - scale_weights_in_exp=scale_weights_in_exp, - fuse_score_scale=fuse_score_scale, - num_warps=warps, - num_stages=stages, - ) + grid = (triton.cdiv(m, query_tile), b * h) # ty: ignore[invalid-argument-type] + with torch.cuda.device(q.device): + cast(Any, _attention_kernel)[grid]( + q, + k, + v, + qs, + ks, + vs, + output, + n_q_heads=h, + n_kv_heads=k.size(1), + n_queries=m, + n_context=n, + head_dim=d, + attn_scale=d**-0.5 if scale is None else scale, + block_q=query_tile, + block_kv=context_tile, + acc_flush_tiles=accumulation_chunk, + scale_weights_in_exp=scale_weights_in_exp, + fuse_score_scale=fuse_score_scale, + num_warps=warps, + num_stages=stages, + ) return output From 65e002458bf53c8079e3f338880efdba6baa7461 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 17 Sep 2026 22:12:56 +0000 Subject: [PATCH 18/44] Name FP8 quantization launch arguments explicitly --- sdm/_kernels/triton/fp8_attention.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index e8d1d16cd7..8abfb5bd1e 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -197,7 +197,7 @@ def quantize(x: Tensor, scale: Tensor | None = None) -> tuple[Tensor, Tensor]: x.abs().amax(dim=(-2, -1), keepdim=True).float().clamp_min(1e-12) / 448.0 ) - output = torch.empty(x.shape, device=x.device, dtype=torch.float8_e4m3fn) + output = x.new_empty(x.shape, dtype=torch.float8_e4m3fn) length = x.size(-2) * x.size(-1) block_size = 1024 grid = ( @@ -209,11 +209,14 @@ def quantize(x: Tensor, scale: Tensor | None = None) -> tuple[Tensor, Tensor]: x, scale, output, - length, - block_size, - x.size(1), - x.size(-1), - *x.stride(), + numel=length, + block_size=block_size, + num_heads=x.size(1), + num_channels=x.size(-1), + stride_b=x.stride(0), + stride_h=x.stride(1), + stride_r=x.stride(2), + stride_c=x.stride(3), ) return output, scale @@ -261,7 +264,7 @@ def quantized_attention( b, h, m, d = q.shape n = k.size(-2) # Q/K are [batch, heads, sequence, channels]; V is transposed. - output = torch.empty((b, h, m, d), device=q.device, dtype=dtype) + output = q.new_empty(q.shape, dtype=dtype) grid = (triton.cdiv(m, query_tile), b * h) # ty: ignore[invalid-argument-type] with torch.cuda.device(q.device): cast(Any, _attention_kernel)[grid]( From dfdf94c238111274d2f273709fd72450375a28b1 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:05:27 +0000 Subject: [PATCH 19/44] Include FP8 wrapper and cache preparation with kernels --- sdm/_kernels/fp8_attention.py | 70 +++++++++++++++++++++++++ sdm/_kernels/triton/fp8_attention.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 sdm/_kernels/fp8_attention.py diff --git a/sdm/_kernels/fp8_attention.py b/sdm/_kernels/fp8_attention.py new file mode 100644 index 0000000000..672a3cbab1 --- /dev/null +++ b/sdm/_kernels/fp8_attention.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Callable +from typing import TypeAlias + +import torch +from torch import Tensor + +from sdm.cache import QuantizedKVCacheEntry + +_FP8Attention: TypeAlias = Callable[ + [Tensor, Tensor, Tensor, QuantizedKVCacheEntry | None, float | None], + tuple[Tensor, QuantizedKVCacheEntry], +] + +_triton_fp8_attention: _FP8Attention | None = None +try: + from sdm._kernels.triton.fp8_attention import ( + _fp8_attention as _triton_fp8_attention_impl, + ) +except ImportError: + pass +else: + _triton_fp8_attention = _triton_fp8_attention_impl + + +def supports_fp8(query: Tensor) -> bool: + """Check whether the query and execution mode support the FP8 kernel. + + Args: + query: Query tensor with channels in its final dimension. + + Returns: + Whether Triton, hardware, head width, and inference mode are supported. + """ + return ( + _triton_fp8_attention is not None + and not torch.is_grad_enabled() + and query.is_cuda + and query.size(-1) in {32, 64, 128, 256} + and torch.cuda.get_device_capability(query.device) + in {(8, 9), (9, 0), (12, 0)} + ) + + +def fp8_attention( + query: Tensor, + key: Tensor, + value: Tensor, + cache: QuantizedKVCacheEntry | None = None, + scale: float | None = None, +) -> tuple[Tensor, QuantizedKVCacheEntry]: + """Compute FP8 attention and create or reuse a quantized context cache. + + Args: + query: Queries shaped ``[..., rows, query_heads, channels]``. When + creating a cache, the leading rows must include the context. + key: Keys shaped ``[..., context_rows, kv_heads, channels]``. + value: Values with the same shape as the keys. + cache: Fitted quantized K/V and scales to reuse, or ``None`` to create + them from the supplied context. Inputs must support FP8 inference. + scale: Attention score multiplier; defaults to inverse square-root + head width. + + Returns: + The attention output and its quantized context cache. + """ + assert _triton_fp8_attention is not None + return _triton_fp8_attention(query, key, value, cache, scale) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index 8abfb5bd1e..9aa7a9ad71 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -13,6 +13,8 @@ import triton import triton.language as tl +from sdm.cache import QuantizedKVCacheEntry + @triton.jit def _attention_kernel( @@ -290,3 +292,77 @@ def quantized_attention( num_stages=stages, ) return output + + +def _fp8_attention( + query: Tensor, + key: Tensor, + value: Tensor, + cache: QuantizedKVCacheEntry | None = None, + scale: float | None = None, +) -> tuple[Tensor, QuantizedKVCacheEntry]: + """Quantize attention inputs and create or reuse their context cache.""" + if torch.is_autocast_enabled("cuda"): + dtype = torch.get_autocast_dtype("cuda") + query = query.to(dtype) + if cache is None: + key, value = key.to(dtype), value.to(dtype) + batch_shape = torch.broadcast_shapes( + query.shape[:-3], key.shape[:-3], value.shape[:-3] + ) + + def heads(x: Tensor) -> Tensor: + return ( + x.expand(batch_shape + x.shape[-3:]) + .reshape(-1, *x.shape[-3:]) + .transpose(1, 2) + ) + + def rows(x: Tensor) -> Tensor: + x = x.transpose(1, 2) + return x.reshape(batch_shape + x.shape[-3:]) + + q = heads(query) + if cache is None: + context = q[..., : key.size(-3), :] + qs = context.abs().amax((-2, -1), keepdim=True).float() + qs = qs.clamp_min(1e-12) / 448.0 + k, ks = quantize(heads(key)) + v, vs = quantize(heads(value)) + else: + k = heads(cache.key).contiguous() + v = heads(cache.value).contiguous() + ks, vs, qs = ( + heads(t).contiguous() + for t in (cache.key_scale, cache.value_scale, cache.query_scale) + ) + q, qs = quantize(q, qs) + blackwell = torch.cuda.get_device_capability(query.device) == (12, 0) + wide = query.size(-1) == 256 + result = quantized_attention( + q, + k, + v.transpose(-1, -2).contiguous(), + qs, + ks, + vs, + dtype=query.dtype, + query_tile=64 if wide else 128, + scale=scale, + context_tile=64 if wide or blackwell or k.size(-2) < 32768 else 128, + warps=8 if wide else 4, + stages=4 if blackwell and not wide else 2, + accumulation_chunk=0 if blackwell else 16, + scale_weights_in_exp=blackwell, + fuse_score_scale=blackwell, + ) + if cache is None: + cache = QuantizedKVCacheEntry( + key=rows(k), + value=rows(v), + key_scale=rows(ks), + value_scale=rows(vs), + query_scale=rows(qs), + dtype=query.dtype, + ) + return rows(result), cache From 2eee07da6748b740d6f8f7476868c30e9d6832a6 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:09:51 +0000 Subject: [PATCH 20/44] Explain FP8 compute capability check --- sdm/_kernels/fp8_attention.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdm/_kernels/fp8_attention.py b/sdm/_kernels/fp8_attention.py index 672a3cbab1..af4a6586d8 100644 --- a/sdm/_kernels/fp8_attention.py +++ b/sdm/_kernels/fp8_attention.py @@ -39,6 +39,8 @@ def supports_fp8(query: Tensor) -> bool: and not torch.is_grad_enabled() and query.is_cuda and query.size(-1) in {32, 64, 128, 256} + # CUDA compute capability (major, minor): Ada, Hopper, RTX Blackwell. + # Restrict dispatch to architectures supported by this FP8 kernel. and torch.cuda.get_device_capability(query.device) in {(8, 9), (9, 0), (12, 0)} ) From fefe251f49cd5bbdf8f4fa126b373c996735974c Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:15:51 +0000 Subject: [PATCH 21/44] Use common FP8 launch settings across supported GPUs --- sdm/_kernels/triton/fp8_attention.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index 9aa7a9ad71..cdbd15463c 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -337,7 +337,6 @@ def rows(x: Tensor) -> Tensor: for t in (cache.key_scale, cache.value_scale, cache.query_scale) ) q, qs = quantize(q, qs) - blackwell = torch.cuda.get_device_capability(query.device) == (12, 0) wide = query.size(-1) == 256 result = quantized_attention( q, @@ -349,12 +348,12 @@ def rows(x: Tensor) -> Tensor: dtype=query.dtype, query_tile=64 if wide else 128, scale=scale, - context_tile=64 if wide or blackwell or k.size(-2) < 32768 else 128, + context_tile=64 if wide or k.size(-2) < 32768 else 128, warps=8 if wide else 4, - stages=4 if blackwell and not wide else 2, - accumulation_chunk=0 if blackwell else 16, - scale_weights_in_exp=blackwell, - fuse_score_scale=blackwell, + stages=2, + accumulation_chunk=16, + scale_weights_in_exp=False, + fuse_score_scale=False, ) if cache is None: cache = QuantizedKVCacheEntry( From 36cec0a7e65f2a0089fc4ba7b5458c00288f1bd4 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Fri, 18 Sep 2026 03:28:49 +0000 Subject: [PATCH 22/44] Validate FP8 launch dimensions and stabilize numerical tests --- sdm/_kernels/triton/fp8_attention.py | 13 +++++- test/conftest.py | 8 ++++ test/nn/test_fp8_attention.py | 60 +++++++++++++++++++++++++--- 3 files changed, 74 insertions(+), 7 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index cdbd15463c..2a153bc4aa 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -250,9 +250,9 @@ def quantized_attention( ks: Key scales shaped ``[batch, kv_heads, 1, 1]``. vs: Value scales shaped ``[batch, kv_heads, 1, 1]``. dtype: Output dtype. - query_tile: Query rows handled by each program. + query_tile: Query rows per program; a positive power of two. scale: Attention score multiplier. Defaults to ``1 / sqrt(D)``. - context_tile: Context rows processed per loop iteration. + context_tile: Context rows per loop iteration; a positive power of two. warps: Cooperating warps per program. stages: Software pipeline stages. accumulation_chunk: Tiles per FP32 accumulator flush; zero disables it. @@ -264,6 +264,15 @@ def quantized_attention( Attention output with the query shape and requested dtype. """ b, h, m, d = q.shape + for name, size in ( + ("query_tile", query_tile), + ("context_tile", context_tile), + ("head dimension", d), + ): + if size <= 0 or size & (size - 1): + raise ValueError(f"{name} must be a positive power of two") + if k.size(1) == 0 or h % k.size(1): + raise ValueError("The number of K/V heads must divide the query heads") n = k.size(-2) # Q/K are [batch, heads, sequence, channels]; V is transposed. output = q.new_empty(q.shape, dtype=dtype) diff --git a/test/conftest.py b/test/conftest.py index 561ab8d002..d13294dbf9 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -10,6 +10,14 @@ from sdm import RelationalData, Stype, TableTensor, infer_stypes +@pytest.fixture +def fp8_rng() -> Iterator[None]: + """Reproduce FP8 comparisons while preserving the surrounding RNG state.""" + with torch.random.fork_rng(): + torch.manual_seed(0) + yield + + @pytest.fixture def device() -> torch.device: return torch.device("cpu") diff --git a/test/nn/test_fp8_attention.py b/test/nn/test_fp8_attention.py index c89bf8bdcf..198c89eea6 100644 --- a/test/nn/test_fp8_attention.py +++ b/test/nn/test_fp8_attention.py @@ -12,11 +12,14 @@ quantized_attention, ) -pytestmark = pytest.mark.skipif( - not torch.cuda.is_available() - or torch.cuda.get_device_capability() < (8, 9), - reason="FP8 attention requires an Ada or newer CUDA GPU", -) +pytestmark = [ + pytest.mark.usefixtures("fp8_rng"), + pytest.mark.skipif( + not torch.cuda.is_available() + or torch.cuda.get_device_capability() < (8, 9), + reason="FP8 attention requires an Ada or newer CUDA GPU", + ), +] @pytest.mark.parametrize( @@ -151,3 +154,50 @@ def test_quantization_reads_strided_heads_without_fp16_copy() -> None: actual.float(), expected.float(), rtol=0, atol=0 ) torch.testing.assert_close(actual_scale, expected_scale, rtol=0, atol=0) + + +@pytest.mark.parametrize( + ( + "query_heads", + "kv_heads", + "channels", + "query_tile", + "context_tile", + "message", + ), + [ + (3, 2, 64, 64, 64, "K/V heads"), + (1, 2, 64, 64, 64, "K/V heads"), + (2, 0, 64, 64, 64, "K/V heads"), + (2, 2, 96, 64, 64, "head dimension"), + (2, 2, 64, 96, 64, "query_tile"), + (2, 2, 64, 64, 96, "context_tile"), + (2, 2, 64, 0, 64, "query_tile"), + ], +) +def test_invalid_attention_launch( + query_heads: int, + kv_heads: int, + channels: int, + query_tile: int, + context_tile: int, + message: str, +) -> None: + q = torch.empty( + 1, query_heads, 1, channels, device="cuda", dtype=torch.float8_e4m3fn + ) + k = q.new_empty(1, kv_heads, 1, channels) + qs = torch.ones(1, query_heads, 1, 1, device="cuda") + ks = qs.new_ones(1, kv_heads, 1, 1) + with pytest.raises(ValueError, match=message): + quantized_attention( + q, + k, + k.transpose(-1, -2).contiguous(), + qs, + ks, + ks, + dtype=torch.float16, + query_tile=query_tile, + context_tile=context_tile, + ) From a1975a3d9eb1c22eb8a66748ddcb7ef1491d7fdc Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 1 Oct 2026 05:59:24 +0000 Subject: [PATCH 23/44] Use the shared Triton platform guard --- sdm/_kernels/triton/fp8_attention.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index 2a153bc4aa..080945668b 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -1,17 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import sys from typing import Any, cast import torch -from torch import Tensor - -if sys.platform != "linux": - raise ImportError("Triton kernels are only available on Linux") - import triton import triton.language as tl +from torch import Tensor from sdm.cache import QuantizedKVCacheEntry From aee9244ec486c62b73eb6c86d15b345fa64cfb6b Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:22:53 +0000 Subject: [PATCH 24/44] Centralize FP8 backend support checks in dispatch wrapper --- sdm/_kernels/fp8_attention.py | 52 +++++++++++++-------------- test/_kernels/test_fp8_attention.py | 56 +++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 test/_kernels/test_fp8_attention.py diff --git a/sdm/_kernels/fp8_attention.py b/sdm/_kernels/fp8_attention.py index af4a6586d8..c484dd94eb 100644 --- a/sdm/_kernels/fp8_attention.py +++ b/sdm/_kernels/fp8_attention.py @@ -25,34 +25,13 @@ _triton_fp8_attention = _triton_fp8_attention_impl -def supports_fp8(query: Tensor) -> bool: - """Check whether the query and execution mode support the FP8 kernel. - - Args: - query: Query tensor with channels in its final dimension. - - Returns: - Whether Triton, hardware, head width, and inference mode are supported. - """ - return ( - _triton_fp8_attention is not None - and not torch.is_grad_enabled() - and query.is_cuda - and query.size(-1) in {32, 64, 128, 256} - # CUDA compute capability (major, minor): Ada, Hopper, RTX Blackwell. - # Restrict dispatch to architectures supported by this FP8 kernel. - and torch.cuda.get_device_capability(query.device) - in {(8, 9), (9, 0), (12, 0)} - ) - - def fp8_attention( query: Tensor, key: Tensor, value: Tensor, cache: QuantizedKVCacheEntry | None = None, scale: float | None = None, -) -> tuple[Tensor, QuantizedKVCacheEntry]: +) -> tuple[Tensor, QuantizedKVCacheEntry] | None: """Compute FP8 attention and create or reuse a quantized context cache. Args: @@ -61,12 +40,33 @@ def fp8_attention( key: Keys shaped ``[..., context_rows, kv_heads, channels]``. value: Values with the same shape as the keys. cache: Fitted quantized K/V and scales to reuse, or ``None`` to create - them from the supplied context. Inputs must support FP8 inference. + them from the supplied context. scale: Attention score multiplier; defaults to inverse square-root head width. Returns: - The attention output and its quantized context cache. + The attention output and its quantized context cache, or ``None`` + when unsupported inputs require the caller's regular attention path. + + Raises: + ValueError: A fitted quantized cache cannot be used with the current + device, head width, backend, or execution mode. """ - assert _triton_fp8_attention is not None - return _triton_fp8_attention(query, key, value, cache, scale) + if ( + _triton_fp8_attention is not None + and not torch.is_grad_enabled() + and query.is_cuda + and query.size(-1) in {32, 64, 128, 256} + # CUDA compute capability (major, minor): Ada, Hopper, RTX Blackwell. + # Restrict dispatch to architectures supported by this FP8 kernel. + and torch.cuda.get_device_capability(query.device) + in {(8, 9), (9, 0), (12, 0)} + ): + if cache is not None and query.numel() == 0: + return query, cache + return _triton_fp8_attention(query, key, value, cache, scale) + if cache is not None: + raise ValueError( + "Fitted FP8 attention requires supported CUDA inference" + ) + return None diff --git a/test/_kernels/test_fp8_attention.py b/test/_kernels/test_fp8_attention.py new file mode 100644 index 0000000000..328b2adaf8 --- /dev/null +++ b/test/_kernels/test_fp8_attention.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import torch + +from sdm._kernels.fp8_attention import fp8_attention +from sdm.cache import QuantizedKVCacheEntry +from sdm.testing import withCUDA + + +@withCUDA +@pytest.mark.parametrize("channels", [32, 48]) +@pytest.mark.parametrize("grad", [False, True]) +def test_fp8_dispatch(device: torch.device, channels: int, grad: bool) -> None: + if device.type == "cuda" and torch.cuda.get_device_capability( + device + ) not in {(8, 9), (9, 0), (12, 0)}: + pytest.skip("FP8 dispatch requires Ada, Hopper, or RTX Blackwell") + x = torch.ones(1, 17, 2, channels, device=device, dtype=torch.float16) + with torch.set_grad_enabled(grad): + result = fp8_attention(x, x, x) + if device.type == "cpu" or grad or channels == 48: + assert result is None + return + assert result is not None + output, cache = result + torch.testing.assert_close(output, x) + assert cache.key.dtype == torch.float8_e4m3fn + cached = fp8_attention(x[:, :3], cache.key, cache.value, cache) + assert cached is not None + torch.testing.assert_close(cached[0], x[:, :3]) + empty = fp8_attention(x[:, :0], cache.key, cache.value, cache) + assert empty is not None + assert empty[0].shape == x[:, :0].shape + assert empty[1] is cache + + +@withCUDA +def test_fp8_cache_rejects_unsupported_execution(device: torch.device) -> None: + x = torch.ones(1, 17, 2, 32, device=device, dtype=torch.float8_e4m3fn) + scale = torch.ones(1, 1, 2, 1, device=device) + cache = QuantizedKVCacheEntry( + key=x, + value=x, + key_scale=scale, + value_scale=scale, + query_scale=scale, + dtype=torch.float16, + ) + query = torch.ones(1, 3, 2, 32, device=device, dtype=torch.float16) + with ( + torch.enable_grad(), + pytest.raises(ValueError, match="CUDA inference"), + ): + fp8_attention(query, cache.key, cache.value, cache) From 144cf8f4de488685d615dc21b8dc0a0d7322ffc4 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:05:23 +0000 Subject: [PATCH 25/44] Keep attention eligibility and query scaling in FP8 dispatch --- sdm/_kernels/fp8_attention.py | 32 ++++++++++++++++++++++++++--- test/_kernels/test_fp8_attention.py | 2 +- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/sdm/_kernels/fp8_attention.py b/sdm/_kernels/fp8_attention.py index c484dd94eb..073f12f121 100644 --- a/sdm/_kernels/fp8_attention.py +++ b/sdm/_kernels/fp8_attention.py @@ -7,7 +7,7 @@ import torch from torch import Tensor -from sdm.cache import QuantizedKVCacheEntry +from sdm.cache import KVCacheEntry, QuantizedKVCacheEntry _FP8Attention: TypeAlias = Callable[ [Tensor, Tensor, Tensor, QuantizedKVCacheEntry | None, float | None], @@ -29,8 +29,13 @@ def fp8_attention( query: Tensor, key: Tensor, value: Tensor, - cache: QuantizedKVCacheEntry | None = None, + cache: KVCacheEntry | None = None, scale: float | None = None, + *, + query_scaling: Callable[..., Tensor] | None = None, + seqused_key_value: Tensor | None = None, + attn_mask: Tensor | None = None, + is_causal: bool = False, ) -> tuple[Tensor, QuantizedKVCacheEntry] | None: """Compute FP8 attention and create or reuse a quantized context cache. @@ -43,6 +48,11 @@ def fp8_attention( them from the supplied context. scale: Attention score multiplier; defaults to inverse square-root head width. + query_scaling: Optional query transformation applied before FP8 + attention, called with ``query`` and ``key_len``. + seqused_key_value: Valid context lengths; require regular attention. + attn_mask: Attention mask; requires regular attention. + is_causal: Causal attention; requires regular attention. Returns: The attention output and its quantized context cache, or ``None`` @@ -50,8 +60,22 @@ def fp8_attention( Raises: ValueError: A fitted quantized cache cannot be used with the current - device, head width, backend, or execution mode. + device, head width, backend, execution mode, or attention mask. """ + if cache is not None and not isinstance(cache, QuantizedKVCacheEntry): + return None + if attn_mask is not None or seqused_key_value is not None or is_causal: + if cache is not None: + raise ValueError( + "Fitted FP8 attention does not support attention masks" + ) + return None + # New caches need enough context work to offset quantization overhead, + # and leading context queries to calibrate Q scales. + if cache is None and ( + key.size(-3) <= 8192 or query.size(-3) < key.size(-3) + ): + return None if ( _triton_fp8_attention is not None and not torch.is_grad_enabled() @@ -64,6 +88,8 @@ def fp8_attention( ): if cache is not None and query.numel() == 0: return query, cache + if query_scaling is not None: + query = query_scaling(query, key_len=key.size(-3)) return _triton_fp8_attention(query, key, value, cache, scale) if cache is not None: raise ValueError( diff --git a/test/_kernels/test_fp8_attention.py b/test/_kernels/test_fp8_attention.py index 328b2adaf8..e8afae969e 100644 --- a/test/_kernels/test_fp8_attention.py +++ b/test/_kernels/test_fp8_attention.py @@ -17,7 +17,7 @@ def test_fp8_dispatch(device: torch.device, channels: int, grad: bool) -> None: device ) not in {(8, 9), (9, 0), (12, 0)}: pytest.skip("FP8 dispatch requires Ada, Hopper, or RTX Blackwell") - x = torch.ones(1, 17, 2, channels, device=device, dtype=torch.float16) + x = torch.ones(1, 8193, 2, channels, device=device, dtype=torch.float16) with torch.set_grad_enabled(grad): result = fp8_attention(x, x, x) if device.type == "cpu" or grad or channels == 48: From ce2a4adf0a785f930d6c4065a37e6a673b29c39f Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:26:20 +0000 Subject: [PATCH 26/44] Use measured attention launches that avoid register spills --- sdm/_kernels/triton/fp8_attention.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py index 080945668b..8a6686bddf 100644 --- a/sdm/_kernels/triton/fp8_attention.py +++ b/sdm/_kernels/triton/fp8_attention.py @@ -341,7 +341,12 @@ def rows(x: Tensor) -> Tensor: for t in (cache.key_scale, cache.value_scale, cache.query_scale) ) q, qs = quantize(q, qs) - wide = query.size(-1) == 256 + head_dim = query.size(-1) + wide = head_dim == 256 or (head_dim == 128 and k.size(-2) >= 32768) + # Measured launches avoid register spills; Ada retains accumulator flushes. + blackwell = head_dim == 64 and torch.cuda.get_device_capability( + query.device + ) == (12, 0) result = quantized_attention( q, k, @@ -350,14 +355,14 @@ def rows(x: Tensor) -> Tensor: ks, vs, dtype=query.dtype, - query_tile=64 if wide else 128, + query_tile=64 if head_dim == 256 else 128, scale=scale, - context_tile=64 if wide or k.size(-2) < 32768 else 128, + context_tile=64 if wide or blackwell or k.size(-2) < 32768 else 128, warps=8 if wide else 4, - stages=2, - accumulation_chunk=16, - scale_weights_in_exp=False, - fuse_score_scale=False, + stages=4 if blackwell else 2, + accumulation_chunk=0 if blackwell else 16, + scale_weights_in_exp=blackwell, + fuse_score_scale=blackwell, ) if cache is None: cache = QuantizedKVCacheEntry( From 6120a259b3ff11b80b747035d4ae18c2318a472a Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:05:38 +0000 Subject: [PATCH 27/44] Enable FP8 attention in shared attention and TabICLv2 --- sdm/models/tabiclv2/model.py | 23 ++++- sdm/nn/attention.py | 106 ++++++++++++++------- test/models/tabiclv2/test_model.py | 40 ++++++++ test/models/test_attention_quantization.py | 33 +++++++ test/nn/test_attention_quantization.py | 100 +++++++++++++++++++ test/nn/test_compile.py | 33 ++++++- 6 files changed, 299 insertions(+), 36 deletions(-) create mode 100644 test/models/test_attention_quantization.py create mode 100644 test/nn/test_attention_quantization.py diff --git a/sdm/models/tabiclv2/model.py b/sdm/models/tabiclv2/model.py index 78d7bda3a5..3d29ea9a30 100644 --- a/sdm/models/tabiclv2/model.py +++ b/sdm/models/tabiclv2/model.py @@ -9,7 +9,7 @@ from __future__ import annotations from collections.abc import Iterable, Mapping -from typing import Any, ClassVar, cast +from typing import Any, ClassVar, Literal, cast import torch from torch import Tensor @@ -19,6 +19,7 @@ from sdm.cache import Cache from sdm.models import ICLModel from sdm.models._huggingface import download_checkpoint +from sdm.models.tabiclv2.block import TabICLv2TransformerBlock from sdm.models.tabiclv2.ckpt import remap_ckpt from sdm.models.tabiclv2.icl import ICLBlock from sdm.models.tabiclv2.recipe import default_recipe @@ -116,6 +117,14 @@ class TabICLv2(ICLModel): model are initialized. pretrained: Whether to load the pretrained checkpoint. device: The device. + attention_quantization: Set to ``"fp8"`` to quantize eligible ICL + attention and its cache during CUDA inference on Ada, Hopper, + and RTX Blackwell GPUs. Requires Triton and more than 8192 + context rows. + Weights and the final ICL layer remain unquantized. Other inputs + use standard attention. Fitted FP8 caches require the same + inference dtype and do not support ``torch.compile``. + Defaults to ``None``. Cached prediction is not always faster. """ supported_feature_stypes: ClassVar[frozenset[Stype]] = frozenset( @@ -157,10 +166,12 @@ def __init__( task: TaskLike | Iterable[TaskLike] | None = None, pretrained: bool = True, device: torch.device | str | None = None, + *, + attention_quantization: Literal["fp8"] | None = None, ) -> None: super().__init__(task=task) - self.models: ModuleDict[TaskLike, torch.nn.Module] = ModuleDict() + self.models: ModuleDict = ModuleDict() for task in self.tasks: self.models[task] = _TabICLv2( num_classes=10 if task == Task.classification else 0, @@ -172,6 +183,14 @@ def __init__( if pretrained: self._load_from_pretrained(device=device) + if attention_quantization == "fp8": + for network in self.models.values(): + for layer in cast(_TabICLv2, network).icl_block.layers[:-1]: + cast( + TabICLv2TransformerBlock, layer + ).attn.attention_quantization = "fp8" + else: + assert attention_quantization is None self.eval() @classmethod diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index e0669e20c5..8811d4528e 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -12,9 +12,10 @@ from torch.nn import Linear from sdm._kernels.flash_attention import flash_attention +from sdm._kernels.fp8_attention import fp8_attention, supports_fp8 from sdm._kernels.sdpa import sdpa from sdm._memory import chunk_memory_limit -from sdm.cache import KVCacheEntry +from sdm.cache import KVCacheEntry, QuantizedKVCacheEntry from sdm.nn import QueryScaling @@ -291,6 +292,8 @@ def __init__( self.q_dim = num_query_heads * self.head_dim # == channels self.kv_dim = num_key_value_heads * self.head_dim + self.attention_quantization: Literal["fp8"] | None = None + self.qkv_lin = Linear( channels, self.q_dim + 2 * self.kv_dim, bias=bias, **factory_kwargs ) @@ -408,7 +411,12 @@ def forward( if self.qkv_lin.bias is not None else None, ) - if ( + if isinstance(key_value, QuantizedKVCacheEntry): + if query.dtype != key_value.dtype: + raise ValueError( + "Query dtype differs from fitted FP8 cache" + ) + elif ( key_value.key.dtype != query.dtype or key_value.value.dtype != query.dtype ): @@ -455,20 +463,61 @@ def forward( else: key = self.key_transform(key, pos=key_pos) - out = self.sdpa( - query=query, # [..., Q, Hq, C // Hq] - key=key, # [..., KV, Hkv, C // Hq] - value=value, # [..., KV, Hkv, C // Hq] - seqused_key_value=seqused_key_value, # [...] - attn_mask=attn_mask, # [..., Hq, Q, KV] - is_causal=is_causal, - ) # [..., Q, Hq, C // Hq] - + quantized_cache = ( + key_value if isinstance(key_value, QuantizedKVCacheEntry) else None + ) + use_fp8 = ( + self.attention_quantization == "fp8" + and not isinstance(key_value, KVCacheEntry) + and query.numel() > 0 + and key.size(-3) > 8192 + and query.size(-3) >= key.size(-3) + and seqused_key_value is None + and attn_mask is None + and supports_fp8(query) + ) + if quantized_cache is not None: + if ( + not supports_fp8(query) + or attn_mask is not None + or seqused_key_value is not None + ): + raise ValueError( + "Fitted FP8 attention requires supported CUDA inference " + "without attention masks" + ) + use_fp8 = True + if quantized_cache is not None and query.numel() == 0: + out = query + elif use_fp8: + scaled_query = query + if self.sdpa.query_scaling is not None: + scaled_query = self.sdpa.query_scaling( + query, key_len=key.size(-3) + ) + out, quantized_cache = fp8_attention( + query=scaled_query, + key=key, + value=value, + cache=quantized_cache, + scale=self.sdpa.scale, + ) + else: + out = self.sdpa( + query=query, # [..., Q, Hq, C // Hq] + key=key, # [..., KV, Hkv, C // Hq] + value=value, # [..., KV, Hkv, C // Hq] + seqused_key_value=seqused_key_value, # [...] + attn_mask=attn_mask, # [..., Q, KV] + is_causal=is_causal, + ) # [..., Q, Hq, C // Hq] out = out.flatten(-2, -1) # [..., Q, C] if gate is not None: out = out * gate out = self.out_lin(out) # [..., Q, C] if return_key_value: + if quantized_cache is not None: + return out, quantized_cache # CUDA autocast runs RMSNorm in fp32, so cast explicitly before # caching: https://github.com/pytorch/pytorch/blob/v2.13.0/aten/src/ATen/autocast_mode.h#L875 return out, KVCacheEntry( @@ -731,8 +780,7 @@ def forward( ) flat_out: Tensor | None = None - flat_key: Tensor | None = None - flat_value: Tensor | None = None + flat_cache: KVCacheEntry | None = None if out is not None and (out.dim() <= 3 or out.is_contiguous()): flat_out = out.view(batch_size, *query.size()[-2:]) @@ -761,18 +809,14 @@ def forward( if isinstance(result, tuple): chunk, chunk_kv = result - if flat_key is None: - flat_key = chunk_kv.key.new_empty( - batch_size, *chunk_kv.key.size()[-3:] - ) - if flat_value is None: - flat_value = chunk_kv.value.new_empty( - batch_size, *chunk_kv.value.size()[-3:] + if flat_cache is None: + flat_cache = chunk_kv._apply_tensor( + lambda t: t.new_empty(batch_size, *t.shape[-3:]) ) - - flat_key[start:end] = chunk_kv.key - flat_value[start:end] = chunk_kv.value - + for dest, source in zip( + flat_cache._tensors(), chunk_kv._tensors() + ): + dest[start:end] = source del chunk_kv else: @@ -798,12 +842,9 @@ def forward( if not return_key_value: return out - assert flat_key is not None - assert flat_value is not None - - return out, KVCacheEntry( - key=flat_key.view(*batch_shape, *flat_key.size()[-3:]), - value=flat_value.view(*batch_shape, *flat_value.size()[-3:]), + assert flat_cache is not None + return out, flat_cache._apply_tensor( + lambda t: t.view(*batch_shape, *t.shape[-3:]) ) def _forward( @@ -956,9 +997,8 @@ def _chunk( return None if isinstance(tensor, KVCacheEntry): - return KVCacheEntry( - key=_chunk(tensor.key, batch_shape, 3, start, end), - value=_chunk(tensor.value, batch_shape, 3, start, end), + return tensor._apply_tensor( + lambda value: _chunk(value, batch_shape, 3, start, end) ) trailing_shape = tensor.size()[-trailing_dims:] if trailing_dims else () diff --git a/test/models/tabiclv2/test_model.py b/test/models/tabiclv2/test_model.py index e33cd6dcf1..a0e3312dac 100644 --- a/test/models/tabiclv2/test_model.py +++ b/test/models/tabiclv2/test_model.py @@ -1,10 +1,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from typing import cast + import pytest import torch from sdm import Recipe +from sdm.cache import Cache, QuantizedKVCacheEntry from sdm.models import TabICLv2 from sdm.models.tabiclv2.model import _TabICLv2 from sdm.models.tabiclv2.row_embedding import RowEmbedding @@ -324,3 +327,40 @@ def test_compile(dtype: torch.dtype) -> None: predicted = model.predict(x_query) assert predicted.allclose(expected, atol=5e-4, rtol=5e-3) assert torch.is_inference(predicted) + + +@onlyCUDA +@pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) +def test_fp8_fit_predict(dtype: torch.dtype) -> None: + if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: + pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") + model = TabICLv2( + task="regression", + pretrained=False, + device="cuda", + attention_quantization="fp8", + ) + x = torch.randn(8193, 3, device="cuda") + y = torch.randn(8193, 1, device="cuda") + query = torch.randn(17, 3, device="cuda") + with ( + torch.inference_mode(), + torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), + ): + expected = model(x, y, query, recipe=Recipe(), num_estimators=1) + model.fit(x, y, recipe=Recipe(), num_estimators=1) + actual = model.predict(query) + assert actual.numerical.isfinite().all() + torch.testing.assert_close( + actual.numerical, expected.numerical, atol=0.01, rtol=0.01 + ) + assert model._cache is not None + # The public lifecycle must retain real quantized tensors, not merely + # accept an option while silently executing ordinary attention. + entries = cast(Cache, model._cache[0]) + assert any( + isinstance(entry, QuantizedKVCacheEntry) + for entry in entries.values() + ) + model.clear() + assert model._cache is None diff --git a/test/models/test_attention_quantization.py b/test/models/test_attention_quantization.py new file mode 100644 index 0000000000..a664caad03 --- /dev/null +++ b/test/models/test_attention_quantization.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import torch + +from sdm.models import TabICLv2 +from sdm.nn import Attention + + +@pytest.mark.parametrize("model_cls", [TabICLv2]) +def test_model_fp8_opt_in(model_cls: type[torch.nn.Module]) -> None: + model = model_cls( + task="regression", + pretrained=False, + device="meta", + attention_quantization="fp8", + ) + ordinary = model_cls(task="regression", pretrained=False, device="meta") + assert ordinary.state_dict().keys() == model.state_dict().keys() + quantized = [ + name + for name, module in model.named_modules() + if isinstance(module, Attention) + and module.attention_quantization == "fp8" + ] + assert quantized + assert all("icl_block.layers" in name for name in quantized) + assert all( + module.attention_quantization is None + for module in ordinary.modules() + if isinstance(module, Attention) + ) diff --git a/test/nn/test_attention_quantization.py b/test/nn/test_attention_quantization.py new file mode 100644 index 0000000000..b7831cbf06 --- /dev/null +++ b/test/nn/test_attention_quantization.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import copy + +import pytest +import torch + +from sdm.cache import Cache, KVCacheEntry, QuantizedKVCacheEntry +from sdm.models.tabiclv2.block import TabICLv2TransformerBlock +from sdm.nn import Attention +from sdm.testing import onlyCUDA, withCUDA + + +@withCUDA +def test_fp8_small_context_fallback(device: torch.device) -> None: + module = Attention(128, 2, device=device) + torch.nn.init.normal_(module.out_lin.weight, std=0.05) + reference = copy.deepcopy(module) + module.attention_quantization = "fp8" + query = torch.randn(2, 19, 128, device=device) + with torch.inference_mode(): + actual, cache = module(query, return_key_value=True) + expected = reference(query) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert isinstance(cache, KVCacheEntry) + assert module.state_dict().keys() == reference.state_dict().keys() + + +@onlyCUDA +@pytest.mark.parametrize( + "dtype", [torch.float16, torch.bfloat16, torch.float32] +) +def test_fp8_context_cache_and_chunking(dtype: torch.dtype) -> None: + if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: + pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") + module = TabICLv2TransformerBlock( + channels=128, + num_heads=2, + norm_bias=True, + qassmax=True, + device="cuda", + ) + torch.nn.init.normal_(module.attn.out_lin.weight, std=0.05) + reference = copy.deepcopy(module) + module.attn.attention_quantization = "fp8" + context = torch.randn(2, 1, 8193, 128, device="cuda") + query = torch.randn(2, 1, 129, 128, device="cuda") + joined = torch.cat([context, query], dim=-2) + with ( + torch.inference_mode(), + torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), + ): + expected = reference(joined, context) + full = module(joined, context) + _, cache = module( + context, context, return_key_value=True, batch_size_limit=1 + ) + assert isinstance(cache, QuantizedKVCacheEntry) + actual = module(query, cache, batch_size_limit=1) + unchunked = module(query, cache) + split = torch.cat( + [ + module(query[..., :64, :], cache), + module(query[..., 64:, :], cache), + ], + dim=-2, + ) + empty = module(query[..., :0, :], cache) + migrated = Cache(kv=cache).cpu().cuda()["kv"] + replay = module(query, migrated) + assert empty.size(-2) == 0 + assert actual.isfinite().all() + torch.testing.assert_close(actual, unchunked, atol=2e-3, rtol=2e-3) + torch.testing.assert_close(actual, split, atol=2e-3, rtol=2e-3) + torch.testing.assert_close(actual, replay, atol=2e-3, rtol=2e-3) + torch.testing.assert_close( + actual, full[..., -129:, :], atol=0.03, rtol=0.03 + ) + error = (full - expected).square().mean() / expected.square().mean() + assert error.sqrt() < 0.03 + assert Cache(kv=cache).size() == sum( + t.numel() * t.element_size() for t in cache._tensors() + ) + assert cache.key.dtype == torch.float8_e4m3fn + assert cache.value.dtype == torch.float8_e4m3fn + assert cache.query_scale.dtype == torch.float32 + with ( + torch.inference_mode(), + torch.autocast( + "cuda", dtype=torch.float16, enabled=dtype == torch.float32 + ), + pytest.raises(ValueError, match="dtype"), + ): + module(query, cache) + with ( + torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), + pytest.raises(ValueError, match="inference"), + ): + module(query, cache) diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index bf85dcbbab..4ccd8a1cf1 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -9,6 +9,7 @@ from torch import Tensor from torch._dynamo.testing import CompileCounter +from sdm.cache import QuantizedKVCacheEntry from sdm.nn import ( SDPA, Attention, @@ -17,7 +18,7 @@ RotaryEmbedding, TransformerBlock, ) -from sdm.testing import withCUDA +from sdm.testing import onlyCUDA, withCUDA # Skip all tests in this test file if it is not a full test run (FULL_TEST=1). pytestmark = [ @@ -300,6 +301,36 @@ def test_attention_compile_key_value_cache(device: torch.device) -> None: torch.testing.assert_close(out, expected) +@onlyCUDA +@pytest.mark.parametrize( + "dtype", [torch.float16, torch.bfloat16, torch.float32] +) +def test_attention_compile_fp8_cache(dtype: torch.dtype) -> None: + if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: + pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") + module = Attention( + channels=128, + num_query_heads=2, + num_key_value_heads=1, + device="cuda", + dtype=dtype, + ).eval() + module.attention_quantization = "fp8" + torch.nn.init.normal_(module.out_lin.weight, std=0.05) + context = torch.randn(1, 8193, 128, device="cuda", dtype=dtype) + compiled = torch.compile(module, fullgraph=True, dynamic=False) + with torch.inference_mode(): + expected, eager_cache = module(context, return_key_value=True) + actual, cache = compiled(context, return_key_value=True) + assert isinstance(cache, QuantizedKVCacheEntry) + torch.testing.assert_close(actual, expected) + for rows in (17, 33, 1, 17): + query = torch.randn(1, rows, 128, device="cuda", dtype=dtype) + expected = module(query, eager_cache) + torch.testing.assert_close(compiled(query, cache), expected) + torch.testing.assert_close(compiled(query, eager_cache), expected) + + @withCUDA @pytest.mark.parametrize( ("qassmax", "masking"), From c6ef246d7ab9fc1feaee18c9189d15838d0ea013 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Fri, 18 Sep 2026 03:11:28 +0000 Subject: [PATCH 28/44] Enable FP8 attention with a scoped optimization policy --- sdm/__init__.py | 2 + sdm/models/tabiclv2/icl.py | 6 +++ sdm/models/tabiclv2/model.py | 23 +-------- sdm/nn/attention.py | 14 +++++- sdm/optimization.py | 47 +++++++++++++++++++ test/models/tabiclv2/test_model.py | 4 +- test/models/test_attention_quantization.py | 33 ------------- test/nn/test_attention_quantization.py | 54 ++++++++++++++-------- test/nn/test_compile.py | 15 +++++- test/test_optimization.py | 48 +++++++++++++++++++ 10 files changed, 166 insertions(+), 80 deletions(-) create mode 100644 sdm/optimization.py delete mode 100644 test/models/test_attention_quantization.py create mode 100644 test/test_optimization.py diff --git a/sdm/__init__.py b/sdm/__init__.py index e5d22b1543..08eb5e7a7c 100644 --- a/sdm/__init__.py +++ b/sdm/__init__.py @@ -12,6 +12,7 @@ from importlib.metadata import PackageNotFoundError, version from sdm._constants import NaT +from sdm.optimization import optimize from sdm.stype import Stype, StypeLike, infer_stypes from sdm.task import Task, TaskLike from sdm.tensor import ( @@ -42,6 +43,7 @@ __all__ = [ "NaT", + "optimize", "Stype", "StypeLike", "Task", diff --git a/sdm/models/tabiclv2/icl.py b/sdm/models/tabiclv2/icl.py index d1903354d1..3f2704a7b9 100644 --- a/sdm/models/tabiclv2/icl.py +++ b/sdm/models/tabiclv2/icl.py @@ -56,6 +56,12 @@ def __init__( for _ in range(num_layers) ) + # The final layer has no context queries to calibrate FP8 scales. + for layer in self.layers[:-1]: + cast( + TabICLv2TransformerBlock, layer + ).attn._supports_quantized_attention = True + self.norm = LayerNorm(channels, bias=norm_bias, **factory_kwargs) self.head = Sequential( Linear(channels, 2 * channels, **factory_kwargs), diff --git a/sdm/models/tabiclv2/model.py b/sdm/models/tabiclv2/model.py index 3d29ea9a30..78d7bda3a5 100644 --- a/sdm/models/tabiclv2/model.py +++ b/sdm/models/tabiclv2/model.py @@ -9,7 +9,7 @@ from __future__ import annotations from collections.abc import Iterable, Mapping -from typing import Any, ClassVar, Literal, cast +from typing import Any, ClassVar, cast import torch from torch import Tensor @@ -19,7 +19,6 @@ from sdm.cache import Cache from sdm.models import ICLModel from sdm.models._huggingface import download_checkpoint -from sdm.models.tabiclv2.block import TabICLv2TransformerBlock from sdm.models.tabiclv2.ckpt import remap_ckpt from sdm.models.tabiclv2.icl import ICLBlock from sdm.models.tabiclv2.recipe import default_recipe @@ -117,14 +116,6 @@ class TabICLv2(ICLModel): model are initialized. pretrained: Whether to load the pretrained checkpoint. device: The device. - attention_quantization: Set to ``"fp8"`` to quantize eligible ICL - attention and its cache during CUDA inference on Ada, Hopper, - and RTX Blackwell GPUs. Requires Triton and more than 8192 - context rows. - Weights and the final ICL layer remain unquantized. Other inputs - use standard attention. Fitted FP8 caches require the same - inference dtype and do not support ``torch.compile``. - Defaults to ``None``. Cached prediction is not always faster. """ supported_feature_stypes: ClassVar[frozenset[Stype]] = frozenset( @@ -166,12 +157,10 @@ def __init__( task: TaskLike | Iterable[TaskLike] | None = None, pretrained: bool = True, device: torch.device | str | None = None, - *, - attention_quantization: Literal["fp8"] | None = None, ) -> None: super().__init__(task=task) - self.models: ModuleDict = ModuleDict() + self.models: ModuleDict[TaskLike, torch.nn.Module] = ModuleDict() for task in self.tasks: self.models[task] = _TabICLv2( num_classes=10 if task == Task.classification else 0, @@ -183,14 +172,6 @@ def __init__( if pretrained: self._load_from_pretrained(device=device) - if attention_quantization == "fp8": - for network in self.models.values(): - for layer in cast(_TabICLv2, network).icl_block.layers[:-1]: - cast( - TabICLv2TransformerBlock, layer - ).attn.attention_quantization = "fp8" - else: - assert attention_quantization is None self.eval() @classmethod diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index 8811d4528e..477ddc5bc5 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -17,6 +17,7 @@ from sdm._memory import chunk_memory_limit from sdm.cache import KVCacheEntry, QuantizedKVCacheEntry from sdm.nn import QueryScaling +from sdm.optimization import _attention_quantization class SDPA(torch.nn.Module): @@ -292,7 +293,7 @@ def __init__( self.q_dim = num_query_heads * self.head_dim # == channels self.kv_dim = num_key_value_heads * self.head_dim - self.attention_quantization: Literal["fp8"] | None = None + self._supports_quantized_attention = False self.qkv_lin = Linear( channels, self.q_dim + 2 * self.kv_dim, bias=bias, **factory_kwargs @@ -402,6 +403,14 @@ def forward( if key_value is None and key_pos is None: key_pos = query_pos + if ( + isinstance(key_value, QuantizedKVCacheEntry) + and _attention_quantization.get() != "fp8" + ): + raise RuntimeError( + 'This cache requires sdm.optimize(attention="fp8"). ' + "Re-enter that scope, or clear the model and refit." + ) if isinstance(key_value, KVCacheEntry): query = F.linear( @@ -467,7 +476,8 @@ def forward( key_value if isinstance(key_value, QuantizedKVCacheEntry) else None ) use_fp8 = ( - self.attention_quantization == "fp8" + _attention_quantization.get() == "fp8" + and self._supports_quantized_attention and not isinstance(key_value, KVCacheEntry) and query.numel() > 0 and key.size(-3) > 8192 diff --git a/sdm/optimization.py b/sdm/optimization.py new file mode 100644 index 0000000000..9deb229302 --- /dev/null +++ b/sdm/optimization.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Literal + +_attention_quantization: ContextVar[Literal["fp8"] | None] = ContextVar( + "sdm_attention_quantization", default=None +) + + +@contextmanager +def optimize( + *, attention: Literal["fp8", "inherit"] | None = "inherit" +) -> Iterator[None]: + """Temporarily configure execution optimizations in the current context. + + Args: + attention: ``"fp8"`` enables quantization in eligible ICL attention + layers during supported CUDA inference with more than 8192 context + rows. ``None`` disables it; ``"inherit"`` preserves the enclosing + setting. Weights and the final ICL layer remain unquantized. + + Yields: + None. Previous settings are restored on exit, including on exceptions. + + Note: + Create and use FP8 caches inside an FP8 scope. Using such a cache + outside an enabled scope raises ``RuntimeError``. Exiting neither + converts nor clears caches; they can be reused in another FP8 scope. + FP8 is not guaranteed to improve prediction latency. + + Example: + >>> with sdm.optimize(attention="fp8"): + ... model.fit(context, labels) + ... predictions = model.predict(queries) + """ + setting = ( + _attention_quantization.get() if attention == "inherit" else attention + ) + token = _attention_quantization.set(setting) + try: + yield + finally: + _attention_quantization.reset(token) diff --git a/test/models/tabiclv2/test_model.py b/test/models/tabiclv2/test_model.py index a0e3312dac..99f0edf7c4 100644 --- a/test/models/tabiclv2/test_model.py +++ b/test/models/tabiclv2/test_model.py @@ -6,7 +6,7 @@ import pytest import torch -from sdm import Recipe +from sdm import Recipe, optimize from sdm.cache import Cache, QuantizedKVCacheEntry from sdm.models import TabICLv2 from sdm.models.tabiclv2.model import _TabICLv2 @@ -338,13 +338,13 @@ def test_fp8_fit_predict(dtype: torch.dtype) -> None: task="regression", pretrained=False, device="cuda", - attention_quantization="fp8", ) x = torch.randn(8193, 3, device="cuda") y = torch.randn(8193, 1, device="cuda") query = torch.randn(17, 3, device="cuda") with ( torch.inference_mode(), + optimize(attention="fp8"), torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), ): expected = model(x, y, query, recipe=Recipe(), num_estimators=1) diff --git a/test/models/test_attention_quantization.py b/test/models/test_attention_quantization.py deleted file mode 100644 index a664caad03..0000000000 --- a/test/models/test_attention_quantization.py +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest -import torch - -from sdm.models import TabICLv2 -from sdm.nn import Attention - - -@pytest.mark.parametrize("model_cls", [TabICLv2]) -def test_model_fp8_opt_in(model_cls: type[torch.nn.Module]) -> None: - model = model_cls( - task="regression", - pretrained=False, - device="meta", - attention_quantization="fp8", - ) - ordinary = model_cls(task="regression", pretrained=False, device="meta") - assert ordinary.state_dict().keys() == model.state_dict().keys() - quantized = [ - name - for name, module in model.named_modules() - if isinstance(module, Attention) - and module.attention_quantization == "fp8" - ] - assert quantized - assert all("icl_block.layers" in name for name in quantized) - assert all( - module.attention_quantization is None - for module in ordinary.modules() - if isinstance(module, Attention) - ) diff --git a/test/nn/test_attention_quantization.py b/test/nn/test_attention_quantization.py index b7831cbf06..fe3aae4b4d 100644 --- a/test/nn/test_attention_quantization.py +++ b/test/nn/test_attention_quantization.py @@ -6,6 +6,7 @@ import pytest import torch +from sdm import optimize from sdm.cache import Cache, KVCacheEntry, QuantizedKVCacheEntry from sdm.models.tabiclv2.block import TabICLv2TransformerBlock from sdm.nn import Attention @@ -17,10 +18,11 @@ def test_fp8_small_context_fallback(device: torch.device) -> None: module = Attention(128, 2, device=device) torch.nn.init.normal_(module.out_lin.weight, std=0.05) reference = copy.deepcopy(module) - module.attention_quantization = "fp8" + module._supports_quantized_attention = True query = torch.randn(2, 19, 128, device=device) with torch.inference_mode(): - actual, cache = module(query, return_key_value=True) + with optimize(attention="fp8"): + actual, cache = module(query, return_key_value=True) expected = reference(query) torch.testing.assert_close(actual, expected, rtol=0, atol=0) assert isinstance(cache, KVCacheEntry) @@ -43,7 +45,7 @@ def test_fp8_context_cache_and_chunking(dtype: torch.dtype) -> None: ) torch.nn.init.normal_(module.attn.out_lin.weight, std=0.05) reference = copy.deepcopy(module) - module.attn.attention_quantization = "fp8" + module.attn._supports_quantized_attention = True context = torch.randn(2, 1, 8193, 128, device="cuda") query = torch.randn(2, 1, 129, 128, device="cuda") joined = torch.cat([context, query], dim=-2) @@ -52,23 +54,24 @@ def test_fp8_context_cache_and_chunking(dtype: torch.dtype) -> None: torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), ): expected = reference(joined, context) - full = module(joined, context) - _, cache = module( - context, context, return_key_value=True, batch_size_limit=1 - ) - assert isinstance(cache, QuantizedKVCacheEntry) - actual = module(query, cache, batch_size_limit=1) - unchunked = module(query, cache) - split = torch.cat( - [ - module(query[..., :64, :], cache), - module(query[..., 64:, :], cache), - ], - dim=-2, - ) - empty = module(query[..., :0, :], cache) - migrated = Cache(kv=cache).cpu().cuda()["kv"] - replay = module(query, migrated) + with optimize(attention="fp8"): + full = module(joined, context) + _, cache = module( + context, context, return_key_value=True, batch_size_limit=1 + ) + assert isinstance(cache, QuantizedKVCacheEntry) + actual = module(query, cache, batch_size_limit=1) + unchunked = module(query, cache) + split = torch.cat( + [ + module(query[..., :64, :], cache), + module(query[..., 64:, :], cache), + ], + dim=-2, + ) + empty = module(query[..., :0, :], cache) + migrated = Cache(kv=cache).cpu().cuda()["kv"] + replay = module(query, migrated) assert empty.size(-2) == 0 assert actual.isfinite().all() torch.testing.assert_close(actual, unchunked, atol=2e-3, rtol=2e-3) @@ -90,11 +93,22 @@ def test_fp8_context_cache_and_chunking(dtype: torch.dtype) -> None: torch.autocast( "cuda", dtype=torch.float16, enabled=dtype == torch.float32 ), + optimize(attention="fp8"), pytest.raises(ValueError, match="dtype"), ): module(query, cache) with ( torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), + optimize(attention="fp8"), pytest.raises(ValueError, match="inference"), ): module(query, cache) + + with ( + torch.inference_mode(), + torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), + ): + with pytest.raises(RuntimeError, match=r"sdm\.optimize"): + module(query, cache) + with optimize(attention="fp8"): + torch.testing.assert_close(module(query, cache), actual) diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index 4ccd8a1cf1..2d6e9d5509 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -9,6 +9,7 @@ from torch import Tensor from torch._dynamo.testing import CompileCounter +from sdm import optimize from sdm.cache import QuantizedKVCacheEntry from sdm.nn import ( SDPA, @@ -315,11 +316,11 @@ def test_attention_compile_fp8_cache(dtype: torch.dtype) -> None: device="cuda", dtype=dtype, ).eval() - module.attention_quantization = "fp8" + module._supports_quantized_attention = True torch.nn.init.normal_(module.out_lin.weight, std=0.05) context = torch.randn(1, 8193, 128, device="cuda", dtype=dtype) compiled = torch.compile(module, fullgraph=True, dynamic=False) - with torch.inference_mode(): + with torch.inference_mode(), optimize(attention="fp8"): expected, eager_cache = module(context, return_key_value=True) actual, cache = compiled(context, return_key_value=True) assert isinstance(cache, QuantizedKVCacheEntry) @@ -330,6 +331,16 @@ def test_attention_compile_fp8_cache(dtype: torch.dtype) -> None: torch.testing.assert_close(compiled(query, cache), expected) torch.testing.assert_close(compiled(query, eager_cache), expected) + with torch.inference_mode(): + expected, _ = module(context, return_key_value=True) + actual, ordinary_cache = compiled(context, return_key_value=True) + assert not isinstance(ordinary_cache, QuantizedKVCacheEntry) + torch.testing.assert_close(actual, expected) + with optimize(attention="fp8"): + torch.testing.assert_close( + compiled(query, cache), module(query, cache) + ) + @withCUDA @pytest.mark.parametrize( diff --git a/test/test_optimization.py b/test/test_optimization.py new file mode 100644 index 0000000000..eef6866e77 --- /dev/null +++ b/test/test_optimization.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from sdm import optimize +from sdm.optimization import _attention_quantization + + +def test_nested_optimization_restores_on_exception() -> None: + assert _attention_quantization.get() is None + with optimize(attention="fp8"): + with optimize(): + assert _attention_quantization.get() == "fp8" + with optimize(attention=None): + assert _attention_quantization.get() is None + with ( + pytest.raises(ValueError, match="test exception"), + optimize(attention=None), + ): + raise ValueError("test exception") + assert _attention_quantization.get() == "fp8" + assert _attention_quantization.get() is None + + +def test_optimization_is_thread_local() -> None: + with optimize(attention="fp8"), ThreadPoolExecutor(max_workers=1) as pool: + assert pool.submit(_attention_quantization.get).result() is None + assert _attention_quantization.get() == "fp8" + + +def test_optimization_is_task_local() -> None: + async def enabled() -> None: + with optimize(attention="fp8"): + await asyncio.sleep(0) + assert _attention_quantization.get() == "fp8" + + async def disabled() -> None: + await asyncio.sleep(0) + assert _attention_quantization.get() is None + + async def run() -> None: + await asyncio.gather(enabled(), disabled()) + + asyncio.run(run()) From b6e64d7abf019e060b010133603c34d9b9dc2a37 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Fri, 18 Sep 2026 03:12:57 +0000 Subject: [PATCH 29/44] Test model cache reuse across optimization scopes --- sdm/nn/attention.py | 1 + test/models/tabiclv2/test_model.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index 477ddc5bc5..4d9d57d178 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -293,6 +293,7 @@ def __init__( self.q_dim = num_query_heads * self.head_dim # == channels self.kv_dim = num_key_value_heads * self.head_dim + # ICL blocks enable this where queries include context for calibration. self._supports_quantized_attention = False self.qkv_lin = Linear( diff --git a/test/models/tabiclv2/test_model.py b/test/models/tabiclv2/test_model.py index 99f0edf7c4..de15774174 100644 --- a/test/models/tabiclv2/test_model.py +++ b/test/models/tabiclv2/test_model.py @@ -362,5 +362,12 @@ def test_fp8_fit_predict(dtype: torch.dtype) -> None: isinstance(entry, QuantizedKVCacheEntry) for entry in entries.values() ) - model.clear() - assert model._cache is None + with torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32): + with pytest.raises(RuntimeError, match=r"sdm\.optimize"): + model.predict(query) + with optimize(attention="fp8"): + torch.testing.assert_close( + model.predict(query).numerical, actual.numerical + ) + model.clear() + assert model._cache is None From 2d32a5bfc0f7383afce5b900dd77ea08f5fb1664 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Fri, 18 Sep 2026 03:28:49 +0000 Subject: [PATCH 30/44] Make FP8 fallback and numerical regressions explicit --- test/models/tabiclv2/test_model.py | 1 + test/nn/test_attention_quantization.py | 3 +++ test/nn/test_compile.py | 1 + 3 files changed, 5 insertions(+) diff --git a/test/models/tabiclv2/test_model.py b/test/models/tabiclv2/test_model.py index de15774174..066a7cb638 100644 --- a/test/models/tabiclv2/test_model.py +++ b/test/models/tabiclv2/test_model.py @@ -331,6 +331,7 @@ def test_compile(dtype: torch.dtype) -> None: @onlyCUDA @pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) +@pytest.mark.usefixtures("fp8_rng") def test_fp8_fit_predict(dtype: torch.dtype) -> None: if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") diff --git a/test/nn/test_attention_quantization.py b/test/nn/test_attention_quantization.py index fe3aae4b4d..027d04eac9 100644 --- a/test/nn/test_attention_quantization.py +++ b/test/nn/test_attention_quantization.py @@ -12,6 +12,8 @@ from sdm.nn import Attention from sdm.testing import onlyCUDA, withCUDA +pytestmark = pytest.mark.usefixtures("fp8_rng") + @withCUDA def test_fp8_small_context_fallback(device: torch.device) -> None: @@ -26,6 +28,7 @@ def test_fp8_small_context_fallback(device: torch.device) -> None: expected = reference(query) torch.testing.assert_close(actual, expected, rtol=0, atol=0) assert isinstance(cache, KVCacheEntry) + assert not isinstance(cache, QuantizedKVCacheEntry) assert module.state_dict().keys() == reference.state_dict().keys() diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index 2d6e9d5509..bf771bd137 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -306,6 +306,7 @@ def test_attention_compile_key_value_cache(device: torch.device) -> None: @pytest.mark.parametrize( "dtype", [torch.float16, torch.bfloat16, torch.float32] ) +@pytest.mark.usefixtures("fp8_rng") def test_attention_compile_fp8_cache(dtype: torch.dtype) -> None: if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") From d881778bbf66efedc06ef372f278969f07b002ea Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 1 Oct 2026 06:01:19 +0000 Subject: [PATCH 31/44] Cover FP8 automatic chunking and estimator cache offloading --- test/models/tabiclv2/test_model.py | 26 +++++++++++++++++++++++--- test/nn/test_attention_quantization.py | 10 ++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/test/models/tabiclv2/test_model.py b/test/models/tabiclv2/test_model.py index 066a7cb638..3ce052c488 100644 --- a/test/models/tabiclv2/test_model.py +++ b/test/models/tabiclv2/test_model.py @@ -332,7 +332,10 @@ def test_compile(dtype: torch.dtype) -> None: @onlyCUDA @pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) @pytest.mark.usefixtures("fp8_rng") -def test_fp8_fit_predict(dtype: torch.dtype) -> None: +@pytest.mark.parametrize("estimator_batch_size", [1, 2]) +def test_fp8_fit_predict( + dtype: torch.dtype, estimator_batch_size: int +) -> None: if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") model = TabICLv2( @@ -343,19 +346,36 @@ def test_fp8_fit_predict(dtype: torch.dtype) -> None: x = torch.randn(8193, 3, device="cuda") y = torch.randn(8193, 1, device="cuda") query = torch.randn(17, 3, device="cuda") + num_estimators = 3 if estimator_batch_size == 2 else 1 with ( torch.inference_mode(), optimize(attention="fp8"), torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), ): - expected = model(x, y, query, recipe=Recipe(), num_estimators=1) - model.fit(x, y, recipe=Recipe(), num_estimators=1) + expected = model( + x, + y, + query, + recipe=Recipe(), + num_estimators=num_estimators, + estimator_batch_size=1, + ) + model.fit( + x, + y, + recipe=Recipe(), + num_estimators=num_estimators, + estimator_batch_size=estimator_batch_size, + ) actual = model.predict(query) assert actual.numerical.isfinite().all() torch.testing.assert_close( actual.numerical, expected.numerical, atol=0.01, rtol=0.01 ) assert model._cache is not None + if num_estimators > 1: + assert model._cache.is_cpu + assert model._cache["num_batches"] == 2 # The public lifecycle must retain real quantized tensors, not merely # accept an option while silently executing ordinary attention. entries = cast(Cache, model._cache[0]) diff --git a/test/nn/test_attention_quantization.py b/test/nn/test_attention_quantization.py index 027d04eac9..9743567618 100644 --- a/test/nn/test_attention_quantization.py +++ b/test/nn/test_attention_quantization.py @@ -36,9 +36,12 @@ def test_fp8_small_context_fallback(device: torch.device) -> None: @pytest.mark.parametrize( "dtype", [torch.float16, torch.bfloat16, torch.float32] ) -def test_fp8_context_cache_and_chunking(dtype: torch.dtype) -> None: +def test_fp8_context_cache_and_chunking( + dtype: torch.dtype, monkeypatch: pytest.MonkeyPatch +) -> None: if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") + monkeypatch.setenv("SDM_CHUNK_MEMORY_FRACTION", "0.00001") module = TabICLv2TransformerBlock( channels=128, num_heads=2, @@ -60,7 +63,10 @@ def test_fp8_context_cache_and_chunking(dtype: torch.dtype) -> None: with optimize(attention="fp8"): full = module(joined, context) _, cache = module( - context, context, return_key_value=True, batch_size_limit=1 + context, + context, + return_key_value=True, + batch_size_limit="auto", ) assert isinstance(cache, QuantizedKVCacheEntry) actual = module(query, cache, batch_size_limit=1) From 5eb1f999cad1b70baf0ee8ad6b7b473efc8681a3 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:23:51 +0000 Subject: [PATCH 32/44] Use centralized FP8 dispatch with existing SDPA fallback --- sdm/nn/attention.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index 4d9d57d178..2bca95ca27 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -12,7 +12,7 @@ from torch.nn import Linear from sdm._kernels.flash_attention import flash_attention -from sdm._kernels.fp8_attention import fp8_attention, supports_fp8 +from sdm._kernels.fp8_attention import fp8_attention from sdm._kernels.sdpa import sdpa from sdm._memory import chunk_memory_limit from sdm.cache import KVCacheEntry, QuantizedKVCacheEntry @@ -485,35 +485,28 @@ def forward( and query.size(-3) >= key.size(-3) and seqused_key_value is None and attn_mask is None - and supports_fp8(query) ) if quantized_cache is not None: - if ( - not supports_fp8(query) - or attn_mask is not None - or seqused_key_value is not None - ): + if attn_mask is not None or seqused_key_value is not None: raise ValueError( - "Fitted FP8 attention requires supported CUDA inference " - "without attention masks" + "Fitted FP8 attention does not support attention masks" ) use_fp8 = True - if quantized_cache is not None and query.numel() == 0: - out = query - elif use_fp8: + result = None + if use_fp8: scaled_query = query - if self.sdpa.query_scaling is not None: + if query.numel() > 0 and self.sdpa.query_scaling is not None: scaled_query = self.sdpa.query_scaling( query, key_len=key.size(-3) ) - out, quantized_cache = fp8_attention( + result = fp8_attention( query=scaled_query, key=key, value=value, cache=quantized_cache, scale=self.sdpa.scale, ) - else: + if result is None: out = self.sdpa( query=query, # [..., Q, Hq, C // Hq] key=key, # [..., KV, Hkv, C // Hq] @@ -522,6 +515,8 @@ def forward( attn_mask=attn_mask, # [..., Q, KV] is_causal=is_causal, ) # [..., Q, Hq, C // Hq] + else: + out, quantized_cache = result out = out.flatten(-2, -1) # [..., Q, C] if gate is not None: out = out * gate From a002bcfe2e4675b7a0fdeb467a32cf5cc9c1b649 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:10:57 +0000 Subject: [PATCH 33/44] Explain why final query-only attention stays on SDPA --- sdm/models/tabiclv2/icl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdm/models/tabiclv2/icl.py b/sdm/models/tabiclv2/icl.py index 3f2704a7b9..1e431b7cfa 100644 --- a/sdm/models/tabiclv2/icl.py +++ b/sdm/models/tabiclv2/icl.py @@ -56,7 +56,7 @@ def __init__( for _ in range(num_layers) ) - # The final layer has no context queries to calibrate FP8 scales. + # Keep final attention on SDPA to avoid FP8 overhead on small batches. for layer in self.layers[:-1]: cast( TabICLv2TransformerBlock, layer From a8245eb73aa86c7c89918084a69330facf4c5ef8 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:43:52 +0000 Subject: [PATCH 34/44] Preserve causal attention behavior when FP8 is enabled --- sdm/nn/attention.py | 7 ++++++- test/nn/test_attention_quantization.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index 2bca95ca27..23b2f8d4bb 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -485,9 +485,14 @@ def forward( and query.size(-3) >= key.size(-3) and seqused_key_value is None and attn_mask is None + and not is_causal ) if quantized_cache is not None: - if attn_mask is not None or seqused_key_value is not None: + if ( + attn_mask is not None + or seqused_key_value is not None + or is_causal + ): raise ValueError( "Fitted FP8 attention does not support attention masks" ) diff --git a/test/nn/test_attention_quantization.py b/test/nn/test_attention_quantization.py index 9743567618..cb1bf68a3d 100644 --- a/test/nn/test_attention_quantization.py +++ b/test/nn/test_attention_quantization.py @@ -32,6 +32,26 @@ def test_fp8_small_context_fallback(device: torch.device) -> None: assert module.state_dict().keys() == reference.state_dict().keys() +@onlyCUDA +def test_fp8_causal_attention_fallback() -> None: + module = Attention(64, 1, device="cuda") + torch.nn.init.normal_(module.out_lin.weight, std=0.05) + module._supports_quantized_attention = True + query = torch.randn(1, 8193, 64, device="cuda") + with torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16): + expected = module(query, is_causal=True) + with optimize(attention="fp8"): + actual, cache = module( + query, is_causal=True, return_key_value=True + ) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert not isinstance(cache, QuantizedKVCacheEntry) + _, cache = module(query, return_key_value=True) + assert isinstance(cache, QuantizedKVCacheEntry) + with pytest.raises(ValueError, match="attention masks"): + module(query[:, :1], cache, is_causal=True) + + @onlyCUDA @pytest.mark.parametrize( "dtype", [torch.float16, torch.bfloat16, torch.float32] From 2a545f84d1d8afd85eca566f865e64f4eefba4cf Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:48:35 +0000 Subject: [PATCH 35/44] Simplify FP8 eligibility and keep focused integration coverage --- sdm/nn/attention.py | 32 +++--- test/models/tabiclv2/test_model.py | 24 ++--- test/nn/test_attention_quantization.py | 136 +++++-------------------- test/nn/test_compile.py | 9 +- test/test_optimization.py | 25 ----- 5 files changed, 50 insertions(+), 176 deletions(-) diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index 23b2f8d4bb..524816ff59 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -476,29 +476,25 @@ def forward( quantized_cache = ( key_value if isinstance(key_value, QuantizedKVCacheEntry) else None ) - use_fp8 = ( - _attention_quantization.get() == "fp8" - and self._supports_quantized_attention + masked = ( + attn_mask is not None or seqused_key_value is not None or is_causal + ) + if quantized_cache is not None and masked: + raise ValueError( + "Fitted FP8 attention does not support attention masks" + ) + # New caches need unmasked context queries to calibrate Q scales. + can_quantize = ( + self._supports_quantized_attention and not isinstance(key_value, KVCacheEntry) - and query.numel() > 0 and key.size(-3) > 8192 and query.size(-3) >= key.size(-3) - and seqused_key_value is None - and attn_mask is None - and not is_causal + and not masked ) - if quantized_cache is not None: - if ( - attn_mask is not None - or seqused_key_value is not None - or is_causal - ): - raise ValueError( - "Fitted FP8 attention does not support attention masks" - ) - use_fp8 = True result = None - if use_fp8: + if _attention_quantization.get() == "fp8" and ( + quantized_cache is not None or can_quantize + ): scaled_query = query if query.numel() > 0 and self.sdpa.query_scaling is not None: scaled_query = self.sdpa.query_scaling( diff --git a/test/models/tabiclv2/test_model.py b/test/models/tabiclv2/test_model.py index 3ce052c488..cdaad6a115 100644 --- a/test/models/tabiclv2/test_model.py +++ b/test/models/tabiclv2/test_model.py @@ -330,12 +330,8 @@ def test_compile(dtype: torch.dtype) -> None: @onlyCUDA -@pytest.mark.parametrize("dtype", [torch.float16, torch.float32]) @pytest.mark.usefixtures("fp8_rng") -@pytest.mark.parametrize("estimator_batch_size", [1, 2]) -def test_fp8_fit_predict( - dtype: torch.dtype, estimator_batch_size: int -) -> None: +def test_fp8_fit_predict() -> None: if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") model = TabICLv2( @@ -346,44 +342,38 @@ def test_fp8_fit_predict( x = torch.randn(8193, 3, device="cuda") y = torch.randn(8193, 1, device="cuda") query = torch.randn(17, 3, device="cuda") - num_estimators = 3 if estimator_batch_size == 2 else 1 with ( torch.inference_mode(), optimize(attention="fp8"), - torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), + torch.autocast("cuda", dtype=torch.float16), ): expected = model( x, y, query, recipe=Recipe(), - num_estimators=num_estimators, + num_estimators=3, estimator_batch_size=1, ) model.fit( x, y, recipe=Recipe(), - num_estimators=num_estimators, - estimator_batch_size=estimator_batch_size, + num_estimators=3, + estimator_batch_size=2, ) actual = model.predict(query) - assert actual.numerical.isfinite().all() torch.testing.assert_close( actual.numerical, expected.numerical, atol=0.01, rtol=0.01 ) assert model._cache is not None - if num_estimators > 1: - assert model._cache.is_cpu - assert model._cache["num_batches"] == 2 - # The public lifecycle must retain real quantized tensors, not merely - # accept an option while silently executing ordinary attention. + assert model._cache.is_cpu entries = cast(Cache, model._cache[0]) assert any( isinstance(entry, QuantizedKVCacheEntry) for entry in entries.values() ) - with torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32): + with torch.autocast("cuda", dtype=torch.float16): with pytest.raises(RuntimeError, match=r"sdm\.optimize"): model.predict(query) with optimize(attention="fp8"): diff --git a/test/nn/test_attention_quantization.py b/test/nn/test_attention_quantization.py index cb1bf68a3d..da88681ec1 100644 --- a/test/nn/test_attention_quantization.py +++ b/test/nn/test_attention_quantization.py @@ -1,143 +1,59 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import copy - import pytest import torch from sdm import optimize -from sdm.cache import Cache, KVCacheEntry, QuantizedKVCacheEntry +from sdm.cache import QuantizedKVCacheEntry from sdm.models.tabiclv2.block import TabICLv2TransformerBlock from sdm.nn import Attention -from sdm.testing import onlyCUDA, withCUDA +from sdm.testing import onlyCUDA pytestmark = pytest.mark.usefixtures("fp8_rng") -@withCUDA -def test_fp8_small_context_fallback(device: torch.device) -> None: - module = Attention(128, 2, device=device) - torch.nn.init.normal_(module.out_lin.weight, std=0.05) - reference = copy.deepcopy(module) - module._supports_quantized_attention = True - query = torch.randn(2, 19, 128, device=device) - with torch.inference_mode(): - with optimize(attention="fp8"): - actual, cache = module(query, return_key_value=True) - expected = reference(query) - torch.testing.assert_close(actual, expected, rtol=0, atol=0) - assert isinstance(cache, KVCacheEntry) - assert not isinstance(cache, QuantizedKVCacheEntry) - assert module.state_dict().keys() == reference.state_dict().keys() - - @onlyCUDA -def test_fp8_causal_attention_fallback() -> None: +@pytest.mark.parametrize(("rows", "is_causal"), [(19, False), (8193, True)]) +def test_fp8_attention_fallback(rows: int, is_causal: bool) -> None: module = Attention(64, 1, device="cuda") torch.nn.init.normal_(module.out_lin.weight, std=0.05) module._supports_quantized_attention = True - query = torch.randn(1, 8193, 64, device="cuda") + query = torch.randn(1, rows, 64, device="cuda") with torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16): - expected = module(query, is_causal=True) + expected = module(query, is_causal=is_causal) with optimize(attention="fp8"): actual, cache = module( - query, is_causal=True, return_key_value=True + query, is_causal=is_causal, return_key_value=True ) - torch.testing.assert_close(actual, expected, rtol=0, atol=0) - assert not isinstance(cache, QuantizedKVCacheEntry) - _, cache = module(query, return_key_value=True) - assert isinstance(cache, QuantizedKVCacheEntry) - with pytest.raises(ValueError, match="attention masks"): - module(query[:, :1], cache, is_causal=True) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert not isinstance(cache, QuantizedKVCacheEntry) @onlyCUDA -@pytest.mark.parametrize( - "dtype", [torch.float16, torch.bfloat16, torch.float32] -) -def test_fp8_context_cache_and_chunking( - dtype: torch.dtype, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_fp8_cache_chunking(monkeypatch: pytest.MonkeyPatch) -> None: if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") monkeypatch.setenv("SDM_CHUNK_MEMORY_FRACTION", "0.00001") - module = TabICLv2TransformerBlock( - channels=128, - num_heads=2, - norm_bias=True, - qassmax=True, - device="cuda", + block = TabICLv2TransformerBlock( + channels=128, num_heads=2, norm_bias=True, qassmax=True, device="cuda" ) - torch.nn.init.normal_(module.attn.out_lin.weight, std=0.05) - reference = copy.deepcopy(module) - module.attn._supports_quantized_attention = True + torch.nn.init.normal_(block.attn.out_lin.weight, std=0.05) + block.attn._supports_quantized_attention = True context = torch.randn(2, 1, 8193, 128, device="cuda") - query = torch.randn(2, 1, 129, 128, device="cuda") - joined = torch.cat([context, query], dim=-2) - with ( - torch.inference_mode(), - torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), - ): - expected = reference(joined, context) - with optimize(attention="fp8"): - full = module(joined, context) - _, cache = module( - context, - context, - return_key_value=True, - batch_size_limit="auto", - ) - assert isinstance(cache, QuantizedKVCacheEntry) - actual = module(query, cache, batch_size_limit=1) - unchunked = module(query, cache) - split = torch.cat( - [ - module(query[..., :64, :], cache), - module(query[..., 64:, :], cache), - ], - dim=-2, - ) - empty = module(query[..., :0, :], cache) - migrated = Cache(kv=cache).cpu().cuda()["kv"] - replay = module(query, migrated) - assert empty.size(-2) == 0 - assert actual.isfinite().all() - torch.testing.assert_close(actual, unchunked, atol=2e-3, rtol=2e-3) - torch.testing.assert_close(actual, split, atol=2e-3, rtol=2e-3) - torch.testing.assert_close(actual, replay, atol=2e-3, rtol=2e-3) - torch.testing.assert_close( - actual, full[..., -129:, :], atol=0.03, rtol=0.03 - ) - error = (full - expected).square().mean() / expected.square().mean() - assert error.sqrt() < 0.03 - assert Cache(kv=cache).size() == sum( - t.numel() * t.element_size() for t in cache._tensors() - ) - assert cache.key.dtype == torch.float8_e4m3fn - assert cache.value.dtype == torch.float8_e4m3fn - assert cache.query_scale.dtype == torch.float32 + query = torch.randn(2, 1, 17, 128, device="cuda") with ( torch.inference_mode(), - torch.autocast( - "cuda", dtype=torch.float16, enabled=dtype == torch.float32 - ), + torch.autocast("cuda", dtype=torch.float16), optimize(attention="fp8"), - pytest.raises(ValueError, match="dtype"), ): - module(query, cache) - with ( - torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), - optimize(attention="fp8"), - pytest.raises(ValueError, match="inference"), - ): - module(query, cache) - - with ( - torch.inference_mode(), - torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), - ): - with pytest.raises(RuntimeError, match=r"sdm\.optimize"): - module(query, cache) - with optimize(attention="fp8"): - torch.testing.assert_close(module(query, cache), actual) + _, cache = block( + context, context, return_key_value=True, batch_size_limit="auto" + ) + assert isinstance(cache, QuantizedKVCacheEntry) + expected = block(query, cache) + actual = block(query, cache, batch_size_limit=1) + torch.testing.assert_close(actual, expected, atol=2e-3, rtol=2e-3) + assert block(query[..., :0, :], cache).size(-2) == 0 + with pytest.raises(ValueError, match="attention masks"): + block.attn(query, cache, is_causal=True) diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index bf771bd137..6727680976 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -303,11 +303,9 @@ def test_attention_compile_key_value_cache(device: torch.device) -> None: @onlyCUDA -@pytest.mark.parametrize( - "dtype", [torch.float16, torch.bfloat16, torch.float32] -) @pytest.mark.usefixtures("fp8_rng") -def test_attention_compile_fp8_cache(dtype: torch.dtype) -> None: +def test_attention_compile_fp8_cache() -> None: + dtype = torch.float16 if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") module = Attention( @@ -326,11 +324,10 @@ def test_attention_compile_fp8_cache(dtype: torch.dtype) -> None: actual, cache = compiled(context, return_key_value=True) assert isinstance(cache, QuantizedKVCacheEntry) torch.testing.assert_close(actual, expected) - for rows in (17, 33, 1, 17): + for rows in (17, 1): query = torch.randn(1, rows, 128, device="cuda", dtype=dtype) expected = module(query, eager_cache) torch.testing.assert_close(compiled(query, cache), expected) - torch.testing.assert_close(compiled(query, eager_cache), expected) with torch.inference_mode(): expected, _ = module(context, return_key_value=True) diff --git a/test/test_optimization.py b/test/test_optimization.py index eef6866e77..7c48eb35a1 100644 --- a/test/test_optimization.py +++ b/test/test_optimization.py @@ -1,9 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import asyncio -from concurrent.futures import ThreadPoolExecutor - import pytest from sdm import optimize @@ -24,25 +21,3 @@ def test_nested_optimization_restores_on_exception() -> None: raise ValueError("test exception") assert _attention_quantization.get() == "fp8" assert _attention_quantization.get() is None - - -def test_optimization_is_thread_local() -> None: - with optimize(attention="fp8"), ThreadPoolExecutor(max_workers=1) as pool: - assert pool.submit(_attention_quantization.get).result() is None - assert _attention_quantization.get() == "fp8" - - -def test_optimization_is_task_local() -> None: - async def enabled() -> None: - with optimize(attention="fp8"): - await asyncio.sleep(0) - assert _attention_quantization.get() == "fp8" - - async def disabled() -> None: - await asyncio.sleep(0) - assert _attention_quantization.get() is None - - async def run() -> None: - await asyncio.gather(enabled(), disabled()) - - asyncio.run(run()) From a1390b0c73d7a4c591793f3f3e11096d6c6e8ac0 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:05:59 +0000 Subject: [PATCH 36/44] Keep the attention forward path focused on projection and dispatch --- sdm/nn/attention.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index 524816ff59..e0dc6224a0 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -476,36 +476,22 @@ def forward( quantized_cache = ( key_value if isinstance(key_value, QuantizedKVCacheEntry) else None ) - masked = ( - attn_mask is not None or seqused_key_value is not None or is_causal - ) - if quantized_cache is not None and masked: - raise ValueError( - "Fitted FP8 attention does not support attention masks" - ) - # New caches need unmasked context queries to calibrate Q scales. - can_quantize = ( - self._supports_quantized_attention - and not isinstance(key_value, KVCacheEntry) - and key.size(-3) > 8192 - and query.size(-3) >= key.size(-3) - and not masked - ) result = None if _attention_quantization.get() == "fp8" and ( - quantized_cache is not None or can_quantize + self._supports_quantized_attention or quantized_cache is not None ): - scaled_query = query - if query.numel() > 0 and self.sdpa.query_scaling is not None: - scaled_query = self.sdpa.query_scaling( - query, key_len=key.size(-3) - ) result = fp8_attention( - query=scaled_query, + query=query, key=key, value=value, - cache=quantized_cache, + cache=key_value + if isinstance(key_value, KVCacheEntry) + else None, scale=self.sdpa.scale, + query_scaling=self.sdpa.query_scaling, + seqused_key_value=seqused_key_value, + attn_mask=attn_mask, + is_causal=is_causal, ) if result is None: out = self.sdpa( From 22912e02d70985262a174f8cf6d2be5e9ec64b5f Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:56:44 +0000 Subject: [PATCH 37/44] Enable FP8 attention on CUDA architectures with native FP8 support --- sdm/_kernels/fp8_attention.py | 7 +++--- sdm/optimization.py | 7 +++--- test/_kernels/test_fp8_attention.py | 33 ++++++++++++++++++++++---- test/nn/test_attention_quantization.py | 4 ++-- 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/sdm/_kernels/fp8_attention.py b/sdm/_kernels/fp8_attention.py index 073f12f121..6b745041a3 100644 --- a/sdm/_kernels/fp8_attention.py +++ b/sdm/_kernels/fp8_attention.py @@ -80,11 +80,10 @@ def fp8_attention( _triton_fp8_attention is not None and not torch.is_grad_enabled() and query.is_cuda + and torch.version.cuda is not None and query.size(-1) in {32, 64, 128, 256} - # CUDA compute capability (major, minor): Ada, Hopper, RTX Blackwell. - # Restrict dispatch to architectures supported by this FP8 kernel. - and torch.cuda.get_device_capability(query.device) - in {(8, 9), (9, 0), (12, 0)} + # Native FP8 Tensor Cores require CUDA compute capability 8.9 or newer. + and torch.cuda.get_device_capability(query.device) >= (8, 9) ): if cache is not None and query.numel() == 0: return query, cache diff --git a/sdm/optimization.py b/sdm/optimization.py index 9deb229302..1356d33bd2 100644 --- a/sdm/optimization.py +++ b/sdm/optimization.py @@ -19,9 +19,10 @@ def optimize( Args: attention: ``"fp8"`` enables quantization in eligible ICL attention - layers during supported CUDA inference with more than 8192 context - rows. ``None`` disables it; ``"inherit"`` preserves the enclosing - setting. Weights and the final ICL layer remain unquantized. + layers during CUDA inference (compute capability 8.9 or newer) + with more than 8192 context rows. ``None`` disables it; + ``"inherit"`` preserves the enclosing setting. Weights and the + final ICL layer remain unquantized. Yields: None. Previous settings are restored on exit, including on exceptions. diff --git a/test/_kernels/test_fp8_attention.py b/test/_kernels/test_fp8_attention.py index e8afae969e..984a8df614 100644 --- a/test/_kernels/test_fp8_attention.py +++ b/test/_kernels/test_fp8_attention.py @@ -6,17 +6,18 @@ from sdm._kernels.fp8_attention import fp8_attention from sdm.cache import QuantizedKVCacheEntry -from sdm.testing import withCUDA +from sdm.testing import onlyCUDA, withCUDA @withCUDA @pytest.mark.parametrize("channels", [32, 48]) @pytest.mark.parametrize("grad", [False, True]) def test_fp8_dispatch(device: torch.device, channels: int, grad: bool) -> None: - if device.type == "cuda" and torch.cuda.get_device_capability( - device - ) not in {(8, 9), (9, 0), (12, 0)}: - pytest.skip("FP8 dispatch requires Ada, Hopper, or RTX Blackwell") + if device.type == "cuda" and ( + torch.version.cuda is None + or torch.cuda.get_device_capability(device) < (8, 9) + ): + pytest.skip("FP8 requires CUDA compute capability 8.9 or newer") x = torch.ones(1, 8193, 2, channels, device=device, dtype=torch.float16) with torch.set_grad_enabled(grad): result = fp8_attention(x, x, x) @@ -36,6 +37,28 @@ def test_fp8_dispatch(device: torch.device, channels: int, grad: bool) -> None: assert empty[1] is cache +@onlyCUDA +@pytest.mark.parametrize( + "capability", [(8, 6), (10, 0), (10, 3), (11, 0), (12, 1)] +) +def test_fp8_device_capability( + capability: tuple[int, int], monkeypatch: pytest.MonkeyPatch +) -> None: + if not torch.version.cuda or torch.cuda.get_device_capability() < (8, 9): + pytest.skip("FP8 requires CUDA compute capability 8.9 or newer") + monkeypatch.setattr( + torch.cuda, "get_device_capability", lambda _: capability + ) + x = torch.ones(1, 8193, 1, 32, device="cuda", dtype=torch.float16) + with torch.inference_mode(): + result = fp8_attention(x, x, x) + if capability < (8, 9): + assert result is None + else: + assert result is not None + torch.testing.assert_close(result[0], x) + + @withCUDA def test_fp8_cache_rejects_unsupported_execution(device: torch.device) -> None: x = torch.ones(1, 17, 2, 32, device=device, dtype=torch.float8_e4m3fn) diff --git a/test/nn/test_attention_quantization.py b/test/nn/test_attention_quantization.py index da88681ec1..5c3b174353 100644 --- a/test/nn/test_attention_quantization.py +++ b/test/nn/test_attention_quantization.py @@ -32,8 +32,8 @@ def test_fp8_attention_fallback(rows: int, is_causal: bool) -> None: @onlyCUDA def test_fp8_cache_chunking(monkeypatch: pytest.MonkeyPatch) -> None: - if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: - pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") + if not torch.version.cuda or torch.cuda.get_device_capability() < (8, 9): + pytest.skip("FP8 requires CUDA compute capability 8.9 or newer") monkeypatch.setenv("SDM_CHUNK_MEMORY_FRACTION", "0.00001") block = TabICLv2TransformerBlock( channels=128, num_heads=2, norm_bias=True, qassmax=True, device="cuda" From 532ce25cb8f54af12f72173ae15487a2b22ae462 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:06:58 +0000 Subject: [PATCH 38/44] Enable FP8 attention in Kumo and TabFM --- sdm/models/kumo/tabular/icl.py | 6 + sdm/models/tabfm/icl.py | 6 + test/models/test_attention_quantization.py | 212 +++++++++++++++++++++ 3 files changed, 224 insertions(+) create mode 100644 test/models/test_attention_quantization.py diff --git a/sdm/models/kumo/tabular/icl.py b/sdm/models/kumo/tabular/icl.py index 066001790f..c3d97c6ac3 100644 --- a/sdm/models/kumo/tabular/icl.py +++ b/sdm/models/kumo/tabular/icl.py @@ -50,6 +50,12 @@ def __init__( for _ in range(num_layers) ) + # The final layer has no context queries to calibrate FP8 scales. + for layer in self.layers[:-1]: + cast( + KumoTabularTransformerBlock, layer + ).attn._supports_quantized_attention = True + self.norm = RMSNorm(channels, **factory_kwargs) self.head = Sequential( Linear(channels, 2 * channels, **factory_kwargs), diff --git a/sdm/models/tabfm/icl.py b/sdm/models/tabfm/icl.py index 8efa1a7049..01ea4b0389 100644 --- a/sdm/models/tabfm/icl.py +++ b/sdm/models/tabfm/icl.py @@ -57,6 +57,12 @@ def __init__( for _ in range(num_layers) ) + # The final layer has no context queries to calibrate FP8 scales. + for layer in self.layers[:-1]: + cast( + TabFMTransformerBlock, layer + ).attn._supports_quantized_attention = True + self.norm = RMSNorm(channels, eps=1e-6, **factory_kwargs) self.head = Sequential( Linear(channels, 2 * channels, **factory_kwargs), diff --git a/test/models/test_attention_quantization.py b/test/models/test_attention_quantization.py new file mode 100644 index 0000000000..78af7ea62e --- /dev/null +++ b/test/models/test_attention_quantization.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import copy +import functools +from typing import Any, cast + +import pytest +import torch + +from sdm import ColumnarTensor, Recipe, RelatedTables, TableTensor, optimize +from sdm.cache import Cache, QuantizedKVCacheEntry +from sdm.models import KumoRelational, KumoTabular, TabFM +from sdm.models.kumo.tabular.icl import ICLBlock as KumoICLBlock +from sdm.models.tabfm import model as tabfm_module +from sdm.models.tabfm.icl import ICLBlock as TabFMICLBlock +from sdm.testing import onlyCUDA + + +@onlyCUDA +@pytest.mark.parametrize( + "dtype", [torch.float16, torch.bfloat16, torch.float32] +) +@pytest.mark.parametrize( + ("kind", "heads"), + [("kumo", None), ("kumo", 1), ("kumo", 2), ("tabfm", None)], +) +def test_fp8_icl_cache( + kind: str, heads: int | None, dtype: torch.dtype +) -> None: + if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: + pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") + if kind == "kumo": + block = KumoICLBlock( + num_classes=0, + out_channels=5, + channels=256, + num_layers=2, + num_heads=4, + num_key_value_heads_for_query=heads, + device="cuda", + ) + else: + # Stock TabFM has 256 channels per ICL attention head. + block = TabFMICLBlock( + num_classes=0, + out_channels=5, + channels=512, + num_layers=2, + num_heads=2, + device="cuda", + ) + for parameter in block.parameters(): + if not parameter.any(): + torch.nn.init.normal_(parameter, std=0.02) + reference = copy.deepcopy(block) + channels = 256 if kind == "kumo" else 512 + x = torch.randn(2, 8226, channels, device="cuda") + y = torch.randn(2, 8193, device="cuda") + with ( + torch.inference_mode(), + torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), + ): + expected = reference(x.clone(), y) + with optimize(attention="fp8"): + actual = block(x.clone(), y) + cache = Cache() + block(x[:, :8193].clone(), y, cache=cache) + replayed = block( + x[:, 8193:].clone(), y[:, :0], cache=cache.freeze() + ) + split = torch.cat( + [ + block(x[:, 8193:8209].clone(), y[:, :0], cache=cache), + block(x[:, 8209:].clone(), y[:, :0], cache=cache), + ], + dim=-2, + ) + entry = cache["icl_block.layer0"] + assert isinstance(entry, QuantizedKVCacheEntry) + assert not isinstance(cache["icl_block.layer1"], QuantizedKVCacheEntry) + assert entry.key.dtype == torch.float8_e4m3fn + assert entry.value.dtype == torch.float8_e4m3fn + assert entry.query_scale.size(-2) == (4 if kind == "kumo" else 2) + if heads is not None: + assert entry.key.size(-2) == heads + assert entry.key.untyped_storage().nbytes() == entry.key.numel() + assert entry.value.untyped_storage().nbytes() == entry.value.numel() + assert actual.isfinite().all() + torch.testing.assert_close(replayed, actual, atol=0.01, rtol=0.03) + torch.testing.assert_close(split, replayed, atol=0.01, rtol=0.03) + relative_rms = ( + (actual.float() - expected.float()).square().mean() + / expected.float().square().mean() + ).sqrt() + assert relative_rms < 0.08 + + +@onlyCUDA +@pytest.mark.parametrize("kind", ["kumo-small", "kumo-large", "tabfm"]) +def test_fp8_model_fit_predict( + kind: str, monkeypatch: pytest.MonkeyPatch +) -> None: + if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: + pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") + kwargs: dict[str, Any] = { + "task": "regression", + "pretrained": False, + "device": "cuda", + } + if kind.startswith("kumo-"): + model = KumoTabular( + size="small" if kind == "kumo-small" else "large", **kwargs + ) + else: + # Keep the stock 256-channel ICL heads while reducing model depth. + monkeypatch.setattr( + tabfm_module, + "_TabFM", + functools.partial( + tabfm_module._TabFM, + num_icl_layers=2, + num_embedding_layers=1, + num_embedding_repeats=1, + ), + ) + model = TabFM(**kwargs) + x = torch.randn(8193, 3, device="cuda") + y = torch.randn(8193, 1, device="cuda") + query = torch.randn(17, 3, device="cuda") + with ( + torch.inference_mode(), + torch.autocast("cuda", dtype=torch.float16), + optimize(attention="fp8"), + ): + expected = model(x, y, query, recipe=Recipe(), num_estimators=1) + model.fit(x, y, recipe=Recipe(), num_estimators=1) + actual = model.predict(query) + assert actual.numerical.isfinite().all() + torch.testing.assert_close( + actual.numerical, expected.numerical, atol=0.01, rtol=0.03 + ) + assert model._cache is not None + entries = cast(Cache, model._cache[0]) + quantized = [ + entry + for entry in entries.values() + if isinstance(entry, QuantizedKVCacheEntry) + ] + assert quantized + if kind == "kumo-large": + assert all(entry.key.size(-2) == 2 for entry in quantized) + with torch.autocast("cuda", dtype=torch.float16): + with pytest.raises(RuntimeError, match=r"sdm\.optimize"): + model.predict(query) + with optimize(attention="fp8"): + torch.testing.assert_close( + model.predict(query).numerical, actual.numerical + ) + model.clear() + assert model._cache is None + + +@onlyCUDA +def test_fp8_relational_fit_predict() -> None: + if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: + pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") + model = KumoRelational( + task="regression", + pretrained=False, + device="cuda", + ) + table = TableTensor( + id=ColumnarTensor((torch.arange(8210, device="cuda"),)), + numerical=torch.randn(8210, 1, device="cuda"), + columns={"id": ["id"], "numerical": ["value"]}, + ) + related = RelatedTables( + tables={"entities": table}, + relationships=[], + task_links=[ + {"task_column": "id", "table": "entities", "table_column": "id"} + ], + ) + x, query = table[:8193], table[8193:] + y = torch.randn(8193, 1, device="cuda") + with ( + torch.inference_mode(), + torch.autocast("cuda", dtype=torch.float16), + optimize(attention="fp8"), + ): + expected = model( + x, + y, + query, + related_context_tables=related, + related_query_tables=related, + num_hops=0, + recipe=Recipe(), + num_estimators=1, + ) + model.fit(x, y, related, num_hops=0, recipe=Recipe(), num_estimators=1) + actual = model.predict(query, related_tables=related) + assert actual.numerical.isfinite().all() + torch.testing.assert_close( + actual.numerical, expected.numerical, atol=0.01, rtol=0.03 + ) + assert model._cache is not None + entries = cast(Cache, model._cache[0]) + assert any( + isinstance(entry, QuantizedKVCacheEntry) for entry in entries.values() + ) From 13d8f88fd5e1a0bb16f127cc5a512211a3bdc551 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Fri, 18 Sep 2026 03:28:49 +0000 Subject: [PATCH 39/44] Control randomness in FP8 model comparisons --- test/models/test_attention_quantization.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/models/test_attention_quantization.py b/test/models/test_attention_quantization.py index 78af7ea62e..172c338ee3 100644 --- a/test/models/test_attention_quantization.py +++ b/test/models/test_attention_quantization.py @@ -16,6 +16,8 @@ from sdm.models.tabfm.icl import ICLBlock as TabFMICLBlock from sdm.testing import onlyCUDA +pytestmark = pytest.mark.usefixtures("fp8_rng") + @onlyCUDA @pytest.mark.parametrize( From a85eec550200cf5b0be4592523038b49472979d6 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 1 Oct 2026 06:01:20 +0000 Subject: [PATCH 40/44] Cover current Kumo sizes and batched FP8 cache replay --- test/models/test_attention_quantization.py | 92 ++++++++++++++++++++-- 1 file changed, 84 insertions(+), 8 deletions(-) diff --git a/test/models/test_attention_quantization.py b/test/models/test_attention_quantization.py index 172c338ee3..3e9eb19e0b 100644 --- a/test/models/test_attention_quantization.py +++ b/test/models/test_attention_quantization.py @@ -3,12 +3,19 @@ import copy import functools -from typing import Any, cast +from typing import Any, Literal, cast import pytest import torch -from sdm import ColumnarTensor, Recipe, RelatedTables, TableTensor, optimize +from sdm import ( + CategoricalTensor, + ColumnarTensor, + Recipe, + RelatedTables, + TableTensor, + optimize, +) from sdm.cache import Cache, QuantizedKVCacheEntry from sdm.models import KumoRelational, KumoTabular, TabFM from sdm.models.kumo.tabular.icl import ICLBlock as KumoICLBlock @@ -99,9 +106,12 @@ def test_fp8_icl_cache( @onlyCUDA -@pytest.mark.parametrize("kind", ["kumo-small", "kumo-large", "tabfm"]) +@pytest.mark.parametrize( + "kind", ["kumo-small", "kumo-medium", "kumo-large", "tabfm"] +) +@pytest.mark.parametrize("estimator_batch_size", [1, 2]) def test_fp8_model_fit_predict( - kind: str, monkeypatch: pytest.MonkeyPatch + kind: str, estimator_batch_size: int, monkeypatch: pytest.MonkeyPatch ) -> None: if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") @@ -112,7 +122,10 @@ def test_fp8_model_fit_predict( } if kind.startswith("kumo-"): model = KumoTabular( - size="small" if kind == "kumo-small" else "large", **kwargs + size=cast( + Literal["small", "medium", "large"], kind.removeprefix("kumo-") + ), + **kwargs, ) else: # Keep the stock 256-channel ICL heads while reducing model depth. @@ -130,19 +143,36 @@ def test_fp8_model_fit_predict( x = torch.randn(8193, 3, device="cuda") y = torch.randn(8193, 1, device="cuda") query = torch.randn(17, 3, device="cuda") + num_estimators = 3 if estimator_batch_size == 2 else 1 with ( torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16), optimize(attention="fp8"), ): - expected = model(x, y, query, recipe=Recipe(), num_estimators=1) - model.fit(x, y, recipe=Recipe(), num_estimators=1) + expected = model( + x, + y, + query, + recipe=Recipe(), + num_estimators=num_estimators, + estimator_batch_size=1, + ) + model.fit( + x, + y, + recipe=Recipe(), + num_estimators=num_estimators, + estimator_batch_size=estimator_batch_size, + ) actual = model.predict(query) assert actual.numerical.isfinite().all() torch.testing.assert_close( actual.numerical, expected.numerical, atol=0.01, rtol=0.03 ) assert model._cache is not None + if num_estimators > 1: + assert model._cache.is_cpu + assert model._cache["num_batches"] == 2 entries = cast(Cache, model._cache[0]) quantized = [ entry @@ -150,7 +180,7 @@ def test_fp8_model_fit_predict( if isinstance(entry, QuantizedKVCacheEntry) ] assert quantized - if kind == "kumo-large": + if kind in {"kumo-medium", "kumo-large"}: assert all(entry.key.size(-2) == 2 for entry in quantized) with torch.autocast("cuda", dtype=torch.float16): with pytest.raises(RuntimeError, match=r"sdm\.optimize"): @@ -212,3 +242,49 @@ def test_fp8_relational_fit_predict() -> None: assert any( isinstance(entry, QuantizedKVCacheEntry) for entry in entries.values() ) + + +@onlyCUDA +def test_fp8_ecoc_cache() -> None: + if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: + pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") + model = KumoTabular( + task="classification", size="small", pretrained=False, device="cuda" + ) + x = torch.randn(8193, 3, device="cuda") + query = torch.randn(17, 3, device="cuda") + y = TableTensor( + columns={"categorical": ["target"]}, + categorical=CategoricalTensor.from_tensor( + (torch.arange(8193, device="cuda") % 12).unsqueeze(-1) + ), + ) + with ( + torch.autocast("cuda", dtype=torch.float16), + optimize(attention="fp8"), + ): + expected = model( + x, + y, + query, + recipe=Recipe(), + num_estimators=1, + generator=torch.Generator(device="cuda").manual_seed(0), + ) + model.fit( + x, + y, + recipe=Recipe(), + num_estimators=1, + generator=torch.Generator(device="cuda").manual_seed(0), + ) + actual = model.predict(query) + torch.testing.assert_close( + actual.numerical, expected.numerical, atol=0.01, rtol=0.03 + ) + assert actual.size() == (1, 17, 12) + assert model._cache is not None + nested = cast(Cache, cast(Cache, model._cache[0])["ecoc_model"]) + assert any( + isinstance(entry, QuantizedKVCacheEntry) for entry in nested.values() + ) From 4eb92d1c9488ab2279ed06c7a9788fee0d953157 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:10:58 +0000 Subject: [PATCH 41/44] Explain final attention precision in Kumo and TabFM --- sdm/models/kumo/tabular/icl.py | 2 +- sdm/models/tabfm/icl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdm/models/kumo/tabular/icl.py b/sdm/models/kumo/tabular/icl.py index c3d97c6ac3..7001701675 100644 --- a/sdm/models/kumo/tabular/icl.py +++ b/sdm/models/kumo/tabular/icl.py @@ -50,7 +50,7 @@ def __init__( for _ in range(num_layers) ) - # The final layer has no context queries to calibrate FP8 scales. + # Keep final attention on SDPA to avoid FP8 overhead on small batches. for layer in self.layers[:-1]: cast( KumoTabularTransformerBlock, layer diff --git a/sdm/models/tabfm/icl.py b/sdm/models/tabfm/icl.py index 01ea4b0389..23dd53acc9 100644 --- a/sdm/models/tabfm/icl.py +++ b/sdm/models/tabfm/icl.py @@ -57,7 +57,7 @@ def __init__( for _ in range(num_layers) ) - # The final layer has no context queries to calibrate FP8 scales. + # Keep final attention on SDPA to avoid FP8 overhead on small batches. for layer in self.layers[:-1]: cast( TabFMTransformerBlock, layer From 80c3ea0006c228d1badcbc79f0b42e26d92a08b2 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:48:35 +0000 Subject: [PATCH 42/44] Trim redundant FP8 model test combinations --- test/models/test_attention_quantization.py | 41 ++++++---------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/test/models/test_attention_quantization.py b/test/models/test_attention_quantization.py index 3e9eb19e0b..bd0cd5817a 100644 --- a/test/models/test_attention_quantization.py +++ b/test/models/test_attention_quantization.py @@ -28,11 +28,15 @@ @onlyCUDA @pytest.mark.parametrize( - "dtype", [torch.float16, torch.bfloat16, torch.float32] -) -@pytest.mark.parametrize( - ("kind", "heads"), - [("kumo", None), ("kumo", 1), ("kumo", 2), ("tabfm", None)], + ("kind", "heads", "dtype"), + [ + ("kumo", None, torch.float16), + ("kumo", None, torch.bfloat16), + ("kumo", None, torch.float32), + ("kumo", 1, torch.float16), + ("kumo", 2, torch.float16), + ("tabfm", None, torch.float16), + ], ) def test_fp8_icl_cache( kind: str, heads: int | None, dtype: torch.dtype @@ -78,26 +82,13 @@ def test_fp8_icl_cache( replayed = block( x[:, 8193:].clone(), y[:, :0], cache=cache.freeze() ) - split = torch.cat( - [ - block(x[:, 8193:8209].clone(), y[:, :0], cache=cache), - block(x[:, 8209:].clone(), y[:, :0], cache=cache), - ], - dim=-2, - ) entry = cache["icl_block.layer0"] assert isinstance(entry, QuantizedKVCacheEntry) assert not isinstance(cache["icl_block.layer1"], QuantizedKVCacheEntry) - assert entry.key.dtype == torch.float8_e4m3fn - assert entry.value.dtype == torch.float8_e4m3fn - assert entry.query_scale.size(-2) == (4 if kind == "kumo" else 2) if heads is not None: assert entry.key.size(-2) == heads - assert entry.key.untyped_storage().nbytes() == entry.key.numel() - assert entry.value.untyped_storage().nbytes() == entry.value.numel() assert actual.isfinite().all() torch.testing.assert_close(replayed, actual, atol=0.01, rtol=0.03) - torch.testing.assert_close(split, replayed, atol=0.01, rtol=0.03) relative_rms = ( (actual.float() - expected.float()).square().mean() / expected.float().square().mean() @@ -107,9 +98,9 @@ def test_fp8_icl_cache( @onlyCUDA @pytest.mark.parametrize( - "kind", ["kumo-small", "kumo-medium", "kumo-large", "tabfm"] + ("kind", "estimator_batch_size"), + [("kumo-small", 1), ("kumo-medium", 2), ("kumo-large", 2), ("tabfm", 1)], ) -@pytest.mark.parametrize("estimator_batch_size", [1, 2]) def test_fp8_model_fit_predict( kind: str, estimator_batch_size: int, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -172,7 +163,6 @@ def test_fp8_model_fit_predict( assert model._cache is not None if num_estimators > 1: assert model._cache.is_cpu - assert model._cache["num_batches"] == 2 entries = cast(Cache, model._cache[0]) quantized = [ entry @@ -182,15 +172,6 @@ def test_fp8_model_fit_predict( assert quantized if kind in {"kumo-medium", "kumo-large"}: assert all(entry.key.size(-2) == 2 for entry in quantized) - with torch.autocast("cuda", dtype=torch.float16): - with pytest.raises(RuntimeError, match=r"sdm\.optimize"): - model.predict(query) - with optimize(attention="fp8"): - torch.testing.assert_close( - model.predict(query).numerical, actual.numerical - ) - model.clear() - assert model._cache is None @onlyCUDA From b83b47668a5d82e2d03f0e4f1f46e1fffa41d931 Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Wed, 18 Nov 2026 18:53:06 -0800 Subject: [PATCH 43/44] update --- sdm/__init__.py | 2 - sdm/_kernels/fp8_attention.py | 97 ------ sdm/_kernels/triton/fp8_attention.py | 376 --------------------- sdm/cache.py | 99 +----- sdm/models/kumo/tabular/icl.py | 44 ++- sdm/models/tabfm/icl.py | 6 - sdm/models/tabiclv2/icl.py | 6 - sdm/nn/attention.py | 99 ++---- sdm/optimization.py | 48 --- test/_kernels/test_fp8_attention.py | 79 ----- test/conftest.py | 8 - test/models/tabiclv2/test_model.py | 60 +--- test/models/test_attention_quantization.py | 271 --------------- test/nn/test_attention_quantization.py | 59 ---- test/nn/test_compile.py | 42 +-- test/nn/test_fp8_attention.py | 203 ----------- test/test_cache.py | 44 +-- test/test_optimization.py | 23 -- 18 files changed, 72 insertions(+), 1494 deletions(-) delete mode 100644 sdm/_kernels/fp8_attention.py delete mode 100644 sdm/_kernels/triton/fp8_attention.py delete mode 100644 sdm/optimization.py delete mode 100644 test/_kernels/test_fp8_attention.py delete mode 100644 test/models/test_attention_quantization.py delete mode 100644 test/nn/test_attention_quantization.py delete mode 100644 test/nn/test_fp8_attention.py delete mode 100644 test/test_optimization.py diff --git a/sdm/__init__.py b/sdm/__init__.py index 08eb5e7a7c..e5d22b1543 100644 --- a/sdm/__init__.py +++ b/sdm/__init__.py @@ -12,7 +12,6 @@ from importlib.metadata import PackageNotFoundError, version from sdm._constants import NaT -from sdm.optimization import optimize from sdm.stype import Stype, StypeLike, infer_stypes from sdm.task import Task, TaskLike from sdm.tensor import ( @@ -43,7 +42,6 @@ __all__ = [ "NaT", - "optimize", "Stype", "StypeLike", "Task", diff --git a/sdm/_kernels/fp8_attention.py b/sdm/_kernels/fp8_attention.py deleted file mode 100644 index 6b745041a3..0000000000 --- a/sdm/_kernels/fp8_attention.py +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from collections.abc import Callable -from typing import TypeAlias - -import torch -from torch import Tensor - -from sdm.cache import KVCacheEntry, QuantizedKVCacheEntry - -_FP8Attention: TypeAlias = Callable[ - [Tensor, Tensor, Tensor, QuantizedKVCacheEntry | None, float | None], - tuple[Tensor, QuantizedKVCacheEntry], -] - -_triton_fp8_attention: _FP8Attention | None = None -try: - from sdm._kernels.triton.fp8_attention import ( - _fp8_attention as _triton_fp8_attention_impl, - ) -except ImportError: - pass -else: - _triton_fp8_attention = _triton_fp8_attention_impl - - -def fp8_attention( - query: Tensor, - key: Tensor, - value: Tensor, - cache: KVCacheEntry | None = None, - scale: float | None = None, - *, - query_scaling: Callable[..., Tensor] | None = None, - seqused_key_value: Tensor | None = None, - attn_mask: Tensor | None = None, - is_causal: bool = False, -) -> tuple[Tensor, QuantizedKVCacheEntry] | None: - """Compute FP8 attention and create or reuse a quantized context cache. - - Args: - query: Queries shaped ``[..., rows, query_heads, channels]``. When - creating a cache, the leading rows must include the context. - key: Keys shaped ``[..., context_rows, kv_heads, channels]``. - value: Values with the same shape as the keys. - cache: Fitted quantized K/V and scales to reuse, or ``None`` to create - them from the supplied context. - scale: Attention score multiplier; defaults to inverse square-root - head width. - query_scaling: Optional query transformation applied before FP8 - attention, called with ``query`` and ``key_len``. - seqused_key_value: Valid context lengths; require regular attention. - attn_mask: Attention mask; requires regular attention. - is_causal: Causal attention; requires regular attention. - - Returns: - The attention output and its quantized context cache, or ``None`` - when unsupported inputs require the caller's regular attention path. - - Raises: - ValueError: A fitted quantized cache cannot be used with the current - device, head width, backend, execution mode, or attention mask. - """ - if cache is not None and not isinstance(cache, QuantizedKVCacheEntry): - return None - if attn_mask is not None or seqused_key_value is not None or is_causal: - if cache is not None: - raise ValueError( - "Fitted FP8 attention does not support attention masks" - ) - return None - # New caches need enough context work to offset quantization overhead, - # and leading context queries to calibrate Q scales. - if cache is None and ( - key.size(-3) <= 8192 or query.size(-3) < key.size(-3) - ): - return None - if ( - _triton_fp8_attention is not None - and not torch.is_grad_enabled() - and query.is_cuda - and torch.version.cuda is not None - and query.size(-1) in {32, 64, 128, 256} - # Native FP8 Tensor Cores require CUDA compute capability 8.9 or newer. - and torch.cuda.get_device_capability(query.device) >= (8, 9) - ): - if cache is not None and query.numel() == 0: - return query, cache - if query_scaling is not None: - query = query_scaling(query, key_len=key.size(-3)) - return _triton_fp8_attention(query, key, value, cache, scale) - if cache is not None: - raise ValueError( - "Fitted FP8 attention requires supported CUDA inference" - ) - return None diff --git a/sdm/_kernels/triton/fp8_attention.py b/sdm/_kernels/triton/fp8_attention.py deleted file mode 100644 index 8a6686bddf..0000000000 --- a/sdm/_kernels/triton/fp8_attention.py +++ /dev/null @@ -1,376 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from typing import Any, cast - -import torch -import triton -import triton.language as tl -from torch import Tensor - -from sdm.cache import QuantizedKVCacheEntry - - -@triton.jit -def _attention_kernel( - q_ptr, - k_ptr, - v_ptr, - q_scale_ptr, - k_scale_ptr, - v_scale_ptr, - out_ptr, - n_q_heads: tl.constexpr, - n_kv_heads: tl.constexpr, - n_queries: tl.constexpr, - n_context: tl.constexpr, - head_dim: tl.constexpr, - attn_scale: tl.constexpr, - block_q: tl.constexpr, - block_kv: tl.constexpr, - acc_flush_tiles: tl.constexpr, - scale_weights_in_exp: tl.constexpr, - fuse_score_scale: tl.constexpr, -) -> None: - """Write attention outputs for one query tile and head per program. - - Each program keeps a ``[block_q, head_dim]`` query tile and loops over - context tiles of ``block_kv`` rows. Both matrix multiplications use - the full head width. - - """ - query_block, batch_query_head = tl.program_id(0), tl.program_id(1) - batch = batch_query_head // n_q_heads - query_head = batch_query_head % n_q_heads - # Grouped-query attention shares K/V heads when n_q_heads > n_kv_heads. - batch_kv_head = batch * n_kv_heads + query_head // ( - n_q_heads // n_kv_heads - ) - rows = query_block * block_q + tl.arange(0, block_q) - cols = tl.arange(0, block_kv) - dims = tl.arange(0, head_dim) - q = tl.load( - q_ptr - + batch_query_head * n_queries * head_dim - + rows[:, None] * head_dim - + dims[None, :], - mask=rows[:, None] < n_queries, - other=0.0, - ) - # Restore Q/K magnitudes and convert natural-exponential scores to base 2. - score_scale = ( - tl.load(q_scale_ptr + batch_query_head) - * tl.load(k_scale_ptr + batch_kv_head) - * attn_scale - * 1.4426950408889634 - ) - value_scale = tl.load(v_scale_ptr + batch_kv_head) - row_max = tl.full((block_q,), -float("inf"), tl.float32) - weight_sum = tl.full((block_q,), 0, tl.float32) - acc = tl.full((block_q, head_dim), 0, tl.float32) - if acc_flush_tiles > 0: - outer_acc = tl.full((block_q, head_dim), 0, tl.float32) - outer_max = tl.full((block_q,), -float("inf"), tl.float32) - for start in range(tl.cdiv(n_context, block_kv)): # ty: ignore[invalid-argument-type] - # 1. Tiled QK^T: reduce over channels to get [block_q, block_kv]. - indices = start * block_kv + cols - k = tl.load( - k_ptr - + batch_kv_head * n_context * head_dim - + indices[None, :] * head_dim - + dims[:, None], - mask=indices[None, :] < n_context, - other=0.0, - ) - score = tl.dot(q, k, max_num_imprecise_acc=32) - - # 2. Online softmax: update row maxima and weight sums across tiles. - if fuse_score_scale and attn_scale > 0: - score = tl.where( - indices[None, :] < n_context, score, -float("inf") - ) - new_max = tl.maximum(row_max, tl.max(score, 1) * score_scale) # ty: ignore[invalid-argument-type] - shift = new_max - 8.0 if scale_weights_in_exp else new_max - weights = tl.exp2(score * score_scale - shift[:, None]) - else: - score = tl.where( - indices[None, :] < n_context, - score * score_scale, - -float("inf"), - ) - new_max = tl.maximum(row_max, tl.max(score, 1)) # ty: ignore[invalid-argument-type] - if scale_weights_in_exp: - weights = tl.exp2(score - (new_max[:, None] - 8.0)) - else: - weights = tl.exp2(score - new_max[:, None]) - correction = tl.exp2(row_max - new_max) - weight_sum = weight_sum * correction + tl.sum(weights, 1) # ty: ignore[invalid-argument-type] - acc *= correction[:, None] - - # 3. Tiled weights-V: reduce over context to get [block_q, head_dim]. - v_offsets = indices[:, None] + dims[None, :] * n_context - v = tl.load( - v_ptr + batch_kv_head * n_context * head_dim + v_offsets, - mask=indices[:, None] < n_context, - other=0.0, - ) - acc = tl.dot( - (weights if scale_weights_in_exp else weights * 256.0).to( - tl.float8e4nv - ), - v, - acc, - max_num_imprecise_acc=32, - ) - # Periodically flush the dot accumulator to FP32 for long L4 contexts. - if acc_flush_tiles > 0: # noqa: SIM102 - constexpr guard avoids modulo zero. - if ((start + 1) % acc_flush_tiles == 0) | ( # ty: ignore[unsupported-operator] - start + 1 == tl.cdiv(n_context, block_kv) # ty: ignore[invalid-argument-type] - ): - outer_acc = ( - outer_acc * tl.exp2(outer_max - new_max)[:, None] + acc - ) - outer_max = new_max - acc = tl.full((block_q, head_dim), 0, tl.float32) - row_max = new_max - if acc_flush_tiles > 0: - acc = outer_acc - # After all tiles, divide accumulated weighted values by the weight sum. - out = acc * value_scale / weight_sum[:, None] - if not scale_weights_in_exp: - out *= 1.0 / 256.0 - tl.store( - out_ptr - + batch_query_head * n_queries * head_dim - + rows[:, None] * head_dim - + dims[None, :], - out, - mask=rows[:, None] < n_queries, - ) - - -@triton.jit -def _quantize_kernel( - x_ptr, - scale_ptr, - out_ptr, - numel: tl.constexpr, - block_size: tl.constexpr, - num_heads: tl.constexpr, - num_channels: tl.constexpr, - stride_b: tl.constexpr, - stride_h: tl.constexpr, - stride_r: tl.constexpr, - stride_c: tl.constexpr, -) -> None: - """Write scaled, clamped input values into a contiguous FP8 buffer.""" - offset = tl.program_id(0) * block_size + tl.arange(0, block_size) - head = tl.program_id(1) - source = ( - (head // num_heads) * stride_b - + (head % num_heads) * stride_h - + (offset // num_channels) * stride_r - + (offset % num_channels) * stride_c - ) - x = tl.load(x_ptr + source, mask=offset < numel, other=0.0).to(tl.float32) - scale = tl.load(scale_ptr + head) - y = tl.minimum(tl.maximum(x / scale, -448.0), 448.0) - tl.store(out_ptr + head * numel + offset, y, mask=offset < numel) - - -def quantize(x: Tensor, scale: Tensor | None = None) -> tuple[Tensor, Tensor]: - """Quantize a tensor to FP8 using per-head scales. - - Args: - x: Input with shape ``[batch, heads, rows, channels]``. - scale: Contiguous scales shaped ``[batch, heads, 1, 1]``. If omitted, - derive scales from each head's maximum absolute input value. - - Returns: - The contiguous FP8 tensor and its dequantization scales. - """ - if scale is None: - scale = ( - x.abs().amax(dim=(-2, -1), keepdim=True).float().clamp_min(1e-12) - / 448.0 - ) - output = x.new_empty(x.shape, dtype=torch.float8_e4m3fn) - length = x.size(-2) * x.size(-1) - block_size = 1024 - grid = ( - triton.cdiv(length, block_size), # ty: ignore[invalid-argument-type] - x.size(0) * x.size(1), - ) - with torch.cuda.device(x.device): - cast(Any, _quantize_kernel)[grid]( - x, - scale, - output, - numel=length, - block_size=block_size, - num_heads=x.size(1), - num_channels=x.size(-1), - stride_b=x.stride(0), - stride_h=x.stride(1), - stride_r=x.stride(2), - stride_c=x.stride(3), - ) - return output, scale - - -def quantized_attention( - q: Tensor, - k: Tensor, - v: Tensor, - qs: Tensor, - ks: Tensor, - vs: Tensor, - dtype: torch.dtype, - query_tile: int, - scale: float | None = None, - context_tile: int = 64, - warps: int = 4, - stages: int = 2, - accumulation_chunk: int = 0, - scale_weights_in_exp: bool = False, - fuse_score_scale: bool = False, -) -> Tensor: - """Run tiled attention on per-head scaled FP8 inputs. - - Args: - q: Contiguous FP8 queries shaped ``[batch, query_heads, queries, D]``. - k: Contiguous FP8 keys shaped ``[batch, kv_heads, context, D]``. - v: Contiguous FP8 values shaped ``[batch, kv_heads, D, context]``. - qs: Query scales shaped ``[batch, query_heads, 1, 1]``. - ks: Key scales shaped ``[batch, kv_heads, 1, 1]``. - vs: Value scales shaped ``[batch, kv_heads, 1, 1]``. - dtype: Output dtype. - query_tile: Query rows per program; a positive power of two. - scale: Attention score multiplier. Defaults to ``1 / sqrt(D)``. - context_tile: Context rows per loop iteration; a positive power of two. - warps: Cooperating warps per program. - stages: Software pipeline stages. - accumulation_chunk: Tiles per FP32 accumulator flush; zero disables it. - scale_weights_in_exp: Apply the weight factor 256 inside exp2 instead - of before the weighted-value multiplication. - fuse_score_scale: Permit fused score scaling and maximum subtraction. - - Returns: - Attention output with the query shape and requested dtype. - """ - b, h, m, d = q.shape - for name, size in ( - ("query_tile", query_tile), - ("context_tile", context_tile), - ("head dimension", d), - ): - if size <= 0 or size & (size - 1): - raise ValueError(f"{name} must be a positive power of two") - if k.size(1) == 0 or h % k.size(1): - raise ValueError("The number of K/V heads must divide the query heads") - n = k.size(-2) - # Q/K are [batch, heads, sequence, channels]; V is transposed. - output = q.new_empty(q.shape, dtype=dtype) - grid = (triton.cdiv(m, query_tile), b * h) # ty: ignore[invalid-argument-type] - with torch.cuda.device(q.device): - cast(Any, _attention_kernel)[grid]( - q, - k, - v, - qs, - ks, - vs, - output, - n_q_heads=h, - n_kv_heads=k.size(1), - n_queries=m, - n_context=n, - head_dim=d, - attn_scale=d**-0.5 if scale is None else scale, - block_q=query_tile, - block_kv=context_tile, - acc_flush_tiles=accumulation_chunk, - scale_weights_in_exp=scale_weights_in_exp, - fuse_score_scale=fuse_score_scale, - num_warps=warps, - num_stages=stages, - ) - return output - - -def _fp8_attention( - query: Tensor, - key: Tensor, - value: Tensor, - cache: QuantizedKVCacheEntry | None = None, - scale: float | None = None, -) -> tuple[Tensor, QuantizedKVCacheEntry]: - """Quantize attention inputs and create or reuse their context cache.""" - if torch.is_autocast_enabled("cuda"): - dtype = torch.get_autocast_dtype("cuda") - query = query.to(dtype) - if cache is None: - key, value = key.to(dtype), value.to(dtype) - batch_shape = torch.broadcast_shapes( - query.shape[:-3], key.shape[:-3], value.shape[:-3] - ) - - def heads(x: Tensor) -> Tensor: - return ( - x.expand(batch_shape + x.shape[-3:]) - .reshape(-1, *x.shape[-3:]) - .transpose(1, 2) - ) - - def rows(x: Tensor) -> Tensor: - x = x.transpose(1, 2) - return x.reshape(batch_shape + x.shape[-3:]) - - q = heads(query) - if cache is None: - context = q[..., : key.size(-3), :] - qs = context.abs().amax((-2, -1), keepdim=True).float() - qs = qs.clamp_min(1e-12) / 448.0 - k, ks = quantize(heads(key)) - v, vs = quantize(heads(value)) - else: - k = heads(cache.key).contiguous() - v = heads(cache.value).contiguous() - ks, vs, qs = ( - heads(t).contiguous() - for t in (cache.key_scale, cache.value_scale, cache.query_scale) - ) - q, qs = quantize(q, qs) - head_dim = query.size(-1) - wide = head_dim == 256 or (head_dim == 128 and k.size(-2) >= 32768) - # Measured launches avoid register spills; Ada retains accumulator flushes. - blackwell = head_dim == 64 and torch.cuda.get_device_capability( - query.device - ) == (12, 0) - result = quantized_attention( - q, - k, - v.transpose(-1, -2).contiguous(), - qs, - ks, - vs, - dtype=query.dtype, - query_tile=64 if head_dim == 256 else 128, - scale=scale, - context_tile=64 if wide or blackwell or k.size(-2) < 32768 else 128, - warps=8 if wide else 4, - stages=4 if blackwell else 2, - accumulation_chunk=0 if blackwell else 16, - scale_weights_in_exp=blackwell, - fuse_score_scale=blackwell, - ) - if cache is None: - cache = QuantizedKVCacheEntry( - key=rows(k), - value=rows(v), - key_scale=rows(ks), - value_scale=rows(vs), - query_scale=rows(qs), - dtype=query.dtype, - ) - return rows(result), cache diff --git a/sdm/cache.py b/sdm/cache.py index a11e375716..59c6c0db06 100644 --- a/sdm/cache.py +++ b/sdm/cache.py @@ -9,9 +9,8 @@ Mapping, MutableMapping, ) -from dataclasses import dataclass, replace from enum import StrEnum -from typing import Self, cast +from typing import NamedTuple, Self, cast import torch from torch import Tensor @@ -20,8 +19,12 @@ from sdm.tensor.mixin import DeviceMixin -@dataclass -class KVCacheEntry(DeviceMixin): +class _KVCacheEntry(NamedTuple): + key: Tensor + value: Tensor + + +class KVCacheEntry(_KVCacheEntry, DeviceMixin): r"""Cached key/value projections for a single transformer block. Args: @@ -29,85 +32,12 @@ class KVCacheEntry(DeviceMixin): value: Cached value projection tensor. """ - key: Tensor - value: Tensor - - def select_heads(self, num_heads: int) -> Self: - """Retain the leading key/value heads for grouped-query attention. - - Args: - num_heads: Number of leading K/V heads to retain. - - Returns: - An entry with contiguous K/V copies when reducing the head count. - """ - if num_heads >= self.key.size(-2): - return self - key, value = _copy_key_value( - key=self.key[..., :num_heads, :], - value=self.value[..., :num_heads, :], - ) - return replace(self, key=key, value=value) - def _tensors(self) -> Iterator[Tensor]: yield self.key yield self.value def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self: - return replace(self, key=fn(self.key), value=fn(self.value)) - - -@dataclass -class QuantizedKVCacheEntry(KVCacheEntry): - """Quantized attention projections with per-head scaling metadata. - - Keys and values have shape ``[..., rows, heads, channels]``. Scales have - shape ``[..., 1, heads, 1]``. The query scale is learned from context - queries and reused for prediction. The key/value tensor dtypes identify - their storage formats; ``dtype`` is the attention output dtype. - - Args: - key: Quantized key projections. - value: Quantized value projections. - key_scale: Per-head key dequantization scales. - value_scale: Per-head value dequantization scales. - query_scale: Per-query-head scales derived from the context. - dtype: Attention output dtype. - """ - - key_scale: Tensor - value_scale: Tensor - query_scale: Tensor - dtype: torch.dtype - - def select_heads(self, num_heads: int) -> Self: - """Retain leading K/V heads and scales, keeping all query scales. - - Args: - num_heads: Number of leading K/V heads to retain. - - Returns: - A quantized entry with selected K/V heads and matching scales. - """ - return replace( - super().select_heads(num_heads), - key_scale=self.key_scale[..., :num_heads, :].contiguous(), - value_scale=self.value_scale[..., :num_heads, :].contiguous(), - ) - - def _tensors(self) -> Iterator[Tensor]: - yield from super()._tensors() - yield self.key_scale - yield self.value_scale - yield self.query_scale - - def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self: - return replace( - super()._apply_tensor(fn), - key_scale=fn(self.key_scale), - value_scale=fn(self.value_scale), - query_scale=fn(self.query_scale), - ) + return self.__class__(key=fn(self.key), value=fn(self.value)) class Cache(MutableMapping[Hashable, object], DeviceMixin): @@ -268,16 +198,3 @@ def _apply(value: object) -> object: ) out._mode = self._mode return out - - -def _copy_key_value(key: Tensor, value: Tensor) -> tuple[Tensor, Tensor]: - return ( - key.clone(memory_format=torch.contiguous_format), - value.clone(memory_format=torch.contiguous_format), - ) - - -# Keep the guard for PyTorch 2.7; 2.8+ can use the decorator directly. -if hasattr(torch.compiler, "nested_compile_region"): - # Compile cache copies separately to free each layer's full K/V tensors. - _copy_key_value = torch.compiler.nested_compile_region(_copy_key_value) diff --git a/sdm/models/kumo/tabular/icl.py b/sdm/models/kumo/tabular/icl.py index 7001701675..7f76b6dd86 100644 --- a/sdm/models/kumo/tabular/icl.py +++ b/sdm/models/kumo/tabular/icl.py @@ -50,12 +50,6 @@ def __init__( for _ in range(num_layers) ) - # Keep final attention on SDPA to avoid FP8 overhead on small batches. - for layer in self.layers[:-1]: - cast( - KumoTabularTransformerBlock, layer - ).attn._supports_quantized_attention = True - self.norm = RMSNorm(channels, **factory_kwargs) self.head = Sequential( Linear(channels, 2 * channels, **factory_kwargs), @@ -115,19 +109,21 @@ def forward( ) if cache is not None and cache.is_recording: - x, entry = result - cache[cache_key] = ( - entry.select_heads(self.kv_heads) - if self.kv_heads is not None - else entry - ) - del entry + x, (key, value) = result + kv_heads = self.kv_heads + if kv_heads is not None and kv_heads < key.size(-2): + key, value = _copy_key_value( + key=key[..., :kv_heads, :], + value=value[..., :kv_heads, :], + ) + cache[cache_key] = KVCacheEntry(key, value) + del key, value else: x = result del result continue - x_context, entry = layer( + x_context, (key, value) = layer( query=x if train_size is not None else x[..., :0, :] @@ -142,8 +138,11 @@ def forward( if last_layer else x[..., :R_train, :], ) - key_value = entry.select_heads(self.kv_heads) - del entry + key_value = KVCacheEntry( + key=key[..., : self.kv_heads, :].contiguous(), + value=value[..., : self.kv_heads, :].contiguous(), + ) + del key, value x_query = layer( query=x if train_size is not None else x[..., R_train:, :], key_value=key_value, @@ -160,3 +159,16 @@ def forward( x = torch.cat([x_context, x_query], -2) return self.head(self.norm(x)) + + +def _copy_key_value(key: Tensor, value: Tensor) -> tuple[Tensor, Tensor]: + return ( + key.clone(memory_format=torch.contiguous_format), + value.clone(memory_format=torch.contiguous_format), + ) + + +# Keep the guard for PyTorch 2.7; 2.8+ can use the decorator directly. +if hasattr(torch.compiler, "nested_compile_region"): + # Compile cache copies separately to free each layer's full K/V tensors. + _copy_key_value = torch.compiler.nested_compile_region(_copy_key_value) diff --git a/sdm/models/tabfm/icl.py b/sdm/models/tabfm/icl.py index 23dd53acc9..8efa1a7049 100644 --- a/sdm/models/tabfm/icl.py +++ b/sdm/models/tabfm/icl.py @@ -57,12 +57,6 @@ def __init__( for _ in range(num_layers) ) - # Keep final attention on SDPA to avoid FP8 overhead on small batches. - for layer in self.layers[:-1]: - cast( - TabFMTransformerBlock, layer - ).attn._supports_quantized_attention = True - self.norm = RMSNorm(channels, eps=1e-6, **factory_kwargs) self.head = Sequential( Linear(channels, 2 * channels, **factory_kwargs), diff --git a/sdm/models/tabiclv2/icl.py b/sdm/models/tabiclv2/icl.py index 1e431b7cfa..d1903354d1 100644 --- a/sdm/models/tabiclv2/icl.py +++ b/sdm/models/tabiclv2/icl.py @@ -56,12 +56,6 @@ def __init__( for _ in range(num_layers) ) - # Keep final attention on SDPA to avoid FP8 overhead on small batches. - for layer in self.layers[:-1]: - cast( - TabICLv2TransformerBlock, layer - ).attn._supports_quantized_attention = True - self.norm = LayerNorm(channels, bias=norm_bias, **factory_kwargs) self.head = Sequential( Linear(channels, 2 * channels, **factory_kwargs), diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index e0dc6224a0..e0669e20c5 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -12,12 +12,10 @@ from torch.nn import Linear from sdm._kernels.flash_attention import flash_attention -from sdm._kernels.fp8_attention import fp8_attention from sdm._kernels.sdpa import sdpa from sdm._memory import chunk_memory_limit -from sdm.cache import KVCacheEntry, QuantizedKVCacheEntry +from sdm.cache import KVCacheEntry from sdm.nn import QueryScaling -from sdm.optimization import _attention_quantization class SDPA(torch.nn.Module): @@ -293,9 +291,6 @@ def __init__( self.q_dim = num_query_heads * self.head_dim # == channels self.kv_dim = num_key_value_heads * self.head_dim - # ICL blocks enable this where queries include context for calibration. - self._supports_quantized_attention = False - self.qkv_lin = Linear( channels, self.q_dim + 2 * self.kv_dim, bias=bias, **factory_kwargs ) @@ -404,14 +399,6 @@ def forward( if key_value is None and key_pos is None: key_pos = query_pos - if ( - isinstance(key_value, QuantizedKVCacheEntry) - and _attention_quantization.get() != "fp8" - ): - raise RuntimeError( - 'This cache requires sdm.optimize(attention="fp8"). ' - "Re-enter that scope, or clear the model and refit." - ) if isinstance(key_value, KVCacheEntry): query = F.linear( @@ -421,12 +408,7 @@ def forward( if self.qkv_lin.bias is not None else None, ) - if isinstance(key_value, QuantizedKVCacheEntry): - if query.dtype != key_value.dtype: - raise ValueError( - "Query dtype differs from fitted FP8 cache" - ) - elif ( + if ( key_value.key.dtype != query.dtype or key_value.value.dtype != query.dtype ): @@ -473,44 +455,20 @@ def forward( else: key = self.key_transform(key, pos=key_pos) - quantized_cache = ( - key_value if isinstance(key_value, QuantizedKVCacheEntry) else None - ) - result = None - if _attention_quantization.get() == "fp8" and ( - self._supports_quantized_attention or quantized_cache is not None - ): - result = fp8_attention( - query=query, - key=key, - value=value, - cache=key_value - if isinstance(key_value, KVCacheEntry) - else None, - scale=self.sdpa.scale, - query_scaling=self.sdpa.query_scaling, - seqused_key_value=seqused_key_value, - attn_mask=attn_mask, - is_causal=is_causal, - ) - if result is None: - out = self.sdpa( - query=query, # [..., Q, Hq, C // Hq] - key=key, # [..., KV, Hkv, C // Hq] - value=value, # [..., KV, Hkv, C // Hq] - seqused_key_value=seqused_key_value, # [...] - attn_mask=attn_mask, # [..., Q, KV] - is_causal=is_causal, - ) # [..., Q, Hq, C // Hq] - else: - out, quantized_cache = result + out = self.sdpa( + query=query, # [..., Q, Hq, C // Hq] + key=key, # [..., KV, Hkv, C // Hq] + value=value, # [..., KV, Hkv, C // Hq] + seqused_key_value=seqused_key_value, # [...] + attn_mask=attn_mask, # [..., Hq, Q, KV] + is_causal=is_causal, + ) # [..., Q, Hq, C // Hq] + out = out.flatten(-2, -1) # [..., Q, C] if gate is not None: out = out * gate out = self.out_lin(out) # [..., Q, C] if return_key_value: - if quantized_cache is not None: - return out, quantized_cache # CUDA autocast runs RMSNorm in fp32, so cast explicitly before # caching: https://github.com/pytorch/pytorch/blob/v2.13.0/aten/src/ATen/autocast_mode.h#L875 return out, KVCacheEntry( @@ -773,7 +731,8 @@ def forward( ) flat_out: Tensor | None = None - flat_cache: KVCacheEntry | None = None + flat_key: Tensor | None = None + flat_value: Tensor | None = None if out is not None and (out.dim() <= 3 or out.is_contiguous()): flat_out = out.view(batch_size, *query.size()[-2:]) @@ -802,14 +761,18 @@ def forward( if isinstance(result, tuple): chunk, chunk_kv = result - if flat_cache is None: - flat_cache = chunk_kv._apply_tensor( - lambda t: t.new_empty(batch_size, *t.shape[-3:]) + if flat_key is None: + flat_key = chunk_kv.key.new_empty( + batch_size, *chunk_kv.key.size()[-3:] + ) + if flat_value is None: + flat_value = chunk_kv.value.new_empty( + batch_size, *chunk_kv.value.size()[-3:] ) - for dest, source in zip( - flat_cache._tensors(), chunk_kv._tensors() - ): - dest[start:end] = source + + flat_key[start:end] = chunk_kv.key + flat_value[start:end] = chunk_kv.value + del chunk_kv else: @@ -835,9 +798,12 @@ def forward( if not return_key_value: return out - assert flat_cache is not None - return out, flat_cache._apply_tensor( - lambda t: t.view(*batch_shape, *t.shape[-3:]) + assert flat_key is not None + assert flat_value is not None + + return out, KVCacheEntry( + key=flat_key.view(*batch_shape, *flat_key.size()[-3:]), + value=flat_value.view(*batch_shape, *flat_value.size()[-3:]), ) def _forward( @@ -990,8 +956,9 @@ def _chunk( return None if isinstance(tensor, KVCacheEntry): - return tensor._apply_tensor( - lambda value: _chunk(value, batch_shape, 3, start, end) + return KVCacheEntry( + key=_chunk(tensor.key, batch_shape, 3, start, end), + value=_chunk(tensor.value, batch_shape, 3, start, end), ) trailing_shape = tensor.size()[-trailing_dims:] if trailing_dims else () diff --git a/sdm/optimization.py b/sdm/optimization.py deleted file mode 100644 index 1356d33bd2..0000000000 --- a/sdm/optimization.py +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from collections.abc import Iterator -from contextlib import contextmanager -from contextvars import ContextVar -from typing import Literal - -_attention_quantization: ContextVar[Literal["fp8"] | None] = ContextVar( - "sdm_attention_quantization", default=None -) - - -@contextmanager -def optimize( - *, attention: Literal["fp8", "inherit"] | None = "inherit" -) -> Iterator[None]: - """Temporarily configure execution optimizations in the current context. - - Args: - attention: ``"fp8"`` enables quantization in eligible ICL attention - layers during CUDA inference (compute capability 8.9 or newer) - with more than 8192 context rows. ``None`` disables it; - ``"inherit"`` preserves the enclosing setting. Weights and the - final ICL layer remain unquantized. - - Yields: - None. Previous settings are restored on exit, including on exceptions. - - Note: - Create and use FP8 caches inside an FP8 scope. Using such a cache - outside an enabled scope raises ``RuntimeError``. Exiting neither - converts nor clears caches; they can be reused in another FP8 scope. - FP8 is not guaranteed to improve prediction latency. - - Example: - >>> with sdm.optimize(attention="fp8"): - ... model.fit(context, labels) - ... predictions = model.predict(queries) - """ - setting = ( - _attention_quantization.get() if attention == "inherit" else attention - ) - token = _attention_quantization.set(setting) - try: - yield - finally: - _attention_quantization.reset(token) diff --git a/test/_kernels/test_fp8_attention.py b/test/_kernels/test_fp8_attention.py deleted file mode 100644 index 984a8df614..0000000000 --- a/test/_kernels/test_fp8_attention.py +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest -import torch - -from sdm._kernels.fp8_attention import fp8_attention -from sdm.cache import QuantizedKVCacheEntry -from sdm.testing import onlyCUDA, withCUDA - - -@withCUDA -@pytest.mark.parametrize("channels", [32, 48]) -@pytest.mark.parametrize("grad", [False, True]) -def test_fp8_dispatch(device: torch.device, channels: int, grad: bool) -> None: - if device.type == "cuda" and ( - torch.version.cuda is None - or torch.cuda.get_device_capability(device) < (8, 9) - ): - pytest.skip("FP8 requires CUDA compute capability 8.9 or newer") - x = torch.ones(1, 8193, 2, channels, device=device, dtype=torch.float16) - with torch.set_grad_enabled(grad): - result = fp8_attention(x, x, x) - if device.type == "cpu" or grad or channels == 48: - assert result is None - return - assert result is not None - output, cache = result - torch.testing.assert_close(output, x) - assert cache.key.dtype == torch.float8_e4m3fn - cached = fp8_attention(x[:, :3], cache.key, cache.value, cache) - assert cached is not None - torch.testing.assert_close(cached[0], x[:, :3]) - empty = fp8_attention(x[:, :0], cache.key, cache.value, cache) - assert empty is not None - assert empty[0].shape == x[:, :0].shape - assert empty[1] is cache - - -@onlyCUDA -@pytest.mark.parametrize( - "capability", [(8, 6), (10, 0), (10, 3), (11, 0), (12, 1)] -) -def test_fp8_device_capability( - capability: tuple[int, int], monkeypatch: pytest.MonkeyPatch -) -> None: - if not torch.version.cuda or torch.cuda.get_device_capability() < (8, 9): - pytest.skip("FP8 requires CUDA compute capability 8.9 or newer") - monkeypatch.setattr( - torch.cuda, "get_device_capability", lambda _: capability - ) - x = torch.ones(1, 8193, 1, 32, device="cuda", dtype=torch.float16) - with torch.inference_mode(): - result = fp8_attention(x, x, x) - if capability < (8, 9): - assert result is None - else: - assert result is not None - torch.testing.assert_close(result[0], x) - - -@withCUDA -def test_fp8_cache_rejects_unsupported_execution(device: torch.device) -> None: - x = torch.ones(1, 17, 2, 32, device=device, dtype=torch.float8_e4m3fn) - scale = torch.ones(1, 1, 2, 1, device=device) - cache = QuantizedKVCacheEntry( - key=x, - value=x, - key_scale=scale, - value_scale=scale, - query_scale=scale, - dtype=torch.float16, - ) - query = torch.ones(1, 3, 2, 32, device=device, dtype=torch.float16) - with ( - torch.enable_grad(), - pytest.raises(ValueError, match="CUDA inference"), - ): - fp8_attention(query, cache.key, cache.value, cache) diff --git a/test/conftest.py b/test/conftest.py index d13294dbf9..561ab8d002 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -10,14 +10,6 @@ from sdm import RelationalData, Stype, TableTensor, infer_stypes -@pytest.fixture -def fp8_rng() -> Iterator[None]: - """Reproduce FP8 comparisons while preserving the surrounding RNG state.""" - with torch.random.fork_rng(): - torch.manual_seed(0) - yield - - @pytest.fixture def device() -> torch.device: return torch.device("cpu") diff --git a/test/models/tabiclv2/test_model.py b/test/models/tabiclv2/test_model.py index cdaad6a115..e33cd6dcf1 100644 --- a/test/models/tabiclv2/test_model.py +++ b/test/models/tabiclv2/test_model.py @@ -1,13 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import cast - import pytest import torch -from sdm import Recipe, optimize -from sdm.cache import Cache, QuantizedKVCacheEntry +from sdm import Recipe from sdm.models import TabICLv2 from sdm.models.tabiclv2.model import _TabICLv2 from sdm.models.tabiclv2.row_embedding import RowEmbedding @@ -327,58 +324,3 @@ def test_compile(dtype: torch.dtype) -> None: predicted = model.predict(x_query) assert predicted.allclose(expected, atol=5e-4, rtol=5e-3) assert torch.is_inference(predicted) - - -@onlyCUDA -@pytest.mark.usefixtures("fp8_rng") -def test_fp8_fit_predict() -> None: - if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: - pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") - model = TabICLv2( - task="regression", - pretrained=False, - device="cuda", - ) - x = torch.randn(8193, 3, device="cuda") - y = torch.randn(8193, 1, device="cuda") - query = torch.randn(17, 3, device="cuda") - with ( - torch.inference_mode(), - optimize(attention="fp8"), - torch.autocast("cuda", dtype=torch.float16), - ): - expected = model( - x, - y, - query, - recipe=Recipe(), - num_estimators=3, - estimator_batch_size=1, - ) - model.fit( - x, - y, - recipe=Recipe(), - num_estimators=3, - estimator_batch_size=2, - ) - actual = model.predict(query) - torch.testing.assert_close( - actual.numerical, expected.numerical, atol=0.01, rtol=0.01 - ) - assert model._cache is not None - assert model._cache.is_cpu - entries = cast(Cache, model._cache[0]) - assert any( - isinstance(entry, QuantizedKVCacheEntry) - for entry in entries.values() - ) - with torch.autocast("cuda", dtype=torch.float16): - with pytest.raises(RuntimeError, match=r"sdm\.optimize"): - model.predict(query) - with optimize(attention="fp8"): - torch.testing.assert_close( - model.predict(query).numerical, actual.numerical - ) - model.clear() - assert model._cache is None diff --git a/test/models/test_attention_quantization.py b/test/models/test_attention_quantization.py deleted file mode 100644 index bd0cd5817a..0000000000 --- a/test/models/test_attention_quantization.py +++ /dev/null @@ -1,271 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import copy -import functools -from typing import Any, Literal, cast - -import pytest -import torch - -from sdm import ( - CategoricalTensor, - ColumnarTensor, - Recipe, - RelatedTables, - TableTensor, - optimize, -) -from sdm.cache import Cache, QuantizedKVCacheEntry -from sdm.models import KumoRelational, KumoTabular, TabFM -from sdm.models.kumo.tabular.icl import ICLBlock as KumoICLBlock -from sdm.models.tabfm import model as tabfm_module -from sdm.models.tabfm.icl import ICLBlock as TabFMICLBlock -from sdm.testing import onlyCUDA - -pytestmark = pytest.mark.usefixtures("fp8_rng") - - -@onlyCUDA -@pytest.mark.parametrize( - ("kind", "heads", "dtype"), - [ - ("kumo", None, torch.float16), - ("kumo", None, torch.bfloat16), - ("kumo", None, torch.float32), - ("kumo", 1, torch.float16), - ("kumo", 2, torch.float16), - ("tabfm", None, torch.float16), - ], -) -def test_fp8_icl_cache( - kind: str, heads: int | None, dtype: torch.dtype -) -> None: - if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: - pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") - if kind == "kumo": - block = KumoICLBlock( - num_classes=0, - out_channels=5, - channels=256, - num_layers=2, - num_heads=4, - num_key_value_heads_for_query=heads, - device="cuda", - ) - else: - # Stock TabFM has 256 channels per ICL attention head. - block = TabFMICLBlock( - num_classes=0, - out_channels=5, - channels=512, - num_layers=2, - num_heads=2, - device="cuda", - ) - for parameter in block.parameters(): - if not parameter.any(): - torch.nn.init.normal_(parameter, std=0.02) - reference = copy.deepcopy(block) - channels = 256 if kind == "kumo" else 512 - x = torch.randn(2, 8226, channels, device="cuda") - y = torch.randn(2, 8193, device="cuda") - with ( - torch.inference_mode(), - torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32), - ): - expected = reference(x.clone(), y) - with optimize(attention="fp8"): - actual = block(x.clone(), y) - cache = Cache() - block(x[:, :8193].clone(), y, cache=cache) - replayed = block( - x[:, 8193:].clone(), y[:, :0], cache=cache.freeze() - ) - entry = cache["icl_block.layer0"] - assert isinstance(entry, QuantizedKVCacheEntry) - assert not isinstance(cache["icl_block.layer1"], QuantizedKVCacheEntry) - if heads is not None: - assert entry.key.size(-2) == heads - assert actual.isfinite().all() - torch.testing.assert_close(replayed, actual, atol=0.01, rtol=0.03) - relative_rms = ( - (actual.float() - expected.float()).square().mean() - / expected.float().square().mean() - ).sqrt() - assert relative_rms < 0.08 - - -@onlyCUDA -@pytest.mark.parametrize( - ("kind", "estimator_batch_size"), - [("kumo-small", 1), ("kumo-medium", 2), ("kumo-large", 2), ("tabfm", 1)], -) -def test_fp8_model_fit_predict( - kind: str, estimator_batch_size: int, monkeypatch: pytest.MonkeyPatch -) -> None: - if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: - pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") - kwargs: dict[str, Any] = { - "task": "regression", - "pretrained": False, - "device": "cuda", - } - if kind.startswith("kumo-"): - model = KumoTabular( - size=cast( - Literal["small", "medium", "large"], kind.removeprefix("kumo-") - ), - **kwargs, - ) - else: - # Keep the stock 256-channel ICL heads while reducing model depth. - monkeypatch.setattr( - tabfm_module, - "_TabFM", - functools.partial( - tabfm_module._TabFM, - num_icl_layers=2, - num_embedding_layers=1, - num_embedding_repeats=1, - ), - ) - model = TabFM(**kwargs) - x = torch.randn(8193, 3, device="cuda") - y = torch.randn(8193, 1, device="cuda") - query = torch.randn(17, 3, device="cuda") - num_estimators = 3 if estimator_batch_size == 2 else 1 - with ( - torch.inference_mode(), - torch.autocast("cuda", dtype=torch.float16), - optimize(attention="fp8"), - ): - expected = model( - x, - y, - query, - recipe=Recipe(), - num_estimators=num_estimators, - estimator_batch_size=1, - ) - model.fit( - x, - y, - recipe=Recipe(), - num_estimators=num_estimators, - estimator_batch_size=estimator_batch_size, - ) - actual = model.predict(query) - assert actual.numerical.isfinite().all() - torch.testing.assert_close( - actual.numerical, expected.numerical, atol=0.01, rtol=0.03 - ) - assert model._cache is not None - if num_estimators > 1: - assert model._cache.is_cpu - entries = cast(Cache, model._cache[0]) - quantized = [ - entry - for entry in entries.values() - if isinstance(entry, QuantizedKVCacheEntry) - ] - assert quantized - if kind in {"kumo-medium", "kumo-large"}: - assert all(entry.key.size(-2) == 2 for entry in quantized) - - -@onlyCUDA -def test_fp8_relational_fit_predict() -> None: - if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: - pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") - model = KumoRelational( - task="regression", - pretrained=False, - device="cuda", - ) - table = TableTensor( - id=ColumnarTensor((torch.arange(8210, device="cuda"),)), - numerical=torch.randn(8210, 1, device="cuda"), - columns={"id": ["id"], "numerical": ["value"]}, - ) - related = RelatedTables( - tables={"entities": table}, - relationships=[], - task_links=[ - {"task_column": "id", "table": "entities", "table_column": "id"} - ], - ) - x, query = table[:8193], table[8193:] - y = torch.randn(8193, 1, device="cuda") - with ( - torch.inference_mode(), - torch.autocast("cuda", dtype=torch.float16), - optimize(attention="fp8"), - ): - expected = model( - x, - y, - query, - related_context_tables=related, - related_query_tables=related, - num_hops=0, - recipe=Recipe(), - num_estimators=1, - ) - model.fit(x, y, related, num_hops=0, recipe=Recipe(), num_estimators=1) - actual = model.predict(query, related_tables=related) - assert actual.numerical.isfinite().all() - torch.testing.assert_close( - actual.numerical, expected.numerical, atol=0.01, rtol=0.03 - ) - assert model._cache is not None - entries = cast(Cache, model._cache[0]) - assert any( - isinstance(entry, QuantizedKVCacheEntry) for entry in entries.values() - ) - - -@onlyCUDA -def test_fp8_ecoc_cache() -> None: - if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: - pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") - model = KumoTabular( - task="classification", size="small", pretrained=False, device="cuda" - ) - x = torch.randn(8193, 3, device="cuda") - query = torch.randn(17, 3, device="cuda") - y = TableTensor( - columns={"categorical": ["target"]}, - categorical=CategoricalTensor.from_tensor( - (torch.arange(8193, device="cuda") % 12).unsqueeze(-1) - ), - ) - with ( - torch.autocast("cuda", dtype=torch.float16), - optimize(attention="fp8"), - ): - expected = model( - x, - y, - query, - recipe=Recipe(), - num_estimators=1, - generator=torch.Generator(device="cuda").manual_seed(0), - ) - model.fit( - x, - y, - recipe=Recipe(), - num_estimators=1, - generator=torch.Generator(device="cuda").manual_seed(0), - ) - actual = model.predict(query) - torch.testing.assert_close( - actual.numerical, expected.numerical, atol=0.01, rtol=0.03 - ) - assert actual.size() == (1, 17, 12) - assert model._cache is not None - nested = cast(Cache, cast(Cache, model._cache[0])["ecoc_model"]) - assert any( - isinstance(entry, QuantizedKVCacheEntry) for entry in nested.values() - ) diff --git a/test/nn/test_attention_quantization.py b/test/nn/test_attention_quantization.py deleted file mode 100644 index 5c3b174353..0000000000 --- a/test/nn/test_attention_quantization.py +++ /dev/null @@ -1,59 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest -import torch - -from sdm import optimize -from sdm.cache import QuantizedKVCacheEntry -from sdm.models.tabiclv2.block import TabICLv2TransformerBlock -from sdm.nn import Attention -from sdm.testing import onlyCUDA - -pytestmark = pytest.mark.usefixtures("fp8_rng") - - -@onlyCUDA -@pytest.mark.parametrize(("rows", "is_causal"), [(19, False), (8193, True)]) -def test_fp8_attention_fallback(rows: int, is_causal: bool) -> None: - module = Attention(64, 1, device="cuda") - torch.nn.init.normal_(module.out_lin.weight, std=0.05) - module._supports_quantized_attention = True - query = torch.randn(1, rows, 64, device="cuda") - with torch.inference_mode(), torch.autocast("cuda", dtype=torch.float16): - expected = module(query, is_causal=is_causal) - with optimize(attention="fp8"): - actual, cache = module( - query, is_causal=is_causal, return_key_value=True - ) - torch.testing.assert_close(actual, expected, rtol=0, atol=0) - assert not isinstance(cache, QuantizedKVCacheEntry) - - -@onlyCUDA -def test_fp8_cache_chunking(monkeypatch: pytest.MonkeyPatch) -> None: - if not torch.version.cuda or torch.cuda.get_device_capability() < (8, 9): - pytest.skip("FP8 requires CUDA compute capability 8.9 or newer") - monkeypatch.setenv("SDM_CHUNK_MEMORY_FRACTION", "0.00001") - block = TabICLv2TransformerBlock( - channels=128, num_heads=2, norm_bias=True, qassmax=True, device="cuda" - ) - torch.nn.init.normal_(block.attn.out_lin.weight, std=0.05) - block.attn._supports_quantized_attention = True - context = torch.randn(2, 1, 8193, 128, device="cuda") - query = torch.randn(2, 1, 17, 128, device="cuda") - with ( - torch.inference_mode(), - torch.autocast("cuda", dtype=torch.float16), - optimize(attention="fp8"), - ): - _, cache = block( - context, context, return_key_value=True, batch_size_limit="auto" - ) - assert isinstance(cache, QuantizedKVCacheEntry) - expected = block(query, cache) - actual = block(query, cache, batch_size_limit=1) - torch.testing.assert_close(actual, expected, atol=2e-3, rtol=2e-3) - assert block(query[..., :0, :], cache).size(-2) == 0 - with pytest.raises(ValueError, match="attention masks"): - block.attn(query, cache, is_causal=True) diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index 6727680976..bf85dcbbab 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -9,8 +9,6 @@ from torch import Tensor from torch._dynamo.testing import CompileCounter -from sdm import optimize -from sdm.cache import QuantizedKVCacheEntry from sdm.nn import ( SDPA, Attention, @@ -19,7 +17,7 @@ RotaryEmbedding, TransformerBlock, ) -from sdm.testing import onlyCUDA, withCUDA +from sdm.testing import withCUDA # Skip all tests in this test file if it is not a full test run (FULL_TEST=1). pytestmark = [ @@ -302,44 +300,6 @@ def test_attention_compile_key_value_cache(device: torch.device) -> None: torch.testing.assert_close(out, expected) -@onlyCUDA -@pytest.mark.usefixtures("fp8_rng") -def test_attention_compile_fp8_cache() -> None: - dtype = torch.float16 - if torch.cuda.get_device_capability() not in {(8, 9), (9, 0), (12, 0)}: - pytest.skip("FP8 integration supports Ada, Hopper, and RTX Blackwell") - module = Attention( - channels=128, - num_query_heads=2, - num_key_value_heads=1, - device="cuda", - dtype=dtype, - ).eval() - module._supports_quantized_attention = True - torch.nn.init.normal_(module.out_lin.weight, std=0.05) - context = torch.randn(1, 8193, 128, device="cuda", dtype=dtype) - compiled = torch.compile(module, fullgraph=True, dynamic=False) - with torch.inference_mode(), optimize(attention="fp8"): - expected, eager_cache = module(context, return_key_value=True) - actual, cache = compiled(context, return_key_value=True) - assert isinstance(cache, QuantizedKVCacheEntry) - torch.testing.assert_close(actual, expected) - for rows in (17, 1): - query = torch.randn(1, rows, 128, device="cuda", dtype=dtype) - expected = module(query, eager_cache) - torch.testing.assert_close(compiled(query, cache), expected) - - with torch.inference_mode(): - expected, _ = module(context, return_key_value=True) - actual, ordinary_cache = compiled(context, return_key_value=True) - assert not isinstance(ordinary_cache, QuantizedKVCacheEntry) - torch.testing.assert_close(actual, expected) - with optimize(attention="fp8"): - torch.testing.assert_close( - compiled(query, cache), module(query, cache) - ) - - @withCUDA @pytest.mark.parametrize( ("qassmax", "masking"), diff --git a/test/nn/test_fp8_attention.py b/test/nn/test_fp8_attention.py deleted file mode 100644 index 198c89eea6..0000000000 --- a/test/nn/test_fp8_attention.py +++ /dev/null @@ -1,203 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from typing import Any - -import pytest -import torch - -pytest.importorskip("triton") -from sdm._kernels.triton.fp8_attention import ( - quantize, - quantized_attention, -) - -pytestmark = [ - pytest.mark.usefixtures("fp8_rng"), - pytest.mark.skipif( - not torch.cuda.is_available() - or torch.cuda.get_device_capability() < (8, 9), - reason="FP8 attention requires an Ada or newer CUDA GPU", - ), -] - - -@pytest.mark.parametrize( - ("batch", "queries", "context"), [(2, 129, 513), (1, 129, 8192)] -) -def test_fp8_partial_tiles(batch: int, queries: int, context: int) -> None: - with torch.inference_mode(): - q = torch.randn( - batch, 8, queries, 64, device="cuda", dtype=torch.float16 - ) - k = torch.randn( - batch, 8, context, 64, device="cuda", dtype=torch.float16 - ) - v = torch.randn_like(k) - q8, qs = quantize(q) - k8, ks = quantize(k) - v8, vs = quantize(v) - actual = quantized_attention( - q8, - k8, - v8.transpose(-1, -2).contiguous(), - qs, - ks, - vs, - q.dtype, - 128, - ) - expected = torch.nn.functional.scaled_dot_product_attention(q, k, v) - assert actual.isfinite().all() - relative_rms = ( - actual.float() - expected.float() - ).square().mean().sqrt() / expected.float().square().mean().sqrt() - assert relative_rms < 0.08 - - -@pytest.mark.parametrize("scale", [None, 1.0]) -@pytest.mark.parametrize("context_tile", [64, 128]) -def test_grouped_query_attention_and_model_scale( - scale: float | None, context_tile: int -) -> None: - with torch.inference_mode(): - q = ( - torch.randn(2, 8, 129, 64, device="cuda", dtype=torch.float16) - * 0.3 - ) - k = ( - torch.randn(2, 2, 8192, 64, device="cuda", dtype=torch.float16) - * 0.3 - ) - v = torch.randn_like(k) - q8, qs = quantize(q) - k8, ks = quantize(k) - v8, vs = quantize(v) - actual = quantized_attention( - q8, - k8, - v8.transpose(-1, -2).contiguous(), - qs, - ks, - vs, - q.dtype, - 128, - scale, - context_tile=context_tile, - ) - expected = torch.nn.functional.scaled_dot_product_attention( - q, k, v, enable_gqa=True, scale=scale - ) - assert actual.isfinite().all() - relative_rms = ( - actual.float() - expected.float() - ).square().mean().sqrt() / expected.float().square().mean().sqrt() - assert relative_rms < 0.08 - - -@pytest.mark.parametrize("accumulation_chunk", [0, 16]) -@pytest.mark.parametrize("optimized", [False, True]) -@pytest.mark.parametrize("context_tile", [64, 128]) -@pytest.mark.parametrize("context", [8193, 65536]) -def test_periodic_attention_matches_cached_query_slice( - context: int, context_tile: int, optimized: bool, accumulation_chunk: int -) -> None: - with torch.inference_mode(): - q = torch.randn(1, 1, context, 64, device="cuda", dtype=torch.float16) - k, v = torch.randn_like(q), torch.randn_like(q) - q8, qs = quantize(q) - k8, ks = quantize(k) - v8, vs = quantize(v) - v8 = v8.transpose(-1, -2).contiguous() - opts: dict[str, Any] = { - "accumulation_chunk": accumulation_chunk, - "context_tile": context_tile, - "scale_weights_in_exp": optimized, - "fuse_score_scale": optimized, - } - full = quantized_attention( - q8, k8, v8, qs, ks, vs, q.dtype, 128, **opts - ) - cached = quantized_attention( - q8[:, :, -129:].contiguous(), - k8, - v8, - qs, - ks, - vs, - q.dtype, - 128, - **opts, - ) - torch.testing.assert_close( - cached, full[:, :, -129:], rtol=1e-3, atol=2e-5 - ) - reference = torch.nn.functional.scaled_dot_product_attention( - q[:, :, -129:], k, v - ) - relative_rms = ( - (cached.float() - reference.float()).square().mean() - / reference.float().square().mean() - ).sqrt() - assert relative_rms < 0.08 - assert cached.isfinite().all() - - -def test_quantization_reads_strided_heads_without_fp16_copy() -> None: - x = torch.randn( - 2, 513, 8, 64, device="cuda", dtype=torch.float16 - ).transpose(1, 2) - actual, actual_scale = quantize(x) - expected, expected_scale = quantize(x.contiguous()) - assert actual.is_contiguous() - torch.testing.assert_close( - actual.float(), expected.float(), rtol=0, atol=0 - ) - torch.testing.assert_close(actual_scale, expected_scale, rtol=0, atol=0) - - -@pytest.mark.parametrize( - ( - "query_heads", - "kv_heads", - "channels", - "query_tile", - "context_tile", - "message", - ), - [ - (3, 2, 64, 64, 64, "K/V heads"), - (1, 2, 64, 64, 64, "K/V heads"), - (2, 0, 64, 64, 64, "K/V heads"), - (2, 2, 96, 64, 64, "head dimension"), - (2, 2, 64, 96, 64, "query_tile"), - (2, 2, 64, 64, 96, "context_tile"), - (2, 2, 64, 0, 64, "query_tile"), - ], -) -def test_invalid_attention_launch( - query_heads: int, - kv_heads: int, - channels: int, - query_tile: int, - context_tile: int, - message: str, -) -> None: - q = torch.empty( - 1, query_heads, 1, channels, device="cuda", dtype=torch.float8_e4m3fn - ) - k = q.new_empty(1, kv_heads, 1, channels) - qs = torch.ones(1, query_heads, 1, 1, device="cuda") - ks = qs.new_ones(1, kv_heads, 1, 1) - with pytest.raises(ValueError, match=message): - quantized_attention( - q, - k, - k.transpose(-1, -2).contiguous(), - qs, - ks, - ks, - dtype=torch.float16, - query_tile=query_tile, - context_tile=context_tile, - ) diff --git a/test/test_cache.py b/test/test_cache.py index 0edbcaff0b..d05102f090 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -7,7 +7,7 @@ import pytest import torch -from sdm.cache import Cache, KVCacheEntry, QuantizedKVCacheEntry +from sdm.cache import Cache, KVCacheEntry from sdm.testing import onlyCUDA, withCUDA @@ -118,45 +118,3 @@ def test_cache_serialization_without_cuda( torch.testing.assert_close( restored["tensor"], cache["tensor"], rtol=0, atol=0 ) - - -@withCUDA -def test_quantized_cache_head_selection_and_transfer( - device: torch.device, -) -> None: - key = torch.arange(192).reshape(2, 3, 4, 8).to(torch.float8_e4m3fn) - scale = torch.arange(8).reshape(2, 1, 4, 1).float() + 1 - entry = QuantizedKVCacheEntry( - key=key, - value=key, - key_scale=scale, - value_scale=scale, - query_scale=torch.ones(2, 1, 8, 1), - dtype=torch.float32, - ) - selected = entry.select_heads(2).to(device) - assert isinstance(selected, QuantizedKVCacheEntry) - assert selected.dtype == entry.dtype - assert entry.key.size(-2) == 4 - for name in ("key", "value", "key_scale", "value_scale", "query_scale"): - expected = getattr(entry, name) - if name != "query_scale": - expected = expected[..., :2, :] - actual = getattr(selected, name) - assert actual.device == device - assert actual.dtype == expected.dtype - assert actual.is_contiguous() - torch.testing.assert_close(actual.cpu().float(), expected.float()) - # 192 K/V bytes + 32 K/V scale bytes + 64 query scale bytes. - assert Cache(entry=selected).size() == 288 - - -def test_cache_head_selection_releases_unused_heads() -> None: - entry = KVCacheEntry(torch.ones(1, 1, 4, 8), torch.zeros(1, 1, 4, 8)) - selected = entry.select_heads(2) - assert selected.key.shape == selected.value.shape == (1, 1, 2, 8) - assert selected.key.data_ptr() != entry.key.data_ptr() - assert selected.value.data_ptr() != entry.value.data_ptr() - assert selected.key.is_contiguous() - assert selected.value.is_contiguous() - assert entry.select_heads(4) is entry diff --git a/test/test_optimization.py b/test/test_optimization.py deleted file mode 100644 index 7c48eb35a1..0000000000 --- a/test/test_optimization.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import pytest - -from sdm import optimize -from sdm.optimization import _attention_quantization - - -def test_nested_optimization_restores_on_exception() -> None: - assert _attention_quantization.get() is None - with optimize(attention="fp8"): - with optimize(): - assert _attention_quantization.get() == "fp8" - with optimize(attention=None): - assert _attention_quantization.get() is None - with ( - pytest.raises(ValueError, match="test exception"), - optimize(attention=None), - ): - raise ValueError("test exception") - assert _attention_quantization.get() == "fp8" - assert _attention_quantization.get() is None From 2110c1257f10cbe4759463cf6df6161df2079a0e Mon Sep 17 00:00:00 2001 From: Akihiro Nitta Date: Wed, 18 Nov 2026 20:33:17 -0800 Subject: [PATCH 44/44] update --- sdm/cache.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/sdm/cache.py b/sdm/cache.py index 59c6c0db06..4967882251 100644 --- a/sdm/cache.py +++ b/sdm/cache.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import abc from collections.abc import ( Callable, Hashable, @@ -40,6 +41,45 @@ def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self: return self.__class__(key=fn(self.key), value=fn(self.value)) +class KVCacheDType(abc.ABC): + r"""Base class for the storage dtype of cached key/value projections.""" + + @abc.abstractmethod + def quantize(self, entry: KVCacheEntry) -> KVCacheEntry: + r"""Returns ``entry`` in this storage dtype. + + Args: + entry: Key/value projections as computed by attention. + """ + + @abc.abstractmethod + def dequantize( + self, + entry: KVCacheEntry, + dtype: torch.dtype, + ) -> KVCacheEntry: + r"""Returns ``entry`` with key/value projections in ``dtype``. + + Args: + entry: Key/value projections in this storage dtype. + dtype: The target dtype. + """ + + +class Unquantized(KVCacheDType): + r"""Stores key/value projections as computed.""" + + def quantize(self, entry: KVCacheEntry) -> KVCacheEntry: # noqa: D102 + return entry + + def dequantize( # noqa: D102 + self, + entry: KVCacheEntry, + dtype: torch.dtype, + ) -> KVCacheEntry: + return entry + + class Cache(MutableMapping[Hashable, object], DeviceMixin): r"""A mutable mapping of model cache values.""" @@ -65,6 +105,8 @@ def __init__( **kwargs: object, ) -> None: self._mode = Cache.Mode.record + # TODO: Support different dtypes given str in kwargs: + self._kv_cache_dtype: KVCacheDType = Unquantized() self._items: dict[Hashable, object] = dict(*args, **kwargs) @property @@ -107,6 +149,8 @@ def __setitem__(self, key: Hashable, value: object) -> None: raise RuntimeError( "'__setitem__' requires the cache to be in 'record' mode" ) + if isinstance(value, KVCacheEntry): + value = self._kv_cache_dtype.quantize(value) self._items[key] = value def __delitem__(self, key: Hashable) -> None: @@ -196,5 +240,6 @@ def _apply(value: object) -> object: out = self.__class__( {key: _apply(value) for key, value in self.items()} ) + out._kv_cache_dtype = self._kv_cache_dtype out._mode = self._mode return out