diff --git a/sdm/_memory.py b/sdm/_memory.py index ae55f6da4..27d36fd73 100644 --- a/sdm/_memory.py +++ b/sdm/_memory.py @@ -63,9 +63,15 @@ def split_size(num_items: int, item_bytes: int, device: torch.device) -> int: On CUDA devices, chunks fit :func:`chunk_memory_limit`, unless a single item exceeds the limit. Autograd keeps the memory of every chunk, so one - chunk holds all items elsewhere or while gradients are enabled. + chunk holds all items elsewhere or while gradients are enabled. Compiled + code also uses one chunk: a new number of chunks would force a recompile. """ - if device.type != "cuda" or torch.is_grad_enabled() or item_bytes == 0: + if ( + device.type != "cuda" + or torch.is_grad_enabled() + or torch.compiler.is_dynamo_compiling() + or item_bytes == 0 + ): return max(num_items, 1) limit = max(chunk_memory_limit(device), 1) capacity = max(limit // item_bytes, 1) diff --git a/sdm/models/base/compile.py b/sdm/models/base/compile.py new file mode 100644 index 000000000..6e949a4fb --- /dev/null +++ b/sdm/models/base/compile.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dynamic shapes of a compiled :meth:`~sdm.models.ICLModel.predict`.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, TypeVar + +import torch +from torch import Tensor + +from sdm import TableTensor + +if TYPE_CHECKING: + from sdm.models.base.icl import ICLModel + +ModelT = TypeVar("ModelT", bound="ICLModel") + + +@dataclass(frozen=True) +class DynamicShapes: + r"""The dimensions that a compiled graph of a model keeps dynamic. + + A model declares them with :func:`dynamic_shapes`, and + :func:`compile_predict` marks them. + """ + + #: The dimensions of the query. Rows (``-2``) require that the model and + #: its recipe predict every query row independently of the other rows. + query: tuple[int, ...] = () + + +def dynamic_shapes(**dims: Any) -> Callable[[type[ModelT]], type[ModelT]]: + r"""Declare the :class:`DynamicShapes` of a model. + + Args: + **dims: The fields of :class:`DynamicShapes`. + """ + + def decorator(cls: type[ModelT]) -> type[ModelT]: + cls._dynamic_shapes = DynamicShapes(**dims) + return cls + + return decorator + + +def compile_predict( + model: ICLModel, + dynamic_shapes: bool = True, + **kwargs: Any, +) -> Callable[..., TableTensor]: + r"""Compile ``model.predict`` using :func:`torch.compile`. + + One graph serves queries of any size along the :class:`DynamicShapes` + that ``model`` declares. Empty queries are predicted without the graph. + + Args: + model: The model. + dynamic_shapes: If set to ``False``, no dimension is dynamic, for + callers whose queries always have the same shape. + **kwargs: Keyword arguments passed to :func:`torch.compile`. + ``isolate_recompiles`` defaults to ``True``, since all models + would otherwise share one recompile limit. + """ + kwargs = {"isolate_recompiles": True} | kwargs + compiled = torch.compile(model.predict, **kwargs) + shapes = model._dynamic_shapes if dynamic_shapes else DynamicShapes() + # Without dynamic shapes, none of the steps below are needed. + if shapes == DynamicShapes(): + return compiled + + def predict( + x: Tensor | TableTensor, # [..., R, D] + *args: Any, + **kwargs: Any, + ) -> TableTensor: + rows = x.size(-2) + if rows == 0: + return model.predict(x, *args, **kwargs) + # A graph specializes on sizes of zero and one, so a single row is + # repeated if the rows are dynamic, unless callbacks see the rows. A + # contiguous copy has one memory layout for any view. + repeat = ( + rows == 1 and -2 in shapes.query and not kwargs.get("callbacks") + ) + x = ( + torch.cat([x, x], dim=-2) + if repeat + else x.clone(memory_format=torch.contiguous_format) + ) + _mark_dynamic(x, shapes.query) + out = compiled(x, *args, **kwargs) + return out[..., :1, :] if repeat else out + + return predict + + +def _mark_dynamic(tensor: Tensor, dims: tuple[int, ...]) -> None: + for dim in dims: + dim %= tensor.dim() + if isinstance(tensor, TableTensor): + tensor._mark_dynamic(dim) + else: + torch._dynamo.mark_dynamic(tensor, dim) diff --git a/sdm/models/base/icl.py b/sdm/models/base/icl.py index b0ac1c754..9ac829dc9 100644 --- a/sdm/models/base/icl.py +++ b/sdm/models/base/icl.py @@ -22,6 +22,7 @@ from sdm._memory import set_pinned_max_round_threshold from sdm.cache import Cache from sdm.models.base import Callback, Model +from sdm.models.base.compile import DynamicShapes, compile_predict from sdm.processing.execution import ( MemberContext, MemberQuery, @@ -57,6 +58,9 @@ class ICLModel(Model): #: Whether this model supports additional related context. supports_related_tables: ClassVar[bool] + #: The dimensions that a compiled :meth:`predict` keeps dynamic. + _dynamic_shapes: ClassVar[DynamicShapes] = DynamicShapes() + def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) @@ -483,16 +487,14 @@ def _compile_predict(self, **kwargs: Any) -> Callable[..., TableTensor]: r"""Compile :meth:`predict` using :func:`torch.compile`. Args: - kwargs: Keyword arguments passed to :func:`torch.compile`. - ``isolate_recompiles`` defaults to ``True``, since all models - would otherwise share one recompile limit. + kwargs: Keyword arguments passed to + :func:`~sdm.models.base.compile.compile_predict`. Returns: The compiled function, which is called like :meth:`predict`. :meth:`predict` itself is not compiled. """ - kwargs = {"isolate_recompiles": True} | kwargs - return torch.compile(self.predict, **kwargs) + return compile_predict(self, **kwargs) # Abstract Methods ######################################################## diff --git a/sdm/models/kumo/tabular/model.py b/sdm/models/kumo/tabular/model.py index 45490093a..de89fd27c 100644 --- a/sdm/models/kumo/tabular/model.py +++ b/sdm/models/kumo/tabular/model.py @@ -16,6 +16,7 @@ from sdm.cache import Cache from sdm.models import ECOC, ICLModel from sdm.models._huggingface import download_checkpoint +from sdm.models.base.compile import dynamic_shapes from sdm.models.kumo.tabular.icl import ICLBlock from sdm.models.kumo.tabular.recipe import default_recipe from sdm.models.kumo.tabular.row_embedding import RowEmbedding @@ -63,6 +64,7 @@ } +@dynamic_shapes(query=(-2,)) class KumoTabular(ICLModel): r"""The tabular foundation model from the `"NVIDIA Kumo Tabular: Pushing the Accuracy-Efficiency Frontier for Tabular Prediction" diff --git a/sdm/models/tabfm/cell_embedding.py b/sdm/models/tabfm/cell_embedding.py index babe20c5d..ffd8f60c1 100644 --- a/sdm/models/tabfm/cell_embedding.py +++ b/sdm/models/tabfm/cell_embedding.py @@ -135,7 +135,7 @@ def forward( if batch_size_limit == "auto": batch_size_limit = None - if x.is_cuda: + if x.is_cuda and not torch.compiler.is_dynamo_compiling(): if torch.is_autocast_enabled(x.device.type): element_size = torch.empty( size=(), diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index d3fc05065..e0669e20c 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -636,7 +636,8 @@ def forward( return_key_value: Whether to return the computed key and value projections alongside the block output. batch_size_limit: Maximum number of batch elements processed at - once. + once. Only an integer limit applies in compiled code, which + would otherwise recompile when the number of chunks changes. out: The output tensor. Returns: @@ -658,7 +659,10 @@ def forward( "already cached" ) - if torch.is_grad_enabled(): + if torch.is_grad_enabled() or ( + not isinstance(batch_size_limit, int) + and torch.compiler.is_dynamo_compiling() + ): return self._forward( query=query, key_value=key_value, diff --git a/sdm/tensor/table.py b/sdm/tensor/table.py index 6993db416..9833a13dc 100644 --- a/sdm/tensor/table.py +++ b/sdm/tensor/table.py @@ -967,6 +967,16 @@ def schema(self) -> TableSchema: r"""The schema of this table.""" return TableSchema(columns=self._columns) + def _mark_dynamic(self, dim: int) -> None: + r"""Mark a dimension as dynamic for :func:`torch.compile`. + + Args: + dim: The non-negative dimension. + """ + # The mark reaches the blocks, but not the layout of the text block. + torch._dynamo.mark_dynamic(self, dim) + self._text._mark_dynamic(dim) + def is_same_schema(self, other: TableTensor) -> bool: r"""Whether ``other`` has the same schema layout. diff --git a/sdm/tensor/var_len.py b/sdm/tensor/var_len.py index 42cb4f893..03ccd660b 100644 --- a/sdm/tensor/var_len.py +++ b/sdm/tensor/var_len.py @@ -538,6 +538,20 @@ def is_nullable(self) -> bool: r"""Whether this tensor has a validity mask.""" return self._valid is not None + def _mark_dynamic(self, dim: int) -> None: + r"""Mark a dimension as dynamic for :func:`torch.compile`. + + Args: + dim: The non-negative dimension. + """ + # Marks reach inner tensors of the same rank only: the layout has one + # more dimension, and the storage varies in size with any dimension. + for tensor in (self, self._layout): + torch._dynamo.mark_dynamic(tensor, dim) + for storage in (self._data, self._offset, self._valid): + if storage is not None: + torch._dynamo.maybe_mark_dynamic(storage, 0) + # Decorators ############################################################## @classmethod diff --git a/test/models/kumo/tabular/test_model.py b/test/models/kumo/tabular/test_model.py index 437eedf2f..3473d92d3 100644 --- a/test/models/kumo/tabular/test_model.py +++ b/test/models/kumo/tabular/test_model.py @@ -9,6 +9,7 @@ import sdm.processing as sp from sdm import CategoricalTensor, StringTensor, Stype, TableTensor +from sdm.explain.gradient import CaptureInputs from sdm.models import KumoTabular from sdm.testing import onlyMPS @@ -360,7 +361,7 @@ def forward(size: int | None) -> TableTensor: @pytest.mark.usefixtures("compilation") @pytest.mark.parametrize("task", ["classification", "regression"]) -def test_compile_predict_static_shapes( +def test_compile_predict( task: Literal["classification", "regression"], ) -> None: model = _build(task, "small") @@ -396,16 +397,28 @@ def test_compile_predict_static_shapes( fullgraph=True, dynamic=False, ) - # Queries of the same size reuse one graph. - for _ in range(2): + # Queries of any number of rows, including one and none, reuse one graph. + for query in (x[40:], x[41:44], x[42:43], x[40:40]): torch.testing.assert_close( - actual=predict(x[40:]).numerical, - expected=model.predict(x[40:]).numerical, + actual=predict(query).numerical, + expected=model.predict(query).numerical, atol=1e-4, rtol=1e-4, ) assert counter.frame_count == 1 + # Callbacks see a single row as it is. + capture = CaptureInputs() + predict(x[42:43], callbacks=[capture]) + assert capture.inputs[0][0].size(-2) == 1 + + # Without dynamic shapes, every number of rows compiles its own graph. + counter = CompileCounter() + predict = model._compile_predict(backend=counter, dynamic_shapes=False) + for query in (x[40:], x[41:44]): + predict(query) + assert counter.frame_count == 2 + @onlyMPS @pytest.mark.parametrize("task", ["classification", "regression"]) diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index 3fa6dc004..bf85dcbba 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -226,6 +226,22 @@ def test_sdpa_compile_dynamic_batch_autocast( torch.testing.assert_close(out, expected) +def test_transformer_block_compile_dynamic_batch() -> None: + module = TransformerBlock( + channels=8, num_query_heads=2, mlp=torch.nn.Identity() + ).eval() + counter = CompileCounter() + compiled = torch.compile(module, fullgraph=True, backend=counter) + with torch.inference_mode(): + for batch_size in [3, 65_536]: + query = torch.randn(batch_size, 4, 8) + torch._dynamo.mark_dynamic(query, 0) + expected = module(query=query, batch_size_limit="auto") + out = compiled(query=query, batch_size_limit="auto") + torch.testing.assert_close(out, expected) + assert counter.frame_count == 1 + + @withCUDA @pytest.mark.parametrize( ("num_key_value_heads", "qassmax", "self_attn"),