From 5ee91d5c79195f95ae5bff712308c3ea0f74a065 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:55:41 -0800 Subject: [PATCH 1/5] Skip automatic chunking in compiled code A compiled graph records the number of chunks of the inputs it was traced with, so it is only valid for inputs that need the same number. Compiled code now processes its inputs in one chunk wherever the chunk size is derived from the memory budget: `TransformerBlock` without an explicit `batch_size_limit`, `CellEmbedding` with `batch_size_limit="auto"` and `split_size`. Explicit limits are honoured as before, and eager code chunks as before. The check is `torch.compiler.is_dynamo_compiling()`: PyTorch 2.14 sets `torch.compiler.is_compiling()` for the whole process while any thread compiles, which would switch chunking off for eager calls in other threads. --- sdm/_memory.py | 10 ++++++++-- sdm/models/tabfm/cell_embedding.py | 2 +- sdm/nn/attention.py | 8 ++++++-- test/nn/test_compile.py | 16 ++++++++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/sdm/_memory.py b/sdm/_memory.py index ae55f6da4..27d36fd73 100644 --- a/sdm/_memory.py +++ b/sdm/_memory.py @@ -63,9 +63,15 @@ def split_size(num_items: int, item_bytes: int, device: torch.device) -> int: On CUDA devices, chunks fit :func:`chunk_memory_limit`, unless a single item exceeds the limit. Autograd keeps the memory of every chunk, so one - chunk holds all items elsewhere or while gradients are enabled. + chunk holds all items elsewhere or while gradients are enabled. Compiled + code also uses one chunk: a new number of chunks would force a recompile. """ - if device.type != "cuda" or torch.is_grad_enabled() or item_bytes == 0: + if ( + device.type != "cuda" + or torch.is_grad_enabled() + or torch.compiler.is_dynamo_compiling() + or item_bytes == 0 + ): return max(num_items, 1) limit = max(chunk_memory_limit(device), 1) capacity = max(limit // item_bytes, 1) diff --git a/sdm/models/tabfm/cell_embedding.py b/sdm/models/tabfm/cell_embedding.py index babe20c5d..ffd8f60c1 100644 --- a/sdm/models/tabfm/cell_embedding.py +++ b/sdm/models/tabfm/cell_embedding.py @@ -135,7 +135,7 @@ def forward( if batch_size_limit == "auto": batch_size_limit = None - if x.is_cuda: + if x.is_cuda and not torch.compiler.is_dynamo_compiling(): if torch.is_autocast_enabled(x.device.type): element_size = torch.empty( size=(), diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index d3fc05065..e0669e20c 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -636,7 +636,8 @@ def forward( return_key_value: Whether to return the computed key and value projections alongside the block output. batch_size_limit: Maximum number of batch elements processed at - once. + once. Only an integer limit applies in compiled code, which + would otherwise recompile when the number of chunks changes. out: The output tensor. Returns: @@ -658,7 +659,10 @@ def forward( "already cached" ) - if torch.is_grad_enabled(): + if torch.is_grad_enabled() or ( + not isinstance(batch_size_limit, int) + and torch.compiler.is_dynamo_compiling() + ): return self._forward( query=query, key_value=key_value, diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index 3fa6dc004..bf85dcbba 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -226,6 +226,22 @@ def test_sdpa_compile_dynamic_batch_autocast( torch.testing.assert_close(out, expected) +def test_transformer_block_compile_dynamic_batch() -> None: + module = TransformerBlock( + channels=8, num_query_heads=2, mlp=torch.nn.Identity() + ).eval() + counter = CompileCounter() + compiled = torch.compile(module, fullgraph=True, backend=counter) + with torch.inference_mode(): + for batch_size in [3, 65_536]: + query = torch.randn(batch_size, 4, 8) + torch._dynamo.mark_dynamic(query, 0) + expected = module(query=query, batch_size_limit="auto") + out = compiled(query=query, batch_size_limit="auto") + torch.testing.assert_close(out, expected) + assert counter.frame_count == 1 + + @withCUDA @pytest.mark.parametrize( ("num_key_value_heads", "qassmax", "self_attn"), From 9da709d694fe223dcface345d8f2e94c669132f8 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:16:31 -0800 Subject: [PATCH 2/5] Mark the dynamic dimensions of compiled predictions in one place A model declares the dimensions that one compiled graph keeps dynamic with the `dynamic_shapes` class decorator, and `compile_predict` marks them on every call of the compiled function. `ICLModel._compile_predict` calls it. `KumoTabular` declares its query rows. One graph then serves queries of any number of rows: a single row is repeated, since a graph specializes on sizes of zero and one, and an empty query is predicted without the graph. The compiled function receives a contiguous copy of the query, which has one memory layout for any view of a table. `TableTensor` and `VarLenTensor` mark their own inner tensors. --- sdm/models/base/compile.py | 99 ++++++++++++++++++++++++++ sdm/models/base/icl.py | 12 ++-- sdm/models/kumo/tabular/model.py | 2 + sdm/tensor/table.py | 10 +++ sdm/tensor/var_len.py | 14 ++++ test/models/kumo/tabular/test_model.py | 10 +-- 6 files changed, 137 insertions(+), 10 deletions(-) create mode 100644 sdm/models/base/compile.py diff --git a/sdm/models/base/compile.py b/sdm/models/base/compile.py new file mode 100644 index 000000000..a679a1f86 --- /dev/null +++ b/sdm/models/base/compile.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dynamic shapes of a compiled :meth:`~sdm.models.ICLModel.predict`.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, TypeVar + +import torch +from torch import Tensor + +from sdm import TableTensor + +if TYPE_CHECKING: + from sdm.models.base.icl import ICLModel + +ModelT = TypeVar("ModelT", bound="ICLModel") + + +@dataclass(frozen=True) +class DynamicShapes: + r"""The dimensions that a compiled graph of a model keeps dynamic. + + A model declares them with :func:`dynamic_shapes`, and + :func:`compile_predict` marks them. + """ + + #: The dimensions of the query. Rows (``-2``) require that the model and + #: its recipe predict every query row independently of the other rows. + query: tuple[int, ...] = () + + +def dynamic_shapes(**dims: Any) -> Callable[[type[ModelT]], type[ModelT]]: + r"""Declare the :class:`DynamicShapes` of a model. + + Args: + **dims: The fields of :class:`DynamicShapes`. + """ + + def decorator(cls: type[ModelT]) -> type[ModelT]: + cls._dynamic_shapes = DynamicShapes(**dims) + return cls + + return decorator + + +def compile_predict( + model: ICLModel, + **kwargs: Any, +) -> Callable[..., TableTensor]: + r"""Compile ``model.predict`` using :func:`torch.compile`. + + One graph serves queries of any size along the :class:`DynamicShapes` + that ``model`` declares. Empty queries are predicted without the graph. + + Args: + model: The model. + **kwargs: Keyword arguments passed to :func:`torch.compile`. + ``isolate_recompiles`` defaults to ``True``, since all models + would otherwise share one recompile limit. + """ + kwargs = {"isolate_recompiles": True} | kwargs + compiled = torch.compile(model.predict, **kwargs) + dims = model._dynamic_shapes.query + if not dims: + return compiled + + def predict( + x: Tensor | TableTensor, # [..., R, D] + *args: Any, + **kwargs: Any, + ) -> TableTensor: + rows = x.size(-2) + if rows == 0: + return model.predict(x, *args, **kwargs) + # A graph specializes on sizes of zero and one, so a single row is + # repeated. A contiguous copy has one memory layout for any view. + x = ( + torch.cat([x, x], dim=-2) + if rows == 1 + else x.clone(memory_format=torch.contiguous_format) + ) + _mark_dynamic(x, dims) + out = compiled(x, *args, **kwargs) + return out[..., :1, :] if rows == 1 else out + + return predict + + +def _mark_dynamic(tensor: Tensor, dims: tuple[int, ...]) -> None: + for dim in dims: + dim %= tensor.dim() + if isinstance(tensor, TableTensor): + tensor._mark_dynamic(dim) + else: + torch._dynamo.mark_dynamic(tensor, dim) diff --git a/sdm/models/base/icl.py b/sdm/models/base/icl.py index b0ac1c754..9ac829dc9 100644 --- a/sdm/models/base/icl.py +++ b/sdm/models/base/icl.py @@ -22,6 +22,7 @@ from sdm._memory import set_pinned_max_round_threshold from sdm.cache import Cache from sdm.models.base import Callback, Model +from sdm.models.base.compile import DynamicShapes, compile_predict from sdm.processing.execution import ( MemberContext, MemberQuery, @@ -57,6 +58,9 @@ class ICLModel(Model): #: Whether this model supports additional related context. supports_related_tables: ClassVar[bool] + #: The dimensions that a compiled :meth:`predict` keeps dynamic. + _dynamic_shapes: ClassVar[DynamicShapes] = DynamicShapes() + def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) @@ -483,16 +487,14 @@ def _compile_predict(self, **kwargs: Any) -> Callable[..., TableTensor]: r"""Compile :meth:`predict` using :func:`torch.compile`. Args: - kwargs: Keyword arguments passed to :func:`torch.compile`. - ``isolate_recompiles`` defaults to ``True``, since all models - would otherwise share one recompile limit. + kwargs: Keyword arguments passed to + :func:`~sdm.models.base.compile.compile_predict`. Returns: The compiled function, which is called like :meth:`predict`. :meth:`predict` itself is not compiled. """ - kwargs = {"isolate_recompiles": True} | kwargs - return torch.compile(self.predict, **kwargs) + return compile_predict(self, **kwargs) # Abstract Methods ######################################################## diff --git a/sdm/models/kumo/tabular/model.py b/sdm/models/kumo/tabular/model.py index 45490093a..de89fd27c 100644 --- a/sdm/models/kumo/tabular/model.py +++ b/sdm/models/kumo/tabular/model.py @@ -16,6 +16,7 @@ from sdm.cache import Cache from sdm.models import ECOC, ICLModel from sdm.models._huggingface import download_checkpoint +from sdm.models.base.compile import dynamic_shapes from sdm.models.kumo.tabular.icl import ICLBlock from sdm.models.kumo.tabular.recipe import default_recipe from sdm.models.kumo.tabular.row_embedding import RowEmbedding @@ -63,6 +64,7 @@ } +@dynamic_shapes(query=(-2,)) class KumoTabular(ICLModel): r"""The tabular foundation model from the `"NVIDIA Kumo Tabular: Pushing the Accuracy-Efficiency Frontier for Tabular Prediction" diff --git a/sdm/tensor/table.py b/sdm/tensor/table.py index 6993db416..9833a13dc 100644 --- a/sdm/tensor/table.py +++ b/sdm/tensor/table.py @@ -967,6 +967,16 @@ def schema(self) -> TableSchema: r"""The schema of this table.""" return TableSchema(columns=self._columns) + def _mark_dynamic(self, dim: int) -> None: + r"""Mark a dimension as dynamic for :func:`torch.compile`. + + Args: + dim: The non-negative dimension. + """ + # The mark reaches the blocks, but not the layout of the text block. + torch._dynamo.mark_dynamic(self, dim) + self._text._mark_dynamic(dim) + def is_same_schema(self, other: TableTensor) -> bool: r"""Whether ``other`` has the same schema layout. diff --git a/sdm/tensor/var_len.py b/sdm/tensor/var_len.py index 42cb4f893..03ccd660b 100644 --- a/sdm/tensor/var_len.py +++ b/sdm/tensor/var_len.py @@ -538,6 +538,20 @@ def is_nullable(self) -> bool: r"""Whether this tensor has a validity mask.""" return self._valid is not None + def _mark_dynamic(self, dim: int) -> None: + r"""Mark a dimension as dynamic for :func:`torch.compile`. + + Args: + dim: The non-negative dimension. + """ + # Marks reach inner tensors of the same rank only: the layout has one + # more dimension, and the storage varies in size with any dimension. + for tensor in (self, self._layout): + torch._dynamo.mark_dynamic(tensor, dim) + for storage in (self._data, self._offset, self._valid): + if storage is not None: + torch._dynamo.maybe_mark_dynamic(storage, 0) + # Decorators ############################################################## @classmethod diff --git a/test/models/kumo/tabular/test_model.py b/test/models/kumo/tabular/test_model.py index 437eedf2f..97a5f4d4a 100644 --- a/test/models/kumo/tabular/test_model.py +++ b/test/models/kumo/tabular/test_model.py @@ -360,7 +360,7 @@ def forward(size: int | None) -> TableTensor: @pytest.mark.usefixtures("compilation") @pytest.mark.parametrize("task", ["classification", "regression"]) -def test_compile_predict_static_shapes( +def test_compile_predict( task: Literal["classification", "regression"], ) -> None: model = _build(task, "small") @@ -396,11 +396,11 @@ def test_compile_predict_static_shapes( fullgraph=True, dynamic=False, ) - # Queries of the same size reuse one graph. - for _ in range(2): + # Queries of any number of rows, including one and none, reuse one graph. + for query in (x[40:], x[41:44], x[42:43], x[40:40]): torch.testing.assert_close( - actual=predict(x[40:]).numerical, - expected=model.predict(x[40:]).numerical, + actual=predict(query).numerical, + expected=model.predict(query).numerical, atol=1e-4, rtol=1e-4, ) From 5c0e2f9dea648d89ba816a1f026cd431cef1dcd5 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:22:43 -0800 Subject: [PATCH 3/5] Apply the compiled prediction wrapper to any declared dynamic shapes `compile_predict` returned the plain compiled function for a model that declares no dynamic query dimensions, which would ignore any other declared dimension. The wrapper now applies whenever a model declares dynamic shapes, and repeats a single row only if the query rows are dynamic, since that requires independent query rows. --- sdm/models/base/compile.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sdm/models/base/compile.py b/sdm/models/base/compile.py index a679a1f86..449f898ca 100644 --- a/sdm/models/base/compile.py +++ b/sdm/models/base/compile.py @@ -64,8 +64,9 @@ def compile_predict( """ kwargs = {"isolate_recompiles": True} | kwargs compiled = torch.compile(model.predict, **kwargs) - dims = model._dynamic_shapes.query - if not dims: + shapes = model._dynamic_shapes + # A model without declared dynamic shapes needs none of the steps below. + if shapes == DynamicShapes(): return compiled def predict( @@ -77,15 +78,17 @@ def predict( if rows == 0: return model.predict(x, *args, **kwargs) # A graph specializes on sizes of zero and one, so a single row is - # repeated. A contiguous copy has one memory layout for any view. + # repeated if the rows are dynamic. A contiguous copy has one memory + # layout for any view. + repeat = rows == 1 and -2 in shapes.query x = ( torch.cat([x, x], dim=-2) - if rows == 1 + if repeat else x.clone(memory_format=torch.contiguous_format) ) - _mark_dynamic(x, dims) + _mark_dynamic(x, shapes.query) out = compiled(x, *args, **kwargs) - return out[..., :1, :] if rows == 1 else out + return out[..., :1, :] if repeat else out return predict From 66b7b581a74d3d4fa611214e7ed92ba4d317437c Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:30:03 -0800 Subject: [PATCH 4/5] Keep single-row queries as they are when callbacks are passed `compile_predict` repeats a single query row, which the model predicts independently of the other rows. Callbacks are not part of that promise: a callback that depends on the number of rows would see two. A single row is not repeated when callbacks are passed. --- sdm/models/base/compile.py | 8 +++++--- test/models/kumo/tabular/test_model.py | 6 ++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sdm/models/base/compile.py b/sdm/models/base/compile.py index 449f898ca..d9f2d6178 100644 --- a/sdm/models/base/compile.py +++ b/sdm/models/base/compile.py @@ -78,9 +78,11 @@ def predict( if rows == 0: return model.predict(x, *args, **kwargs) # A graph specializes on sizes of zero and one, so a single row is - # repeated if the rows are dynamic. A contiguous copy has one memory - # layout for any view. - repeat = rows == 1 and -2 in shapes.query + # repeated if the rows are dynamic, unless callbacks see the rows. A + # contiguous copy has one memory layout for any view. + repeat = ( + rows == 1 and -2 in shapes.query and not kwargs.get("callbacks") + ) x = ( torch.cat([x, x], dim=-2) if repeat diff --git a/test/models/kumo/tabular/test_model.py b/test/models/kumo/tabular/test_model.py index 97a5f4d4a..f5954695a 100644 --- a/test/models/kumo/tabular/test_model.py +++ b/test/models/kumo/tabular/test_model.py @@ -9,6 +9,7 @@ import sdm.processing as sp from sdm import CategoricalTensor, StringTensor, Stype, TableTensor +from sdm.explain.gradient import CaptureInputs from sdm.models import KumoTabular from sdm.testing import onlyMPS @@ -406,6 +407,11 @@ def test_compile_predict( ) assert counter.frame_count == 1 + # Callbacks see a single row as it is. + capture = CaptureInputs() + predict(x[42:43], callbacks=[capture]) + assert capture.inputs[0][0].size(-2) == 1 + @onlyMPS @pytest.mark.parametrize("task", ["classification", "regression"]) From 901aa7d199c906c8da18be51326a5d97d8ed4cf4 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:47:27 -0800 Subject: [PATCH 5/5] Let callers compile predictions without dynamic shapes --- sdm/models/base/compile.py | 7 +++++-- test/models/kumo/tabular/test_model.py | 7 +++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sdm/models/base/compile.py b/sdm/models/base/compile.py index d9f2d6178..6e949a4fb 100644 --- a/sdm/models/base/compile.py +++ b/sdm/models/base/compile.py @@ -49,6 +49,7 @@ def decorator(cls: type[ModelT]) -> type[ModelT]: def compile_predict( model: ICLModel, + dynamic_shapes: bool = True, **kwargs: Any, ) -> Callable[..., TableTensor]: r"""Compile ``model.predict`` using :func:`torch.compile`. @@ -58,14 +59,16 @@ def compile_predict( Args: model: The model. + dynamic_shapes: If set to ``False``, no dimension is dynamic, for + callers whose queries always have the same shape. **kwargs: Keyword arguments passed to :func:`torch.compile`. ``isolate_recompiles`` defaults to ``True``, since all models would otherwise share one recompile limit. """ kwargs = {"isolate_recompiles": True} | kwargs compiled = torch.compile(model.predict, **kwargs) - shapes = model._dynamic_shapes - # A model without declared dynamic shapes needs none of the steps below. + shapes = model._dynamic_shapes if dynamic_shapes else DynamicShapes() + # Without dynamic shapes, none of the steps below are needed. if shapes == DynamicShapes(): return compiled diff --git a/test/models/kumo/tabular/test_model.py b/test/models/kumo/tabular/test_model.py index f5954695a..3473d92d3 100644 --- a/test/models/kumo/tabular/test_model.py +++ b/test/models/kumo/tabular/test_model.py @@ -412,6 +412,13 @@ def test_compile_predict( predict(x[42:43], callbacks=[capture]) assert capture.inputs[0][0].size(-2) == 1 + # Without dynamic shapes, every number of rows compiles its own graph. + counter = CompileCounter() + predict = model._compile_predict(backend=counter, dynamic_shapes=False) + for query in (x[40:], x[41:44]): + predict(query) + assert counter.frame_count == 2 + @onlyMPS @pytest.mark.parametrize("task", ["classification", "regression"])