From 835eed065d4248046e1add373ca5e01fc9c4043b Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Wed, 18 Nov 2026 12:12:28 -0800 Subject: [PATCH] Keep the lists of categories dynamic in compiled predictions A graph specializes on the size of every list of categories of its query, so a query with its own categories compiled a new graph. `TableTensor._mark_dynamic` now marks the lists of categories as well. PyTorch specializes a marked size of zero or one, and a list of a single category is common, so `compile_predict` traces with `backed_size_oblivious`, which keeps such a size symbolic. A query with an empty list of categories is predicted without the graph, like an empty query: the alignment of categories skips an empty list in a branch that the graph does not hold. A repeated single row is copied as well, so the marks stay off the tensors of the caller. --- sdm/models/base/compile.py | 25 +++++++++++------- sdm/tensor/table.py | 13 +++++++++- test/models/kumo/tabular/test_model.py | 35 ++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/sdm/models/base/compile.py b/sdm/models/base/compile.py index 6e949a4fb..6473d4534 100644 --- a/sdm/models/base/compile.py +++ b/sdm/models/base/compile.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, TypeVar import torch +import torch.fx.experimental._config as fx_config from torch import Tensor from sdm import TableTensor @@ -55,7 +56,8 @@ def compile_predict( r"""Compile ``model.predict`` using :func:`torch.compile`. One graph serves queries of any size along the :class:`DynamicShapes` - that ``model`` declares. Empty queries are predicted without the graph. + that ``model`` declares, with any lists of categories. A query that is + empty or holds an empty list of categories is predicted without the graph. Args: model: The model. @@ -71,6 +73,9 @@ def compile_predict( # Without dynamic shapes, none of the steps below are needed. if shapes == DynamicShapes(): return compiled + # With this option PyTorch does not specialize every marked size of zero + # or one, which lets one graph serve a list of a single category. + compiled = fx_config.patch(backed_size_oblivious=True)(compiled) # ty: ignore[unresolved-attribute] def predict( x: Tensor | TableTensor, # [..., R, D] @@ -78,19 +83,21 @@ def predict( **kwargs: Any, ) -> TableTensor: rows = x.size(-2) - if rows == 0: + if rows == 0 or ( + isinstance(x, TableTensor) + and any(c.size(0) == 0 for c in x.categorical.categories) + ): return model.predict(x, *args, **kwargs) - # A graph specializes on sizes of zero and one, so a single row is + # Operations on the rows still specialize on a single row, so it is # repeated if the rows are dynamic, unless callbacks see the rows. A - # contiguous copy has one memory layout for any view. + # contiguous copy has one memory layout for any view, and keeps the + # marks off the tensors of the caller. repeat = ( rows == 1 and -2 in shapes.query and not kwargs.get("callbacks") ) - x = ( - torch.cat([x, x], dim=-2) - if repeat - else x.clone(memory_format=torch.contiguous_format) - ) + if repeat: + x = torch.cat([x, x], dim=-2) + x = x.clone(memory_format=torch.contiguous_format) _mark_dynamic(x, shapes.query) out = compiled(x, *args, **kwargs) return out[..., :1, :] if repeat else out diff --git a/sdm/tensor/table.py b/sdm/tensor/table.py index 9833a13dc..763c5c67b 100644 --- a/sdm/tensor/table.py +++ b/sdm/tensor/table.py @@ -19,7 +19,12 @@ from typing_extensions import override from sdm import NaT, Stype, StypeLike -from sdm.tensor import CategoricalTensor, ColumnarTensor, StringTensor +from sdm.tensor import ( + CategoricalTensor, + ColumnarTensor, + StringTensor, + VarLenTensor, +) from sdm.tensor.io import arrow_as_tensor, to_arrow, to_cudf from sdm.tensor.mixin import _record_compiled_stream, _resolve_device @@ -976,6 +981,12 @@ def _mark_dynamic(self, dim: int) -> None: # The mark reaches the blocks, but not the layout of the text block. torch._dynamo.mark_dynamic(self, dim) self._text._mark_dynamic(dim) + # A table of other rows brings other lists of categories. + for categories in self._categorical.categories: + if isinstance(categories, VarLenTensor): + categories._mark_dynamic(0) + else: + torch._dynamo.maybe_mark_dynamic(categories, 0) def is_same_schema(self, other: TableTensor) -> bool: r"""Whether ``other`` has the same schema layout. diff --git a/test/models/kumo/tabular/test_model.py b/test/models/kumo/tabular/test_model.py index 3473d92d3..0791988ca 100644 --- a/test/models/kumo/tabular/test_model.py +++ b/test/models/kumo/tabular/test_model.py @@ -420,6 +420,41 @@ def test_compile_predict( assert counter.frame_count == 2 +@pytest.mark.usefixtures("compilation") +def test_compile_predict_query_categories() -> None: + def table(categories: list[str]) -> TableTensor: + code = torch.arange(6, dtype=torch.int32) + code = code % len(categories) if categories else code.fill_(-1) + return TableTensor( + columns={ + Stype.numerical: ("x", "y"), + Stype.categorical: ("color",), + }, + numerical=torch.randn(6, 2), + categorical=CategoricalTensor( + code=code.unsqueeze(-1), + categories=(StringTensor.from_list(categories),), + ), + ) + + model = _build("regression", "small") + model.fit(table(["a", "b"]), torch.randn(6, 1)) + + counter = CompileCounter() + predict = model._compile_predict(backend=counter) + # Every query has its own categories; "c" is not a fitted one. A query + # with an empty list is predicted without the graph. + for categories in (["b", "a"], ["a", "c", "b"], ["c"], []): + query = table(categories) + torch.testing.assert_close( + actual=predict(query).numerical, + expected=model.predict(query).numerical, + atol=1e-4, + rtol=1e-4, + ) + assert counter.frame_count == 1 + + @onlyMPS @pytest.mark.parametrize("task", ["classification", "regression"]) def test_default_recipe_on_mps(