From 5ee91d5c79195f95ae5bff712308c3ea0f74a065 Mon Sep 17 00:00:00 2001 From: Ard <88337265+aw471@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:55:41 -0800 Subject: [PATCH] Skip automatic chunking in compiled code A compiled graph records the number of chunks of the inputs it was traced with, so it is only valid for inputs that need the same number. Compiled code now processes its inputs in one chunk wherever the chunk size is derived from the memory budget: `TransformerBlock` without an explicit `batch_size_limit`, `CellEmbedding` with `batch_size_limit="auto"` and `split_size`. Explicit limits are honoured as before, and eager code chunks as before. The check is `torch.compiler.is_dynamo_compiling()`: PyTorch 2.14 sets `torch.compiler.is_compiling()` for the whole process while any thread compiles, which would switch chunking off for eager calls in other threads. --- sdm/_memory.py | 10 ++++++++-- sdm/models/tabfm/cell_embedding.py | 2 +- sdm/nn/attention.py | 8 ++++++-- test/nn/test_compile.py | 16 ++++++++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/sdm/_memory.py b/sdm/_memory.py index ae55f6da4..27d36fd73 100644 --- a/sdm/_memory.py +++ b/sdm/_memory.py @@ -63,9 +63,15 @@ def split_size(num_items: int, item_bytes: int, device: torch.device) -> int: On CUDA devices, chunks fit :func:`chunk_memory_limit`, unless a single item exceeds the limit. Autograd keeps the memory of every chunk, so one - chunk holds all items elsewhere or while gradients are enabled. + chunk holds all items elsewhere or while gradients are enabled. Compiled + code also uses one chunk: a new number of chunks would force a recompile. """ - if device.type != "cuda" or torch.is_grad_enabled() or item_bytes == 0: + if ( + device.type != "cuda" + or torch.is_grad_enabled() + or torch.compiler.is_dynamo_compiling() + or item_bytes == 0 + ): return max(num_items, 1) limit = max(chunk_memory_limit(device), 1) capacity = max(limit // item_bytes, 1) diff --git a/sdm/models/tabfm/cell_embedding.py b/sdm/models/tabfm/cell_embedding.py index babe20c5d..ffd8f60c1 100644 --- a/sdm/models/tabfm/cell_embedding.py +++ b/sdm/models/tabfm/cell_embedding.py @@ -135,7 +135,7 @@ def forward( if batch_size_limit == "auto": batch_size_limit = None - if x.is_cuda: + if x.is_cuda and not torch.compiler.is_dynamo_compiling(): if torch.is_autocast_enabled(x.device.type): element_size = torch.empty( size=(), diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index d3fc05065..e0669e20c 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -636,7 +636,8 @@ def forward( return_key_value: Whether to return the computed key and value projections alongside the block output. batch_size_limit: Maximum number of batch elements processed at - once. + once. Only an integer limit applies in compiled code, which + would otherwise recompile when the number of chunks changes. out: The output tensor. Returns: @@ -658,7 +659,10 @@ def forward( "already cached" ) - if torch.is_grad_enabled(): + if torch.is_grad_enabled() or ( + not isinstance(batch_size_limit, int) + and torch.compiler.is_dynamo_compiling() + ): return self._forward( query=query, key_value=key_value, diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index 3fa6dc004..bf85dcbba 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -226,6 +226,22 @@ def test_sdpa_compile_dynamic_batch_autocast( torch.testing.assert_close(out, expected) +def test_transformer_block_compile_dynamic_batch() -> None: + module = TransformerBlock( + channels=8, num_query_heads=2, mlp=torch.nn.Identity() + ).eval() + counter = CompileCounter() + compiled = torch.compile(module, fullgraph=True, backend=counter) + with torch.inference_mode(): + for batch_size in [3, 65_536]: + query = torch.randn(batch_size, 4, 8) + torch._dynamo.mark_dynamic(query, 0) + expected = module(query=query, batch_size_limit="auto") + out = compiled(query=query, batch_size_limit="auto") + torch.testing.assert_close(out, expected) + assert counter.frame_count == 1 + + @withCUDA @pytest.mark.parametrize( ("num_key_value_heads", "qassmax", "self_attn"),