diff --git a/sdm/_memory.py b/sdm/_memory.py index ae55f6da4..27d36fd73 100644 --- a/sdm/_memory.py +++ b/sdm/_memory.py @@ -63,9 +63,15 @@ def split_size(num_items: int, item_bytes: int, device: torch.device) -> int: On CUDA devices, chunks fit :func:`chunk_memory_limit`, unless a single item exceeds the limit. Autograd keeps the memory of every chunk, so one - chunk holds all items elsewhere or while gradients are enabled. + chunk holds all items elsewhere or while gradients are enabled. Compiled + code also uses one chunk: a new number of chunks would force a recompile. """ - if device.type != "cuda" or torch.is_grad_enabled() or item_bytes == 0: + if ( + device.type != "cuda" + or torch.is_grad_enabled() + or torch.compiler.is_dynamo_compiling() + or item_bytes == 0 + ): return max(num_items, 1) limit = max(chunk_memory_limit(device), 1) capacity = max(limit // item_bytes, 1) diff --git a/sdm/models/tabfm/cell_embedding.py b/sdm/models/tabfm/cell_embedding.py index babe20c5d..ffd8f60c1 100644 --- a/sdm/models/tabfm/cell_embedding.py +++ b/sdm/models/tabfm/cell_embedding.py @@ -135,7 +135,7 @@ def forward( if batch_size_limit == "auto": batch_size_limit = None - if x.is_cuda: + if x.is_cuda and not torch.compiler.is_dynamo_compiling(): if torch.is_autocast_enabled(x.device.type): element_size = torch.empty( size=(), diff --git a/sdm/nn/attention.py b/sdm/nn/attention.py index d3fc05065..e0669e20c 100644 --- a/sdm/nn/attention.py +++ b/sdm/nn/attention.py @@ -636,7 +636,8 @@ def forward( return_key_value: Whether to return the computed key and value projections alongside the block output. batch_size_limit: Maximum number of batch elements processed at - once. + once. Only an integer limit applies in compiled code, which + would otherwise recompile when the number of chunks changes. out: The output tensor. Returns: @@ -658,7 +659,10 @@ def forward( "already cached" ) - if torch.is_grad_enabled(): + if torch.is_grad_enabled() or ( + not isinstance(batch_size_limit, int) + and torch.compiler.is_dynamo_compiling() + ): return self._forward( query=query, key_value=key_value, diff --git a/test/nn/test_compile.py b/test/nn/test_compile.py index 3fa6dc004..bf85dcbba 100644 --- a/test/nn/test_compile.py +++ b/test/nn/test_compile.py @@ -226,6 +226,22 @@ def test_sdpa_compile_dynamic_batch_autocast( torch.testing.assert_close(out, expected) +def test_transformer_block_compile_dynamic_batch() -> None: + module = TransformerBlock( + channels=8, num_query_heads=2, mlp=torch.nn.Identity() + ).eval() + counter = CompileCounter() + compiled = torch.compile(module, fullgraph=True, backend=counter) + with torch.inference_mode(): + for batch_size in [3, 65_536]: + query = torch.randn(batch_size, 4, 8) + torch._dynamo.mark_dynamic(query, 0) + expected = module(query=query, batch_size_limit="auto") + out = compiled(query=query, batch_size_limit="auto") + torch.testing.assert_close(out, expected) + assert counter.frame_count == 1 + + @withCUDA @pytest.mark.parametrize( ("num_key_value_heads", "qassmax", "self_attn"),