Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions sdm/_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,15 @@ def split_size(num_items: int, item_bytes: int, device: torch.device) -> int:

On CUDA devices, chunks fit :func:`chunk_memory_limit`, unless a single
item exceeds the limit. Autograd keeps the memory of every chunk, so one
chunk holds all items elsewhere or while gradients are enabled.
chunk holds all items elsewhere or while gradients are enabled. Compiled
code also uses one chunk: a new number of chunks would force a recompile.
"""
if device.type != "cuda" or torch.is_grad_enabled() or item_bytes == 0:
if (
device.type != "cuda"
or torch.is_grad_enabled()
or torch.compiler.is_dynamo_compiling()
or item_bytes == 0
):
return max(num_items, 1)
limit = max(chunk_memory_limit(device), 1)
capacity = max(limit // item_bytes, 1)
Expand Down
2 changes: 1 addition & 1 deletion sdm/models/tabfm/cell_embedding.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ def forward(

if batch_size_limit == "auto":
batch_size_limit = None
if x.is_cuda:
if x.is_cuda and not torch.compiler.is_dynamo_compiling():
if torch.is_autocast_enabled(x.device.type):
element_size = torch.empty(
size=(),
Expand Down
8 changes: 6 additions & 2 deletions sdm/nn/attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -636,7 +636,8 @@ def forward(
return_key_value: Whether to return the computed key and value
projections alongside the block output.
batch_size_limit: Maximum number of batch elements processed at
once.
once. Only an integer limit applies in compiled code, which
would otherwise recompile when the number of chunks changes.
out: The output tensor.

Returns:
Expand All @@ -658,7 +659,10 @@ def forward(
"already cached"
)

if torch.is_grad_enabled():
if torch.is_grad_enabled() or (
not isinstance(batch_size_limit, int)
and torch.compiler.is_dynamo_compiling()
):
return self._forward(
query=query,
key_value=key_value,
Expand Down
16 changes: 16 additions & 0 deletions test/nn/test_compile.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,22 @@ def test_sdpa_compile_dynamic_batch_autocast(
torch.testing.assert_close(out, expected)


def test_transformer_block_compile_dynamic_batch() -> None:
module = TransformerBlock(
channels=8, num_query_heads=2, mlp=torch.nn.Identity()
).eval()
counter = CompileCounter()
compiled = torch.compile(module, fullgraph=True, backend=counter)
with torch.inference_mode():
for batch_size in [3, 65_536]:
query = torch.randn(batch_size, 4, 8)
torch._dynamo.mark_dynamic(query, 0)
expected = module(query=query, batch_size_limit="auto")
out = compiled(query=query, batch_size_limit="auto")
torch.testing.assert_close(out, expected)
assert counter.frame_count == 1


@withCUDA
@pytest.mark.parametrize(
("num_key_value_heads", "qassmax", "self_attn"),
Expand Down