diff --git a/sdm/cache.py b/sdm/cache.py index 59c6c0db0..94a86eca6 100644 --- a/sdm/cache.py +++ b/sdm/cache.py @@ -22,22 +22,37 @@ class _KVCacheEntry(NamedTuple): key: Tensor value: Tensor + key_scale: Tensor | None = None + value_scale: Tensor | None = None class KVCacheEntry(_KVCacheEntry, DeviceMixin): r"""Cached key/value projections for a single transformer block. Args: - key: Cached key projection tensor. - value: Cached value projection tensor. + key: Key projections with shape ``[..., KV, H, C]``. + value: Value projections with shape ``[..., KV, H, C]``. + key_scale: Optional scales broadcastable to ``key``. + value_scale: Optional scales broadcastable to ``value``. """ def _tensors(self) -> Iterator[Tensor]: yield self.key yield self.value + if self.key_scale is not None: + yield self.key_scale + if self.value_scale is not None: + yield self.value_scale def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self: - return self.__class__(key=fn(self.key), value=fn(self.value)) + return self.__class__( + key=fn(self.key), + value=fn(self.value), + key_scale=None if self.key_scale is None else fn(self.key_scale), + value_scale=None + if self.value_scale is None + else fn(self.value_scale), + ) class Cache(MutableMapping[Hashable, object], DeviceMixin): diff --git a/sdm/models/kumo/tabular/icl.py b/sdm/models/kumo/tabular/icl.py index 7f76b6dd8..783dd66f1 100644 --- a/sdm/models/kumo/tabular/icl.py +++ b/sdm/models/kumo/tabular/icl.py @@ -109,7 +109,9 @@ def forward( ) if cache is not None and cache.is_recording: - x, (key, value) = result + x, entry = result + key, value = entry.key, entry.value + del entry kv_heads = self.kv_heads if kv_heads is not None and kv_heads < key.size(-2): key, value = _copy_key_value( @@ -123,7 +125,7 @@ def forward( del result continue - x_context, (key, value) = layer( + x_context, entry = layer( query=x if train_size is not None else x[..., :0, :] @@ -139,10 +141,10 @@ def forward( else x[..., :R_train, :], ) key_value = KVCacheEntry( - key=key[..., : self.kv_heads, :].contiguous(), - value=value[..., : self.kv_heads, :].contiguous(), + key=entry.key[..., : self.kv_heads, :].contiguous(), + value=entry.value[..., : self.kv_heads, :].contiguous(), ) - del key, value + del entry x_query = layer( query=x if train_size is not None else x[..., R_train:, :], key_value=key_value, diff --git a/test/test_cache.py b/test/test_cache.py index d05102f09..f82ef89cb 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -118,3 +118,16 @@ def test_cache_serialization_without_cuda( torch.testing.assert_close( restored["tensor"], cache["tensor"], rtol=0, atol=0 ) + + +@withCUDA +def test_kv_cache_entry_scales(device: torch.device) -> None: + entry = KVCacheEntry( + key=torch.ones(2, 3, dtype=torch.int8), + value=torch.ones(2, 3, dtype=torch.int8), + key_scale=torch.ones(2, 1), + value_scale=torch.ones(2, 1), + ) + cache = Cache(entry=entry).to(device) + assert cast(KVCacheEntry, cache["entry"]).device == device + assert cache.size() == 2 * 6 * 1 + 2 * 2 * 4