Skip to content
21 changes: 18 additions & 3 deletions sdm/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,22 +22,37 @@
class _KVCacheEntry(NamedTuple):
key: Tensor
value: Tensor
key_scale: Tensor | None = None
value_scale: Tensor | None = None


class KVCacheEntry(_KVCacheEntry, DeviceMixin):
r"""Cached key/value projections for a single transformer block.

Args:
key: Cached key projection tensor.
value: Cached value projection tensor.
key: Key projections with shape ``[..., KV, H, C]``.
value: Value projections with shape ``[..., KV, H, C]``.
key_scale: Optional scales broadcastable to ``key``.
value_scale: Optional scales broadcastable to ``value``.
"""

def _tensors(self) -> Iterator[Tensor]:
yield self.key
yield self.value
if self.key_scale is not None:
yield self.key_scale
if self.value_scale is not None:
yield self.value_scale

def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self:
return self.__class__(key=fn(self.key), value=fn(self.value))
return self.__class__(
key=fn(self.key),
value=fn(self.value),
key_scale=None if self.key_scale is None else fn(self.key_scale),
value_scale=None
if self.value_scale is None
else fn(self.value_scale),
)


class Cache(MutableMapping[Hashable, object], DeviceMixin):
Expand Down
12 changes: 7 additions & 5 deletions sdm/models/kumo/tabular/icl.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,9 @@ def forward(
)

if cache is not None and cache.is_recording:
x, (key, value) = result
x, entry = result
key, value = entry.key, entry.value
del entry
kv_heads = self.kv_heads
if kv_heads is not None and kv_heads < key.size(-2):
key, value = _copy_key_value(
Expand All @@ -123,7 +125,7 @@ def forward(
del result
continue

x_context, (key, value) = layer(
x_context, entry = layer(
query=x
if train_size is not None
else x[..., :0, :]
Expand All @@ -139,10 +141,10 @@ def forward(
else x[..., :R_train, :],
)
key_value = KVCacheEntry(
key=key[..., : self.kv_heads, :].contiguous(),
value=value[..., : self.kv_heads, :].contiguous(),
key=entry.key[..., : self.kv_heads, :].contiguous(),
value=entry.value[..., : self.kv_heads, :].contiguous(),
)
del key, value
del entry
x_query = layer(
query=x if train_size is not None else x[..., R_train:, :],
key_value=key_value,
Expand Down
13 changes: 13 additions & 0 deletions test/test_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,3 +118,16 @@ def test_cache_serialization_without_cuda(
torch.testing.assert_close(
restored["tensor"], cache["tensor"], rtol=0, atol=0
)


@withCUDA
def test_kv_cache_entry_scales(device: torch.device) -> None:
entry = KVCacheEntry(
key=torch.ones(2, 3, dtype=torch.int8),
value=torch.ones(2, 3, dtype=torch.int8),
key_scale=torch.ones(2, 1),
value_scale=torch.ones(2, 1),
)
cache = Cache(entry=entry).to(device)
assert cast(KVCacheEntry, cache["entry"]).device == device
assert cache.size() == 2 * 6 * 1 + 2 * 2 * 4