diff --git a/sdm/cache.py b/sdm/cache.py index 59c6c0db0..496788225 100644 --- a/sdm/cache.py +++ b/sdm/cache.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import abc from collections.abc import ( Callable, Hashable, @@ -40,6 +41,45 @@ def _apply_tensor(self, fn: Callable[[Tensor], Tensor]) -> Self: return self.__class__(key=fn(self.key), value=fn(self.value)) +class KVCacheDType(abc.ABC): + r"""Base class for the storage dtype of cached key/value projections.""" + + @abc.abstractmethod + def quantize(self, entry: KVCacheEntry) -> KVCacheEntry: + r"""Returns ``entry`` in this storage dtype. + + Args: + entry: Key/value projections as computed by attention. + """ + + @abc.abstractmethod + def dequantize( + self, + entry: KVCacheEntry, + dtype: torch.dtype, + ) -> KVCacheEntry: + r"""Returns ``entry`` with key/value projections in ``dtype``. + + Args: + entry: Key/value projections in this storage dtype. + dtype: The target dtype. + """ + + +class Unquantized(KVCacheDType): + r"""Stores key/value projections as computed.""" + + def quantize(self, entry: KVCacheEntry) -> KVCacheEntry: # noqa: D102 + return entry + + def dequantize( # noqa: D102 + self, + entry: KVCacheEntry, + dtype: torch.dtype, + ) -> KVCacheEntry: + return entry + + class Cache(MutableMapping[Hashable, object], DeviceMixin): r"""A mutable mapping of model cache values.""" @@ -65,6 +105,8 @@ def __init__( **kwargs: object, ) -> None: self._mode = Cache.Mode.record + # TODO: Support different dtypes given str in kwargs: + self._kv_cache_dtype: KVCacheDType = Unquantized() self._items: dict[Hashable, object] = dict(*args, **kwargs) @property @@ -107,6 +149,8 @@ def __setitem__(self, key: Hashable, value: object) -> None: raise RuntimeError( "'__setitem__' requires the cache to be in 'record' mode" ) + if isinstance(value, KVCacheEntry): + value = self._kv_cache_dtype.quantize(value) self._items[key] = value def __delitem__(self, key: Hashable) -> None: @@ -196,5 +240,6 @@ def _apply(value: object) -> object: out = self.__class__( {key: _apply(value) for key, value in self.items()} ) + out._kv_cache_dtype = self._kv_cache_dtype out._mode = self._mode return out