From 1fd74c856c14f4c83624e5749a8a4db9879105f8 Mon Sep 17 00:00:00 2001 From: Debasis Mandal Date: Wed, 29 Jul 2026 02:39:18 -0400 Subject: [PATCH] bench(rocm,rope): use a real cos/sin cache in bench_rope _shared_cos_sin_cache() built the cos||sin table with torch.randn, so values were unbounded and did not satisfy cos^2+sin^2=1. That made the --accuracy error numbers non-representative and let inplace runs drift, since repeated rotations were not norm-preserving. Build a proper RoPE cos||sin table instead (same construction as benchmarks/bench_rope.py / tests/test_helpers/rope_reference.py): entries are bounded in [-1, 1] and rotations are norm-preserving. Shape, dtype (float32) and ~32 MiB footprint are unchanged. Addresses Copilot review on PR #267 (discussion r3468044946). Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmarks/rocm_benchmarks/bench_rope.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/benchmarks/rocm_benchmarks/bench_rope.py b/benchmarks/rocm_benchmarks/bench_rope.py index bac8ae192b..2c1cfa584b 100644 --- a/benchmarks/rocm_benchmarks/bench_rope.py +++ b/benchmarks/rocm_benchmarks/bench_rope.py @@ -62,6 +62,7 @@ _HEAD_SIZE = 128 _ROTARY_DIM = 128 _MAX_SEQ_LEN = 65536 +_ROPE_THETA = 1e4 # RoPE base; standard Llama-family value. _DTYPES = [(torch.float16, "f16"), (torch.bfloat16, "bf16")] _BACKENDS = ["native", "aiter"] # Both op modes: inplace rotates q/k in place; out-of-place returns fresh @@ -75,14 +76,26 @@ def _shared_cos_sin_cache() -> torch.Tensor: + # A real RoPE cos||sin table (same construction as + # benchmarks/bench_rope.py / tests/test_helpers/rope_reference.py), not + # random values: entries are bounded in [-1, 1] and satisfy cos^2+sin^2=1, + # so rotations are norm-preserving. Random data would leave inplace runs to + # drift over repeated rotations and make the --accuracy numbers meaningless. # cos||sin cache is float32 on the HIP path (see rope_aiter.cu) and is # read-only, so a single ~32 MiB tensor is shared across all configs # rather than reallocated per config. global _COS_SIN_CACHE if _COS_SIN_CACHE is None: - _COS_SIN_CACHE = torch.randn( - _MAX_SEQ_LEN, _ROTARY_DIM, device="cuda", dtype=torch.float32 + inv_freq = 1.0 / ( + _ROPE_THETA + ** ( + torch.arange(0, _ROTARY_DIM, 2, device="cuda", dtype=torch.float32) + / _ROTARY_DIM + ) ) + t = torch.arange(_MAX_SEQ_LEN, device="cuda", dtype=torch.float32) + freqs = torch.einsum("i,j -> ij", t, inv_freq) + _COS_SIN_CACHE = torch.cat((freqs.cos(), freqs.sin()), dim=-1) return _COS_SIN_CACHE