From 3bfcb96c184b5358272a1f32f87a725e97db0620 Mon Sep 17 00:00:00 2001 From: Kislay Kishore Date: Wed, 30 Sep 2026 00:29:36 +0530 Subject: [PATCH] feat(lru): productionize v0.0.1 API, unify pressure reclamation, and harden engines - Standardize module/package as github.com/google/go-lru (package lru) with unified lru.New constructor, WithBackend selector, and StringValue, BytesValue, and SizedValue[T] wrappers. - Extract shared lock-free, reentrancy-safe, epoch-tracked pressureState (pressure.go) across MapCache, RadixCache, and ArenaRadixCache with 0-alloc pooled runtime/metrics sampling and unified Tier 1 compaction / Tier 2 LRU shedding. - Harden all three engines with substring key cloning, O(1)-stack iterative tree traversals, incremental FNV-1a subtree hashing, hash-collision nodeMap healing, and bidirectional UpdateSize bounds checks. - Add SHA-pinned GitHub Actions CI/benchmark workflows, golangci-lint config, architecture/performance docs, runnable examples, and comprehensive compaction, pressure, concurrency, and differential test suites. TAG=agy CONV=673fe62d-db71-4fe3-b421-d9b7ef6cce6e --- .github/workflows/benchmarks.yml | 95 + .github/workflows/ci.yml | 118 ++ .gitignore | 29 + .golangci.yml | 19 + CHANGELOG.md | 49 + CONTRIBUTING.md | 2 +- README.md | 266 +-- arena_radix.go | 379 ++-- arena_radix_lru.go | 441 +++-- arena_radix_test.go | 1846 ++++++++++--------- benchmarks_test.go | 332 ++-- cache.go | 43 +- compaction_test.go | 1515 ++++++++++++++++ concurrency_test.go | 307 ++-- differential_test.go | 313 +++- docs/architecture.md | 79 + docs/performance.md | 78 + errors.go | 2 +- example_test.go | 119 ++ go.mod | 2 +- map_lru.go | 469 +++-- map_lru_test.go | 907 ++++++---- options.go | 160 +- options_test.go | 532 ++++-- pressure.go | 550 ++++++ pressure_test.go | 2888 ++++++++++++++++++++++++++++++ radix_lru.go | 514 ++++-- radix_lru_test.go | 1046 ++++++----- values.go | 101 ++ values_test.go | 139 ++ 30 files changed, 10398 insertions(+), 2942 deletions(-) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .golangci.yml create mode 100644 CHANGELOG.md create mode 100644 compaction_test.go create mode 100644 docs/architecture.md create mode 100644 docs/performance.md create mode 100644 example_test.go create mode 100644 pressure.go create mode 100644 pressure_test.go create mode 100644 values.go create mode 100644 values_test.go diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000..812ae29 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,95 @@ +name: Benchmarks & Resource Usage + +on: + push: + branches: + - main + - master + - "feat/**" + pull_request: + branches: + - main + - master + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: benchmarks-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + benchmark: + name: Throughput, Memory & Arena Footprint + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout PR / head commit + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + + - name: Install benchstat + run: go install golang.org/x/perf/cmd/benchstat@latest + + - name: Run Performance & Memory Allocation Benchmarks (Head) + run: | + go test -run=^$ -bench=. -benchmem -benchtime=10ms -count=6 ./... | tee benchmark-head.txt + + - name: Run Base Branch Benchmarks for Regression Comparison (PR only) + if: github.event_name == 'pull_request' + continue-on-error: true + run: | + HEAD_SHA=$(git rev-parse HEAD) + trap 'git checkout "$HEAD_SHA"' EXIT + git checkout "${{ github.event.pull_request.base.sha }}" + go test -run=^$ -bench=. -benchmem -benchtime=10ms -count=6 ./... | tee benchmark-base.txt + git checkout "$HEAD_SHA" + trap - EXIT + sed -i 's|^pkg: .*|pkg: github.com/google/go-lru|' benchmark-base.txt + benchstat benchmark-base.txt benchmark-head.txt | tee benchstat-comparison.txt + + - name: Publish Benchmark & Resource Footprint Summary + run: | + { + echo "## 📊 Performance & Resource Usage Summary" + echo "" + echo "| Benchmark | Iterations | Throughput (\`ns/op\`) | Memory (\`B/op\`) | Allocations (\`allocs/op\`) | Additional Resource Metrics |" + echo "|---|---:|---:|---:|---:|---|" + awk '/^Benchmark_/ && !seen[$1]++ { + bench=$1; iters=$2; ns=$3 " " $4; + b_op="-"; allocs="-"; extra=""; + for (i=5; i<=NF; i++) { + if ($(i+1) == "B/op") { b_op = $i " B/op"; i++; } + else if ($(i+1) == "allocs/op") { allocs = $i " allocs/op"; i++; } + else if (i < NF) { extra = extra $i " " $(i+1) " "; i++; } + } + printf "| \`%s\` | %s | %s | %s | %s | %s |\n", bench, iters, ns, b_op, allocs, extra + }' benchmark-head.txt + echo "" + if [ -f benchstat-comparison.txt ]; then + echo "### 🔍 PR vs. Base Regression Comparison (\`benchstat\`)" + echo "" + echo '```text' + cat benchstat-comparison.txt + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Benchmark Artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: benchmark-results + path: | + benchmark-head.txt + benchmark-base.txt + benchstat-comparison.txt + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..41fe697 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,118 @@ +name: CI + +on: + push: + branches: + - main + - master + - "feat/**" + tags: + - "v*" + pull_request: + branches: + - main + - master + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + lint: + name: Format & Vet + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + + - name: Check formatting (gofmt -s) + run: | + UNFORMATTED=$(gofmt -s -l .) + if [ -n "$UNFORMATTED" ]; then + echo "The following files are not properly formatted with 'gofmt -s':" + echo "$UNFORMATTED" + exit 1 + fi + + - name: Check imports (goimports) + run: | + go install golang.org/x/tools/cmd/goimports@latest + UNFORMATTED_IMPORTS=$("$(go env GOPATH)/bin/goimports" -l .) + if [ -n "$UNFORMATTED_IMPORTS" ]; then + echo "The following files have unformatted imports ('goimports -l .'):" + echo "$UNFORMATTED_IMPORTS" + exit 1 + fi + + - name: Run go vet + run: go vet ./... + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@9fae48acfc02a90574d7c304a1758ef9895495fa # v7 + with: + version: v2.13.1 + + - name: Verify go.mod and go.sum are tidy + run: | + go mod tidy + git diff --exit-code go.mod go.sum + + test: + name: Correctness, Concurrency & Coverage + runs-on: ubuntu-latest + timeout-minutes: 15 + needs: lint + steps: + - name: Checkout repository + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + + - name: Run Unit, Differential, and Concurrency Tests with Race Detector & Coverage + run: go test -v -race -covermode=atomic -coverprofile=coverage.out ./... + + - name: Run Documentation Examples + run: go test -v -run=^Example ./... + + - name: Generate Code Coverage Report & Summary + run: | + go tool cover -func=coverage.out | tee coverage.txt + TOTAL_COV=$(tail -n 1 coverage.txt | awk '{print $3}') + { + echo "## ✅ Correctness, Concurrency & Coverage Summary" + echo "" + echo "- **Race Detector (\`-race\`)**: Passed" + echo "- **Unit, Differential & Concurrency Suites**: Passed" + echo "- **Total Statement Coverage**: **${TOTAL_COV}**" + echo "" + echo "
Per-Function Coverage Breakdown" + echo "" + echo '```text' + cat coverage.txt + echo '```' + echo "
" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: coverage-report + path: | + coverage.out + coverage.txt + retention-days: 14 diff --git a/.gitignore b/.gitignore index efc6c44..03cce68 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,32 @@ +# Compiled test binaries and build outputs *.test *.out +*.exe +*.dll +*.so +*.dylib + +# Coverage and profiling artifacts +coverage.txt +coverage.html +*.prof +*.pprof +benchmark-*.txt +benchstat-*.txt +bench*.txt + +# Agent workspaces and review/sentinel artifacts .agents/ +BRIEFING.md +BRIEFING_ARCHIVE.md +ORIGINAL_REQUEST.md +REVIEW_REPORT.md +handoff.md + +# Editor and OS scratch files +.DS_Store +.idea/ +.vscode/ +*.swp +*.swo +*~ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..8423b56 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,19 @@ +version: "2" + +linters: + default: standard + enable: + - errcheck + - gocritic + - govet + - ineffassign + - revive + - staticcheck + - testifylint + - thelper + - unused + +formatters: + enable: + - gofmt + - goimports diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..6fbbd36 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,49 @@ +# Changelog + +All notable changes to `github.com/google/go-lru` (`package lru`) are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + +## [0.0.1] - 2026-04-20 + +### Added +- **Standalone Module & Package Identity**: Published under `module github.com/google/go-lru` with idiomatic `package lru` import surface. +- **Unified Constructor & Configurable Backends**: + - `lru.New(maxSize uint64, opts ...Option) Cache` entry point supporting `lru.WithBackend(backend)` across `BackendMap` (default), `BackendRadix`, and `BackendArenaRadix`. + - Direct engine constructors `lru.NewMapCache`, `lru.NewRadixCache`, and `lru.NewArenaRadixCache`, adapted from Google Cloud Storage FUSE (`gcsfuse`) with differential state-machine tests, multi-goroutine race stress tests, and `WithInvariantChecking` runtime structural verification. +- **Ergonomic `ValueType` Wrappers (`values.go`)**: + - `StringValue` (`NewStringValue`), `BytesValue` (`NewBytesValue`), and generic `SizedValue[T]` (`NewSizedValue[T]`, `NewValue[T]`) so callers can cache `string`, `[]byte`, and arbitrary types without boilerplate struct definitions. +- **Two-Tier Memory-Pressure Reclamation (`PressureAwareCache`)**: + - All three backends (`MapCache`, `RadixCache`, and `ArenaRadixCache`) implement `PressureAwareCache` (`Compact()` and `EvaluateMemoryPressure()`). + - Configurable via `WithPressureFunc`, `WithMemoryBudget`, `WithCompactionThreshold` (Tier 1 lossless compaction), `WithEvictionThreshold` (Tier 2 proactive LRU tail shedding + compaction), `WithEvictionRetentionRatio`, and `DefaultRuntimePressureFunc`. +- **Documentation & Runnable Examples**: + - `example_test.go` with `pkg.go.dev` runnable examples (`ExampleNew`, `ExampleNew_eviction`, `ExampleNew_backendsAndPrefixErase`, `ExampleNew_memoryPressure`). + - Concise user guide in `README.md` backed by `docs/architecture.md` and `docs/performance.md`. +- **Automated GitHub Actions CI**: + - `.github/workflows/ci.yml`: formatting (`gofmt -s`, `goimports`), static analysis (`go vet`, `golangci-lint`), `go mod tidy` verification, `-race` unit/differential/concurrency tests, runnable examples, and statement coverage reporting. + - `.github/workflows/benchmarks.yml`: automated performance and resource footprint benchmarking (`ns/op`, `B/op`, `allocs/op`, `heap-B/entry`, `reclaimed-B/op`) with `benchstat` PR regression comparison. + +--- + +## Release & Versioning Workflow + +To cut a new semantic version release (e.g. `v0.0.1`): + +```bash +# 1. Verify formatting, static analysis, race-enabled test suite, and examples +test -z "$(gofmt -s -l .)" +test -z "$(goimports -l .)" +go vet ./... +golangci-lint run +go test -race ./... +go test -v -run=^Example ./... + +# 2. Verify benchmark & resource usage suite +go test -run=^$ -bench=. -benchmem -benchtime=10ms ./... + +# 3. Create and push annotated semantic version tag +git tag -a v0.0.1 -m "Release v0.0.1" +git push origin --tags +``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fa69ea3..654a071 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,4 +25,4 @@ information on using pull requests. ## Community Guidelines This project follows [Google's Open Source Community -Guidelines](https://opensource.google/conduct/). \ No newline at end of file +Guidelines](https://opensource.google/conduct/). diff --git a/README.md b/README.md index 19ba706..794f686 100644 --- a/README.md +++ b/README.md @@ -1,255 +1,113 @@ -# High-Performance Go LRU Cache Suite +# go-lru -[![Go Reference](https://pkg.go.dev/badge/github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru.svg)](https://pkg.go.dev/github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru) +[![Go Reference](https://pkg.go.dev/badge/github.com/google/go-lru.svg)](https://pkg.go.dev/github.com/google/go-lru) +[![CI](https://github.com/google/go-lru/actions/workflows/ci.yml/badge.svg)](https://github.com/google/go-lru/actions/workflows/ci.yml) +[![Benchmarks](https://github.com/google/go-lru/actions/workflows/benchmarks.yml/badge.svg)](https://github.com/google/go-lru/actions/workflows/benchmarks.yml) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](LICENSE) [![Zero Dependencies](https://img.shields.io/badge/dependencies-zero-brightgreen.svg)]() -`lrus` is a standalone, high-concurrency, zero-dependency Go library providing three specialized Least Recently Used (LRU) cache engines satisfying a unified interface. Directly adapted and evolved from Google Cloud Storage FUSE ([GCSFuse](https://github.com/GoogleCloudPlatform/gcsfuse)), `lrus` delivers production-grade caching optimized for diverse workloads: flat key-value lookups, hierarchical file system trees, and massive scale in-memory caches. +`package lru` (`github.com/google/go-lru`) is a high-concurrency, zero-dependency Go library providing size-aware Least Recently Used (LRU) cache implementations behind a unified `Cache` interface. Adapted and evolved from Google Cloud Storage FUSE ([GCSFuse](https://github.com/GoogleCloudPlatform/gcsfuse)), `go-lru` supports flat key-value caching (`MapCache`), hierarchical prefix-eviction trees (`RadixCache`), and arena-allocated `uint32`-indexed trees with two-tier memory-pressure reclamation (`ArenaRadixCache`). --- -## Key Features +## Installation -- **Zero External Dependencies**: Implemented strictly using the Go standard library (`sync`, `container/list`, `math`, `strings`). -- **Unified Interface**: Seamlessly switch between cache implementations (`MapCache`, `RadixCache`, `ArenaRadixCache`) without changing application logic. -- **Size-Aware Eviction**: Eviction is driven by actual byte or logical size (`ValueType.Size()`), not just entry count. -- **Subtree Prefix Eviction**: `O(prefix_len + subtree_size)` prefix deletion via radix tree engines—up to **20x–70x faster** than map scans. -- **Memory Compactness**: Radix engines reduce heap footprint by **20% to 31%** compared to standard map-based LRUs. -- **Zero GC Pointer Scanning**: The arena-backed engine allocates nodes in contiguous slice memory with 32-bit indices, eliminating garbage collector pointer graph traversals. -- **Concurrent Safety**: Full multi-goroutine thread safety protected by standard synchronization (`sync.RWMutex` / `sync.Mutex`). -- **Invariant Checking**: Configurable debug integrity checks validate tree consistency, bidirectional pointers, and size parity. - ---- - -## Architectural Comparison Matrix - -| Dimension | `MapCache` (`NewMapCache`) | `RadixCache` (`NewRadixCache`) | `ArenaRadixCache` (`NewArenaRadixCache`) | -| :--- | :--- | :--- | :--- | -| **Primary Data Structure** | `map[string]*list.Element` + `container/list.List` | Compressed Radix Tree (LCRS) + Intrusive Pointer LRU List | Contiguous Slice Arena `[]arenaRadixNode` + Freelist + Hash Index | -| **Node Representation** | `list.Element` (48 B) + `entry` (32 B) + Map Buckets | `radixNode` struct (72 B heap object) | `arenaRadixNode` struct (56 B contiguous array entry) | -| **Per-Node Heap Allocations** | 2–3 allocations per entry | 1 allocation per node | **0 allocations** (recycled via free-list) | -| **Node Pointer Width** | 64-bit pointers | 64-bit pointers (5 pointers / node) | **32-bit indices** (`uint32`) | -| **Max Entry Capacity** | Memory / Heap limited | Memory / Heap limited | **`2^32 - 1` entries** (~4.29 billion nodes) | -| **Point Lookup Latency** | **~50 ns/op** (`O(1)` Hash Map) | ~170 ns/op (`O(K)` Trie Descent) | **~100 ns/op** (`O(1)` Hash Map + Zero-Alloc Path Check) | -| **Sequential Insert Latency** | ~98 ns/op | ~175 ns/op (0 allocs) | ~270 ns/op | -| **Point Update Latency** | ~82 ns/op | ~97 ns/op (0 allocs) | ~88 ns/op (0 allocs) | -| **Prefix Erase (100 items)** | ~160 µs (`O(N)` full scan) | **~2.2 µs** (**71x faster**) | **~27.6 µs** (**6x faster**) | -| **Prefix Erase (50K/100K items)** | ~21.6 ms | **~1.05 ms** (**20.5x faster**) | ~13.6 ms | -| **1M Items Heap Memory** | ~129.6 MB (~135.9 B/entry) | **~89.8 MB** (**~30.7% reduction**) | **~101.9 MB** (**~21.4% reduction**) | -| **GC Pressure & Overhead** | High (millions of distinct heap objects) | Moderate (heap nodes with pointers) | **Ultra-Low** (flat slice; indices invisible to GC) | -| **Concurrency Lock** | `sync.RWMutex` | `sync.RWMutex` | `sync.Mutex` | -| **Best Used For** | Flat keys, maximum read/write throughput | File systems, directory trees, prefix purges | Large-scale hierarchical caches (1M+ items) | - ---- - -## Engine Deep Dive - -### 1. `MapCache` (Standard Map LRU) -The classic LRU implementation combining Go's built-in `map[string]*list.Element` with `container/list.List`. -- **Strengths**: Fastest point lookups (~50 ns) and inserts (~98 ns). -- **Trade-offs**: Prefix deletion requires a linear `O(N)` scan across all map keys. Highest heap overhead per entry (~136 B/entry). - -### 2. `RadixCache` (Pointer-Based Radix Tree LRU) -A compact Radix Tree with Left-Child Right-Sibling (LCRS) tree layout and embedded intrusive doubly-linked list pointers (`prev`, `next`, `parent`, `child`, `sibling`). -- **Strengths**: Subtree detachment allows deleting entire directory subtrees in microseconds. Uses ~30.7% less memory than `MapCache` due to prefix path compression. Zero-allocation node updates. -- **Trade-offs**: Point lookups require traversing tree edges (`O(K)` where `K` is key length). - -### 3. `ArenaRadixCache` (Flat-Slice Arena Radix Tree LRU) -An arena-allocated radix tree storing all nodes in a contiguous slice `[]arenaRadixNode` indexed by 32-bit integers (`uint32`). Integrates an `O(1)` FNV-1a hash map accelerator with zero-allocation bottom-up key verification (`verifyKey`). -- **Strengths**: Constant-time lookup acceleration (~100 ns). Zero per-node heap allocations after warm-up via `O(1)` singly-linked `freeHead` free-list reuse. Zero garbage collector pointer tracing overhead. -- **Trade-offs**: Slightly higher insert latency due to hash table maintenance and array index lookups. +```bash +go get github.com/google/go-lru +``` --- ## Quickstart -### Installation - -```bash -go get github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru -``` - -### Implementing `ValueType` - -Any cached item must implement the `lrus.ValueType` interface by exposing `Size() uint64`: +Use `lru.New` with built-in `ValueType` wrappers (`lru.StringValue`, `lru.NewBytesValue`, `lru.NewValue`)—no custom struct boilerplate required: ```go package main import ( "fmt" - "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" -) -type FileMetadata struct { - Inode uint64 - ByteSize uint64 -} - -func (f *FileMetadata) Size() uint64 { - return f.ByteSize -} -``` - -### Basic Usage + "github.com/google/go-lru" +) -```go func main() { - // Create a Radix LRU cache with a 100 MB capacity - cache := lrus.NewRadixCache(100 * 1024 * 1024) - - // Insert entries - file := &FileMetadata{Inode: 1001, ByteSize: 4096} - evicted, err := cache.Insert("projects/app/logs/server.log", file) - if err != nil { - panic(err) + // 1. Initialize a 64 MiB LRU cache (defaults to MapCache; use WithBackend for Radix/Arena). + cache := lru.New( + 64*1024*1024, + lru.WithBackend(lru.BackendArenaRadix), + ) + + // 2. Insert common types directly using built-in ValueType helpers. + _, _ = cache.Insert("bucket/dirA/config.json", lru.StringValue(`{"version":1}`)) + _, _ = cache.Insert("bucket/dirA/chunk.bin", lru.NewBytesValue([]byte{0xDE, 0xAD, 0xBE, 0xEF})) + _, _ = cache.Insert("bucket/dirB/inode-42", lru.NewValue(uint64(42), 128)) + + // 3. Point Lookup (promotes entry to MRU). + if v := cache.LookUp("bucket/dirA/config.json"); v != nil { + fmt.Printf("config: %s (%d bytes)\n", v.(lru.StringValue), v.Size()) } - fmt.Printf("Inserted entry; evicted %d items\n", len(evicted)) - - // Point Lookup (updates LRU order) - if val := cache.LookUp("projects/app/logs/server.log"); val != nil { - meta := val.(*FileMetadata) - fmt.Printf("Found inode: %d\n", meta.Inode) - } - - // Read without altering LRU order - _ = cache.LookUpWithoutChangingOrder("projects/app/logs/server.log") - - // Update existing value payload without modifying LRU order - newFile := &FileMetadata{Inode: 1001, ByteSize: 4096} - _ = cache.UpdateWithoutChangingOrder("projects/app/logs/server.log", newFile) - // Increment size for growing files (evicts LRU items if capacity exceeded) - _ = cache.UpdateSize("projects/app/logs/server.log", 1024) + // 4. Inspect or update without altering LRU recency order. + _ = cache.LookUpWithoutChangingOrder("bucket/dirB/inode-42") + _ = cache.UpdateWithoutChangingOrder("bucket/dirB/inode-42", lru.NewValue(uint64(43), 128)) - // Subtree Prefix Deletion: instantaneous purging of all logs under projects/app/ - cache.EraseEntriesWithGivenPrefix("projects/app/logs/") + // 5. Incrementally grow entry size (automatically evicts LRU entries if capacity is exceeded). + _ = cache.UpdateSize("bucket/dirA/chunk.bin", 4096) - // Individual Erase - _ = cache.Erase("projects/app/logs/server.log") + // 6. Fast O(prefix + subtree) prefix eviction and individual key erasure. + cache.EraseEntriesWithGivenPrefix("bucket/dirA/") + _ = cache.Erase("bucket/dirB/inode-42") } ``` ---- - -## Unified `Cache` Interface - -```go -type Cache interface { - // Insert inserts or updates a key-value entry in the cache. - // Returns evicted entries if capacity was exceeded, or an error. - Insert(key string, value ValueType) ([]ValueType, error) - - // Erase removes the entry associated with key, returning its value (or nil). - Erase(key string) (value ValueType) - - // LookUp retrieves the value for key and updates its LRU position. - LookUp(key string) (value ValueType) - - // LookUpWithoutChangingOrder retrieves the value for key without modifying LRU order. - LookUpWithoutChangingOrder(key string) (value ValueType) - - // UpdateWithoutChangingOrder updates the value of an existing key without modifying LRU order. - UpdateWithoutChangingOrder(key string, value ValueType) error - - // UpdateSize updates the size of an existing key by sizeDelta and evicts excess if needed. - UpdateSize(key string, sizeDelta uint64) error - - // EraseEntriesWithGivenPrefix deletes all entries whose keys start with prefix. - EraseEntriesWithGivenPrefix(prefix string) -} -``` +Custom domain types can also implement `lru.ValueType` directly by defining `Size() uint64`. --- -## Configuration & Invariant Checking +## Choosing a Cache Backend -You can configure internal debugging and data integrity invariants via functional options: - -```go -// Create a MapCache with runtime invariant verification enabled -cache := lrus.NewMapCache(1000000, lrus.WithInvariantChecking(true)) -``` +Select an engine via `lru.New(maxSize, lru.WithBackend(...))` or call its dedicated constructor directly: -When `WithInvariantChecking(true)` is enabled: -- **`MapCache`**: Verifies `len(map) == entries.Len()`, total aggregated size matches `currentSize <= maxSize`, and bidirectional linked list pointer bijection. -- **`RadixCache`**: Executes an `O(1)`-space non-recursive tree walk validating pre-order LCRS child/sibling links, LRU linked list consistency, and size accounting. -- **`ArenaRadixCache`**: Validates 32-bit array index references, free-list integrity, hash index 1:1 mapping, and tree-to-LRU bijection. +| Backend | Constructor / Option | Lookup / Insert | Prefix Erase (`EraseEntriesWithGivenPrefix`) | Memory & GC Profile | Best Workload Fit | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **`MapCache`** *(default)* | `lru.NewMapCache` / `WithBackend(BackendMap)` | `O(1)` (~50 ns / ~98 ns in-place, ~383–466 ns turnover) | `O(N)` full map scan | ~136–163 B/entry, standard GC pointers | Flat keys, maximum point read/write throughput | +| **`RadixCache`** | `lru.NewRadixCache` / `WithBackend(BackendRadix)` | `O(K)` (~170 ns / ~172–230 ns in-place, ~472–511 ns turnover) | `O(P + S)` subtree (**20x–88x faster**) | ~94–96 B/entry (**~30–41% less heap**), 0-alloc updates | File paths, object storage namespaces, frequent prefix purges | +| **`ArenaRadixCache`** | `lru.NewArenaRadixCache` / `WithBackend(BackendArenaRadix)` | `O(1)` hash-accelerated (~100 ns / ~271–472 ns in-place, ~515–822 ns turnover) | `O(P + S)` subtree (**4x–29x faster**) | ~107–111 B/entry, `uint32` slice indices, **two-tier pressure compaction** | 1M+ hierarchical entries, strict GC latency & `GOMEMLIMIT` budgets | -*Note: Invariant validation is designed for testing/debugging and should be disabled (`false`, the default) in production for maximum throughput.* +All three backends (`MapCache`, `RadixCache`, and `ArenaRadixCache`) also implement `lru.PressureAwareCache`, exposing `Compact()` and `EvaluateMemoryPressure()`. --- -## Empirical Benchmark Results +## Configuration Options -Benchmarks executed on an **Intel Xeon CPU @ 2.60GHz (96 cores)**: - -### 1. Point Operation Latency & Throughput - -| Operation | `MapCache` | `RadixCache` | `ArenaRadixCache` | +| Option | Default | Applies To | Description | | :--- | :--- | :--- | :--- | -| **Lookup (Flat)** | **50.6 ns/op** (0 B, 0 allocs) | 174.0 ns/op (0 B, 0 allocs) | 100.7 ns/op (0 B, 0 allocs) | -| **Lookup (Nested)** | **49.5 ns/op** (0 B, 0 allocs) | 169.6 ns/op (0 B, 0 allocs) | 107.8 ns/op (0 B, 0 allocs) | -| **Lookup (Deeply Nested)** | **64.3 ns/op** (0 B, 0 allocs) | 185.2 ns/op (0 B, 0 allocs) | 194.1 ns/op (0 B, 0 allocs) | -| **Lookup (No Order Change)** | **33.2 ns/op** (0 B, 0 allocs) | 163.3 ns/op (0 B, 0 allocs) | 103.8 ns/op (0 B, 0 allocs) | -| **Insert (Flat)** | **97.7 ns/op** (32 B, 1 alloc) | 178.2 ns/op (0 B, 0 allocs) | 270.6 ns/op (1 B, 0 allocs) | -| **Insert (Nested)** | **98.1 ns/op** (32 B, 1 alloc) | 172.3 ns/op (0 B, 0 allocs) | 270.9 ns/op (1 B, 0 allocs) | -| **Insert (Deeply Nested)** | **119.0 ns/op** (32 B, 1 alloc) | 229.9 ns/op (0 B, 0 allocs) | 471.8 ns/op (2 B, 0 allocs) | -| **Update Value** | **82.5 ns/op** (32 B, 1 alloc) | 97.3 ns/op (0 B, 0 allocs) | 88.6 ns/op (0 B, 0 allocs) | -| **Individual Erase** | 237.9 ns/op (0 B, 0 allocs) | **147.7 ns/op** (0 B, 0 allocs) | 310.6 ns/op (0 B, 0 allocs) | - -### 2. Prefix Deletion Latency (Subtree Eviction) - -| Prefix Topology | `MapCache` (`O(N)` Scan) | `RadixCache` (`O(P + S)` Subtree) | Speedup vs Map | -| :--- | :--- | :--- | :--- | -| **Flat Prefix (100 items)** | 160.6 µs/op | **2.25 µs/op** | **71.3x faster** | -| **Nested Prefix (100 items)** | 151.2 µs/op | **2.30 µs/op** | **65.7x faster** | -| **Deeply Nested (100 items)** | 219.9 µs/op | **2.50 µs/op** | **88.0x faster** | -| **100K Scale (50K purged items)** | 21.6 ms/op | **1.05 ms/op** | **20.5x faster** | - -### 3. True Heap Memory Footprint (1,000,000 Keys) - -Measured via `runtime.ReadMemStats` with isolated GC sweep (`debug.FreeOSMemory()`): - -| Topology (1M Items) | `MapCache` Heap | `RadixCache` Heap | `ArenaRadixCache` Heap | Memory Reduction | -| :--- | :--- | :--- | :--- | :--- | -| **Flat** (`file_%d.txt`) | 129.6 MB (135.9 B/entry) | **89.8 MB** (94.2 B/entry) | 102.0 MB (106.9 B/entry) | **30.7% less memory** | -| **Nested** (`dir_%04d/file_%04d.txt`) | 129.6 MB (135.9 B/entry) | **90.5 MB** (94.9 B/entry) | 101.9 MB (106.9 B/entry) | **30.2% less memory** | -| **Deeply Nested** (`projects/...`) | 129.6 MB (135.9 B/entry) | **90.9 MB** (95.4 B/entry) | 101.9 MB (106.9 B/entry) | **29.9% less memory** | +| `WithBackend(backend)` | `BackendMap` | `lru.New` | Selects underlying cache engine (`BackendMap`, `BackendRadix`, `BackendArenaRadix`) | +| `WithInvariantChecking(bool)` | `false` | All engines | Enables runtime structural & size parity invariant verification (for tests/debugging) | +| `WithMemoryBudget(bytes)` | `0` (`GOMEMLIMIT`) | All engines | Explicit memory ceiling in bytes for the built-in `runtime/metrics` pressure probe | +| `WithPressureFunc(fn)` | `DefaultRuntimePressureFunc` | All engines | Custom callback `func() float64` returning normalized memory pressure in `[0.0, 1.0+]` | +| `WithCompactionThreshold(float64)` | `0.75` (`DefaultCompactionThreshold`) | All engines | Tier 1 Moderate Pressure threshold triggering lossless backing structure & hash map compaction | +| `WithEvictionThreshold(float64)` | `0.90` (`DefaultEvictionThreshold`) | All engines | Tier 2 Critical Pressure threshold triggering proactive LRU tail shedding + compaction | +| `WithEvictionRetentionRatio(float64)` | `0.50` (`DefaultEvictionRetentionRatio`) | All engines | Target fraction `[0.0, 1.0]` of `maxSize` retained during Tier 2 Critical Pressure shedding | --- -## Testing & Verification +## Documentation & Verification -Run the entire test suite including functional tests, parameterized tests, and high-concurrency race stress tests: +- **[Architecture & Memory-Pressure Reclamation (`docs/architecture.md`)](docs/architecture.md)**: Deep dive into `MapCache`, `RadixCache`, and `ArenaRadixCache` node layouts, 32-bit slice arena indexing, FNV-1a lookup acceleration, and Two-Tier Memory-Pressure Reclamation. +- **[Benchmarks & Heap Footprint (`docs/performance.md`)](docs/performance.md)**: Empirical latency, prefix deletion speedups, 1M-key true heap footprint tables, and CLI reproduction commands. +- **[Changelog & Versioning (`CHANGELOG.md`)](CHANGELOG.md)**: Release history and semantic versioning guide. ```bash -# Run all tests -go test -v ./... - -# Run race detector stress tests (0 data races) -go test -v -race -run=TestConcurrency_ ./... - -# Run static analysis -go vet ./... - -# Run memory footprint profiling -go test -v -run=TestMemoryFootprint_ ./... - -# Run full benchmark suite -go test -bench=. -benchmem -run=^$ ./... +# Run unit, differential, concurrency (-race), and runnable Example tests +go test -race ./... +go test -v -run=^Example ./... ``` --- -## License - -`lrus` is released under the Apache 2.0 License. See [LICENSE](LICENSE) for details. -Originally developed by Google LLC as part of [GoogleCloudPlatform/gcsfuse](https://github.com/GoogleCloudPlatform/gcsfuse). - ---- - -## Disclaimer - -This is not an officially supported Google product. This project is not eligible for the [Google Open Source Software Vulnerability Rewards Program](https://bughunters.google.com/open-source-security) +## License & Disclaimer +Released under the [Apache 2.0 License](LICENSE). See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution guidelines. +This is not an officially supported Google product. This project is not eligible for the [Google Open Source Software Vulnerability Rewards Program](https://bughunters.google.com/open-source-security). diff --git a/arena_radix.go b/arena_radix.go index b299a21..b065eb5 100644 --- a/arena_radix.go +++ b/arena_radix.go @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus +package lru import ( "math" "strings" "sync" - "sync/atomic" ) // nilNode represents the sentinel null reference for 32-bit node indices. @@ -45,13 +44,13 @@ type arenaRadix struct { maxSize uint64 currentSize uint64 mu sync.RWMutex - options Options nodes []arenaRadixNode freeHead uint32 freeCount uint32 - nodeMap map[uint64]uint32 + nodeMap map[uint64]uint32 + nodeMapDirty bool root uint32 @@ -60,11 +59,7 @@ type arenaRadix struct { len int - samplingPressure atomic.Bool - pressureNeedsRefresh atomic.Bool - pressureSampleSeq atomic.Uint64 - cachedPressureBits atomic.Uint64 - reclaimEpoch atomic.Uint64 + pressureState } // FNV-1a 64-bit hashing constants. @@ -75,7 +70,11 @@ const ( // hashString computes the 64-bit FNV-1a hash of string s. func hashString(s string) uint64 { - h := offset64 + return hashStringCont(offset64, s) +} + +// hashStringCont continues a 64-bit FNV-1a hash calculation from h over string s. +func hashStringCont(h uint64, s string) uint64 { for i := 0; i < len(s); i++ { h ^= uint64(s[i]) h *= prime64 @@ -84,7 +83,8 @@ func hashString(s string) uint64 { } // hashNodeKey computes the 64-bit FNV-1a hash of the full key for nodeID -// by walking the ancestor chain with a stack-allocated buffer (0 heap allocations). +// by walking the ancestor chain. Uses a 64-level stack buffer for common depths +// and seamlessly handles arbitrary depths > 64. func (c *arenaRadix) hashNodeKey(nodeID uint32) uint64 { var stackBuf [64]uint32 path := stackBuf[:0] @@ -93,24 +93,11 @@ func (c *arenaRadix) hashNodeKey(nodeID uint32) uint64 { } h := offset64 for i := len(path) - 1; i >= 0; i-- { - prefix := c.nodes[path[i]].prefix - for j := 0; j < len(prefix); j++ { - h ^= uint64(prefix[j]) - h *= prime64 - } + h = hashStringCont(h, c.nodes[path[i]].prefix) } return h } -// longestCommonPrefix finds the length of the longest common prefix of a and b. -func (c *arenaRadix) longestCommonPrefix(a, b string) int { - i := 0 - for i < len(a) && i < len(b) && a[i] == b[i] { - i++ - } - return i -} - // allocateNode allocates a node index from the free-list or appends to the arena slice. func (c *arenaRadix) allocateNode() uint32 { if c.freeHead != nilNode { @@ -127,7 +114,7 @@ func (c *arenaRadix) allocateNode() uint32 { return id } id := uint32(len(c.nodes)) - if id >= nilNode { + if id >= foregroundNoProtect { panic("arena radix capacity exceeded limit") } c.nodes = append(c.nodes, arenaRadixNode{ @@ -203,6 +190,7 @@ func (c *arenaRadix) replaceChild(nID uint32, oldChildID uint32, newChildID uint if *pcurr != oldChildID { continue } + c.nodes[newChildID].parent = nID c.nodes[newChildID].sibling = c.nodes[oldChildID].sibling *pcurr = newChildID c.nodes[oldChildID].sibling = nilNode @@ -212,10 +200,10 @@ func (c *arenaRadix) replaceChild(nID uint32, oldChildID uint32, newChildID uint panic("replaceChild: requested child not found in sibling list") } -// insertNode inserts key-value into the radix trie and returns the node index and any previous value. -func (c *arenaRadix) insertNode(key string, value ValueType) (uint32, ValueType) { +// insertNode inserts key-value into the radix trie and returns the node index. +func (c *arenaRadix) insertNode(key string, value ValueType) uint32 { if value == nil { - return nilNode, nil + return nilNode } nodeID := c.root @@ -223,21 +211,20 @@ func (c *arenaRadix) insertNode(key string, value ValueType) (uint32, ValueType) for { if len(search) == 0 { - oldValue := c.nodes[nodeID].value c.nodes[nodeID].value = value - return nodeID, oldValue + return nodeID } childID := c.getChild(nodeID, search[0]) if childID == nilNode { newLeafID := c.allocateNode() - c.nodes[newLeafID].prefix = strings.Clone(search) + c.nodes[newLeafID].prefix = clonePrefix(search) c.nodes[newLeafID].value = value c.addChild(nodeID, newLeafID) - return newLeafID, nil + return newLeafID } - lcp := c.longestCommonPrefix(search, c.nodes[childID].prefix) + lcp := longestCommonPrefix(search, c.nodes[childID].prefix) if lcp == len(c.nodes[childID].prefix) { search = search[lcp:] @@ -245,26 +232,28 @@ func (c *arenaRadix) insertNode(key string, value ValueType) (uint32, ValueType) continue } + // Clone both split prefix halves so surviving intermediate routing nodes never pin + // the underlying backing arrays of large evicted leaf keys. + oldPrefix := c.nodes[childID].prefix splitNodeID := c.allocateNode() - c.nodes[splitNodeID].prefix = strings.Clone(c.nodes[childID].prefix[:lcp]) + c.nodes[splitNodeID].prefix = clonePrefix(oldPrefix[:lcp]) c.nodes[splitNodeID].parent = nodeID c.replaceChild(nodeID, childID, splitNodeID) - c.nodes[childID].prefix = strings.Clone(c.nodes[childID].prefix[lcp:]) + c.nodes[childID].prefix = clonePrefix(oldPrefix[lcp:]) c.addChild(splitNodeID, childID) if lcp == len(search) { - oldValue := c.nodes[splitNodeID].value c.nodes[splitNodeID].value = value - return splitNodeID, oldValue + return splitNodeID } newLeafID := c.allocateNode() - c.nodes[newLeafID].prefix = strings.Clone(search[lcp:]) + c.nodes[newLeafID].prefix = clonePrefix(search[lcp:]) c.nodes[newLeafID].value = value c.addChild(splitNodeID, newLeafID) - return newLeafID, nil + return newLeafID } } @@ -290,16 +279,7 @@ func (c *arenaRadix) verifyKey(nodeID uint32, key string) bool { return end == 0 && curr == c.root } -// getNodeKey finds a value-bearing node index for key using O(1) FNV-1a hash lookup with zero-alloc verification, -// falling back to top-down trie traversal. -func (c *arenaRadix) getNodeKey(key string) (uint32, bool) { - nodeID, ok := c.nodeMap[hashString(key)] - if ok && c.nodes[nodeID].value != nil { - if c.verifyKey(nodeID, key) { - return nodeID, true - } - } - +func (c *arenaRadix) findNodeByTrieWalk(key string) (uint32, bool) { curr := c.root search := key @@ -328,6 +308,52 @@ func (c *arenaRadix) getNodeKey(key string) (uint32, bool) { return nilNode, false } +// getNodeKey finds a value-bearing node index for key using O(1) FNV-1a hash lookup with zero-alloc verification, +// falling back to top-down trie traversal only when a hash collision or dirty nodeMap state exists. +// It does not mutate c.nodeMap, making it safe under RLock. +func (c *arenaRadix) getNodeKey(key string) (uint32, bool) { + return c.lookupNodeKeyWithHash(key, hashString(key)) +} + +// lookupNodeKeyWithHash finds a value-bearing node index for key using a precomputed FNV-1a hash +// without mutating c.nodeMap. +func (c *arenaRadix) lookupNodeKeyWithHash(key string, keyHash uint64) (uint32, bool) { + nodeID, ok := c.nodeMap[keyHash] + if ok { + if nodeID < uint32(len(c.nodes)) && c.nodes[nodeID].value != nil && c.verifyKey(nodeID, key) { + return nodeID, true + } + } else if len(c.nodeMap) == c.len { + // By Invariant 6 and the Pigeonhole Principle, when len(c.nodeMap) == c.len, + // there is an exact 1:1 bijection between c.nodeMap and all live value-bearing nodes, + // so a map miss is a guaranteed cache miss. + return nilNode, false + } + return c.findNodeByTrieWalk(key) +} + +// getNodeKeyWithHash finds a value-bearing node index for key under exclusive write lock, +// healing c.nodeMap[keyHash] if resolved via the slow-path trie walk. +func (c *arenaRadix) getNodeKeyWithHash(key string, keyHash uint64) (uint32, bool) { + nodeID, ok := c.nodeMap[keyHash] + if ok { + if nodeID < uint32(len(c.nodes)) && c.nodes[nodeID].value != nil && c.verifyKey(nodeID, key) { + return nodeID, true + } + } else if len(c.nodeMap) == c.len { + // By Invariant 6 and the Pigeonhole Principle, when len(c.nodeMap) == c.len, + // there is an exact 1:1 bijection between c.nodeMap and all live value-bearing nodes, + // so a map miss is a guaranteed cache miss. + return nilNode, false + } + + curr, found := c.findNodeByTrieWalk(key) + if found { + c.nodeMap[keyHash] = curr + } + return curr, found +} + // deleteNode clears the value at nodeID and compresses the parent path if needed. func (c *arenaRadix) deleteNode(nodeID uint32) { if nodeID == nilNode || c.nodes[nodeID].value == nil { @@ -362,8 +388,7 @@ func (c *arenaRadix) compressPathUpwards(currID uint32) { c.replaceChild(parentID, currID, onlyChildID) c.freeNode(currID) - currID = parentID - continue + return } break } @@ -441,15 +466,27 @@ func (c *arenaRadix) evictOne() ValueType { return c.eraseInternal(nodeID) } +const foregroundNoProtect uint32 = nilNode - 1 + +func (c *arenaRadix) isDirtyLocked() bool { + return c.freeHead != nilNode || len(c.nodes) < cap(c.nodes) || c.nodeMapDirty +} + // eraseInternal handles unlinking from LRU, cleaning up nodeMap, and deleting from the tree. func (c *arenaRadix) eraseInternal(nodeID uint32) ValueType { + return c.eraseInternalWithHash(nodeID, c.hashNodeKey(nodeID)) +} + +func (c *arenaRadix) eraseInternalWithHash(nodeID uint32, hash uint64) ValueType { deletedEntry := c.nodes[nodeID].value + if deletedEntry != nil { + c.onEntryDeleted(c.nodes[nodeID].size) + c.nodeMapDirty = true + } c.currentSize -= c.nodes[nodeID].size c.nodes[nodeID].size = 0 - // Prevent hash collision cross-deletions with 0 heap string allocations. - hash := c.hashNodeKey(nodeID) - if c.nodeMap[hash] == nodeID { + if mappedID, ok := c.nodeMap[hash]; ok && mappedID == nodeID { delete(c.nodeMap, hash) } @@ -459,88 +496,71 @@ func (c *arenaRadix) eraseInternal(nodeID uint32) ValueType { return deletedEntry } -// samplePressureFresh reads normalized memory pressure lock-free outside c.mu.Lock() -// with an atomic re-entrancy guard that prevents infinite mutual recursion if a -// custom PressureFunc calls back into cache methods, while returning the latest -// cached pressure reading to concurrent goroutines instead of a false 0.0. -func (c *arenaRadix) samplePressureFresh() float64 { - if !c.samplingPressure.CompareAndSwap(false, true) { - return math.Float64frombits(c.cachedPressureBits.Load()) - } - defer c.samplingPressure.Store(false) - - p := c.options.PressureFunc() - if math.IsNaN(p) || p < 0.0 { - p = 0.0 - } - c.cachedPressureBits.Store(math.Float64bits(p)) - c.pressureNeedsRefresh.Store(false) - return p -} +func (c *arenaRadix) clearEmptyArenaStateLocked() { + c.freeHead = nilNode + c.freeCount = 0 -// samplePressure returns the current memory pressure for foreground cache operations. -// Custom PressureFunc callbacks are invoked on every call; the default runtime/metrics -// probe is amortized across a 256-operation window (and immediately refreshed after any -// reclamation/compaction cycle) to eliminate runtime.metricsLock contention on hot-path writes. -func (c *arenaRadix) samplePressure() float64 { - if c.options.hasCustomPressureFunc { - return c.samplePressureFresh() - } - seq := c.pressureSampleSeq.Add(1) - if (seq&255) == 1 || c.pressureNeedsRefresh.CompareAndSwap(true, false) { - return c.samplePressureFresh() + if cap(c.nodes) == 1 { + c.nodes = c.nodes[:1] + c.nodes[0] = arenaRadixNode{parent: nilNode, child: nilNode, sibling: nilNode, prev: nilNode, next: nilNode} + c.root = 0 + } else { + clear(c.nodes) + c.nodes = make([]arenaRadixNode, 1) + c.nodes[0] = arenaRadixNode{parent: nilNode, child: nilNode, sibling: nilNode, prev: nilNode, next: nilNode} + c.root = 0 + } + + c.head = nilNode + c.tail = nilNode + c.currentSize = 0 + c.len = 0 + if c.nodeMap != nil && c.peakEntryLen <= 8 && c.deletedSinceCompact < 64 { + clear(c.nodeMap) + } else { + c.nodeMap = make(map[uint64]uint32) } - return math.Float64frombits(c.cachedPressureBits.Load()) + c.nodeMapDirty = false + c.resetWatermarks() } -// computeTargetSize safely computes uint64(float64(maxSize) * retention) without -// float64-to-uint64 overflow at math.MaxUint64 or zero-truncation when maxSize == 1 and retention > 0. -func computeTargetSize(maxSize uint64, retention float64) uint64 { - if retention <= 0.0 { - return 0 - } - if retention >= 1.0 { - return maxSize - } - f := float64(maxSize) * retention - if f >= float64(math.MaxUint64) { - return math.MaxUint64 - } - target := uint64(f) - if target == 0 && maxSize > 0 { - return 1 - } - return target +// resetEmptyArenaLocked releases peak arena slice and hash-map allocations when the cache transitions to empty. +// Caller MUST hold c.mu.Lock(). +func (c *arenaRadix) resetEmptyArenaLocked() { + c.clearEmptyArenaStateLocked() + c.markReclaimedLocked() } // shouldAutoCompactLocked determines whether an automatic compaction should run. // Explicit EvaluateMemoryPressure calls (protectedNodeID == nilNode) compact whenever any // free slots, slice slack, or unhealed hash-collision slots exist. Inline foreground mutations -// require at least 25% of arena slots to be on the free-list so O(N) compaction is amortized to O(1) per write. +// (protectedNodeID != nilNode) only count recycled free-list fragmentation (>= 25% of len(c.nodes)) +// so Go's geometric slice growth capacity slack never triggers O(N^2) compaction thrashing. func (c *arenaRadix) shouldAutoCompactLocked(protectedNodeID uint32) bool { if protectedNodeID == nilNode { - return c.freeHead != nilNode || len(c.nodes) < cap(c.nodes) || len(c.nodeMap) < c.len + return c.isDirtyLocked() + } + if c.freeHead != nilNode && c.peakEntryLen > 8 && len(c.nodes) > 8 && c.freeCount >= 2 && uint64(c.freeCount)*4 >= uint64(len(c.nodes)) { + return true } - return c.freeHead != nilNode && uint64(c.freeCount)*4 >= uint64(len(c.nodes)) + return c.shouldAutoCompactEntryCounts(c.nodeMapDirty, false, c.len) } -// compactLocked performs lossless O(N) compaction of the node arena slice and hash lookup map. -// It eliminates all free-list slots so len(c.nodes) == cap(c.nodes) == liveCount and c.freeHead == nilNode, -// remaps all 8 uint32 index pointers (root, head, tail, parent, child, sibling, prev, next), -// and reallocates c.nodeMap to eliminate Go map bucket slack while preserving 100% of live entries and LRU order. +// compactDataStructuresLocked performs the physical slice and map compaction without updating +// the reclamation epoch, returning true if any backing structure was reallocated. // Caller MUST hold c.mu.Lock(). -func (c *arenaRadix) compactLocked() { +func (c *arenaRadix) compactDataStructuresLocked() bool { // Fast-path: when no nodes are on the free-list, node indices are already contiguous [0..len-1]. if c.freeHead == nilNode { - if len(c.nodes) == cap(c.nodes) && len(c.nodeMap) >= c.len { - return + if len(c.nodes) == cap(c.nodes) && !c.nodeMapDirty { + return false } if len(c.nodes) < cap(c.nodes) { newNodes := make([]arenaRadixNode, len(c.nodes)) copy(newNodes, c.nodes) c.nodes = newNodes } - if len(c.nodeMap) < c.len { + if c.nodeMapDirty { newNodeMap := make(map[uint64]uint32, c.len) for id := uint32(0); id < uint32(len(c.nodes)); id++ { if c.nodes[id].value != nil { @@ -548,10 +568,10 @@ func (c *arenaRadix) compactLocked() { } } c.nodeMap = newNodeMap + c.nodeMapDirty = false } - c.pressureNeedsRefresh.Store(true) - c.reclaimEpoch.Add(1) - return + c.onCompacted(c.len) + return true } oldLen := uint32(len(c.nodes)) @@ -607,88 +627,113 @@ func (c *arenaRadix) compactLocked() { c.freeCount = 0 c.nodes = newNodes c.nodeMap = newNodeMap - c.pressureNeedsRefresh.Store(true) - c.reclaimEpoch.Add(1) + c.nodeMapDirty = false + c.onCompacted(c.len) + return true +} + +// compactLocked performs lossless O(N) compaction of the node arena slice and hash lookup map. +// It eliminates all free-list slots so len(c.nodes) == cap(c.nodes) == liveCount and c.freeHead == nilNode, +// remaps all 8 uint32 index pointers (root, head, tail, parent, child, sibling, prev, next), +// and reallocates c.nodeMap to eliminate Go map bucket slack while preserving 100% of live entries and LRU order. +// Caller MUST hold c.mu.Lock(). +func (c *arenaRadix) compactLocked() { + if c.compactDataStructuresLocked() { + c.markReclaimedLocked() + } } -// shedAndCompactLocked evicts least-recently-used entries strictly from c.tail until -// c.currentSize <= targetSize (or until the cache is completely empty when retention == 0.0), -// always protecting protectedNodeID (the newly inserted/updated entry during foreground writes), +// shedAndCompactLocked evicts least-recently-used entries strictly from c.tail in a single O(N) pass +// until c.currentSize <= targetSize (and proportionally sheds zero-size entries down to targetZeroCount), +// protecting protectedNodeID only when it resides at the MRU head (c.head), // then performs lossless arena and map compaction if fragmentation warrants it. // Caller MUST hold c.mu.Lock(). func (c *arenaRadix) shedAndCompactLocked(targetSize uint64, retention float64, protectedNodeID uint32) []ValueType { - effectiveTarget := targetSize - if protectedNodeID != nilNode && retention > 0.0 && c.nodes[protectedNodeID].size > effectiveTarget { - if math.MaxUint64-c.nodes[protectedNodeID].size >= targetSize && c.nodes[protectedNodeID].size+targetSize <= c.maxSize { - effectiveTarget = c.nodes[protectedNodeID].size + targetSize - } else { - effectiveTarget = c.nodes[protectedNodeID].size - } + autoCompactID := protectedNodeID + if protectedNodeID == foregroundNoProtect || (protectedNodeID != nilNode && (protectedNodeID != c.head || c.nodes[protectedNodeID].prev != nilNode || c.nodes[protectedNodeID].value == nil)) { + protectedNodeID = nilNode } - targetLen := 0 - if c.currentSize == 0 && c.len > 0 && retention > 0.0 { - targetLen = int(float64(c.len) * retention) - if protectedNodeID != nilNode && targetLen < 1 { - targetLen = 1 - } + var protectedSize uint64 + hasProtected := protectedNodeID != nilNode + if hasProtected { + protectedSize = c.nodes[protectedNodeID].size } + effectiveTarget, targetLen, targetZeroCount, unprotectedZeroTarget := c.computeShedTargets(targetSize, retention, c.len, hasProtected, protectedSize) + needFullFlush := retention == 0.0 var evicted []ValueType - for c.tail != nilNode { - needByteShed := c.currentSize > effectiveTarget - needZeroSizeShed := c.currentSize == 0 && c.len > targetLen - needFullFlush := retention == 0.0 - if !needByteShed && !needZeroSizeShed && !needFullFlush { + victimID := c.tail + for victimID != nilNode { + needByteShed := c.currentSize > effectiveTarget || needFullFlush + needZeroShed := !needFullFlush && c.zeroSizeCount > targetZeroCount && c.len > targetLen + if !needByteShed && !needZeroShed { break } - if protectedNodeID != nilNode && c.tail == protectedNodeID { + switch { + case needFullFlush || (needByteShed && c.zeroSizeCount > targetZeroCount): + for victimID != nilNode && victimID == protectedNodeID { + victimID = c.nodes[victimID].prev + } + case needByteShed: + for victimID != nilNode && (victimID == protectedNodeID || c.nodes[victimID].size == 0) { + victimID = c.nodes[victimID].prev + } + default: + for victimID != nilNode && (victimID == protectedNodeID || c.nodes[victimID].size > 0) { + victimID = c.nodes[victimID].prev + } + } + if victimID == nilNode { break } - if val := c.evictOne(); val != nil { + nextVictimID := c.nodes[victimID].prev + if val := c.eraseInternal(victimID); val != nil { evicted = append(evicted, val) } + victimID = nextVictimID } - c.pressureNeedsRefresh.Store(true) - c.reclaimEpoch.Add(1) - if c.shouldAutoCompactLocked(protectedNodeID) { - c.compactLocked() + if len(evicted) > 0 && c.len == 0 { + c.resetEmptyArenaLocked() + return evicted } - return evicted -} -// maybeCompactUnderPressureLocked executes lossless Tier 1 arena/map compaction -// when pressure >= CompactionThreshold and freed slots exceed the fragmentation threshold. -// Caller MUST hold c.mu.Lock(). -func (c *arenaRadix) maybeCompactUnderPressureLocked(pressure float64) { - if pressure >= c.options.CompactionThreshold && c.shouldAutoCompactLocked(c.root) { - c.compactLocked() + if len(evicted) > 0 { + c.updateZeroWatermarkAfterShed(retention, c.len, !hasProtected || effectiveTarget <= targetSize, unprotectedZeroTarget, targetLen) + } + compacted := false + if c.shouldAutoCompactLocked(autoCompactID) { + compacted = c.compactDataStructuresLocked() + } + if len(evicted) > 0 || compacted { + c.markReclaimedLocked() } + return evicted } // maybeReclaimUnderPressureLocked evaluates the sampled pressure against configured thresholds: // - If pressure >= EvictionThreshold (Tier 2 Critical Pressure): evict from LRU tail down to -// c.maxSize * EvictionRetentionRatio (always preserving protectedNodeID), then compact if needed. +// c.maxSize * EvictionRetentionRatio (preserving protectedNodeID if at MRU head), then compact if needed. // - Else if pressure >= CompactionThreshold (Tier 1 Moderate Pressure): perform lossless arena and map compaction. // // Caller MUST hold c.mu.Lock(). func (c *arenaRadix) maybeReclaimUnderPressureLocked(pressure float64, protectedNodeID uint32) []ValueType { + if c.isSamplingGoroutine() { + return nil + } + var evicted []ValueType if pressure >= c.options.EvictionThreshold { retention := c.options.EvictionRetentionRatio targetSize := computeTargetSize(c.maxSize, retention) - if c.currentSize > targetSize || (c.currentSize == 0 && c.len > 0) || (retention == 0.0 && c.tail != nilNode) { - return c.shedAndCompactLocked(targetSize, retention, protectedNodeID) - } - if c.shouldAutoCompactLocked(protectedNodeID) { - c.compactLocked() - } - return nil - } - if pressure >= c.options.CompactionThreshold { - if c.shouldAutoCompactLocked(protectedNodeID) { - c.compactLocked() + evicted = c.shedAndCompactLocked(targetSize, retention, protectedNodeID) + } else { + c.resetZeroWatermarkBelowTier2(pressure) + if pressure >= c.options.CompactionThreshold { + if c.shouldAutoCompactLocked(protectedNodeID) { + c.compactLocked() + } } } - return nil + return evicted } diff --git a/arena_radix_lru.go b/arena_radix_lru.go index bcc35e4..2fdb62e 100644 --- a/arena_radix_lru.go +++ b/arena_radix_lru.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus +package lru import ( "fmt" @@ -27,18 +27,16 @@ func NewArenaRadixCache(maxSize uint64, opts ...Option) Cache { if maxSize == 0 { panic("maxSize must be greater than zero") } - options := ApplyOptions(opts...) + return newArenaRadixCacheWithOptions(maxSize, ApplyOptions(opts...)) +} +func newArenaRadixCacheWithOptions(maxSize uint64, options Options) Cache { c := &arenaRadix{ - maxSize: maxSize, - freeHead: nilNode, - head: nilNode, - tail: nilNode, - nodeMap: make(map[uint64]uint32), - options: options, + maxSize: maxSize, + pressureState: pressureState{options: options}, } - c.root = c.allocateNode() + c.clearEmptyArenaStateLocked() if c.options.EnableInvariantChecking { c.checkInvariants() @@ -67,12 +65,22 @@ func (c *arenaRadix) checkInvariants() { // INVARIANT 3: LRU list validation lruCount := 0 + zeroCount := 0 var sumSize uint64 prevID := nilNode for currID := c.head; currID != nilNode; currID = c.nodes[currID].next { + if currID >= uint32(len(c.nodes)) { + panic(fmt.Sprintf("arenaRadix invariant violation: LRU list contains out-of-bounds index %d (len=%d)", currID, len(c.nodes))) + } lruCount++ + if math.MaxUint64-sumSize < c.nodes[currID].size { + panic("arenaRadix invariant violation: sumSize uint64 overflow") + } sumSize += c.nodes[currID].size + if c.nodes[currID].size == 0 { + zeroCount++ + } if c.nodes[currID].value == nil { panic(fmt.Sprintf("arenaRadix invariant violation: unexpected nil value in LRU list for prefix '%s'", c.nodes[currID].prefix)) } @@ -104,6 +112,10 @@ func (c *arenaRadix) checkInvariants() { panic(fmt.Sprintf("arenaRadix invariant violation: LRU list count %d does not match tracked len %d", lruCount, c.len)) } + if zeroCount != c.zeroSizeCount { + panic(fmt.Sprintf("arenaRadix invariant violation: zeroSizeCount %d does not match live zero-size entries %d", c.zeroSizeCount, zeroCount)) + } + if sumSize != c.currentSize { panic(fmt.Sprintf("arenaRadix: currentSize drift: currentSize=%d sumSize=%d", c.currentSize, sumSize)) } @@ -116,6 +128,9 @@ func (c *arenaRadix) checkInvariants() { if c.head == nilNode || c.tail == nilNode { panic("arenaRadix invariant violation: head or tail is nilNode when len > 0") } + if c.head >= uint32(len(c.nodes)) || c.tail >= uint32(len(c.nodes)) { + panic("arenaRadix invariant violation: head or tail index out of bounds") + } if c.nodes[c.head].prev != nilNode { panic("arenaRadix invariant violation: head prev pointer is not nilNode") } @@ -125,8 +140,11 @@ func (c *arenaRadix) checkInvariants() { } // INVARIANT 4: Root structure checks - if c.root == nilNode { - panic("arenaRadix invariant violation: root node is nilNode") + if c.root == nilNode || c.root >= uint32(len(c.nodes)) { + panic("arenaRadix invariant violation: root node is nilNode or out of bounds") + } + if c.nodes[c.root].prefix != "" { + panic("arenaRadix invariant violation: root node must have empty prefix") } if c.nodes[c.root].parent != nilNode { panic("arenaRadix invariant violation: root node must not have a parent") @@ -143,20 +161,37 @@ func (c *arenaRadix) checkInvariants() { var treeSumSize uint64 currID := c.root for currID != nilNode { + if currID >= uint32(len(c.nodes)) { + panic(fmt.Sprintf("arenaRadix invariant violation: tree contains out-of-bounds index %d (len=%d)", currID, len(c.nodes))) + } treeNodeCount++ if c.nodes[currID].value != nil { treeCount++ + if math.MaxUint64-treeSumSize < c.nodes[currID].size { + panic("arenaRadix invariant violation: treeSumSize uint64 overflow") + } treeSumSize += c.nodes[currID].size - // A node is verifiably in the LRU list if it is the head or has a non-nilNode prev pointer. - inLRU := c.head == currID || c.nodes[currID].prev != nilNode + // A node is verifiably in the LRU list iff it is the head (with nilNode prev) or its prev's next points back to it. + prev := c.nodes[currID].prev + inLRU := (c.head == currID && prev == nilNode) || (prev != nilNode && prev < uint32(len(c.nodes)) && c.nodes[prev].next == currID) if !inLRU { panic(fmt.Sprintf("arenaRadix invariant violation: node with prefix '%s' has value but is missing from LRU list", c.nodes[currID].prefix)) } + } else { + if c.nodes[currID].size != 0 { + panic(fmt.Sprintf("arenaRadix invariant violation: routing node %d has non-zero size %d", currID, c.nodes[currID].size)) + } + if c.nodes[currID].prev != nilNode || c.nodes[currID].next != nilNode || c.head == currID || c.tail == currID { + panic(fmt.Sprintf("arenaRadix invariant violation: routing node %d has non-nilNode LRU pointers", currID)) + } } // Validate child pointers and sibling ordering prevSiblingID := nilNode for chID := c.nodes[currID].child; chID != nilNode; chID = c.nodes[chID].sibling { + if chID >= uint32(len(c.nodes)) { + panic(fmt.Sprintf("arenaRadix invariant violation: child index %d out of bounds (len=%d)", chID, len(c.nodes))) + } if c.nodes[chID].parent != currID { panic(fmt.Sprintf("arenaRadix invariant violation: child with prefix '%s' has incorrect parent pointer", c.nodes[chID].prefix)) } @@ -219,12 +254,21 @@ func (c *arenaRadix) checkInvariants() { if freeID >= uint32(len(c.nodes)) { panic(fmt.Sprintf("arenaRadix invariant violation: free-list contains out-of-bounds index %d (len=%d)", freeID, len(c.nodes))) } + if freeID == c.root { + panic("arenaRadix invariant violation: free-list contains root node") + } if c.nodes[freeID].value != nil { panic(fmt.Sprintf("arenaRadix invariant violation: free-list node %d has non-nil value", freeID)) } if c.nodes[freeID].size != 0 { panic(fmt.Sprintf("arenaRadix invariant violation: free-list node %d has non-zero size %d", freeID, c.nodes[freeID].size)) } + if c.nodes[freeID].prefix != "" { + panic(fmt.Sprintf("arenaRadix invariant violation: free-list node %d has non-empty prefix '%s'", freeID, c.nodes[freeID].prefix)) + } + if c.nodes[freeID].parent != nilNode || c.nodes[freeID].child != nilNode || c.nodes[freeID].sibling != nilNode || c.nodes[freeID].prev != nilNode { + panic(fmt.Sprintf("arenaRadix invariant violation: free-list node %d has non-nilNode tree/LRU pointers", freeID)) + } freeCount++ if freeCount > len(c.nodes) { panic("arenaRadix invariant violation: cycle detected in free-list") @@ -240,16 +284,25 @@ func (c *arenaRadix) checkInvariants() { } } +func (c *arenaRadix) unlock() { + if c.options.EnableInvariantChecking { + c.checkInvariants() + } + c.mu.Unlock() +} + +func (c *arenaRadix) rUnlock() { + if c.options.EnableInvariantChecking { + c.checkInvariants() + } + c.mu.RUnlock() +} + // Compact performs lossless O(N) compaction of the arena node slice and hash lookup map // under exclusive write lock while preserving 100% of live entries, sizes, and exact LRU order. func (c *arenaRadix) Compact() { c.mu.Lock() - defer func() { - if c.options.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + defer c.unlock() c.compactLocked() } @@ -258,17 +311,47 @@ func (c *arenaRadix) Compact() { // or Tier 1 (lossless compaction) if pressure meets or exceeds the configured thresholds. // Returns any values evicted during Tier 2 critical-pressure shedding. func (c *arenaRadix) EvaluateMemoryPressure() []ValueType { - pressure := c.samplePressureFresh() + _, pressure := c.lockWithPressure(&c.mu, true) + defer c.unlock() - c.mu.Lock() - defer func() { - if c.options.EnableInvariantChecking { - c.checkInvariants() + return c.maybeReclaimUnderPressureLocked(pressure, nilNode) +} + +func (c *arenaRadix) finishDeleteReclaimLocked(sizeBefore, sampledEpoch uint64, pressure float64) { + if c.len == 0 { + hadSlack := c.hasEmptyDeleteSlack(c.freeCount >= 64) + c.clearEmptyArenaStateLocked() + if (hadSlack || sizeBefore > 0) && c.reclaimEpoch.Load() == sampledEpoch && c.hasElevatedPressureToInvalidate(pressure) { + c.markReclaimedLocked() } - c.mu.Unlock() - }() + return + } + reclaimedSingleSurvivor := false + if c.shouldReclaimSingleSurvivorOnDelete(c.len, c.freeCount > 0 || c.nodeMapDirty, c.freeCount >= 63) { + c.compactDataStructuresLocked() + reclaimedSingleSurvivor = true + } + c.maybeReclaimUnderPressureLocked(pressure, foregroundNoProtect) + if (reclaimedSingleSurvivor || c.currentSize < sizeBefore) && c.reclaimEpoch.Load() == sampledEpoch && c.hasElevatedPressureToInvalidate(pressure) { + c.markReclaimedLocked() + } +} - return c.maybeReclaimUnderPressureLocked(pressure, nilNode) +func (c *arenaRadix) finishMutationReclaimLocked(evictedValues []ValueType, protectedID uint32, reclaimedPre bool, sizeBefore, sampledEpoch uint64, pressure float64) []ValueType { + evictedByPressure := c.maybeReclaimUnderPressureLocked(pressure, protectedID) + if len(evictedValues) == 0 { + evictedValues = evictedByPressure + } else if len(evictedByPressure) > 0 { + evictedValues = append(evictedValues, evictedByPressure...) + } + netByteReduced := c.currentSize < sizeBefore + if c.shouldCompactAfterMutation(reclaimedPre, netByteReduced, c.isDirtyLocked(), pressure) { + c.compactDataStructuresLocked() + } + if c.shouldMarkReclaimedAfterMutation(reclaimedPre, netByteReduced, sampledEpoch, pressure) { + c.markReclaimedLocked() + } + return evictedValues } // Insert inserts or updates the given key and value in the cache. @@ -287,87 +370,85 @@ func (c *arenaRadix) Insert(key string, value ValueType) ([]ValueType, error) { return nil, ErrInvalidEntrySize } - sampledEpoch := c.reclaimEpoch.Load() - pressure := c.samplePressure() - - c.mu.Lock() - defer func() { - if c.options.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) + defer c.unlock() var evictedValues []ValueType - - // A single insert can allocate up to 2 nodes (one routing node, one leaf). - // If the slice has reached its physical uint32 maximum, we must rely entirely on the freelist. - // Since evicting one LRU item guarantees at least 1 leaf node (and potentially 1 routing node) - // is freed to the freelist, we proactively evict until we have at least 2 free nodes. - for uint32(len(c.nodes)) >= nilNode-2 && c.tail != nilNode && (c.freeHead == nilNode || c.nodes[c.freeHead].next == nilNode) { - evictedValues = append(evictedValues, c.evictOne()) - } - - // If inserting a brand-new key would exceed maxSize, evict from the LRU tail before - // allocating new arena nodes so insertNode immediately recycles the freed slot(s) from freeHead. - if _, exists := c.getNodeKey(key); !exists { - for c.currentSize+valueSize > c.maxSize && c.tail != nilNode { - evictedValues = append(evictedValues, c.evictOne()) - } - } - - nodeID, oldValue := c.insertNode(key, value) - if oldValue != nil { + sizeBefore := c.currentSize + reclaimedPreInsert := false + evictedPreInsert := false + keyHash := hashString(key) + + nodeID, exists := c.getNodeKeyWithHash(key, keyHash) + if exists { + // Updating an existing key requires 0 new node allocations and 0 trie walks. + c.onEntrySizeUpdated(c.nodes[nodeID].size, valueSize) + c.moveToFront(nodeID) c.currentSize -= c.nodes[nodeID].size - c.currentSize += valueSize + for valueSize > c.maxSize-c.currentSize && c.tail != nilNode && c.tail != nodeID { + if evicted := c.evictOne(); evicted != nil { + evictedValues = append(evictedValues, evicted) + evictedPreInsert = true + } + } + c.nodes[nodeID].value = value c.nodes[nodeID].size = valueSize - c.nodeMap[hashString(key)] = nodeID - c.moveToFront(nodeID) + c.currentSize += valueSize + if evictedPreInsert { + c.nodeMap[keyHash] = nodeID + } + reclaimedPreInsert = c.shouldReclaimSingleSurvivorOnMutation(c.len, c.freeCount > 0 || c.nodeMapDirty, c.freeCount >= 63, evictedPreInsert, c.currentSize, sizeBefore, pressure) } else { + // A single new-key insert can allocate up to 2 nodes (one routing node, one leaf). + // If the slice has reached its physical uint32 maximum, ensure enough free slots exist. + for uint64(len(c.nodes))-uint64(c.freeCount)+2 > uint64(foregroundNoProtect) && c.tail != nilNode { + if evicted := c.evictOne(); evicted != nil { + evictedValues = append(evictedValues, evicted) + evictedPreInsert = true + } + } + + // Evict from the LRU tail before allocating new arena nodes when valueSize would exceed remaining capacity + // (using subtraction to avoid uint64 addition overflow when maxSize is near math.MaxUint64). + for valueSize > c.maxSize-c.currentSize && c.tail != nilNode { + if evicted := c.evictOne(); evicted != nil { + evictedValues = append(evictedValues, evicted) + evictedPreInsert = true + } + } + if c.shouldReclaimEmptyPreInsert(c.len, c.freeCount >= 64, evictedPreInsert, valueSize, sizeBefore, pressure) { + c.clearEmptyArenaStateLocked() + reclaimedPreInsert = true + } else if c.shouldReclaimSingleSurvivorOnMutation(c.len, c.freeCount > 0 || c.nodeMapDirty, c.freeCount >= 63, evictedPreInsert, c.currentSize+valueSize, sizeBefore, pressure) { + reclaimedPreInsert = true + } + + nodeID = c.insertNode(key, value) c.nodes[nodeID].size = valueSize c.pushFront(nodeID) + c.onEntryInserted(c.len, valueSize) c.currentSize += valueSize - c.nodeMap[hashString(key)] = nodeID - } - - // Evict until we're at or below maxSize - for c.currentSize > c.maxSize && c.tail != nilNode { - evictedValues = append(evictedValues, c.evictOne()) - } - - if !c.options.hasCustomPressureFunc && sampledEpoch != c.reclaimEpoch.Load() { - pressure = c.samplePressure() - } - if evictedByPressure := c.maybeReclaimUnderPressureLocked(pressure, nodeID); len(evictedByPressure) > 0 { - evictedValues = append(evictedValues, evictedByPressure...) + c.nodeMap[keyHash] = nodeID } + evictedValues = c.finishMutationReclaimLocked(evictedValues, nodeID, reclaimedPreInsert, sizeBefore, sampledEpoch, pressure) return evictedValues, nil } // Erase removes the entry associated with key from the cache, returning its value (or nil if not found). func (c *arenaRadix) Erase(key string) (value ValueType) { - sampledEpoch := c.reclaimEpoch.Load() - pressure := c.samplePressure() + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) + defer c.unlock() - c.mu.Lock() - defer func() { - if c.options.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() - - nodeID, ok := c.getNodeKey(key) + keyHash := hashString(key) + nodeID, ok := c.lookupNodeKeyWithHash(key, keyHash) if !ok { return nil } - deleted := c.eraseInternal(nodeID) - if !c.options.hasCustomPressureFunc && sampledEpoch != c.reclaimEpoch.Load() { - pressure = c.samplePressure() - } - c.maybeCompactUnderPressureLocked(pressure) + sizeBefore := c.currentSize + deleted := c.eraseInternalWithHash(nodeID, keyHash) + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) return deleted } @@ -375,14 +456,10 @@ func (c *arenaRadix) Erase(key string) (value ValueType) { // Returns nil if the key is not found in the cache. func (c *arenaRadix) LookUp(key string) (value ValueType) { c.mu.Lock() - defer func() { - if c.options.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + defer c.unlock() - nodeID, ok := c.getNodeKey(key) + keyHash := hashString(key) + nodeID, ok := c.getNodeKeyWithHash(key, keyHash) if !ok { return nil } @@ -395,12 +472,7 @@ func (c *arenaRadix) LookUp(key string) (value ValueType) { // Returns nil if the key is not found in the cache. func (c *arenaRadix) LookUpWithoutChangingOrder(key string) (value ValueType) { c.mu.RLock() - defer func() { - if c.options.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.RUnlock() - }() + defer c.rUnlock() nodeID, ok := c.getNodeKey(key) if !ok { @@ -423,14 +495,10 @@ func (c *arenaRadix) UpdateWithoutChangingOrder(key string, value ValueType) err valueSize := value.Size() c.mu.Lock() - defer func() { - if c.options.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + defer c.unlock() - nodeID, ok := c.getNodeKey(key) + keyHash := hashString(key) + nodeID, ok := c.getNodeKeyWithHash(key, keyHash) if !ok { return ErrEntryNotExist } @@ -444,80 +512,120 @@ func (c *arenaRadix) UpdateWithoutChangingOrder(key string, value ValueType) err } // UpdateSize adjusts the size accounting for an existing key by sizeDelta without altering its LRU position. -// If the updated cache size exceeds maxSize, excess LRU entries are evicted to maintain capacity invariants. +// If node.size + sizeDelta exceeds maxSize (or cannot fit alongside entries more recent than node), +// only the entry itself is evicted without evicting older entries. +// Otherwise, if the updated cache size exceeds maxSize, excess LRU entries are evicted to maintain capacity invariants. // // Returns ErrEntryNotExist if key is not present in the cache. +// Returns ErrInvalidUpdateEntrySize if sizeDelta causes uint64 integer overflow. func (c *arenaRadix) UpdateSize(key string, sizeDelta uint64) error { - sampledEpoch := c.reclaimEpoch.Load() - pressure := c.samplePressure() - - c.mu.Lock() - defer func() { - if c.options.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) + defer c.unlock() - nodeID, ok := c.getNodeKey(key) + keyHash := hashString(key) + nodeID, ok := c.lookupNodeKeyWithHash(key, keyHash) if !ok { return ErrEntryNotExist } - if math.MaxUint64-c.nodes[nodeID].size < sizeDelta || math.MaxUint64-c.currentSize < sizeDelta { + if math.MaxUint64-c.nodes[nodeID].size < sizeDelta { return ErrInvalidUpdateEntrySize } - // Update size accounting - c.nodes[nodeID].size += sizeDelta - c.currentSize += sizeDelta + if c.nodes[nodeID].size+sizeDelta > c.maxSize { + sizeBefore := c.currentSize + c.eraseInternalWithHash(nodeID, keyHash) + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) + return nil + } - // Evict until we're at or below maxSize to maintain invariants - for c.currentSize > c.maxSize && c.tail != nilNode { - c.evictOne() + avail := c.maxSize - c.currentSize + if sizeDelta > avail { + maxNewer := c.maxSize - (c.nodes[nodeID].size + sizeDelta) + var newerSize uint64 + headCurr := c.head + tailCurr := c.tail + canFit := false + for { + if headCurr == nodeID { + canFit = newerSize <= maxNewer + break + } + if headCurr != nilNode { + newerSize += c.nodes[headCurr].size + if newerSize > maxNewer { + canFit = false + break + } + headCurr = c.nodes[headCurr].next + } + if tailCurr == nilNode || tailCurr == nodeID { + canFit = sizeDelta <= avail + break + } + avail += c.nodes[tailCurr].size + if sizeDelta <= avail { + canFit = true + break + } + tailCurr = c.nodes[tailCurr].prev + } + if !canFit { + sizeBefore := c.currentSize + c.eraseInternalWithHash(nodeID, keyHash) + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) + return nil + } } - if !c.options.hasCustomPressureFunc && sampledEpoch != c.reclaimEpoch.Load() { - pressure = c.samplePressure() + sizeBefore := c.currentSize + evictedAny := false + reclaimedPreUpdate := false + for sizeDelta > c.maxSize-c.currentSize && c.tail != nilNode { + if c.tail == nodeID { + break + } + c.evictOne() + evictedAny = true } - if sizeDelta > 0 { - c.maybeReclaimUnderPressureLocked(pressure, nodeID) - } else { - c.maybeCompactUnderPressureLocked(pressure) + + c.onEntrySizeUpdated(c.nodes[nodeID].size, c.nodes[nodeID].size+sizeDelta) + c.nodes[nodeID].size += sizeDelta + c.currentSize += sizeDelta + c.nodeMap[keyHash] = nodeID + reclaimedPreUpdate = c.shouldReclaimSingleSurvivorOnMutation(c.len, c.freeCount > 0 || c.nodeMapDirty, c.freeCount >= 63, evictedAny, c.currentSize, sizeBefore, pressure) + + protectedID := foregroundNoProtect + if nodeID == c.head && c.nodes[nodeID].value != nil { + protectedID = nodeID } + c.finishMutationReclaimLocked(nil, protectedID, reclaimedPreUpdate, sizeBefore, sampledEpoch, pressure) return nil } // EraseEntriesWithGivenPrefix deletes all entries whose keys begin with prefix. // It severs the matching subtree in O(1) and iteratively reclaims all nodes into the free-list. func (c *arenaRadix) EraseEntriesWithGivenPrefix(prefix string) { - pressure := c.samplePressure() + if prefix == "" { + c.mu.Lock() + defer c.unlock() - c.mu.Lock() - defer func() { - if c.options.EnableInvariantChecking { - c.checkInvariants() + hadEntries := c.len > 0 + hadDirtySlack := c.freeCount > 0 || c.nodeMapDirty || c.peakEntryLen > 8 || c.deletedSinceCompact > 0 || (c.len == 0 && cap(c.nodes) > 1) + if !hadEntries && !hadDirtySlack && c.peakEntryLen == 0 && len(c.nodes) <= 1 { + return } - c.mu.Unlock() - }() - - if prefix == "" { - c.nodes = nil - c.freeHead = nilNode - c.freeCount = 0 - - c.root = c.allocateNode() - c.head = nilNode - c.tail = nilNode - c.currentSize = 0 - c.len = 0 - c.nodeMap = make(map[uint64]uint32) - if pressure >= c.options.CompactionThreshold { - c.compactLocked() + hadReclaimable := c.currentSize > 0 || hadDirtySlack + c.clearEmptyArenaStateLocked() + if hadReclaimable && c.hasElevatedPressureToInvalidate(0.0) { + c.markReclaimedLocked() } return } + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) + defer c.unlock() + nodeID := c.root search := prefix @@ -527,9 +635,10 @@ func (c *arenaRadix) EraseEntriesWithGivenPrefix(prefix string) { return // Prefix doesn't exist } - lcp := c.longestCommonPrefix(search, c.nodes[childID].prefix) + lcp := longestCommonPrefix(search, c.nodes[childID].prefix) if lcp == len(search) { + sizeBefore := c.currentSize // We found the exact node where the prefix ends. // Sever it entirely from the tree structure c.removeChild(nodeID, childID) @@ -540,7 +649,7 @@ func (c *arenaRadix) EraseEntriesWithGivenPrefix(prefix string) { // Now sweep the detached subtree to fix LRU, nodeMap, and currentSize c.freeSubtree(childID) c.compressPathUpwards(nodeID) - c.maybeCompactUnderPressureLocked(pressure) + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) return } @@ -554,27 +663,36 @@ func (c *arenaRadix) EraseEntriesWithGivenPrefix(prefix string) { } } -// freeSubtree iteratively reclaims all nodes in a detached subtree using O(1) stack space, +// freeSubtree iteratively reclaims all nodes in a detached subtree using O(1) stack space +// and incremental FNV-1a prefix hashing (avoiding O(leaves * depth) ancestor walks), // removing active values from the LRU list, nodeMap, and accounting, and returning nodes to the free-list. func (c *arenaRadix) freeSubtree(nodeID uint32) { if nodeID == nilNode { return } + baseHash := c.hashNodeKey(c.nodes[nodeID].parent) + var hashStackBuf [64]uint64 + hashStack := hashStackBuf[:0] + currHash := hashStringCont(baseHash, c.nodes[nodeID].prefix) + currID := nodeID for currID != nilNode { if c.nodes[currID].value != nil { + c.onEntryDeleted(c.nodes[currID].size) c.currentSize -= c.nodes[currID].size c.remove(currID) - hash := c.hashNodeKey(currID) - if c.nodeMap[hash] == currID { - delete(c.nodeMap, hash) + c.nodeMapDirty = true + if mappedID, ok := c.nodeMap[currHash]; ok && mappedID == currID { + delete(c.nodeMap, currHash) } c.nodes[currID].value = nil c.nodes[currID].size = 0 } if c.nodes[currID].child != nilNode { + hashStack = append(hashStack, currHash) currID = c.nodes[currID].child + currHash = hashStringCont(currHash, c.nodes[currID].prefix) continue } @@ -582,6 +700,7 @@ func (c *arenaRadix) freeSubtree(nodeID uint32) { parentID := c.nodes[currID].parent c.freeNode(currID) currID = parentID + hashStack = hashStack[:len(hashStack)-1] } if currID == nodeID { @@ -592,5 +711,7 @@ func (c *arenaRadix) freeSubtree(nodeID uint32) { siblingID := c.nodes[currID].sibling c.freeNode(currID) currID = siblingID + parentHash := hashStack[len(hashStack)-1] + currHash = hashStringCont(parentHash, c.nodes[currID].prefix) } } diff --git a/arena_radix_test.go b/arena_radix_test.go index 896bce3..1e6fa5c 100644 --- a/arena_radix_test.go +++ b/arena_radix_test.go @@ -12,528 +12,543 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus +package lru import ( - "errors" "fmt" "math" + "runtime" + "strings" "testing" + "unsafe" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -const arenaTestMaxSize = 50 - -type arenaTestData struct { - Value int64 - DataSize uint64 -} - -func (td arenaTestData) Size() uint64 { - return td.DataSize -} - func setupArenaRadixCacheTest(t *testing.T) Cache { t.Helper() - return NewArenaRadixCache(arenaTestMaxSize, WithInvariantChecking(true)) -} - -func insertAndAssertArena(t *testing.T, cache Cache, key string, val ValueType, evictedValues []int64, expectedError error) { - t.Helper() - ret, err := cache.Insert(key, val) - - if expectedError != nil { - if !errors.Is(err, expectedError) { - t.Fatalf("expected error %v, got %v", expectedError, err) - } - } else if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if len(ret) != len(evictedValues) { - t.Fatalf("expected %d evicted entries, got %d", len(evictedValues), len(ret)) - } - for i, v := range ret { - td, ok := v.(arenaTestData) - if !ok { - t.Fatalf("expected arenaTestData type, got %T", v) - } - if td.Value != evictedValues[i] { - t.Fatalf("expected evicted value at index %d to be %d, got %d", i, evictedValues[i], td.Value) - } - } -} - -func TestArenaRadixCache_Constructor(t *testing.T) { - t.Run("DefaultOptions", func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Fatalf("expected panic on NewArenaRadixCache(0) with default options, got nil") - } - }() - _ = NewArenaRadixCache(0) - }) - - t.Run("InvariantsEnabled", func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Fatalf("expected panic on NewArenaRadixCache(0) with invariant checking enabled, got nil") - } - }() - _ = NewArenaRadixCache(0, WithInvariantChecking(true)) - }) - - t.Run("InvariantsDisabled", func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Fatalf("expected panic on NewArenaRadixCache(0) with invariant checking disabled, got nil") - } - }() - _ = NewArenaRadixCache(0, WithInvariantChecking(false)) - }) + return NewArenaRadixCache(testMaxSize, WithInvariantChecking(true)) } func TestArenaRadixCache_LookUpInEmptyCache(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - if v := cache.LookUp(""); v != nil { - t.Fatalf("expected nil lookup for empty key on empty cache, got %v", v) - } - if v := cache.LookUp("taco"); v != nil { - t.Fatalf("expected nil lookup for 'taco' on empty cache, got %v", v) - } + + // Act + emptyVal := cache.LookUp("") + tacoVal := cache.LookUp("taco") + + // Assert + assert.Nil(t, emptyVal) + assert.Nil(t, tacoVal) } func TestArenaRadixCache_InsertNilValue(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "taco", nil, []int64{}, ErrInvalidEntry) + + // Act + evicted, err := cache.Insert("taco", nil) + + // Assert + require.ErrorIs(t, err, ErrInvalidEntry) + assert.Empty(t, evicted) } func TestArenaRadixCache_InsertEmptyKey(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "", arenaTestData{Value: 42, DataSize: 10}, []int64{}, nil) - + // Act + evicted, err := cache.Insert("", testData{value: 42, dataSize: 10}) val := cache.LookUp("") - if val == nil { - t.Fatalf("expected value for empty key, got nil") - } - if td, ok := val.(arenaTestData); !ok || td.Value != 42 { - t.Fatalf("expected value 42, got %v", val) - } - if v := cache.LookUp("taco"); v != nil { - t.Fatalf("expected nil for 'taco', got %v", v) - } + tacoVal := cache.LookUp("taco") + + // Assert + require.NoError(t, err) + assert.Empty(t, evicted) + require.NotNil(t, val) + assert.Equal(t, int64(42), val.(testData).value) + assert.Nil(t, tacoVal) } func TestArenaRadixCache_LookUpUnknownKey(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "burrito", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) - insertAndAssertArena(t, cache, "taco", arenaTestData{Value: 23, DataSize: 8}, []int64{}, nil) + _, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("taco", testData{value: 23, dataSize: 8}) + require.NoError(t, err) - if v := cache.LookUp(""); v != nil { - t.Fatalf("expected nil for empty key, got %v", v) - } - if v := cache.LookUp("enchilada"); v != nil { - t.Fatalf("expected nil for 'enchilada', got %v", v) - } + // Act + emptyVal := cache.LookUp("") + enchiladaVal := cache.LookUp("enchilada") + + // Assert + assert.Nil(t, emptyVal) + assert.Nil(t, enchiladaVal) } func TestArenaRadixCache_FillUpToCapacity(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "burrito", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) - insertAndAssertArena(t, cache, "taco", arenaTestData{Value: 26, DataSize: 20}, []int64{}, nil) - insertAndAssertArena(t, cache, "enchilada", arenaTestData{Value: 28, DataSize: 26}, []int64{}, nil) - if v := cache.LookUp("burrito"); v == nil || v.(arenaTestData).Value != 23 { - t.Fatalf("expected 23 for 'burrito', got %v", v) - } - if v := cache.LookUp("taco"); v == nil || v.(arenaTestData).Value != 26 { - t.Fatalf("expected 26 for 'taco', got %v", v) - } - if v := cache.LookUp("enchilada"); v == nil || v.(arenaTestData).Value != 28 { - t.Fatalf("expected 28 for 'enchilada', got %v", v) - } + // Act + ev1, err1 := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + ev2, err2 := cache.Insert("taco", testData{value: 26, dataSize: 20}) + ev3, err3 := cache.Insert("enchilada", testData{value: 28, dataSize: 26}) + + // Assert + require.NoError(t, err1) + assert.Empty(t, ev1) + require.NoError(t, err2) + assert.Empty(t, ev2) + require.NoError(t, err3) + assert.Empty(t, ev3) + + v1 := cache.LookUp("burrito") + require.NotNil(t, v1) + assert.Equal(t, int64(23), v1.(testData).value) + + v2 := cache.LookUp("taco") + require.NotNil(t, v2) + assert.Equal(t, int64(26), v2.(testData).value) + + v3 := cache.LookUp("enchilada") + require.NotNil(t, v3) + assert.Equal(t, int64(28), v3.(testData).value) } func TestArenaRadixCache_ExpiresLeastRecentlyUsed(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "burrito", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) - insertAndAssertArena(t, cache, "taco", arenaTestData{Value: 26, DataSize: 20}, []int64{}, nil) - insertAndAssertArena(t, cache, "enchilada", arenaTestData{Value: 28, DataSize: 26}, []int64{}, nil) + _, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("taco", testData{value: 26, dataSize: 20}) + require.NoError(t, err) + _, err = cache.Insert("enchilada", testData{value: 28, dataSize: 26}) + require.NoError(t, err) // Promote burrito to MRU - if v := cache.LookUp("burrito"); v == nil || v.(arenaTestData).Value != 23 { - t.Fatalf("expected 23 for 'burrito', got %v", v) - } + promoted := cache.LookUp("burrito") + require.NotNil(t, promoted) + assert.Equal(t, int64(23), promoted.(testData).value) - // Insert another item; taco (least recent) should be evicted - insertAndAssertArena(t, cache, "queso", arenaTestData{Value: 34, DataSize: 5}, []int64{26}, nil) + // Act: Insert another item; taco (least recent) should be evicted + evicted, err := cache.Insert("queso", testData{value: 34, dataSize: 5}) - if v := cache.LookUp("taco"); v != nil { - t.Fatalf("expected 'taco' to be evicted, got %v", v) - } - if v := cache.LookUp("burrito"); v == nil || v.(arenaTestData).Value != 23 { - t.Fatalf("expected 23 for 'burrito', got %v", v) - } - if v := cache.LookUp("enchilada"); v == nil || v.(arenaTestData).Value != 28 { - t.Fatalf("expected 28 for 'enchilada', got %v", v) - } - if v := cache.LookUp("queso"); v == nil || v.(arenaTestData).Value != 34 { - t.Fatalf("expected 34 for 'queso', got %v", v) - } + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{26}) + assert.Nil(t, cache.LookUp("taco")) + + vBurrito := cache.LookUp("burrito") + require.NotNil(t, vBurrito) + assert.Equal(t, int64(23), vBurrito.(testData).value) + + vEnchilada := cache.LookUp("enchilada") + require.NotNil(t, vEnchilada) + assert.Equal(t, int64(28), vEnchilada.(testData).value) + + vQueso := cache.LookUp("queso") + require.NotNil(t, vQueso) + assert.Equal(t, int64(34), vQueso.(testData).value) } func TestArenaRadixCache_Overwrite(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "burrito", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) - insertAndAssertArena(t, cache, "taco", arenaTestData{Value: 26, DataSize: 20}, []int64{}, nil) - insertAndAssertArena(t, cache, "enchilada", arenaTestData{Value: 28, DataSize: 20}, []int64{}, nil) - insertAndAssertArena(t, cache, "burrito", arenaTestData{Value: 33, DataSize: 6}, []int64{}, nil) + _, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("taco", testData{value: 26, dataSize: 20}) + require.NoError(t, err) + _, err = cache.Insert("enchilada", testData{value: 28, dataSize: 20}) + require.NoError(t, err) + ev1, err := cache.Insert("burrito", testData{value: 33, dataSize: 6}) + require.NoError(t, err) + assert.Empty(t, ev1) - // Increase size during overwrite; taco should be evicted - insertAndAssertArena(t, cache, "burrito", arenaTestData{Value: 33, DataSize: 12}, []int64{26}, nil) + // Act: Increase size during overwrite; taco should be evicted + evicted, err := cache.Insert("burrito", testData{value: 33, dataSize: 12}) - if v := cache.LookUp("taco"); v != nil { - t.Fatalf("expected 'taco' to be evicted, got %v", v) - } - if v := cache.LookUp("burrito"); v == nil || v.(arenaTestData).Value != 33 { - t.Fatalf("expected 33 for 'burrito', got %v", v) - } - if v := cache.LookUp("enchilada"); v == nil || v.(arenaTestData).Value != 28 { - t.Fatalf("expected 28 for 'enchilada', got %v", v) - } + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{26}) + assert.Nil(t, cache.LookUp("taco")) + + vBurrito := cache.LookUp("burrito") + require.NotNil(t, vBurrito) + assert.Equal(t, int64(33), vBurrito.(testData).value) + + vEnchilada := cache.LookUp("enchilada") + require.NotNil(t, vEnchilada) + assert.Equal(t, int64(28), vEnchilada.(testData).value) } func TestArenaRadixCache_MultipleEviction(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "burrito", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) - insertAndAssertArena(t, cache, "taco", arenaTestData{Value: 26, DataSize: 20}, []int64{}, nil) - insertAndAssertArena(t, cache, "enchilada", arenaTestData{Value: 28, DataSize: 20}, []int64{}, nil) + _, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("taco", testData{value: 26, dataSize: 20}) + require.NoError(t, err) + _, err = cache.Insert("enchilada", testData{value: 28, dataSize: 20}) + require.NoError(t, err) - // Large insert requiring all previous entries to be evicted - insertAndAssertArena(t, cache, "large_data", arenaTestData{Value: 33, DataSize: 45}, []int64{23, 26, 28}, nil) + // Act: Large insert requiring all previous entries to be evicted + evicted, err := cache.Insert("large_data", testData{value: 33, dataSize: 45}) - if v := cache.LookUp("taco"); v != nil { - t.Fatalf("expected 'taco' evicted, got %v", v) - } - if v := cache.LookUp("burrito"); v != nil { - t.Fatalf("expected 'burrito' evicted, got %v", v) - } - if v := cache.LookUp("enchilada"); v != nil { - t.Fatalf("expected 'enchilada' evicted, got %v", v) - } - if v := cache.LookUp("large_data"); v == nil || v.(arenaTestData).Value != 33 { - t.Fatalf("expected 33 for 'large_data', got %v", v) - } + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{23, 26, 28}) + assert.Nil(t, cache.LookUp("taco")) + assert.Nil(t, cache.LookUp("burrito")) + assert.Nil(t, cache.LookUp("enchilada")) + + vLarge := cache.LookUp("large_data") + require.NotNil(t, vLarge) + assert.Equal(t, int64(33), vLarge.(testData).value) } func TestArenaRadixCache_WhenEntrySizeMoreThanCacheMaxSize(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "burrito", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) + _, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) - // Attempt inserting item with size > arenaTestMaxSize - insertAndAssertArena(t, cache, "taco", arenaTestData{Value: 26, DataSize: arenaTestMaxSize + 1}, []int64{}, ErrInvalidEntrySize) + // Act: Attempt inserting item with size > testMaxSize + evicted, err := cache.Insert("taco", testData{value: 26, dataSize: testMaxSize + 1}) - if v := cache.LookUp("burrito"); v == nil || v.(arenaTestData).Value != 23 { - t.Fatalf("expected 'burrito' to be retained, got %v", v) - } + // Assert + require.ErrorIs(t, err, ErrInvalidEntrySize) + assert.Empty(t, evicted) + + vBurrito := cache.LookUp("burrito") + require.NotNil(t, vBurrito) + assert.Equal(t, int64(23), vBurrito.(testData).value) } func TestArenaRadixCache_EraseWhenKeyPresent(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "burrito", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) + _, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + // Act deletedEntry := cache.Erase("burrito") - if deletedEntry == nil || deletedEntry.(arenaTestData).Value != 23 { - t.Fatalf("expected erased value 23, got %v", deletedEntry) - } - if v := cache.LookUp("burrito"); v != nil { - t.Fatalf("expected 'burrito' to be nil after erase, got %v", v) - } + + // Assert + require.NotNil(t, deletedEntry) + assert.Equal(t, int64(23), deletedEntry.(testData).value) + assert.Nil(t, cache.LookUp("burrito")) } func TestArenaRadixCache_EraseWhenKeyNotPresent(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "burrito", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) + _, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + // Act deletedEntry := cache.Erase("taco") - if deletedEntry != nil { - t.Fatalf("expected nil for non-existent erase, got %v", deletedEntry) - } - if v := cache.LookUp("burrito"); v == nil || v.(arenaTestData).Value != 23 { - t.Fatalf("expected 'burrito' to remain, got %v", v) - } + + // Assert + assert.Nil(t, deletedEntry) + vBurrito := cache.LookUp("burrito") + require.NotNil(t, vBurrito) + assert.Equal(t, int64(23), vBurrito.(testData).value) } func TestArenaRadixCache_EraseCacheWithGivenPrefix(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "a", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) - insertAndAssertArena(t, cache, "a/b", arenaTestData{Value: 26, DataSize: 5}, []int64{}, nil) - insertAndAssertArena(t, cache, "a/b/d", arenaTestData{Value: 22, DataSize: 6}, []int64{}, nil) - insertAndAssertArena(t, cache, "a/c", arenaTestData{Value: 20, DataSize: 6}, []int64{}, nil) - insertAndAssertArena(t, cache, "b", arenaTestData{Value: 21, DataSize: 2}, []int64{}, nil) + _, err := cache.Insert("a", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("a/b/d", testData{value: 22, dataSize: 6}) + require.NoError(t, err) + _, err = cache.Insert("a/c", testData{value: 20, dataSize: 6}) + require.NoError(t, err) + _, err = cache.Insert("b", testData{value: 21, dataSize: 2}) + require.NoError(t, err) + // Act cache.EraseEntriesWithGivenPrefix("a") - if v := cache.LookUp("a"); v != nil { - t.Fatalf("expected 'a' erased, got %v", v) - } - if v := cache.LookUp("a/b"); v != nil { - t.Fatalf("expected 'a/b' erased, got %v", v) - } - if v := cache.LookUp("a/b/d"); v != nil { - t.Fatalf("expected 'a/b/d' erased, got %v", v) - } - if v := cache.LookUp("a/c"); v != nil { - t.Fatalf("expected 'a/c' erased, got %v", v) - } - if v := cache.LookUp("b"); v == nil || v.Size() != 2 { - t.Fatalf("expected 'b' to remain with size 2, got %v", v) - } + // Assert + assert.Nil(t, cache.LookUp("a")) + assert.Nil(t, cache.LookUp("a/b")) + assert.Nil(t, cache.LookUp("a/b/d")) + assert.Nil(t, cache.LookUp("a/c")) + + vb := cache.LookUp("b") + require.NotNil(t, vb) + assert.Equal(t, uint64(2), vb.Size()) } func TestArenaRadixCache_EraseCacheWithEmptyPrefix(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "a", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) - insertAndAssertArena(t, cache, "a/b", arenaTestData{Value: 26, DataSize: 5}, []int64{}, nil) - insertAndAssertArena(t, cache, "b", arenaTestData{Value: 21, DataSize: 2}, []int64{}, nil) + _, err := cache.Insert("a", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("b", testData{value: 21, dataSize: 2}) + require.NoError(t, err) + // Act cache.EraseEntriesWithGivenPrefix("") - if v := cache.LookUp("a"); v != nil { - t.Fatalf("expected 'a' erased, got %v", v) - } - if v := cache.LookUp("a/b"); v != nil { - t.Fatalf("expected 'a/b' erased, got %v", v) - } - if v := cache.LookUp("b"); v != nil { - t.Fatalf("expected 'b' erased, got %v", v) - } + // Assert + assert.Nil(t, cache.LookUp("a")) + assert.Nil(t, cache.LookUp("a/b")) + assert.Nil(t, cache.LookUp("b")) } func TestArenaRadixCache_EraseCacheWhereNoEntriesExistWithGivenPrefix(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "a", arenaTestData{Value: 23, DataSize: 4}, []int64{}, nil) - insertAndAssertArena(t, cache, "a/b", arenaTestData{Value: 26, DataSize: 5}, []int64{}, nil) - insertAndAssertArena(t, cache, "b", arenaTestData{Value: 21, DataSize: 2}, []int64{}, nil) + _, err := cache.Insert("a", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("b", testData{value: 21, dataSize: 2}) + require.NoError(t, err) + // Act cache.EraseEntriesWithGivenPrefix("c") - if v := cache.LookUp("a"); v == nil || v.Size() != 4 { - t.Fatalf("expected 'a' size 4, got %v", v) - } - if v := cache.LookUp("a/b"); v == nil || v.Size() != 5 { - t.Fatalf("expected 'a/b' size 5, got %v", v) - } - if v := cache.LookUp("b"); v == nil || v.Size() != 2 { - t.Fatalf("expected 'b' size 2, got %v", v) - } + // Assert + va := cache.LookUp("a") + require.NotNil(t, va) + assert.Equal(t, uint64(4), va.Size()) + + vab := cache.LookUp("a/b") + require.NotNil(t, vab) + assert.Equal(t, uint64(5), vab.Size()) + + vb := cache.LookUp("b") + require.NotNil(t, vb) + assert.Equal(t, uint64(2), vb.Size()) } func TestArenaRadixCache_EraseCacheWithGivenPrefixWithSomeEntriesEvictedDueToCacheSize(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - insertAndAssertArena(t, cache, "a", arenaTestData{Value: 23, DataSize: 20}, []int64{}, nil) - insertAndAssertArena(t, cache, "a/b", arenaTestData{Value: 26, DataSize: 10}, []int64{}, nil) - insertAndAssertArena(t, cache, "a/b/d", arenaTestData{Value: 22, DataSize: 5}, []int64{}, nil) - insertAndAssertArena(t, cache, "a/c", arenaTestData{Value: 20, DataSize: 10}, []int64{}, nil) - insertAndAssertArena(t, cache, "b", arenaTestData{Value: 21, DataSize: 15}, []int64{23}, nil) + _, err := cache.Insert("a", testData{value: 23, dataSize: 20}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 10}) + require.NoError(t, err) + _, err = cache.Insert("a/b/d", testData{value: 22, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("a/c", testData{value: 20, dataSize: 10}) + require.NoError(t, err) + evicted, err := cache.Insert("b", testData{value: 21, dataSize: 15}) + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{23}) - // "a" was evicted by "b", remaining "a/b", "a/b/d", "a/c" should be erased + // Act: "a" was evicted by "b", remaining "a/b", "a/b/d", "a/c" should be erased cache.EraseEntriesWithGivenPrefix("a") - if v := cache.LookUp("a"); v != nil { - t.Fatalf("expected 'a' nil, got %v", v) - } - if v := cache.LookUp("a/b"); v != nil { - t.Fatalf("expected 'a/b' nil, got %v", v) - } - if v := cache.LookUp("a/b/d"); v != nil { - t.Fatalf("expected 'a/b/d' nil, got %v", v) - } - if v := cache.LookUp("a/c"); v != nil { - t.Fatalf("expected 'a/c' nil, got %v", v) - } - if v := cache.LookUp("b"); v == nil || v.Size() != 15 { - t.Fatalf("expected 'b' size 15, got %v", v) - } + // Assert + assert.Nil(t, cache.LookUp("a")) + assert.Nil(t, cache.LookUp("a/b")) + assert.Nil(t, cache.LookUp("a/b/d")) + assert.Nil(t, cache.LookUp("a/c")) + + vb := cache.LookUp("b") + require.NotNil(t, vb) + assert.Equal(t, uint64(15), vb.Size()) } func TestArenaRadixCache_UpdateSize(t *testing.T) { t.Run("NonExistentKey", func(t *testing.T) { + // Arrange cache := NewArenaRadixCache(100, WithInvariantChecking(true)) + + // Act err := cache.UpdateSize("key1", 20) - if !errors.Is(err, ErrEntryNotExist) { - t.Fatalf("expected ErrEntryNotExist, got %v", err) - } + + // Assert + require.ErrorIs(t, err, ErrEntryNotExist) }) - t.Run("Immediate Eviction", func(t *testing.T) { + t.Run("ImmediateEviction", func(t *testing.T) { + // Arrange cache := NewArenaRadixCache(100, WithInvariantChecking(true)) - data1 := arenaTestData{Value: 1, DataSize: 10} - data2 := arenaTestData{Value: 2, DataSize: 70} - _, _ = cache.Insert("key1", data1) - _, _ = cache.Insert("key2", data2) + data1 := testData{value: 1, dataSize: 10} + data2 := testData{value: 2, dataSize: 70} + _, err := cache.Insert("key1", data1) + require.NoError(t, err) + _, err = cache.Insert("key2", data2) + require.NoError(t, err) + // Act errUpdate := cache.UpdateSize("key1", 30) - if errUpdate != nil { - t.Fatalf("unexpected error: %v", errUpdate) - } - if v := cache.LookUp("key1"); v != nil { - t.Fatalf("expected 'key1' evicted, got %v", v) - } - if v := cache.LookUp("key2"); v == nil { - t.Fatalf("expected 'key2' present, got nil") - } + + // Assert + require.NoError(t, errUpdate) + assert.Nil(t, cache.LookUp("key1")) + assert.NotNil(t, cache.LookUp("key2")) }) } func TestArenaRadixCache_UpdateSize_ExceedsMaxSize(t *testing.T) { + // Arrange cache := NewArenaRadixCache(100, WithInvariantChecking(true)) - data := &arenaTestData{Value: 1, DataSize: 50} + data := testData{value: 1, dataSize: 50} _, err := cache.Insert("file.txt", data) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + require.NoError(t, err) - data.DataSize = 150 + // Act err = cache.UpdateSize("file.txt", 100) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if v := cache.LookUp("file.txt"); v != nil { - t.Fatalf("expected 'file.txt' evicted, got %v", v) - } + // Assert + require.NoError(t, err) + assert.Nil(t, cache.LookUp("file.txt")) } func TestArenaRadixCache_UpdateSize_MultipleEvictions(t *testing.T) { + // Arrange cache := NewArenaRadixCache(100, WithInvariantChecking(true)) + _, err := cache.Insert("k1", testData{value: 1, dataSize: 20}) + require.NoError(t, err) + _, err = cache.Insert("k2", testData{value: 2, dataSize: 20}) + require.NoError(t, err) + _, err = cache.Insert("k3", testData{value: 3, dataSize: 20}) + require.NoError(t, err) + _, err = cache.Insert("k4", testData{value: 4, dataSize: 20}) + require.NoError(t, err) - _, _ = cache.Insert("k1", arenaTestData{Value: 1, DataSize: 20}) - _, _ = cache.Insert("k2", arenaTestData{Value: 2, DataSize: 20}) - _, _ = cache.Insert("k3", arenaTestData{Value: 3, DataSize: 20}) - _, _ = cache.Insert("k4", arenaTestData{Value: 4, DataSize: 20}) - - err := cache.UpdateSize("k4", 50) - if err != nil { - t.Fatalf("unexpected error updating size: %v", err) - } + // Act + err = cache.UpdateSize("k4", 50) - if v := cache.LookUp("k1"); v != nil { - t.Fatalf("expected k1 evicted, got %v", v) - } - if v := cache.LookUp("k2"); v != nil { - t.Fatalf("expected k2 evicted, got %v", v) - } - if v := cache.LookUp("k3"); v == nil { - t.Fatalf("expected k3 present, got nil") - } - if v := cache.LookUp("k4"); v == nil { - t.Fatalf("expected k4 present, got nil") - } + // Assert + require.NoError(t, err) + assert.Nil(t, cache.LookUp("k1")) + assert.Nil(t, cache.LookUp("k2")) + assert.NotNil(t, cache.LookUp("k3")) + assert.NotNil(t, cache.LookUp("k4")) } func TestArenaRadixCache_UpdateWhenKeyPresent(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) key := "burrito" - data := arenaTestData{Value: 23, DataSize: 4} - insertAndAssertArena(t, cache, key, data, []int64{}, nil) + data := testData{value: 23, dataSize: 4} + _, err := cache.Insert(key, data) + require.NoError(t, err) - newData := arenaTestData{Value: 2, DataSize: 4} - err := cache.UpdateWithoutChangingOrder(key, newData) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if v := cache.LookUp(key); v == nil || v.(arenaTestData).Value != 2 { - t.Fatalf("expected value 2, got %v", v) - } + // Act + newData := testData{value: 2, dataSize: 4} + err = cache.UpdateWithoutChangingOrder(key, newData) + + // Assert + require.NoError(t, err) + v := cache.LookUp(key) + require.NotNil(t, v) + assert.Equal(t, int64(2), v.(testData).value) } func TestArenaRadixCache_UpdateWhenKeyNotPresent(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) key := "burrito" - data := arenaTestData{Value: 23, DataSize: 4} + data := testData{value: 23, dataSize: 4} + // Act err := cache.UpdateWithoutChangingOrder(key, data) - if !errors.Is(err, ErrEntryNotExist) { - t.Fatalf("expected ErrEntryNotExist, got %v", err) - } + + // Assert + require.ErrorIs(t, err, ErrEntryNotExist) } func TestArenaRadixCache_UpdateWhenSizeIsDifferent(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) key := "burrito" - data := arenaTestData{Value: 23, DataSize: 4} - insertAndAssertArena(t, cache, key, data, []int64{}, nil) + data := testData{value: 23, dataSize: 4} + _, err := cache.Insert(key, data) + require.NoError(t, err) - newData := arenaTestData{Value: 2, DataSize: 3} - err := cache.UpdateWithoutChangingOrder(key, newData) - if !errors.Is(err, ErrInvalidUpdateEntrySize) { - t.Fatalf("expected ErrInvalidUpdateEntrySize, got %v", err) - } + // Act + newData := testData{value: 2, dataSize: 3} + err = cache.UpdateWithoutChangingOrder(key, newData) + + // Assert + require.ErrorIs(t, err, ErrInvalidUpdateEntrySize) } func TestArenaRadixCache_UpdateNilValue(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) + + // Act err := cache.UpdateWithoutChangingOrder("key", nil) - if !errors.Is(err, ErrInvalidEntry) { - t.Fatalf("expected ErrInvalidEntry, got %v", err) - } + + // Assert + require.ErrorIs(t, err, ErrInvalidEntry) } func TestArenaRadixCache_UpdateNotChangeOrder(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) key1 := "burrito1" - data1 := arenaTestData{Value: 23, DataSize: 10} - insertAndAssertArena(t, cache, key1, data1, []int64{}, nil) + data1 := testData{value: 23, dataSize: 10} + _, err := cache.Insert(key1, data1) + require.NoError(t, err) key2 := "burrito2" - data2 := arenaTestData{Value: 2, DataSize: 40} - insertAndAssertArena(t, cache, key2, data2, []int64{}, nil) + data2 := testData{value: 2, dataSize: 40} + _, err = cache.Insert(key2, data2) + require.NoError(t, err) - newData := arenaTestData{Value: 7, DataSize: 10} - err := cache.UpdateWithoutChangingOrder(key1, newData) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + // Act: Update key1 without changing order, then insert key3 (size 5) + newData := testData{value: 7, dataSize: 10} + err = cache.UpdateWithoutChangingOrder(key1, newData) + require.NoError(t, err) - // Inserting key3 (size 5) should evict key1 (value 7) because key1 remained the LRU element key3 := "burrito3" - data3 := arenaTestData{Value: 3, DataSize: 5} - insertAndAssertArena(t, cache, key3, data3, []int64{7}, nil) + data3 := testData{value: 3, dataSize: 5} + evicted, err := cache.Insert(key3, data3) + + // Assert: key1 (value 7) is evicted because key1 remained the LRU element + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{7}) } func TestArenaRadixCache_LookUpWithoutChangingOrder(t *testing.T) { + // Arrange cache := setupArenaRadixCacheTest(t) - - // Absent key - if v := cache.LookUpWithoutChangingOrder("absent"); v != nil { - t.Fatalf("expected nil for absent key, got %v", v) - } + assert.Nil(t, cache.LookUpWithoutChangingOrder("absent")) key1 := "burrito1" - data1 := arenaTestData{Value: 23, DataSize: 10} - insertAndAssertArena(t, cache, key1, data1, []int64{}, nil) + data1 := testData{value: 23, dataSize: 10} + _, err := cache.Insert(key1, data1) + require.NoError(t, err) key2 := "burrito2" - data2 := arenaTestData{Value: 2, DataSize: 40} - insertAndAssertArena(t, cache, key2, data2, []int64{}, nil) + data2 := testData{value: 2, dataSize: 40} + _, err = cache.Insert(key2, data2) + require.NoError(t, err) - // LookUpWithoutChangingOrder on key1 + // Act: LookUpWithoutChangingOrder on key1, then insert key3 (size 5) val := cache.LookUpWithoutChangingOrder(key1) - if val == nil || val.(arenaTestData).Value != 23 { - t.Fatalf("expected 23 for key1, got %v", val) - } + require.NotNil(t, val) + assert.Equal(t, int64(23), val.(testData).value) - // Inserting key3 (size 5) should evict key1 because its LRU position was not altered key3 := "burrito3" - data3 := arenaTestData{Value: 3, DataSize: 5} - insertAndAssertArena(t, cache, key3, data3, []int64{23}, nil) + data3 := testData{value: 3, dataSize: 5} + evicted, err := cache.Insert(key3, data3) + + // Assert: key1 is evicted because its LRU position was not altered + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{23}) } func TestArenaRadixCache_RadixEdgeSplitsAndCompression(t *testing.T) { + // Arrange cache := NewArenaRadixCache(1000, WithInvariantChecking(true)) - keys := []string{ "apple", "app", @@ -546,31 +561,29 @@ func TestArenaRadixCache_RadixEdgeSplitsAndCompression(t *testing.T) { } for i, k := range keys { - _, err := cache.Insert(k, arenaTestData{Value: int64(i + 1), DataSize: 10}) - if err != nil { - t.Fatalf("failed inserting key %q: %v", k, err) - } + _, err := cache.Insert(k, testData{value: int64(i + 1), dataSize: 10}) + require.NoError(t, err) } for i, k := range keys { v := cache.LookUp(k) - if v == nil || v.(arenaTestData).Value != int64(i+1) { - t.Fatalf("expected %d for key %q, got %v", i+1, k, v) - } + require.NotNil(t, v) + assert.Equal(t, int64(i+1), v.(testData).value) } - // Erase leaves and intermediate nodes to exercise compressPathUpwards - if v := cache.Erase("application"); v == nil || v.(arenaTestData).Value != 3 { - t.Fatalf("failed erasing 'application', got %v", v) - } - if v := cache.Erase("apply"); v == nil || v.(arenaTestData).Value != 4 { - t.Fatalf("failed erasing 'apply', got %v", v) - } - if v := cache.Erase("app"); v == nil || v.(arenaTestData).Value != 2 { - t.Fatalf("failed erasing 'app', got %v", v) - } + // Act: Erase leaves and intermediate nodes to exercise compressPathUpwards + vAppn := cache.Erase("application") + vApply := cache.Erase("apply") + vApp := cache.Erase("app") + + // Assert + require.NotNil(t, vAppn) + assert.Equal(t, int64(3), vAppn.(testData).value) + require.NotNil(t, vApply) + assert.Equal(t, int64(4), vApply.(testData).value) + require.NotNil(t, vApp) + assert.Equal(t, int64(2), vApp.(testData).value) - // Verify remaining keys still accessible remaining := []struct { key string val int64 @@ -584,35 +597,33 @@ func TestArenaRadixCache_RadixEdgeSplitsAndCompression(t *testing.T) { for _, tc := range remaining { v := cache.LookUp(tc.key) - if v == nil || v.(arenaTestData).Value != tc.val { - t.Fatalf("expected %d for key %q, got %v", tc.val, tc.key, v) - } + require.NotNil(t, v) + assert.Equal(t, tc.val, v.(testData).value) } } func TestArenaRadixCache_FreeListRecycling(t *testing.T) { + // Arrange cache := NewArenaRadixCache(10000, WithInvariantChecking(true)) - for cycle := 0; cycle < 10; cycle++ { - for i := 0; i < 50; i++ { + // Act & Assert + for range 10 { + for i := range 50 { key := fmt.Sprintf("prefix/subdir_%d/file_%d.txt", i%5, i) - _, err := cache.Insert(key, arenaTestData{Value: int64(i), DataSize: 10}) - if err != nil { - t.Fatalf("insert failed at cycle %d, item %d: %v", cycle, i, err) - } + _, err := cache.Insert(key, testData{value: int64(i), dataSize: 10}) + require.NoError(t, err) } - for i := 0; i < 50; i++ { + for i := range 50 { key := fmt.Sprintf("prefix/subdir_%d/file_%d.txt", i%5, i) v := cache.Erase(key) - if v == nil { - t.Fatalf("erase failed at cycle %d, item %d", cycle, i) - } + require.NotNil(t, v) } } } func TestArenaRadixCache_DeepHierarchy(t *testing.T) { + // Arrange cache := NewArenaRadixCache(10000, WithInvariantChecking(true)) path1 := "a/b/c/d/e/f/g/h/i/j/file1.txt" @@ -620,116 +631,215 @@ func TestArenaRadixCache_DeepHierarchy(t *testing.T) { path3 := "a/b/c/d/other/file3.txt" path4 := "x/y/z/file4.txt" - _, _ = cache.Insert(path1, arenaTestData{Value: 1, DataSize: 10}) - _, _ = cache.Insert(path2, arenaTestData{Value: 2, DataSize: 10}) - _, _ = cache.Insert(path3, arenaTestData{Value: 3, DataSize: 10}) - _, _ = cache.Insert(path4, arenaTestData{Value: 4, DataSize: 10}) + _, err := cache.Insert(path1, testData{value: 1, dataSize: 10}) + require.NoError(t, err) + _, err = cache.Insert(path2, testData{value: 2, dataSize: 10}) + require.NoError(t, err) + _, err = cache.Insert(path3, testData{value: 3, dataSize: 10}) + require.NoError(t, err) + _, err = cache.Insert(path4, testData{value: 4, dataSize: 10}) + require.NoError(t, err) + // Act cache.EraseEntriesWithGivenPrefix("a/b/c/d/e/") - if v := cache.LookUp(path1); v != nil { - t.Fatalf("expected path1 erased, got %v", v) - } - if v := cache.LookUp(path2); v != nil { - t.Fatalf("expected path2 erased, got %v", v) - } - if v := cache.LookUp(path3); v == nil || v.(arenaTestData).Value != 3 { - t.Fatalf("expected path3 value 3, got %v", v) - } - if v := cache.LookUp(path4); v == nil || v.(arenaTestData).Value != 4 { - t.Fatalf("expected path4 value 4, got %v", v) - } + // Assert + assert.Nil(t, cache.LookUp(path1)) + assert.Nil(t, cache.LookUp(path2)) + + v3 := cache.LookUp(path3) + require.NotNil(t, v3) + assert.Equal(t, int64(3), v3.(testData).value) + + v4 := cache.LookUp(path4) + require.NotNil(t, v4) + assert.Equal(t, int64(4), v4.(testData).value) } func TestArenaRadixCache_EraseRootValue(t *testing.T) { + // Arrange cache := NewArenaRadixCache(1000, WithInvariantChecking(true)) + _, err := cache.Insert("", testData{value: 100, dataSize: 10}) + require.NoError(t, err) + _, err = cache.Insert("child", testData{value: 200, dataSize: 20}) + require.NoError(t, err) - _, _ = cache.Insert("", arenaTestData{Value: 100, DataSize: 10}) - _, _ = cache.Insert("child", arenaTestData{Value: 200, DataSize: 20}) - + // Act erased := cache.Erase("") - if erased == nil || erased.(arenaTestData).Value != 100 { - t.Fatalf("expected erased root value 100, got %v", erased) - } - if v := cache.LookUp(""); v != nil { - t.Fatalf("expected root nil after erase, got %v", v) - } - if v := cache.LookUp("child"); v == nil || v.(arenaTestData).Value != 200 { - t.Fatalf("expected child value 200 preserved, got %v", v) - } + // Assert + require.NotNil(t, erased) + assert.Equal(t, int64(100), erased.(testData).value) + assert.Nil(t, cache.LookUp("")) + + vChild := cache.LookUp("child") + require.NotNil(t, vChild) + assert.Equal(t, int64(200), vChild.(testData).value) } +// TestArenaRadixCache_CheckInvariants_PanicScenarios verifies that checkInvariants() detects and panics +// on internal structural, nodeMap, freeCount, and zeroSizeCount corruption scenarios. +// +// White-box testing rationale: Correct public Cache operations preserve all tree, LRU list, free-list, +// nodeMap, and byte-accounting invariants by construction. Triggering checkInvariants() failure paths +// therefore requires injecting synthetic corruption into internal struct fields after setup. func TestArenaRadixCache_CheckInvariants_PanicScenarios(t *testing.T) { t.Run("CurrentSizeExceedsMaxSize", func(t *testing.T) { + // Arrange c := NewArenaRadixCache(10).(*arenaRadix) c.currentSize = 20 - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on currentSize > maxSize") - } - }() - c.checkInvariants() + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) t.Run("CorruptLRULinks", func(t *testing.T) { + // Arrange c := NewArenaRadixCache(50).(*arenaRadix) - _, _ = c.Insert("k1", arenaTestData{Value: 1, DataSize: 10}) - _, _ = c.Insert("k2", arenaTestData{Value: 2, DataSize: 10}) + _, err := c.Insert("k1", testData{value: 1, dataSize: 10}) + require.NoError(t, err) + _, err = c.Insert("k2", testData{value: 2, dataSize: 10}) + require.NoError(t, err) secondID := c.nodes[c.head].next c.nodes[secondID].prev = nilNode - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on corrupt prev pointer") - } - }() - c.checkInvariants() + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) t.Run("CorruptTreeParent", func(t *testing.T) { + // Arrange c := NewArenaRadixCache(50).(*arenaRadix) - _, _ = c.Insert("a/b", arenaTestData{Value: 1, DataSize: 10}) + _, err := c.Insert("a/b", testData{value: 1, dataSize: 10}) + require.NoError(t, err) childID := c.nodes[c.root].child if childID != nilNode { c.nodes[childID].parent = nilNode } - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on corrupt parent pointer") - } - }() - c.checkInvariants() + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) t.Run("SizeSumMismatch", func(t *testing.T) { + // Arrange c := NewArenaRadixCache(50).(*arenaRadix) - _, _ = c.Insert("k1", arenaTestData{Value: 1, DataSize: 10}) - c.currentSize += 1 - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on size sum mismatch") + _, err := c.Insert("k1", testData{value: 1, dataSize: 10}) + require.NoError(t, err) + c.currentSize++ + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) + }) + + t.Run("CompactnessViolation", func(t *testing.T) { + // Arrange + c := NewArenaRadixCache(50).(*arenaRadix) + _, err := c.Insert("ab", testData{value: 1, dataSize: 10}) + require.NoError(t, err) + _, err = c.Insert("ac", testData{value: 2, dataSize: 10}) + require.NoError(t, err) + childID := c.nodes[c.root].child + if childID != nilNode && c.nodes[childID].value == nil { + firstGrandchild := c.nodes[childID].child + if firstGrandchild != nilNode { + c.nodes[firstGrandchild].sibling = nilNode } - }() - c.checkInvariants() + } + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) + }) + + t.Run("NodeMapOutOfBounds", func(t *testing.T) { + // Arrange + c := NewArenaRadixCache(50).(*arenaRadix) + _, _ = c.Insert("k1", testData{value: 1, dataSize: 10}) + c.nodeMap[hashString("k1")] = 999 + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) + }) + + t.Run("NodeMapNilValue", func(t *testing.T) { + // Arrange + c := NewArenaRadixCache(50).(*arenaRadix) + _, _ = c.Insert("k1", testData{value: 1, dataSize: 10}) + c.nodeMap[hashString("k1")] = c.root + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) + }) + + t.Run("NodeMapHashMismatch", func(t *testing.T) { + // Arrange + c := NewArenaRadixCache(50).(*arenaRadix) + _, _ = c.Insert("k1", testData{value: 1, dataSize: 10}) + id := c.nodeMap[hashString("k1")] + c.nodeMap[12345] = id + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) + }) + + t.Run("LeakedNodeAccountingMismatch", func(t *testing.T) { + // Arrange + c := NewArenaRadixCache(50).(*arenaRadix) + _, _ = c.Insert("k1", testData{value: 1, dataSize: 10}) + c.nodes = append(c.nodes, arenaRadixNode{ + parent: nilNode, + child: nilNode, + sibling: nilNode, + prev: nilNode, + next: nilNode, + }) + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) + }) + + t.Run("FreeCountDriftMismatch", func(t *testing.T) { + // Arrange + c := NewArenaRadixCache(50).(*arenaRadix) + _, _ = c.Insert("k1", testData{value: 1, dataSize: 10}) + _ = c.Erase("k1") + c.freeCount = 999 + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) - t.Run("CompactnessViolation", func(t *testing.T) { - c := NewArenaRadixCache(50).(*arenaRadix) - _, _ = c.Insert("ab", arenaTestData{Value: 1, DataSize: 10}) - _, _ = c.Insert("ac", arenaTestData{Value: 2, DataSize: 10}) - childID := c.nodes[c.root].child - if childID != nilNode && c.nodes[childID].value == nil { - firstGrandchild := c.nodes[childID].child - if firstGrandchild != nilNode { - c.nodes[firstGrandchild].sibling = nilNode - } - } - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on intermediate node compactness violation") - } - }() - c.checkInvariants() + t.Run("ZeroSizeCountMismatch", func(t *testing.T) { + // Arrange + c := NewArenaRadixCache(100).(*arenaRadix) + _, err := c.Insert("k1", testData{value: 1, dataSize: 0}) + require.NoError(t, err) + c.zeroSizeCount = 99 + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) } @@ -742,27 +852,20 @@ func TestArenaRadixCache_ModeratePressureLosslessCompaction(t *testing.T) { WithPressureFunc(func() float64 { return pressure }), WithCompactionThreshold(0.75), WithEvictionThreshold(0.90), - ).(*arenaRadix) + ).(PressureAwareCache) const totalKeys = 1000 const survivingStart = 900 // Keep keys 900..999 (100 keys) for i := range totalKeys { key := fmt.Sprintf("bucket_%02d/dir_%02d/sub_%02d/obj_%04d.bin", i%10, (i/10)%10, (i/100)%10, i) - _, err := c.Insert(key, arenaTestData{Value: int64(i), DataSize: 10}) + _, err := c.Insert(key, testData{value: int64(i), dataSize: 10}) require.NoError(t, err) } - peakNodeLen := len(c.nodes) - peakNodeCap := cap(c.nodes) - require.Greater(t, peakNodeLen, 1000) - for i := range survivingStart { key := fmt.Sprintf("bucket_%02d/dir_%02d/sub_%02d/obj_%04d.bin", i%10, (i/10)%10, (i/100)%10, i) require.NotNil(t, c.Erase(key)) } - require.NotEqual(t, nilNode, c.freeHead) - require.Greater(t, c.freeCount, uint32(0)) - require.Equal(t, peakNodeLen, len(c.nodes)) // Act pressure = 0.80 @@ -770,28 +873,24 @@ func TestArenaRadixCache_ModeratePressureLosslessCompaction(t *testing.T) { // Assert assert.Empty(t, evicted) - assert.Equal(t, nilNode, c.freeHead) - assert.Equal(t, uint32(0), c.freeCount) - assert.Equal(t, len(c.nodes), cap(c.nodes)) - assert.Less(t, cap(c.nodes), peakNodeCap) - assert.Equal(t, 100, c.len) - assert.Equal(t, uint64(1000), c.currentSize) + assertAlreadyCompacted(t, c) for i := survivingStart; i < totalKeys; i++ { key := fmt.Sprintf("bucket_%02d/dir_%02d/sub_%02d/obj_%04d.bin", i%10, (i/10)%10, (i/100)%10, i) v := c.LookUpWithoutChangingOrder(key) require.NotNil(t, v) - assert.Equal(t, int64(i), v.(arenaTestData).Value) + assert.Equal(t, int64(i), v.(testData).value) } pressure = 0.10 - _, err := c.Insert("filler", arenaTestData{Value: 999999, DataSize: 19000}) + evFiller, err := c.Insert("filler", testData{value: 999999, dataSize: 19000}) require.NoError(t, err) + assert.Empty(t, evFiller) for expectedID := survivingStart; expectedID < totalKeys; expectedID++ { - ev, err := c.Insert(fmt.Sprintf("trigger_%d", expectedID), arenaTestData{Value: -1, DataSize: 10}) + ev, err := c.Insert(fmt.Sprintf("trigger_%d", expectedID), testData{value: -1, dataSize: 10}) require.NoError(t, err) require.Len(t, ev, 1) - assert.Equal(t, int64(expectedID), ev[0].(arenaTestData).Value) + assert.Equal(t, int64(expectedID), ev[0].(testData).value) } } @@ -805,11 +904,11 @@ func TestArenaRadixCache_CriticalPressureLRUShedding(t *testing.T) { WithCompactionThreshold(0.75), WithEvictionThreshold(0.90), WithEvictionRetentionRatio(0.40), - ).(*arenaRadix) + ).(PressureAwareCache) for i := range 100 { key := fmt.Sprintf("dir_%d/item_%03d", i%5, i) - _, err := c.Insert(key, arenaTestData{Value: int64(i), DataSize: 10}) + _, err := c.Insert(key, testData{value: int64(i), dataSize: 10}) require.NoError(t, err) } @@ -820,13 +919,9 @@ func TestArenaRadixCache_CriticalPressureLRUShedding(t *testing.T) { // Assert require.Len(t, evicted, 60) for i, ev := range evicted { - assert.Equal(t, int64(i), ev.(arenaTestData).Value) + assert.Equal(t, int64(i), ev.(testData).value) } - assert.Equal(t, uint64(400), c.currentSize) - assert.Equal(t, 40, c.len) - assert.Equal(t, nilNode, c.freeHead) - assert.Equal(t, uint32(0), c.freeCount) - assert.Equal(t, len(c.nodes), cap(c.nodes)) + assertAlreadyCompacted(t, c) for i := range 60 { key := fmt.Sprintf("dir_%d/item_%03d", i%5, i) @@ -836,7 +931,7 @@ func TestArenaRadixCache_CriticalPressureLRUShedding(t *testing.T) { key := fmt.Sprintf("dir_%d/item_%03d", i%5, i) v := c.LookUpWithoutChangingOrder(key) require.NotNil(t, v) - assert.Equal(t, int64(i), v.(arenaTestData).Value) + assert.Equal(t, int64(i), v.(testData).value) } } @@ -853,7 +948,7 @@ func TestArenaRadixCache_AutomaticPressureTriggersAndReentrancy(t *testing.T) { if cacheRef != nil { _ = cacheRef.LookUpWithoutChangingOrder("probe_key") // Also verify mutating re-entrancy is guarded against infinite recursion. - _, _ = cacheRef.Insert("reentrant_probe", arenaTestData{Value: 1, DataSize: 0}) + _, _ = cacheRef.Insert("reentrant_probe", testData{value: 1, dataSize: 0}) _ = cacheRef.Erase("reentrant_probe") reentrantReads++ } @@ -862,40 +957,45 @@ func TestArenaRadixCache_AutomaticPressureTriggersAndReentrancy(t *testing.T) { WithCompactionThreshold(0.75), WithEvictionThreshold(0.90), WithEvictionRetentionRatio(0.50), - ).(*arenaRadix) + ).(PressureAwareCache) cacheRef = c for i := range 20 { - _, err := c.Insert(fmt.Sprintf("k_%02d", i), arenaTestData{Value: int64(i), DataSize: 10}) + _, err := c.Insert(fmt.Sprintf("k_%02d", i), testData{value: int64(i), dataSize: 10}) require.NoError(t, err) } for i := range 9 { require.NotNil(t, c.Erase(fmt.Sprintf("k_%02d", i))) } - require.NotEqual(t, nilNode, c.freeHead) // Act 1: Automatic Tier 1 compaction on Erase of existing key under moderate pressure. pressure = 0.80 require.NotNil(t, c.Erase("k_09")) // Assert 1 - assert.Equal(t, nilNode, c.freeHead) - assert.Equal(t, len(c.nodes), cap(c.nodes)) - assert.Equal(t, uint64(100), c.currentSize) + assertAlreadyCompacted(t, c) + for i := 10; i < 20; i++ { + require.NotNil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k_%02d", i))) + } // Act 2: Automatic Tier 2 shedding on Insert under critical pressure. // maxSize = 200, retention = 0.50 -> targetSize = 100 bytes. // Before insert: 10 entries (100 bytes). Insert "new_mru" (60 bytes) -> 160 bytes -> sheds 6 oldest entries (60 bytes) down to 100 bytes. pressure = 0.95 - evicted, err := c.Insert("new_mru", arenaTestData{Value: 999, DataSize: 60}) + evicted, err := c.Insert("new_mru", testData{value: 999, dataSize: 60}) // Assert 2 require.NoError(t, err) assert.Len(t, evicted, 6) - assert.Equal(t, uint64(100), c.currentSize) - assert.Equal(t, nilNode, c.freeHead) + assertAlreadyCompacted(t, c) + for i := 10; i < 16; i++ { + assert.Nil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k_%02d", i))) + } + for i := 16; i < 20; i++ { + assert.NotNil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k_%02d", i))) + } assert.NotNil(t, c.LookUpWithoutChangingOrder("new_mru")) - assert.Greater(t, reentrantReads, 0) + assert.Positive(t, reentrantReads) } func TestArenaRadixCache_CompactionAndSheddingEdgeCases(t *testing.T) { @@ -906,7 +1006,7 @@ func TestArenaRadixCache_CompactionAndSheddingEdgeCases(t *testing.T) { 100, WithInvariantChecking(true), WithPressureFunc(func() float64 { return pressure }), - ).(*arenaRadix) + ).(PressureAwareCache) // Act c.Compact() @@ -914,10 +1014,8 @@ func TestArenaRadixCache_CompactionAndSheddingEdgeCases(t *testing.T) { // Assert assert.Empty(t, ev) - assert.Equal(t, 1, len(c.nodes)) - assert.Equal(t, 1, cap(c.nodes)) - assert.Equal(t, uint32(0), c.root) - assert.Equal(t, nilNode, c.freeHead) + assertAlreadyCompacted(t, c) + assert.Nil(t, c.LookUpWithoutChangingOrder("")) }) t.Run("SingleLargeEntryShedding", func(t *testing.T) { @@ -928,8 +1026,8 @@ func TestArenaRadixCache_CompactionAndSheddingEdgeCases(t *testing.T) { WithInvariantChecking(true), WithPressureFunc(func() float64 { return pressure }), WithEvictionRetentionRatio(0.50), - ).(*arenaRadix) - _, err := c.Insert("only_item", arenaTestData{Value: 42, DataSize: 80}) + ).(PressureAwareCache) + _, err := c.Insert("only_item", testData{value: 42, dataSize: 80}) require.NoError(t, err) // Act @@ -938,19 +1036,17 @@ func TestArenaRadixCache_CompactionAndSheddingEdgeCases(t *testing.T) { // Assert require.Len(t, ev, 1) - assert.Equal(t, int64(42), ev[0].(arenaTestData).Value) - assert.Equal(t, 0, c.len) - assert.Equal(t, uint64(0), c.currentSize) - assert.Equal(t, 1, len(c.nodes)) - assert.Equal(t, 1, cap(c.nodes)) + assert.Equal(t, int64(42), ev[0].(testData).value) + assert.Nil(t, c.LookUpWithoutChangingOrder("only_item")) + assertAlreadyCompacted(t, c) }) t.Run("EmptyStringKeyAtRoot", func(t *testing.T) { // Arrange - c := NewArenaRadixCache(100, WithInvariantChecking(true)).(*arenaRadix) - _, err := c.Insert("", arenaTestData{Value: 777, DataSize: 10}) + c := NewArenaRadixCache(100, WithInvariantChecking(true)).(PressureAwareCache) + _, err := c.Insert("", testData{value: 777, dataSize: 10}) require.NoError(t, err) - _, err = c.Insert("a/b/c", arenaTestData{Value: 888, DataSize: 10}) + _, err = c.Insert("a/b/c", testData{value: 888, dataSize: 10}) require.NoError(t, err) _ = c.Erase("a/b/c") @@ -960,7 +1056,8 @@ func TestArenaRadixCache_CompactionAndSheddingEdgeCases(t *testing.T) { // Assert require.NotNil(t, v) - assert.Equal(t, int64(777), v.(arenaTestData).Value) + assert.Equal(t, int64(777), v.(testData).value) + assertAlreadyCompacted(t, c) }) t.Run("ZeroSizeEntriesTerminationAndFullEvictionWhenRetentionZero", func(t *testing.T) { @@ -971,13 +1068,13 @@ func TestArenaRadixCache_CompactionAndSheddingEdgeCases(t *testing.T) { WithInvariantChecking(true), WithPressureFunc(func() float64 { return pressure }), WithEvictionRetentionRatio(0.0), - ).(*arenaRadix) + ).(PressureAwareCache) for i := range 5 { - _, err := c.Insert(fmt.Sprintf("zero_%d", i), arenaTestData{Value: int64(i), DataSize: 0}) + _, err := c.Insert(fmt.Sprintf("zero_%d", i), testData{value: int64(i), dataSize: 0}) require.NoError(t, err) } - _, err := c.Insert("nonzero", arenaTestData{Value: 99, DataSize: 40}) + _, err := c.Insert("nonzero", testData{value: 99, dataSize: 40}) require.NoError(t, err) // Act @@ -986,24 +1083,15 @@ func TestArenaRadixCache_CompactionAndSheddingEdgeCases(t *testing.T) { // Assert assert.Len(t, ev, 6) - assert.Equal(t, uint64(0), c.currentSize) - assert.Equal(t, 0, c.len) + assert.Nil(t, c.LookUpWithoutChangingOrder("nonzero")) + for i := range 5 { + assert.Nil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("zero_%d", i))) + } + assertAlreadyCompacted(t, c) }) } -type deadlockProbeValue struct { - size uint64 - onSize func() -} - -func (v deadlockProbeValue) Size() uint64 { - if v.onSize != nil { - v.onSize() - } - return v.size -} - -func TestArenaRadixCache_AdversarialAuditRegressions(t *testing.T) { +func TestArenaRadixCache_PressureAndConcurrencyEdgeCases(t *testing.T) { t.Run("InsertIntoEmptyOrUnderTargetCacheDoesNotSelfEvict", func(t *testing.T) { // Arrange c := NewArenaRadixCache( @@ -1011,19 +1099,17 @@ func TestArenaRadixCache_AdversarialAuditRegressions(t *testing.T) { WithInvariantChecking(true), WithPressureFunc(func() float64 { return 0.95 }), WithEvictionRetentionRatio(0.50), - ).(*arenaRadix) + ) // Act - evicted, err := c.Insert("first_key", arenaTestData{Value: 42, DataSize: 10}) + evicted, err := c.Insert("first_key", testData{value: 42, dataSize: 10}) lookedUp := c.LookUpWithoutChangingOrder("first_key") // Assert require.NoError(t, err) assert.Empty(t, evicted) require.NotNil(t, lookedUp) - assert.Equal(t, int64(42), lookedUp.(arenaTestData).Value) - assert.Equal(t, uint64(10), c.currentSize) - assert.Equal(t, 1, c.len) + assert.Equal(t, int64(42), lookedUp.(testData).value) }) t.Run("SustainedCriticalPressureConvergesIdempotentlyWithoutCompoundDecay", func(t *testing.T) { @@ -1034,12 +1120,11 @@ func TestArenaRadixCache_AdversarialAuditRegressions(t *testing.T) { WithInvariantChecking(true), WithPressureFunc(func() float64 { return pressure }), WithEvictionRetentionRatio(0.50), - ).(*arenaRadix) + ).(PressureAwareCache) for i := range 100 { - _, err := c.Insert(fmt.Sprintf("k-%03d", i), arenaTestData{Value: int64(i), DataSize: 10}) + _, err := c.Insert(fmt.Sprintf("k-%03d", i), testData{value: int64(i), dataSize: 10}) require.NoError(t, err) } - require.Equal(t, uint64(1000), c.currentSize) pressure = 0.95 // Act: Evaluate critical pressure 10 times consecutively. @@ -1051,271 +1136,86 @@ func TestArenaRadixCache_AdversarialAuditRegressions(t *testing.T) { // Assert: Stabilizes idempotently at maxSize * retentionRatio (500 bytes, 50 entries). assert.Len(t, firstEvicted, 50) - assert.Equal(t, uint64(500), c.currentSize) - assert.Equal(t, 50, c.len) - }) - - t.Run("KeyMissesOnEraseUpdateSizeAndPrefixDoNotEvictLiveEntries", func(t *testing.T) { - // Arrange - pressure := 0.0 - c := NewArenaRadixCache( - 1000, - WithInvariantChecking(true), - WithPressureFunc(func() float64 { return pressure }), - WithEvictionRetentionRatio(0.50), - ).(*arenaRadix) - for i := range 10 { - _, err := c.Insert(fmt.Sprintf("k-%d", i), arenaTestData{Value: int64(i), DataSize: 10}) - require.NoError(t, err) + for i := range 50 { + assert.Nil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k-%03d", i))) + } + for i := 50; i < 100; i++ { + assert.NotNil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k-%03d", i))) } - pressure = 0.95 - - // Act - erased := c.Erase("absent_key") - err := c.UpdateSize("absent_key", 10) - c.EraseEntriesWithGivenPrefix("absent_prefix/") - - // Assert - assert.Nil(t, erased) - assert.ErrorIs(t, err, ErrEntryNotExist) - assert.Equal(t, 10, c.len) - assert.Equal(t, uint64(100), c.currentSize) }) t.Run("CompactFastPathPerformsZeroAllocationsWhenAlreadyCompact", func(t *testing.T) { // Arrange - c := NewArenaRadixCache(10000, WithInvariantChecking(true)).(*arenaRadix) + c := NewArenaRadixCache(10000, WithInvariantChecking(true)).(PressureAwareCache) for i := range 200 { - _, err := c.Insert(fmt.Sprintf("key-%03d", i), arenaTestData{Value: int64(i), DataSize: 10}) + _, err := c.Insert(fmt.Sprintf("key-%03d", i), testData{value: int64(i), dataSize: 10}) require.NoError(t, err) } - c.Compact() - require.Equal(t, nilNode, c.freeHead) - require.Equal(t, len(c.nodes), cap(c.nodes)) - ptrBefore := &c.nodes[0] - - // Act - allocsPerRun := testing.AllocsPerRun(20, func() { - c.Compact() - }) - ptrAfter := &c.nodes[0] - - // Assert - assert.Equal(t, 0.0, allocsPerRun) - assert.Same(t, ptrBefore, ptrAfter) - }) - - t.Run("UpdateWithoutChangingOrderSamplesValueSizeOutsideLock", func(t *testing.T) { - // Arrange - c := NewArenaRadixCache(100, WithInvariantChecking(true)).(*arenaRadix) - _, err := c.Insert("k1", arenaTestData{Value: 1, DataSize: 10}) - require.NoError(t, err) - - readBlockedInsideSize := true - probe := deadlockProbeValue{ - size: 10, - onSize: func() { - if c.mu.TryRLock() { - c.mu.RUnlock() - _ = c.LookUpWithoutChangingOrder("k1") - readBlockedInsideSize = false - } - }, - } - - // Act - err = c.UpdateWithoutChangingOrder("k1", probe) - - // Assert - require.NoError(t, err) - assert.False(t, readBlockedInsideSize) - }) - t.Run("FNV1aHashCollisionPreservedAndHealedAcrossCompaction", func(t *testing.T) { - // Arrange - c := NewArenaRadixCache(1000, WithInvariantChecking(true)).(*arenaRadix) - _, err := c.Insert("alpha/one", arenaTestData{Value: 101, DataSize: 10}) - require.NoError(t, err) - _, err = c.Insert("alpha/two", arenaTestData{Value: 202, DataSize: 10}) - require.NoError(t, err) - _, err = c.Insert("beta/three", arenaTestData{Value: 303, DataSize: 10}) - require.NoError(t, err) - _ = c.Erase("alpha/two") - - // Simulate hash collision deletion removing "alpha/one" from nodeMap. - delete(c.nodeMap, hashString("alpha/one")) - _, inMapBefore := c.nodeMap[hashString("alpha/one")] - require.False(t, inMapBefore) - - // Act + // Act: First Compact eliminates geometric slice growth slack; second Compact via assertAlreadyCompacted + // must take the clean fast-path with zero heap allocations. c.Compact() - // Assert: both trie lookup and O(1) nodeMap entry are healed after compaction. - _, inMapAfter := c.nodeMap[hashString("alpha/one")] - assert.True(t, inMapAfter) - v1 := c.LookUpWithoutChangingOrder("alpha/one") - v3 := c.LookUpWithoutChangingOrder("beta/three") - require.NotNil(t, v1) - require.NotNil(t, v3) - assert.Equal(t, int64(101), v1.(arenaTestData).Value) - assert.Equal(t, int64(303), v3.(arenaTestData).Value) + // Assert + assertAlreadyCompacted(t, c) }) } -func TestArenaRadixCache_PrincipalReviewFixes(t *testing.T) { - t.Run("F1_NoCompactionThrashingOnSteadyStateInsertsUnderPressure", func(t *testing.T) { +func TestArenaRadixCache_PressureSamplingAndReclamationBehaviors(t *testing.T) { + t.Run("NoCompactionThrashingOnSteadyStateInsertsUnderPressure", func(t *testing.T) { // Arrange - c := NewArenaRadixCache( - 2000, - WithInvariantChecking(true), - WithPressureFunc(func() float64 { return 0.95 }), - WithEvictionRetentionRatio(0.50), - ).(*arenaRadix) + var c Cache - // Act: Insert 100 entries (1000 bytes == targetSize) under critical pressure. + // Act: Construct cache and insert 100 entries (1000 bytes == targetSize) inside AllocsPerRun. allocsPerInsert := testing.AllocsPerRun(1, func() { + c = NewArenaRadixCache( + 2000, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return 0.95 }), + WithEvictionRetentionRatio(0.50), + ) for i := range 100 { - _, err := c.Insert(fmt.Sprintf("k-%04d", i), arenaTestData{Value: int64(i), DataSize: 10}) + _, err := c.Insert(fmt.Sprintf("k-%04d", i), testData{value: int64(i), dataSize: 10}) require.NoError(t, err) } }) - // Assert: Without O(N^2) compaction thrashing on every Insert, 100 inserts allocate ~252 objects total (vs > 2000 previously). + // Assert: Without O(N^2) compaction thrashing on every Insert, 100 inserts allocate ~252 objects total and preserve all 100 keys. assert.Less(t, allocsPerInsert, 350.0) - assert.Equal(t, 100, c.len) - assert.Equal(t, uint64(1000), c.currentSize) - }) - - t.Run("F2_ConcurrentPressureSampleReturnsCachedPressureInsteadOfZero", func(t *testing.T) { - // Arrange - shouldBlock := false - inCallback := make(chan struct{}) - releaseCallback := make(chan struct{}) - c := NewArenaRadixCache( - 1000, - WithInvariantChecking(true), - WithPressureFunc(func() float64 { - if shouldBlock { - close(inCallback) - <-releaseCallback - } - return 0.95 - }), - WithEvictionRetentionRatio(0.50), - ).(*arenaRadix) - - for i := range 10 { - _, err := c.Insert(fmt.Sprintf("item-%d", i), arenaTestData{Value: int64(i), DataSize: 100}) - require.NoError(t, err) + for i := range 100 { + require.NotNil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k-%04d", i))) } - - // Act: Block Goroutine A inside PressureFunc while Goroutine B calls samplePressureFresh(). - shouldBlock = true - doneA := make(chan struct{}) - go func() { - _ = c.samplePressureFresh() - close(doneA) - }() - <-inCallback - - concurrentSample := c.samplePressureFresh() - close(releaseCallback) - <-doneA - - // Assert: Concurrent caller receives cached 0.95 instead of false 0.0. - assert.Equal(t, 0.95, concurrentSample) }) - t.Run("F3_ZeroRetentionRatioDoesNotSelfEvictNewlyInsertedKey", func(t *testing.T) { + t.Run("ZeroRetentionRatioDoesNotSelfEvictNewlyInsertedKey", func(t *testing.T) { // Arrange c := NewArenaRadixCache( 1000, WithInvariantChecking(true), WithPressureFunc(func() float64 { return 0.95 }), WithEvictionRetentionRatio(0.0), - ).(*arenaRadix) + ).(PressureAwareCache) - _, err := c.Insert("old_key", arenaTestData{Value: 1, DataSize: 100}) + _, err := c.Insert("old_key", testData{value: 1, dataSize: 100}) require.NoError(t, err) // Act: Insert "new_key" under critical pressure with retention == 0.0. - evicted, err := c.Insert("new_key", arenaTestData{Value: 2, DataSize: 100}) + evicted, err := c.Insert("new_key", testData{value: 2, dataSize: 100}) // Assert: "old_key" is evicted, but "new_key" is preserved. require.NoError(t, err) require.Len(t, evicted, 1) - assert.Equal(t, int64(1), evicted[0].(arenaTestData).Value) + assert.Equal(t, int64(1), evicted[0].(testData).value) assert.Nil(t, c.LookUpWithoutChangingOrder("old_key")) assert.NotNil(t, c.LookUpWithoutChangingOrder("new_key")) // Explicit EvaluateMemoryPressure still flushes 100% of entries when retention == 0.0. flushed := c.EvaluateMemoryPressure() require.Len(t, flushed, 1) - assert.Equal(t, 0, c.len) - }) - - t.Run("F4_PostReclamationImmediatelyRefreshesAmortizedPressure", func(t *testing.T) { - // Arrange: Simulate default amortized sampling path (hasCustomPressureFunc = false). - pressure := 0.95 - c := NewArenaRadixCache(1000, WithInvariantChecking(true)).(*arenaRadix) - c.options.PressureFunc = func() float64 { return pressure } - c.options.hasCustomPressureFunc = false - - for i := range 10 { - _, err := c.Insert(fmt.Sprintf("k-%d", i), arenaTestData{Value: int64(i), DataSize: 100}) - require.NoError(t, err) - } - - // Act: After reclamation resolves pressure to 0.20, the very next samplePressure() refreshes immediately. - pressure = 0.20 - sampled := c.samplePressure() - - // Assert - assert.Equal(t, 0.20, sampled) - }) - - t.Run("F5_And_F10_MapCacheAndRadixCacheSafeSizeCallbackAndPressureAwareCache", func(t *testing.T) { - for _, tc := range []struct { - name string - fn func(uint64, ...Option) Cache - }{ - {"MapCache", NewMapCache}, - {"RadixCache", NewRadixCache}, - } { - t.Run(tc.name, func(t *testing.T) { - pressure := 0.10 - cache := tc.fn( - 100, - WithInvariantChecking(true), - WithPressureFunc(func() float64 { return pressure }), - WithEvictionRetentionRatio(0.50), - ) - for i := range 10 { - _, err := cache.Insert(fmt.Sprintf("k-%d", i), arenaTestData{Value: int64(i), DataSize: 10}) - require.NoError(t, err) - } - - // F5: Re-entrant LookUpWithoutChangingOrder inside ValueType.Size() does not deadlock. - err := cache.UpdateWithoutChangingOrder("k-0", deadlockProbeValue{ - size: 10, - onSize: func() { - _ = cache.LookUpWithoutChangingOrder("k-0") - }, - }) - require.NoError(t, err) - - // F10: Implements PressureAwareCache. - pac, ok := cache.(PressureAwareCache) - require.True(t, ok) - pac.Compact() - pressure = 0.95 - evicted := pac.EvaluateMemoryPressure() - assert.Len(t, evicted, 5) - }) - } + assert.Nil(t, c.LookUpWithoutChangingOrder("new_key")) }) - t.Run("F6_UpdateSizeUnderCriticalPressureTriggersTier2Shedding", func(t *testing.T) { + t.Run("UpdateSizeUnderCriticalPressureTriggersTier2Shedding", func(t *testing.T) { // Arrange pressure := 0.10 c := NewArenaRadixCache( @@ -1323,115 +1223,405 @@ func TestArenaRadixCache_PrincipalReviewFixes(t *testing.T) { WithInvariantChecking(true), WithPressureFunc(func() float64 { return pressure }), WithEvictionRetentionRatio(0.50), - ).(*arenaRadix) + ) for i := range 8 { - _, err := c.Insert(fmt.Sprintf("k-%d", i), arenaTestData{Value: int64(i), DataSize: 100}) + _, err := c.Insert(fmt.Sprintf("k-%d", i), testData{value: int64(i), dataSize: 100}) require.NoError(t, err) } - require.Equal(t, uint64(800), c.currentSize) // Act: Grow k-7 by 100 bytes under critical pressure (0.95). pressure = 0.95 err := c.UpdateSize("k-7", 100) - // Assert: Sheds down to targetSize (500 bytes) while keeping k-7. + // Assert: Sheds 4 oldest entries (k-0..k-3, 400B) down to targetSize (500 bytes: k-4, k-5, k-6 at 100B + k-7 at 200B). require.NoError(t, err) - assert.LessOrEqual(t, c.currentSize, uint64(500)) - assert.NotNil(t, c.LookUpWithoutChangingOrder("k-7")) + for i := range 4 { + assert.Nil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k-%d", i))) + } + for i := 4; i < 8; i++ { + assert.NotNil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k-%d", i))) + } }) +} + +func TestArenaRadixCache_UpdateSizeAtMRUHeadUnderCriticalPressure(t *testing.T) { + // Arrange: Insert k1 (oldest, at tail), k2, k3 (newest, at MRU head). + pressure := 0.10 + c := NewArenaRadixCache( + 1000, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionRetentionRatio(0.50), + ) - t.Run("F7_ComputeTargetSizeBoundsAndZeroSizeEntryShedding", func(t *testing.T) { - // MaxUint64 does not overflow float64 -> uint64 conversion. - assert.Equal(t, uint64(math.MaxUint64), computeTargetSize(math.MaxUint64, 1.0)) - assert.Greater(t, computeTargetSize(math.MaxUint64, 0.5), uint64(math.MaxUint64/4)) - // maxSize == 1 with retention > 0 clamps to 1 instead of truncating to 0. - assert.Equal(t, uint64(1), computeTargetSize(1, 0.50)) + _, err := c.Insert("k1", testData{value: 1, dataSize: 200}) + require.NoError(t, err) + _, err = c.Insert("k2", testData{value: 2, dataSize: 200}) + require.NoError(t, err) + _, err = c.Insert("k3", testData{value: 3, dataSize: 200}) + require.NoError(t, err) - // Zero-size entries (DataSize == 0) are shed down to retention ratio under critical pressure. - pressure := 0.10 - c := NewArenaRadixCache( - 100, - WithInvariantChecking(true), - WithPressureFunc(func() float64 { return pressure }), - WithEvictionRetentionRatio(0.50), - ).(*arenaRadix) - for i := range 10 { - _, err := c.Insert(fmt.Sprintf("zero-%d", i), arenaTestData{Value: int64(i), DataSize: 0}) - require.NoError(t, err) - } - pressure = 0.95 - evicted := c.EvaluateMemoryPressure() - assert.Len(t, evicted, 5) - assert.Equal(t, 5, c.len) - }) + // Act: Grow k3 (the MRU head) by +100 bytes under critical pressure. + pressure = 0.95 + err = c.UpdateSize("k3", 100) + + // Assert: MRU head k3 is protected and preserved (300 bytes), while oldest tail entry k1 (200 bytes) + // is shed so total size drops from 700 bytes down to targetSize (500 bytes: k2=200B + k3=300B). + require.NoError(t, err) + assert.Nil(t, c.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, c.LookUpWithoutChangingOrder("k2")) + assert.NotNil(t, c.LookUpWithoutChangingOrder("k3")) } -func TestArenaRadixCache_CheckInvariants_ExtendedChecks(t *testing.T) { - t.Run("NodeMapOutOfBounds", func(t *testing.T) { - // Arrange - c := NewArenaRadixCache(50).(*arenaRadix) - _, _ = c.Insert("k1", arenaTestData{Value: 1, DataSize: 10}) - c.nodeMap[hashString("k1")] = 999 +// TestArenaRadixCache_FNV1aHashCollisionAndNodeMapHealing verifies that ArenaRadixCache preserves key +// reachability, LRU invariants, and compaction guarantees when 64-bit FNV-1a hash collisions displace +// entries in the O(1) c.nodeMap accelerator. +// +// White-box testing rationale: +// 1. ArenaRadixCache hashes keys internally using unseeded 64-bit FNV-1a (hashString). By the birthday bound, +// finding two distinct variable-length strings with the exact same 64-bit FNV-1a hash in a 2^64 hash space +// requires ~2^32 (~4.29 billion) evaluations, which is computationally infeasible at unit-test runtime even +// though collisions can occur in production across billions of cumulative keys. +// 2. Furthermore, c.nodeMap is an internal O(1) hash accelerator backed by an exact O(K) radix-trie walk fallback +// (findNodeByTrieWalk). If Erase(K1) or a self-evicting/failing UpdateSize(K1, delta) erroneously overwrites +// c.nodeMap[h] = idK1 before eraseInternalWithHash deletes c.nodeMap[h], public LookUp / LookUpWithoutChangingOrder +// calls still return functionally identical values via the slow-path trie walk while silently evicting the +// surviving colliding peer K2 from c.nodeMap and breaking the len(c.nodeMap) == c.len Pigeonhole Principle O(1) +// cache-miss fast-path across the entire cache. +// These subtests therefore seed a synthetic collision/displacement in c.nodeMap in setup and inspect c.nodeMap +// in assertions alongside public API LookUp/ LookUpWithoutChangingOrder/Insert/UpdateWithoutChangingOrder/UpdateSize/Erase/Compact calls. +func TestArenaRadixCache_FNV1aHashCollisionAndNodeMapHealing(t *testing.T) { + const keyA = "prefix/key-alpha" + const keyB = "prefix/key-beta" + + t.Run("HashCollisionDoesNotThrashCompactionAndPreservesInvariant", func(t *testing.T) { + // Arrange: Insert keyA, keyB, and a temporary key, simulate a 64-bit FNV-1a hash collision + // displacing keyA from nodeMap, and trigger a real deletion via Erase. + c := NewArenaRadixCache(10000, WithInvariantChecking(true)).(*arenaRadix) + _, err := c.Insert(keyA, testData{value: 1, dataSize: 10}) + require.NoError(t, err) + _, err = c.Insert(keyB, testData{value: 2, dataSize: 10}) + require.NoError(t, err) + _, err = c.Insert("prefix/key-temp", testData{value: 3, dataSize: 10}) + require.NoError(t, err) - // Act & Assert - assert.Panics(t, func() { - c.checkInvariants() - }) + delete(c.nodeMap, hashString(keyA)) + require.NotNil(t, c.Erase("prefix/key-temp")) + + // Act & Assert: Both keys remain accessible (keyA via trie fallback), and Compact() heals keyA + // back into nodeMap without thrashing on subsequent Compact() calls. + valA := c.LookUpWithoutChangingOrder(keyA) + require.NotNil(t, valA) + assert.Equal(t, int64(1), valA.(testData).value) + + c.Compact() + _, ok := c.nodeMap[hashString(keyA)] + assert.True(t, ok) + assertAlreadyCompacted(t, c) + + valAAfter := c.LookUp(keyA) + valBAfter := c.LookUp(keyB) + require.NotNil(t, valAAfter) + require.NotNil(t, valBAfter) + assert.Equal(t, int64(1), valAAfter.(testData).value) + assert.Equal(t, int64(2), valBAfter.(testData).value) }) - t.Run("NodeMapNilValue", func(t *testing.T) { + t.Run("OverwriteDoesNotCorruptCollidingPeerInNodeMap", func(t *testing.T) { // Arrange - c := NewArenaRadixCache(50).(*arenaRadix) - _, _ = c.Insert("k1", arenaTestData{Value: 1, DataSize: 10}) - c.nodeMap[hashString("k1")] = c.root + c := NewArenaRadixCache(1000, WithInvariantChecking(true)).(*arenaRadix) + _, err := c.Insert("k1", testData{value: 10, dataSize: 10}) + require.NoError(t, err) + _, err = c.Insert("k2", testData{value: 20, dataSize: 10}) + require.NoError(t, err) - // Act & Assert - assert.Panics(t, func() { - c.checkInvariants() - }) + // Simulate hash collision: point k1's hash slot to k2's node ID. + idK2 := c.nodeMap[hashString("k2")] + c.nodeMap[hashString("k1")] = idK2 + + // Act: Overwrite k1 via trie fallback and verify k1 reclaims its nodeMap slot. + evicted, err := c.Insert("k1", testData{value: 11, dataSize: 15}) + require.NoError(t, err) + assert.Empty(t, evicted) + + // Assert + v1 := c.LookUpWithoutChangingOrder("k1") + v2 := c.LookUpWithoutChangingOrder("k2") + require.NotNil(t, v1) + require.NotNil(t, v2) + assert.Equal(t, int64(11), v1.(testData).value) + assert.Equal(t, int64(20), v2.(testData).value) + assert.NotEqual(t, idK2, c.nodeMap[hashString("k1")]) }) - t.Run("NodeMapHashMismatch", func(t *testing.T) { + t.Run("UpdateMethodsHealNodeMapOnCollisionOrMissingSlot", func(t *testing.T) { // Arrange - c := NewArenaRadixCache(50).(*arenaRadix) - _, _ = c.Insert("k1", arenaTestData{Value: 1, DataSize: 10}) - id := c.nodeMap[hashString("k1")] - c.nodeMap[12345] = id + c := NewArenaRadixCache(1000, WithInvariantChecking(true)).(*arenaRadix) + _, err := c.Insert("key-alpha", testData{value: 1, dataSize: 10}) + require.NoError(t, err) + _, err = c.Insert("key-beta", testData{value: 2, dataSize: 10}) + require.NoError(t, err) - // Act & Assert - assert.Panics(t, func() { - c.checkInvariants() - }) + hAlpha := hashString("key-alpha") + hBeta := hashString("key-beta") + idAlpha := c.nodeMap[hAlpha] + idBeta := c.nodeMap[hBeta] + + // Act & Assert 1: Missing slot healed by LookUp. + delete(c.nodeMap, hAlpha) + v := c.LookUp("key-alpha") + require.NotNil(t, v) + assert.Equal(t, idAlpha, c.nodeMap[hAlpha]) + + // Act & Assert 2: Missing slot healed by UpdateWithoutChangingOrder. + delete(c.nodeMap, hAlpha) + err = c.UpdateWithoutChangingOrder("key-alpha", testData{value: 11, dataSize: 10}) + require.NoError(t, err) + assert.Equal(t, idAlpha, c.nodeMap[hAlpha]) + + // Act & Assert 3: Colliding slot pointing to idBeta healed by UpdateSize("key-alpha", ...). + c.nodeMap[hAlpha] = idBeta + err = c.UpdateSize("key-alpha", 5) + require.NoError(t, err) + assert.Equal(t, idAlpha, c.nodeMap[hAlpha]) }) - t.Run("LeakedNodeAccountingMismatch", func(t *testing.T) { - // Arrange - c := NewArenaRadixCache(50).(*arenaRadix) - _, _ = c.Insert("k1", arenaTestData{Value: 1, DataSize: 10}) - c.nodes = append(c.nodes, arenaRadixNode{ - parent: nilNode, - child: nilNode, - sibling: nilNode, - prev: nilNode, - next: nilNode, - }) + t.Run("EraseAndUpdateSizeSelfEvictionOrOverflowPreserveCollidingPeerInNodeMap", func(t *testing.T) { + hA := hashString(keyA) + hB := hashString(keyB) - // Act & Assert - assert.Panics(t, func() { - c.checkInvariants() - }) + // Case 1: Erase(keyA) when collision bucket hA points to surviving peer keyB. + cErase := NewArenaRadixCache(1000).(*arenaRadix) + _, err := cErase.Insert(keyA, testData{value: 1, dataSize: 100}) + require.NoError(t, err) + _, err = cErase.Insert(keyB, testData{value: 2, dataSize: 100}) + require.NoError(t, err) + idB := cErase.nodeMap[hB] + cErase.nodeMap[hA] = idB + + erased := cErase.Erase(keyA) + require.NotNil(t, erased) + assert.Equal(t, int64(1), erased.(testData).value) + assert.Equal(t, idB, cErase.nodeMap[hA], "Erase(keyA) must not overwrite or delete surviving peer keyB from shared hash bucket") + delete(cErase.nodeMap, hA) + assert.Len(t, cErase.nodeMap, cErase.len, "len(nodeMap) == c.len bijection must hold for O(1) cache-miss fast-path") + cErase.checkInvariants() + assert.Nil(t, cErase.LookUpWithoutChangingOrder(keyA)) + assert.Nil(t, cErase.LookUpWithoutChangingOrder("missing-key")) + require.NotNil(t, cErase.LookUpWithoutChangingOrder(keyB)) + + // Case 2: UpdateSize(keyA, math.MaxUint64) returning ErrInvalidUpdateEntrySize and self-eviction via size+delta > maxSize. + cMaxEvict := NewArenaRadixCache(1000).(*arenaRadix) + _, err = cMaxEvict.Insert(keyA, testData{value: 1, dataSize: 100}) + require.NoError(t, err) + _, err = cMaxEvict.Insert(keyB, testData{value: 2, dataSize: 100}) + require.NoError(t, err) + idB = cMaxEvict.nodeMap[hB] + cMaxEvict.nodeMap[hA] = idB + + err = cMaxEvict.UpdateSize(keyA, math.MaxUint64) + require.ErrorIs(t, err, ErrInvalidUpdateEntrySize) + assert.Equal(t, idB, cMaxEvict.nodeMap[hA], "failed UpdateSize returning ErrInvalidUpdateEntrySize must not overwrite colliding peer in nodeMap") + + err = cMaxEvict.UpdateSize(keyA, 1000) + require.NoError(t, err) + assert.Equal(t, idB, cMaxEvict.nodeMap[hA], "UpdateSize self-eviction (size+delta > maxSize) must preserve surviving peer in nodeMap") + delete(cMaxEvict.nodeMap, hA) + assert.Len(t, cMaxEvict.nodeMap, cMaxEvict.len) + cMaxEvict.checkInvariants() + assert.Nil(t, cMaxEvict.LookUpWithoutChangingOrder(keyA)) + assert.Nil(t, cMaxEvict.LookUpWithoutChangingOrder("missing-key")) + require.NotNil(t, cMaxEvict.LookUpWithoutChangingOrder(keyB)) + + // Case 3: UpdateSize(keyA, delta) self-eviction via !canFit (newer entry keyB occupies capacity). + cCannotFit := NewArenaRadixCache(1000).(*arenaRadix) + _, err = cCannotFit.Insert(keyA, testData{value: 1, dataSize: 100}) + require.NoError(t, err) + _, err = cCannotFit.Insert(keyB, testData{value: 2, dataSize: 900}) + require.NoError(t, err) + idB = cCannotFit.nodeMap[hB] + cCannotFit.nodeMap[hA] = idB + + err = cCannotFit.UpdateSize(keyA, 200) + require.NoError(t, err) + assert.Equal(t, idB, cCannotFit.nodeMap[hA], "UpdateSize self-eviction (!canFit) must preserve surviving peer in nodeMap") + delete(cCannotFit.nodeMap, hA) + assert.Len(t, cCannotFit.nodeMap, cCannotFit.len) + cCannotFit.checkInvariants() + assert.Nil(t, cCannotFit.LookUpWithoutChangingOrder(keyA)) + assert.Nil(t, cCannotFit.LookUpWithoutChangingOrder("missing-key")) + require.NotNil(t, cCannotFit.LookUpWithoutChangingOrder(keyB)) }) - t.Run("FreeCountDriftMismatch", func(t *testing.T) { + t.Run("DisplacedNodeMapSlotOnEraseStillTriggersChurnCompactionAndPreservesPeers", func(t *testing.T) { + // Arrange: Keep keyB as the surviving entry while inserting and erasing two batches of 32 keys + // whose nodeMap slots were displaced by simulated hash collisions. Because peak live entries stays + // at 33 (< 64) and free-list length never exceeds 32 (< 63), single-survivor auto-compaction on the + // 64th Erase occurs if and only if erasing displaced keys still accounts for deletion churn. + c := NewArenaRadixCache(10000, WithInvariantChecking(true)).(*arenaRadix) + _, err := c.Insert(keyB, testData{value: 2, dataSize: 10}) + require.NoError(t, err) + + for batch := range 2 { + for i := range 32 { + k := fmt.Sprintf("displaced-%d-%02d", batch, i) + _, err := c.Insert(k, testData{value: int64(i + 10), dataSize: 10}) + require.NoError(t, err) + delete(c.nodeMap, hashString(k)) + } + for i := range 32 { + k := fmt.Sprintf("displaced-%d-%02d", batch, i) + erased := c.Erase(k) + require.NotNil(t, erased) + assert.Nil(t, c.LookUp(k)) + } + } + + // Assert: The 64th displaced Erase auto-compacted the cache down to the single survivor keyB, + // healed nodeMap, and preserved keyB for subsequent public LookUp, Insert, and Erase calls. + assertAlreadyCompacted(t, c) + valB := c.LookUp(keyB) + require.NotNil(t, valB) + assert.Equal(t, int64(2), valB.(testData).value) + + _, err = c.Insert(keyA, testData{value: 1, dataSize: 10}) + require.NoError(t, err) + require.NotNil(t, c.LookUp(keyA)) + require.NotNil(t, c.Erase(keyA)) + assert.Nil(t, c.LookUp(keyA)) + assert.NotNil(t, c.LookUp(keyB)) + }) + + t.Run("RootNodeEraseWithoutMapEntryPreservesNodeMap", func(t *testing.T) { // Arrange - c := NewArenaRadixCache(50).(*arenaRadix) - _, _ = c.Insert("k1", arenaTestData{Value: 1, DataSize: 10}) - _ = c.Erase("k1") - c.freeCount = 999 + ac := NewArenaRadixCache(100, WithInvariantChecking(true)).(*arenaRadix) + _, err := ac.Insert("", NewStringValue("root_val")) + require.NoError(t, err) + _, err = ac.Insert("other", NewStringValue("other_val")) + require.NoError(t, err) + delete(ac.nodeMap, hashString("")) - // Act & Assert - assert.Panics(t, func() { - c.checkInvariants() - }) + // Act: Erase "" (which resides at c.root == 0) while hash("") is not in c.nodeMap. + erased := ac.Erase("") + + // Assert: nodeMap retains the remaining entry intact. + require.NotNil(t, erased) + assert.Len(t, ac.nodeMap, 1) + }) +} + +func TestArenaRadixCache_DeepHierarchyOver64LevelsAndRoutingPrefixCloning(t *testing.T) { + t.Run("DeepRadixTreeOver64LevelsAndFreeSubtree", func(t *testing.T) { + // Arrange: Build a radix tree with > 80 branching levels so both hashNodeKey and freeSubtree exceed 64 levels. + probe := newPressureProbe(0.10) + c := NewArenaRadixCache(100000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + const depth = 80 + for i := 1; i <= depth; i++ { + key := strings.Repeat("a", i) + "b" + _, err := c.Insert(key, testData{value: int64(i), dataSize: 10}) + require.NoError(t, err) + } + + // Act 1: Compact the 80-level tree (exercises hashNodeKey with > 64 segments) and confirm a second Compact is a no-op. + assert.True(t, probe.ObserveEpochAdvance(t, c, func() { + c.Compact() + })) + assert.False(t, probe.ObserveEpochAdvance(t, c, func() { + c.Compact() + })) + for i := 1; i <= depth; i++ { + key := strings.Repeat("a", i) + "b" + val := c.LookUpWithoutChangingOrder(key) + require.NotNil(t, val) + assert.Equal(t, int64(i), val.(testData).value) + } + + // Act 2: Erase all entries under prefix "a" (exercises iterative freeSubtree with > 64 levels). + c.EraseEntriesWithGivenPrefix("a") + + // Assert + for i := 1; i <= depth; i++ { + key := strings.Repeat("a", i) + "b" + assert.Nil(t, c.LookUpWithoutChangingOrder(key)) + } + assertAlreadyCompacted(t, c) }) + + t.Run("RoutingPrefixDoesNotPinLargeKeyBackingArray", func(t *testing.T) { + // Arrange: Insert a 64 KiB key ("dir/" + 64 KiB suffix), then split the edge + // with two sibling keys ("dir/a" and "dir/b") and erase the 64 KiB key so + // the "dir/" intermediate routing node survives. + // + // White-box testing rationale: String backing-array aliasing on internal routing nodes is + // invisible to public LookUp results, so verifying that the 4-byte "dir/" routing prefix does not pin + // the 64 KiB key buffer requires inspecting unsafe.StringData on the internal routing node. + c := NewArenaRadixCache(1<<20, WithInvariantChecking(true)).(*arenaRadix) + largeKey := strings.Clone("dir/" + strings.Repeat("X", 1<<16)) + + _, err := c.Insert(largeKey, NewStringValue("v")) + require.NoError(t, err) + pinnedInitialPrefix := c.nodes[1].prefix + largeStart := uintptr(unsafe.Pointer(unsafe.StringData(pinnedInitialPrefix))) + largeEnd := largeStart + uintptr(len(pinnedInitialPrefix)) + + // Act: Split the edge and erase the 64 KiB key. + _, err = c.Insert("dir/a", NewStringValue("va")) + require.NoError(t, err) + _, err = c.Insert("dir/b", NewStringValue("vb")) + require.NoError(t, err) + _ = c.Erase(largeKey) + + // Assert: Routing node prefix "dir/" does not alias pinnedInitialPrefix's 64 KiB backing array. + routingID := c.nodes[c.root].child + require.NotEqual(t, nilNode, routingID) + assert.Equal(t, "dir/", c.nodes[routingID].prefix) + prefixPtr := uintptr(unsafe.Pointer(unsafe.StringData(c.nodes[routingID].prefix))) + assert.True(t, prefixPtr < largeStart || prefixPtr >= largeEnd, "routing node prefix must not pin 64 KiB backing array") + runtime.KeepAlive(pinnedInitialPrefix) + }) +} + +// TestArenaRadixCache_InsertSplitsPrefixAndRecyclesFreeSlot verifies that normal insertions splitting radix edges +// and recycling free-list slots succeed without falsely triggering the uint32 arena node-limit guard. +// +// White-box testing rationale: +// 1. Why the node-limit boundary cannot be reproduced solely through the public interface on standard hardware: +// In ArenaRadixCache.Insert, the pre-allocation node-limit guard is: +// uint64(len(c.nodes)) - uint64(c.freeCount) + 2 > uint64(foregroundNoProtect) +// where foregroundNoProtect == nilNode - 1 == math.MaxUint32 - 1 (4,294,967,294). +// Reaching this boundary through public Cache.Insert calls requires populating +// len(c.nodes) - int(c.freeCount) == math.MaxUint32 - 2 (4,294,967,293) active arenaRadixNode structs. +// Because unsafe.Sizeof(arenaRadixNode{}) == 64 bytes on 64-bit platforms, backing the c.nodes slice alone +// requires 4,294,967,293 * 64 == 274,877,906,752 bytes (~274.88 GiB) of contiguous heap memory (plus >100 GiB +// for c.nodeMap), which is infeasible to allocate in a unit test on standard hardware. +// 2. Why the guard is nevertheless essential for correctness: +// a) Sentinel Collision & Panic Prevention: Arena node indices are 32-bit unsigned integers where +// nilNode == math.MaxUint32 and foregroundNoProtect == math.MaxUint32 - 1. A new-key Insert that splits an +// existing edge allocates 2 nodes (one intermediate routing node + one leaf node). If the guard checked +// against math.MaxUint32 instead of foregroundNoProtect, an Insert at 4,294,967,293 active nodes would +// attempt to allocate node index foregroundNoProtect (math.MaxUint32 - 1), which panics in allocateNode() +// (if id >= foregroundNoProtect) or, if allocated, collides with the foregroundNoProtect sentinel passed to +// foreground pressure reclamation / shedAndCompactLocked (stripping MRU head protection from the newly +// inserted entry during Tier 2 pressure shedding). +// b) Free-List Recycling Accuracy: Subtracting uint64(c.freeCount) in +// uint64(len(c.nodes)) - uint64(c.freeCount) + 2 > uint64(foregroundNoProtect) ensures that when recycled +// free-list slots exist (c.freeCount >= 2), Insert pops from c.freeHead without falsely evicting live entries. +func TestArenaRadixCache_InsertSplitsPrefixAndRecyclesFreeSlot(t *testing.T) { + // Arrange: Populate entries that split a shared prefix ("alpha/") and recycle free-list slots via Erase. + c := NewArenaRadixCache(1000, WithInvariantChecking(true)) + _, err := c.Insert("alpha/one", testData{value: 1, dataSize: 10}) + require.NoError(t, err) + _, err = c.Insert("beta/two", testData{value: 2, dataSize: 10}) + require.NoError(t, err) + require.NotNil(t, c.Erase("beta/two")) + + // Act: Insert a sibling key that splits "alpha/one" into routing node "alpha/" + two leaves, reusing the free slot. + evicted, err := c.Insert("alpha/three", testData{value: 3, dataSize: 10}) + + // Assert + require.NoError(t, err) + assert.Empty(t, evicted) + assert.NotNil(t, c.LookUpWithoutChangingOrder("alpha/one")) + assert.Nil(t, c.LookUpWithoutChangingOrder("beta/two")) + assert.NotNil(t, c.LookUpWithoutChangingOrder("alpha/three")) } diff --git a/benchmarks_test.go b/benchmarks_test.go index 3558465..53042b3 100644 --- a/benchmarks_test.go +++ b/benchmarks_test.go @@ -12,18 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus_test +package lru_test import ( "fmt" "math/rand" + "runtime" + "sync/atomic" "testing" - "time" - lrus "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/google/go-lru" ) -// benchValue implements lrus.ValueType for benchmarking. +var ( + benchSinkVal lru.ValueType + benchSinkErr error +) + +// benchValue implements lru.ValueType for benchmarking. type benchValue struct { val int64 dataSize uint64 @@ -73,12 +79,13 @@ func generateBenchmarkKeys(prefixCount, itemsPerPrefix, depth int) (keys []strin // 1. Sequential Insertion Benchmarks // ============================================================================ -func runBenchmarkInsert(b *testing.B, constructor func(uint64, ...lrus.Option) lrus.Cache, depth int) { +func runBenchmarkInsert(b *testing.B, constructor func(uint64, ...lru.Option) lru.Cache, depth int) { + b.Helper() const prefixCount = 100 const itemsPerPrefix = 100 keys, _, _ := generateBenchmarkKeys(prefixCount, itemsPerPrefix, depth) data := benchValue{val: 1, dataSize: 10} - capacity := uint64(len(keys) * 100) + capacity := uint64(len(keys) * 5) cache := constructor(capacity) @@ -86,36 +93,39 @@ func runBenchmarkInsert(b *testing.B, constructor func(uint64, ...lrus.Option) l b.ResetTimer() i := 0 + var lastErr error for b.Loop() { key := keys[i%len(keys)] - _, _ = cache.Insert(key, data) + _, lastErr = cache.Insert(key, data) i++ } + benchSinkErr = lastErr } func Benchmark_Insert_MapCache(b *testing.B) { - b.Run("Flat", func(b *testing.B) { runBenchmarkInsert(b, lrus.NewMapCache, 0) }) - b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkInsert(b, lrus.NewMapCache, 2) }) - b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkInsert(b, lrus.NewMapCache, 10) }) + b.Run("Flat", func(b *testing.B) { runBenchmarkInsert(b, lru.NewMapCache, 0) }) + b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkInsert(b, lru.NewMapCache, 2) }) + b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkInsert(b, lru.NewMapCache, 10) }) } func Benchmark_Insert_RadixCache(b *testing.B) { - b.Run("Flat", func(b *testing.B) { runBenchmarkInsert(b, lrus.NewRadixCache, 0) }) - b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkInsert(b, lrus.NewRadixCache, 2) }) - b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkInsert(b, lrus.NewRadixCache, 10) }) + b.Run("Flat", func(b *testing.B) { runBenchmarkInsert(b, lru.NewRadixCache, 0) }) + b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkInsert(b, lru.NewRadixCache, 2) }) + b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkInsert(b, lru.NewRadixCache, 10) }) } func Benchmark_Insert_ArenaRadixCache(b *testing.B) { - b.Run("Flat", func(b *testing.B) { runBenchmarkInsert(b, lrus.NewArenaRadixCache, 0) }) - b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkInsert(b, lrus.NewArenaRadixCache, 2) }) - b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkInsert(b, lrus.NewArenaRadixCache, 10) }) + b.Run("Flat", func(b *testing.B) { runBenchmarkInsert(b, lru.NewArenaRadixCache, 0) }) + b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkInsert(b, lru.NewArenaRadixCache, 2) }) + b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkInsert(b, lru.NewArenaRadixCache, 10) }) } // ============================================================================ // 2. Point Lookup Latency Benchmarks // ============================================================================ -func runBenchmarkLookUp(b *testing.B, constructor func(uint64, ...lrus.Option) lrus.Cache, depth int) { +func runBenchmarkLookUp(b *testing.B, constructor func(uint64, ...lru.Option) lru.Cache, depth int) { + b.Helper() const prefixCount = 100 const itemsPerPrefix = 100 keys, _, _ := generateBenchmarkKeys(prefixCount, itemsPerPrefix, depth) @@ -131,36 +141,39 @@ func runBenchmarkLookUp(b *testing.B, constructor func(uint64, ...lrus.Option) l b.ResetTimer() i := 0 + var lastVal lru.ValueType for b.Loop() { key := keys[i%len(keys)] - _ = cache.LookUp(key) + lastVal = cache.LookUp(key) i++ } + benchSinkVal = lastVal } func Benchmark_LookUp_MapCache(b *testing.B) { - b.Run("Flat", func(b *testing.B) { runBenchmarkLookUp(b, lrus.NewMapCache, 0) }) - b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkLookUp(b, lrus.NewMapCache, 2) }) - b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkLookUp(b, lrus.NewMapCache, 10) }) + b.Run("Flat", func(b *testing.B) { runBenchmarkLookUp(b, lru.NewMapCache, 0) }) + b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkLookUp(b, lru.NewMapCache, 2) }) + b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkLookUp(b, lru.NewMapCache, 10) }) } func Benchmark_LookUp_RadixCache(b *testing.B) { - b.Run("Flat", func(b *testing.B) { runBenchmarkLookUp(b, lrus.NewRadixCache, 0) }) - b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkLookUp(b, lrus.NewRadixCache, 2) }) - b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkLookUp(b, lrus.NewRadixCache, 10) }) + b.Run("Flat", func(b *testing.B) { runBenchmarkLookUp(b, lru.NewRadixCache, 0) }) + b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkLookUp(b, lru.NewRadixCache, 2) }) + b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkLookUp(b, lru.NewRadixCache, 10) }) } func Benchmark_LookUp_ArenaRadixCache(b *testing.B) { - b.Run("Flat", func(b *testing.B) { runBenchmarkLookUp(b, lrus.NewArenaRadixCache, 0) }) - b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkLookUp(b, lrus.NewArenaRadixCache, 2) }) - b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkLookUp(b, lrus.NewArenaRadixCache, 10) }) + b.Run("Flat", func(b *testing.B) { runBenchmarkLookUp(b, lru.NewArenaRadixCache, 0) }) + b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkLookUp(b, lru.NewArenaRadixCache, 2) }) + b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkLookUp(b, lru.NewArenaRadixCache, 10) }) } // ============================================================================ // 3. LookUpWithoutChangingOrder Benchmarks // ============================================================================ -func runBenchmarkLookUpWithoutChangingOrder(b *testing.B, constructor func(uint64, ...lrus.Option) lrus.Cache, depth int) { +func runBenchmarkLookUpWithoutChangingOrder(b *testing.B, constructor func(uint64, ...lru.Option) lru.Cache, depth int) { + b.Helper() const prefixCount = 100 const itemsPerPrefix = 100 keys, _, _ := generateBenchmarkKeys(prefixCount, itemsPerPrefix, depth) @@ -176,30 +189,33 @@ func runBenchmarkLookUpWithoutChangingOrder(b *testing.B, constructor func(uint6 b.ResetTimer() i := 0 + var lastVal lru.ValueType for b.Loop() { key := keys[i%len(keys)] - _ = cache.LookUpWithoutChangingOrder(key) + lastVal = cache.LookUpWithoutChangingOrder(key) i++ } + benchSinkVal = lastVal } func Benchmark_LookUpWithoutChangingOrder_MapCache(b *testing.B) { - runBenchmarkLookUpWithoutChangingOrder(b, lrus.NewMapCache, 2) + runBenchmarkLookUpWithoutChangingOrder(b, lru.NewMapCache, 2) } func Benchmark_LookUpWithoutChangingOrder_RadixCache(b *testing.B) { - runBenchmarkLookUpWithoutChangingOrder(b, lrus.NewRadixCache, 2) + runBenchmarkLookUpWithoutChangingOrder(b, lru.NewRadixCache, 2) } func Benchmark_LookUpWithoutChangingOrder_ArenaRadixCache(b *testing.B) { - runBenchmarkLookUpWithoutChangingOrder(b, lrus.NewArenaRadixCache, 2) + runBenchmarkLookUpWithoutChangingOrder(b, lru.NewArenaRadixCache, 2) } // ============================================================================ // 4. Update Without Changing Order Benchmarks // ============================================================================ -func runBenchmarkUpdate(b *testing.B, constructor func(uint64, ...lrus.Option) lrus.Cache) { +func runBenchmarkUpdate(b *testing.B, constructor func(uint64, ...lru.Option) lru.Cache) { + b.Helper() const numKeys = 10000 keys := make([]string, numKeys) for i := range numKeys { @@ -217,120 +233,134 @@ func runBenchmarkUpdate(b *testing.B, constructor func(uint64, ...lrus.Option) l b.ResetTimer() i := 0 + var lastErr error for b.Loop() { key := keys[i%numKeys] - _ = cache.UpdateWithoutChangingOrder(key, updatedData) + lastErr = cache.UpdateWithoutChangingOrder(key, updatedData) i++ } + benchSinkErr = lastErr } -func Benchmark_Update_MapCache(b *testing.B) { runBenchmarkUpdate(b, lrus.NewMapCache) } -func Benchmark_Update_RadixCache(b *testing.B) { runBenchmarkUpdate(b, lrus.NewRadixCache) } -func Benchmark_Update_ArenaRadixCache(b *testing.B) { runBenchmarkUpdate(b, lrus.NewArenaRadixCache) } +func Benchmark_Update_MapCache(b *testing.B) { runBenchmarkUpdate(b, lru.NewMapCache) } +func Benchmark_Update_RadixCache(b *testing.B) { runBenchmarkUpdate(b, lru.NewRadixCache) } +func Benchmark_Update_ArenaRadixCache(b *testing.B) { runBenchmarkUpdate(b, lru.NewArenaRadixCache) } // ============================================================================ // 5. Individual Erase Benchmarks // ============================================================================ -func runBenchmarkErase(b *testing.B, constructor func(uint64, ...lrus.Option) lrus.Cache) { - const numKeys = 10000 +func runBenchmarkErase(b *testing.B, constructor func(uint64, ...lru.Option) lru.Cache) { + b.Helper() + const batchSize = 10000 + keys := make([]string, batchSize) + for i := range batchSize { + keys[i] = fmt.Sprintf("erase_key_%d", i) + } data := benchValue{val: 1, dataSize: 10} - - cache := constructor(uint64(numKeys * 100)) + cache := constructor(uint64(batchSize * 100)) b.ReportAllocs() b.ResetTimer() + var lastVal lru.ValueType for i := range b.N { - b.StopTimer() - key := fmt.Sprintf("erase_key_%d", i) - _, _ = cache.Insert(key, data) - b.StartTimer() - - _ = cache.Erase(key) + idx := i % batchSize + if idx == 0 { + b.StopTimer() + for _, k := range keys { + _, _ = cache.Insert(k, data) + } + b.StartTimer() + } + lastVal = cache.Erase(keys[idx]) } + benchSinkVal = lastVal } -func Benchmark_Erase_MapCache(b *testing.B) { runBenchmarkErase(b, lrus.NewMapCache) } -func Benchmark_Erase_RadixCache(b *testing.B) { runBenchmarkErase(b, lrus.NewRadixCache) } -func Benchmark_Erase_ArenaRadixCache(b *testing.B) { runBenchmarkErase(b, lrus.NewArenaRadixCache) } +func Benchmark_Erase_MapCache(b *testing.B) { runBenchmarkErase(b, lru.NewMapCache) } +func Benchmark_Erase_RadixCache(b *testing.B) { runBenchmarkErase(b, lru.NewRadixCache) } +func Benchmark_Erase_ArenaRadixCache(b *testing.B) { runBenchmarkErase(b, lru.NewArenaRadixCache) } // ============================================================================ -// 6. Prefix Deletion Benchmarks Across Topologies (Untimed Key Restoration) +// 6. Prefix Deletion Benchmarks Across Topologies (Batched Untimed Restoration) // ============================================================================ -func runBenchmarkErasePrefix(b *testing.B, constructor func(uint64, ...lrus.Option) lrus.Cache, depth int) { +func runBenchmarkErasePrefix(b *testing.B, constructor func(uint64, ...lru.Option) lru.Cache, depth int) { + b.Helper() const prefixCount = 100 const itemsPerPrefix = 100 - keys, prefixMap, prefixes := generateBenchmarkKeys(prefixCount, itemsPerPrefix, depth) + keys, _, prefixes := generateBenchmarkKeys(prefixCount, itemsPerPrefix, depth) data := benchValue{val: 1, dataSize: 10} capacity := uint64(len(keys) * 100) cache := constructor(capacity) - for _, key := range keys { - _, _ = cache.Insert(key, data) - } b.ReportAllocs() b.ResetTimer() - i := 0 - for b.Loop() { - prefix := prefixes[i%len(prefixes)] - cache.EraseEntriesWithGivenPrefix(prefix) - - // UNTIMED: Pause clock, restore erased keys, and restart timer for next iteration - b.StopTimer() - for _, key := range prefixMap[prefix] { - _, _ = cache.Insert(key, data) + for i := range b.N { + idx := i % len(prefixes) + if idx == 0 { + b.StopTimer() + for _, key := range keys { + _, _ = cache.Insert(key, data) + } + b.StartTimer() } - i++ - b.StartTimer() + cache.EraseEntriesWithGivenPrefix(prefixes[idx]) } } func Benchmark_ErasePrefix_MapCache(b *testing.B) { - b.Run("Flat", func(b *testing.B) { runBenchmarkErasePrefix(b, lrus.NewMapCache, 0) }) - b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkErasePrefix(b, lrus.NewMapCache, 2) }) - b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkErasePrefix(b, lrus.NewMapCache, 10) }) + b.Run("Flat", func(b *testing.B) { runBenchmarkErasePrefix(b, lru.NewMapCache, 0) }) + b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkErasePrefix(b, lru.NewMapCache, 2) }) + b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkErasePrefix(b, lru.NewMapCache, 10) }) } func Benchmark_ErasePrefix_RadixCache(b *testing.B) { - b.Run("Flat", func(b *testing.B) { runBenchmarkErasePrefix(b, lrus.NewRadixCache, 0) }) - b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkErasePrefix(b, lrus.NewRadixCache, 2) }) - b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkErasePrefix(b, lrus.NewRadixCache, 10) }) + b.Run("Flat", func(b *testing.B) { runBenchmarkErasePrefix(b, lru.NewRadixCache, 0) }) + b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkErasePrefix(b, lru.NewRadixCache, 2) }) + b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkErasePrefix(b, lru.NewRadixCache, 10) }) } func Benchmark_ErasePrefix_ArenaRadixCache(b *testing.B) { - b.Run("Flat", func(b *testing.B) { runBenchmarkErasePrefix(b, lrus.NewArenaRadixCache, 0) }) - b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkErasePrefix(b, lrus.NewArenaRadixCache, 2) }) - b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkErasePrefix(b, lrus.NewArenaRadixCache, 10) }) + b.Run("Flat", func(b *testing.B) { runBenchmarkErasePrefix(b, lru.NewArenaRadixCache, 0) }) + b.Run("Nested_Depth2", func(b *testing.B) { runBenchmarkErasePrefix(b, lru.NewArenaRadixCache, 2) }) + b.Run("DeeplyNested_Depth10", func(b *testing.B) { runBenchmarkErasePrefix(b, lru.NewArenaRadixCache, 10) }) } // ============================================================================ // 7. Parallel Multi-Core Throughput Benchmarks (b.RunParallel) // ============================================================================ -func runParallelWorkload(b *testing.B, constructor func(uint64, ...lrus.Option) lrus.Cache, insertPct, lookupPct int) { +func runParallelWorkload(b *testing.B, constructor func(uint64, ...lru.Option) lru.Cache, insertPct, lookupPct int) { + b.Helper() const cacheSize = 50000000 const keySpace = 20000 data := benchValue{val: 1, dataSize: 10} + keys := make([]string, keySpace) + for i := range keySpace { + keys[i] = fmt.Sprintf("key_%d", i) + } cache := constructor(cacheSize) // Pre-populate for i := range keySpace / 2 { - _, _ = cache.Insert(fmt.Sprintf("key_%d", i), data) + _, _ = cache.Insert(keys[i], data) } + var workerSeq atomic.Int64 b.ReportAllocs() b.ResetTimer() b.RunParallel(func(pb *testing.PB) { - r := rand.New(rand.NewSource(time.Now().UnixNano())) + workerID := workerSeq.Add(1) + r := rand.New(rand.NewSource(42 + workerID*10007)) for pb.Next() { op := r.Intn(100) - key := fmt.Sprintf("key_%d", r.Intn(keySpace)) + key := keys[r.Intn(keySpace)] switch { case op < insertPct: _, _ = cache.Insert(key, data) @@ -344,21 +374,21 @@ func runParallelWorkload(b *testing.B, constructor func(uint64, ...lrus.Option) } func Benchmark_ParallelThroughput_Mixed(b *testing.B) { - b.Run("MapCache", func(b *testing.B) { runParallelWorkload(b, lrus.NewMapCache, 30, 60) }) - b.Run("RadixCache", func(b *testing.B) { runParallelWorkload(b, lrus.NewRadixCache, 30, 60) }) - b.Run("ArenaRadixCache", func(b *testing.B) { runParallelWorkload(b, lrus.NewArenaRadixCache, 30, 60) }) + b.Run("MapCache", func(b *testing.B) { runParallelWorkload(b, lru.NewMapCache, 30, 60) }) + b.Run("RadixCache", func(b *testing.B) { runParallelWorkload(b, lru.NewRadixCache, 30, 60) }) + b.Run("ArenaRadixCache", func(b *testing.B) { runParallelWorkload(b, lru.NewArenaRadixCache, 30, 60) }) } func Benchmark_ParallelThroughput_ReadHeavy(b *testing.B) { - b.Run("MapCache", func(b *testing.B) { runParallelWorkload(b, lrus.NewMapCache, 5, 90) }) - b.Run("RadixCache", func(b *testing.B) { runParallelWorkload(b, lrus.NewRadixCache, 5, 90) }) - b.Run("ArenaRadixCache", func(b *testing.B) { runParallelWorkload(b, lrus.NewArenaRadixCache, 5, 90) }) + b.Run("MapCache", func(b *testing.B) { runParallelWorkload(b, lru.NewMapCache, 5, 90) }) + b.Run("RadixCache", func(b *testing.B) { runParallelWorkload(b, lru.NewRadixCache, 5, 90) }) + b.Run("ArenaRadixCache", func(b *testing.B) { runParallelWorkload(b, lru.NewArenaRadixCache, 5, 90) }) } func Benchmark_ParallelThroughput_WriteHeavy(b *testing.B) { - b.Run("MapCache", func(b *testing.B) { runParallelWorkload(b, lrus.NewMapCache, 80, 15) }) - b.Run("RadixCache", func(b *testing.B) { runParallelWorkload(b, lrus.NewRadixCache, 80, 15) }) - b.Run("ArenaRadixCache", func(b *testing.B) { runParallelWorkload(b, lrus.NewArenaRadixCache, 80, 15) }) + b.Run("MapCache", func(b *testing.B) { runParallelWorkload(b, lru.NewMapCache, 80, 15) }) + b.Run("RadixCache", func(b *testing.B) { runParallelWorkload(b, lru.NewRadixCache, 80, 15) }) + b.Run("ArenaRadixCache", func(b *testing.B) { runParallelWorkload(b, lru.NewArenaRadixCache, 80, 15) }) } // ============================================================================ @@ -369,62 +399,67 @@ func Benchmark_LargeScale_Insert_100K(b *testing.B) { const numEntries = 100000 data := benchValue{val: 1, dataSize: 10} cacheMaxSize := uint64(numEntries * 20) + keys := make([]string, numEntries) + for j := range numEntries { + keys[j] = fmt.Sprintf("prefix/key-%d", j) + } - b.Run("MapCache", func(b *testing.B) { + runInsert100K := func(b *testing.B, constructor func(uint64, ...lru.Option) lru.Cache) { + b.Helper() b.ReportAllocs() - for range b.N { + var mBefore, mAfter runtime.MemStats + var lastCache lru.Cache + for i := range b.N { b.StopTimer() - cache := lrus.NewMapCache(cacheMaxSize) - b.StartTimer() - for j := range numEntries { - _, _ = cache.Insert(fmt.Sprintf("prefix/key-%d", j), data) + if i == b.N-1 { + runtime.GC() + runtime.ReadMemStats(&mBefore) } - } - }) - - b.Run("RadixCache", func(b *testing.B) { - b.ReportAllocs() - for range b.N { - b.StopTimer() - cache := lrus.NewRadixCache(cacheMaxSize) + cache := constructor(cacheMaxSize) b.StartTimer() for j := range numEntries { - _, _ = cache.Insert(fmt.Sprintf("prefix/key-%d", j), data) + _, _ = cache.Insert(keys[j], data) } - } - }) - - b.Run("ArenaRadixCache", func(b *testing.B) { - b.ReportAllocs() - for range b.N { - b.StopTimer() - cache := lrus.NewArenaRadixCache(cacheMaxSize) - b.StartTimer() - for j := range numEntries { - _, _ = cache.Insert(fmt.Sprintf("prefix/key-%d", j), data) + if i == b.N-1 { + b.StopTimer() + lastCache = cache + runtime.GC() + runtime.ReadMemStats(&mAfter) + runtime.KeepAlive(lastCache) + b.StartTimer() } } - }) + if mAfter.HeapAlloc > mBefore.HeapAlloc { + b.ReportMetric(float64(mAfter.HeapAlloc-mBefore.HeapAlloc)/float64(numEntries), "heap-B/entry") + } + } + + b.Run("MapCache", func(b *testing.B) { runInsert100K(b, lru.NewMapCache) }) + b.Run("RadixCache", func(b *testing.B) { runInsert100K(b, lru.NewRadixCache) }) + b.Run("ArenaRadixCache", func(b *testing.B) { runInsert100K(b, lru.NewArenaRadixCache) }) } func Benchmark_LargeScale_PrefixErase_100K(b *testing.B) { const numEntries = 100000 data := benchValue{val: 1, dataSize: 10} cacheMaxSize := uint64(numEntries * 20) + keys := make([]string, numEntries) + for j := range numEntries { + if j%2 == 0 { + keys[j] = fmt.Sprintf("target_prefix/key-%d", j) + } else { + keys[j] = fmt.Sprintf("other_prefix/key-%d", j) + } + } - runPrefixErase100K := func(b *testing.B, constructor func(uint64, ...lrus.Option) lrus.Cache) { + runPrefixErase100K := func(b *testing.B, constructor func(uint64, ...lru.Option) lru.Cache) { + b.Helper() b.ReportAllocs() for range b.N { b.StopTimer() cache := constructor(cacheMaxSize) for j := range numEntries { - var key string - if j%2 == 0 { - key = fmt.Sprintf("target_prefix/key-%d", j) - } else { - key = fmt.Sprintf("other_prefix/key-%d", j) - } - _, _ = cache.Insert(key, data) + _, _ = cache.Insert(keys[j], data) } b.StartTimer() @@ -433,9 +468,9 @@ func Benchmark_LargeScale_PrefixErase_100K(b *testing.B) { } } - b.Run("MapCache", func(b *testing.B) { runPrefixErase100K(b, lrus.NewMapCache) }) - b.Run("RadixCache", func(b *testing.B) { runPrefixErase100K(b, lrus.NewRadixCache) }) - b.Run("ArenaRadixCache", func(b *testing.B) { runPrefixErase100K(b, lrus.NewArenaRadixCache) }) + b.Run("MapCache", func(b *testing.B) { runPrefixErase100K(b, lru.NewMapCache) }) + b.Run("RadixCache", func(b *testing.B) { runPrefixErase100K(b, lru.NewRadixCache) }) + b.Run("ArenaRadixCache", func(b *testing.B) { runPrefixErase100K(b, lru.NewArenaRadixCache) }) } // ============================================================================ @@ -445,37 +480,64 @@ func Benchmark_LargeScale_PrefixErase_100K(b *testing.B) { func Benchmark_ArenaRadixCache_Compact(b *testing.B) { const numKeys = 10000 data := benchValue{val: 1, dataSize: 10} + keys := make([]string, numKeys) + for i := range numKeys { + keys[i] = fmt.Sprintf("dir_%02d/file_%05d", i%50, i) + } b.ReportAllocs() - for range b.N { + var reclaimedBytes uint64 + for iter := range b.N { b.StopTimer() - cache := lrus.NewArenaRadixCache(uint64(numKeys * 20)).(lrus.PressureAwareCache) + cache := lru.NewArenaRadixCache(uint64(numKeys * 20)).(lru.PressureAwareCache) for i := range numKeys { - _, _ = cache.Insert(fmt.Sprintf("dir_%02d/file_%05d", i%50, i), data) + _, _ = cache.Insert(keys[i], data) } for i := range numKeys / 2 { - _ = cache.Erase(fmt.Sprintf("dir_%02d/file_%05d", i%50, i)) + _ = cache.Erase(keys[i]) + } + var mBefore, mAfter runtime.MemStats + if iter == b.N-1 { + runtime.GC() + runtime.ReadMemStats(&mBefore) } b.StartTimer() cache.Compact() + + if iter == b.N-1 { + b.StopTimer() + runtime.GC() + runtime.ReadMemStats(&mAfter) + runtime.KeepAlive(cache) + if mBefore.HeapAlloc > mAfter.HeapAlloc { + reclaimedBytes = mBefore.HeapAlloc - mAfter.HeapAlloc + } + b.StartTimer() + } } + b.ReportMetric(float64(reclaimedBytes), "reclaimed-B/op") } func Benchmark_ArenaRadixCache_InsertUnderPressure(b *testing.B) { const numKeys = 10000 + const keyPool = numKeys * 2 + keys := make([]string, keyPool) + for i := range keyPool { + keys[i] = fmt.Sprintf("dir_%02d/file_%05d", i%50, i) + } data := benchValue{val: 1, dataSize: 10} - cache := lrus.NewArenaRadixCache( + cache := lru.NewArenaRadixCache( uint64(numKeys*10), - lrus.WithPressureFunc(func() float64 { return 0.92 }), - lrus.WithEvictionRetentionRatio(0.50), + lru.WithPressureFunc(func() float64 { return 0.92 }), + lru.WithEvictionRetentionRatio(0.50), ) b.ReportAllocs() b.ResetTimer() i := 0 for b.Loop() { - _, _ = cache.Insert(fmt.Sprintf("dir_%02d/file_%05d", i%50, i%(numKeys*2)), data) + _, _ = cache.Insert(keys[i%keyPool], data) i++ } } diff --git a/cache.go b/cache.go index 04bdd03..35557ca 100644 --- a/cache.go +++ b/cache.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package lrus provides high-performance, concurrent, zero-dependency LRU cache implementations +// Package lru provides high-performance, concurrent, zero-dependency LRU cache implementations // adapted from Google Cloud Storage FUSE (GCSFuse). // // The package defines a unified Cache interface satisfied by three specialized engines: @@ -23,7 +23,7 @@ // and prefixes retain standard Go GC properties. // // All cache constructors require maxSize > 0 and unconditionally panic if maxSize == 0. -package lrus +package lru // ValueType represents an entry stored in a Cache that reports its logical or memory size. // The cache uses Size() to calculate total capacity and trigger LRU eviction when capacity is exceeded. @@ -66,8 +66,10 @@ type Cache interface { // UpdateSize adjusts the size accounting for an existing key by sizeDelta without altering its LRU position. // Useful for entries whose size grows incrementally (e.g. sparse files). - // If the cache capacity is exceeded as a result of the size adjustment, least recently used (LRU) entries - // are evicted immediately to maintain capacity invariants. + // If the entry's updated size (existingSize + sizeDelta) exceeds maxSize (or cannot fit alongside + // entries more recent than key), the entry itself is evicted immediately without evicting older entries. + // Otherwise, if total cache capacity is exceeded, least recently used (LRU) entries are evicted + // immediately to maintain capacity invariants. // // Returns ErrEntryNotExist if key is not present in the cache. // Returns ErrInvalidUpdateEntrySize if sizeDelta causes uint64 integer overflow. @@ -78,12 +80,14 @@ type Cache interface { EraseEntriesWithGivenPrefix(prefix string) } -// PressureAwareCache extends Cache with explicit arena compaction and memory-pressure reclamation. -// ArenaRadixCache implements this interface. +// PressureAwareCache extends Cache with explicit arena/map compaction and memory-pressure reclamation. +// All three cache backends (MapCache, RadixCache, and ArenaRadixCache) implement this interface +// and perform both automatic amortized foreground reclamation (on Insert, Erase, UpdateSize, +// and EraseEntriesWithGivenPrefix) and explicit reclamation via EvaluateMemoryPressure() and Compact(). type PressureAwareCache interface { Cache - // Compact performs lossless compaction of the internal node arena and lookup index. + // Compact performs lossless compaction of the internal node arena and/or lookup index. Compact() // EvaluateMemoryPressure samples the configured memory-pressure probe and executes @@ -91,3 +95,28 @@ type PressureAwareCache interface { // returning any values evicted during Tier 2 shedding. EvaluateMemoryPressure() []ValueType } + +// New creates and returns a new LRU Cache bounded by maxSize. +// By default, New constructs a MapCache (BackendMap). Callers can select an alternative +// engine via WithBackend(BackendRadix) or WithBackend(BackendArenaRadix), or invoke +// NewMapCache, NewRadixCache, or NewArenaRadixCache directly. +// +// All returned Cache instances also implement PressureAwareCache. +// +// maxSize must be greater than zero; otherwise New panics. +func New(maxSize uint64, opts ...Option) Cache { + if maxSize == 0 { + panic("maxSize must be greater than zero") + } + options := ApplyOptions(opts...) + switch options.Backend { + case BackendRadix: + return newRadixCacheWithOptions(maxSize, options) + case BackendArenaRadix: + return newArenaRadixCacheWithOptions(maxSize, options) + case BackendMap: + fallthrough + default: + return newMapCacheWithOptions(maxSize, options) + } +} diff --git a/compaction_test.go b/compaction_test.go new file mode 100644 index 0000000..d8c5a44 --- /dev/null +++ b/compaction_test.go @@ -0,0 +1,1515 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lru + +import ( + "fmt" + "math" + "runtime" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testGoroutineID parses the calling goroutine's numeric ID from runtime.Stack. +func testGoroutineID() uint64 { + var buf [32]byte + n := runtime.Stack(buf[:], false) + const prefix = "goroutine " + var id uint64 + if n > len(prefix) { + for i := len(prefix); i < n; i++ { + b := buf[i] + if b < '0' || b > '9' { + break + } + id = id*10 + uint64(b-'0') + } + } + return id +} + +// assertAlreadyCompacted verifies that pac.Compact() allocates 0 heap objects on its very first invocation, +// and when a pressureProbe is provided, also verifies via !probe.ObserveEpochAdvance(tb, pac, compactOnce) +// that the first Compact() call itself did not advance reclaimEpoch (deterministically verifying RadixCache +// was already compacted too). +func assertAlreadyCompacted(tb testing.TB, pac PressureAwareCache, probes ...*pressureProbe) { + tb.Helper() + var before, after runtime.MemStats + compactOnce := func() { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) + runtime.Gosched() + runtime.ReadMemStats(&before) + pac.Compact() + runtime.ReadMemStats(&after) + } + hasProbe := false + for _, probe := range probes { + if probe != nil { + assert.False(tb, probe.ObserveEpochAdvance(tb, pac, compactOnce), "expected cache to already be compacted (first Compact call must not advance reclaimEpoch)") + hasProbe = true + } + } + if !hasProbe { + compactOnce() + } + assert.Equal(tb, uint64(0), after.Mallocs-before.Mallocs, "expected cache to already be compacted (0 allocations on Compact)") +} + +// pressureProbe provides a public-API PressureFunc callback that allows tests to dynamically +// update simulated memory pressure and observe via ObserveEpochAdvance whether an +// operation advanced the cache's reclamation epoch. +type pressureProbe struct { + pressureBits atomic.Uint64 + + mu sync.Mutex + hookActive bool + bgGID uint64 + bgCalls int + bgEntered chan struct{} + bgRelease chan struct{} +} + +func newPressureProbe(initial float64) *pressureProbe { + p := &pressureProbe{} + p.pressureBits.Store(math.Float64bits(initial)) + return p +} + +func (p *pressureProbe) Set(val float64) { + p.pressureBits.Store(math.Float64bits(val)) +} + +func (p *pressureProbe) Option() Option { + return WithPressureFunc(p.PressureFunc) +} + +func (p *pressureProbe) PressureFunc() float64 { + p.mu.Lock() + if p.hookActive && testGoroutineID() == p.bgGID { + p.bgCalls++ + callNum := p.bgCalls + entered := p.bgEntered + release := p.bgRelease + p.mu.Unlock() + if callNum == 1 { + close(entered) + <-release + } + return 0.0 + } + p.mu.Unlock() + return math.Float64frombits(p.pressureBits.Load()) +} + +// ObserveEpochAdvance executes op() while a background EvaluateMemoryPressure() call +// holds a pre-operation epoch snapshot inside PressureFunc, returning true if and only if op() +// advanced the cache's reclamation epoch (causing lockWithPressure to re-sample PressureFunc). +func (p *pressureProbe) ObserveEpochAdvance(tb testing.TB, pac PressureAwareCache, op func()) bool { + tb.Helper() + + bgEntered := make(chan struct{}) + bgRelease := make(chan struct{}) + bgDone := make(chan struct{}) + + go func() { + defer close(bgDone) + p.mu.Lock() + p.hookActive = true + p.bgGID = testGoroutineID() + p.bgCalls = 0 + p.bgEntered = bgEntered + p.bgRelease = bgRelease + p.mu.Unlock() + + _ = pac.EvaluateMemoryPressure() + + p.mu.Lock() + p.hookActive = false + p.bgGID = 0 + p.mu.Unlock() + }() + + <-bgEntered + op() + close(bgRelease) + <-bgDone + + p.mu.Lock() + calls := p.bgCalls + p.mu.Unlock() + return calls > 1 +} + +func allBackends() []struct { + name string + fn func(uint64, ...Option) Cache +} { + return []struct { + name string + fn func(uint64, ...Option) Cache + }{ + {"MapCache", NewMapCache}, + {"RadixCache", NewRadixCache}, + {"ArenaRadixCache", NewArenaRadixCache}, + } +} + +func TestCompaction_EmptyDrainAndPreInsertSlackReclamation(t *testing.T) { + t.Run("SequentialEraseDrainToEmptyReleasesPeakSlack", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + // Arrange + probe := newPressureProbe(0.10) + pac := b.fn(100000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + + for i := range 200 { + k := fmt.Sprintf("item/sub/%04d", i) + _, err := pac.Insert(k, NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act: Drain all entries via individual Erase(key) calls down to 0 entries. + for i := range 200 { + k := fmt.Sprintf("item/sub/%04d", i) + require.NotNil(t, pac.Erase(k)) + } + + // Assert: Peak structures are released upon reaching empty state. + assert.Nil(t, pac.LookUpWithoutChangingOrder("item/sub/0000")) + assertAlreadyCompacted(t, pac, probe) + }) + } + }) + + t.Run("PreInsertDrainReleasesPeakSlackAtNormalPressure", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + // Arrange: Populate 100 hierarchical keys (1000B total in 1000B cache) at normal pressure (0.10). + var sampleCount atomic.Int32 + c := b.fn( + 1000, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + sampleCount.Add(1) + return 0.10 + }), + ).(PressureAwareCache) + + for i := range 100 { + key := fmt.Sprintf("dir_%02d/sub_%02d/file_%03d", i%10, (i/10)%10, i) + _, err := c.Insert(key, NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act: Insert a single 1000B jumbo entry that pre-evicts all 100 entries down to empty before inserting. + samplesBefore := sampleCount.Load() + evicted, err := c.Insert("jumbo", NewSizedValue("jumbo_val", 1000)) + + // Assert: All 100 entries were evicted, peak slack was released, and only 1 pressure sample ran. + require.NoError(t, err) + assert.Len(t, evicted, 100) + assert.Equal(t, int32(1), sampleCount.Load()-samplesBefore) + assert.Nil(t, c.LookUpWithoutChangingOrder("dir_00/sub_00/file_000")) + assert.NotNil(t, c.LookUpWithoutChangingOrder("jumbo")) + assertAlreadyCompacted(t, c) + }) + } + }) + + t.Run("PreInsertEmptyDrainUnderTier1PressureAdvancesReclaimEpochAcrossBackends", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + // Arrange: Populate 70 keys (10B each = 700B in 1000B cache) at normal pressure (0.10). + probe := newPressureProbe(0.10) + pac := b.fn( + 1000, + WithInvariantChecking(true), + probe.Option(), + ).(PressureAwareCache) + + for i := range 70 { + _, err := pac.Insert(fmt.Sprintf("dir_%02d/k_%03d", i%10, i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act: Raise pressure to Tier 1 (0.80) and insert a 1000B entry that pre-evicts all 70 keys. + probe.Set(0.80) + var evicted []ValueType + var err error + advanced := probe.ObserveEpochAdvance(t, pac, func() { + evicted, err = pac.Insert("jumbo", NewSizedValue("v", 1000)) + }) + + // Assert + require.NoError(t, err) + assert.Len(t, evicted, 70) + assert.True(t, advanced) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("jumbo")) + assertAlreadyCompacted(t, pac, probe) + }) + } + }) + + t.Run("NetByteReductionPreInsertEvictionUnderTier1PressureAdvancesReclaimEpoch", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + // Arrange: Fill a 100B cache with "k1" (60B) and "k2" (40B) = 100B at Tier 1 pressure (0.80). + probe := newPressureProbe(0.80) + pac := b.fn( + 100, + WithInvariantChecking(true), + probe.Option(), + ).(PressureAwareCache) + _, err := pac.Insert("k1", NewSizedValue("v1", 60)) + require.NoError(t, err) + _, err = pac.Insert("k2", NewSizedValue("v2", 40)) + require.NoError(t, err) + + // Act: Insert "k3" (50B), which pre-evicts "k1" (60B) so net currentSize decreases from 100B to 90B. + var evicted []ValueType + advanced := probe.ObserveEpochAdvance(t, pac, func() { + evicted, err = pac.Insert("k3", NewSizedValue("v3", 50)) + }) + + // Assert: Net byte reduction via pre-insert eviction under elevated pressure advances reclamation epoch and compacts. + require.NoError(t, err) + require.Len(t, evicted, 1) + assert.True(t, advanced) + assert.Nil(t, pac.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("k2")) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("k3")) + assertAlreadyCompacted(t, pac, probe) + }) + } + }) + + t.Run("LargeCacheFullDrainAvoidsPeakOldToNewAllocation", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + // Arrange: Populate 200 entries via public API, then insert a single entry of size == maxSize (200). + const n = 200 + probe := newPressureProbe(0.10) + pac := b.fn(n, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + for i := range n { + _, err := pac.Insert(fmt.Sprintf("k/%06d", i), NewSizedValue("v", 1)) + require.NoError(t, err) + } + + // Act + evicted, err := pac.Insert("jumbo", NewSizedValue("big", n)) + require.NoError(t, err) + + // Assert: All 200 entries were evicted, only "jumbo" remains, and cache is already compacted without peak slack. + assert.Len(t, evicted, n) + assert.Nil(t, pac.LookUp("k/000000")) + assert.NotNil(t, pac.LookUp("jumbo")) + assertAlreadyCompacted(t, pac, probe) + }) + } + }) +} + +func TestCompaction_EraseEmptyPrefixResetsSlack(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + // Arrange + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + for i := range 100 { + _, err := pac.Insert(fmt.Sprintf("dir/sub/%03d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act + pac.EraseEntriesWithGivenPrefix("") + + // Assert: All entries are removed, backing structures are reset/compacted, and full capacity is available. + assert.Nil(t, pac.LookUpWithoutChangingOrder("dir/sub/000")) + assert.Nil(t, pac.LookUpWithoutChangingOrder("dir/sub/099")) + assertAlreadyCompacted(t, pac, probe) + + evicted, err := pac.Insert("full_capacity", NewSizedValue("v", 1000)) + require.NoError(t, err) + assert.Empty(t, evicted) + }) + } +} + +func TestCompaction_SingleSurvivorAndOverwriteSlackReclamation(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("OverwriteEvictingAllOtherEntriesReleasesPeakSlack", func(t *testing.T) { + // Arrange: Populate 70 keys (10B each = 700B in 1000B cache). + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + for i := range 70 { + key := fmt.Sprintf("k-%03d", i) + _, err := pac.Insert(key, NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act: Overwrite existing key "k-000" with a 1000B value, evicting all 69 other entries down to 1 entry. + evicted, err := pac.Insert("k-000", NewSizedValue("jumbo", 1000)) + require.NoError(t, err) + + // Assert + assert.Len(t, evicted, 69) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("k-000")) + assert.Nil(t, pac.LookUpWithoutChangingOrder("k-001")) + assertAlreadyCompacted(t, pac, probe) + }) + + t.Run("UpdateSizeEvictingAllOtherEntriesReleasesPeakSlack", func(t *testing.T) { + // Arrange + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + for i := range 70 { + key := fmt.Sprintf("k-%03d", i) + _, err := pac.Insert(key, NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act: Grow MRU key "k-069" by +990B (to 1000B), evicting all 69 older entries down to 1 entry. + err := pac.UpdateSize("k-069", 990) + + // Assert + require.NoError(t, err) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("k-069")) + assert.Nil(t, pac.LookUpWithoutChangingOrder("k-000")) + assertAlreadyCompacted(t, pac, probe) + }) + + t.Run("InsertNewKeyWithSurvivingZeroSizeMRUEntryReclaimsPeakSlack", func(t *testing.T) { + // Arrange + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + for i := range 70 { + key := fmt.Sprintf("k-%03d", i) + _, err := pac.Insert(key, NewSizedValue("v", 10)) + require.NoError(t, err) + } + _, err := pac.Insert("z_head", NewSizedValue("z", 0)) + require.NoError(t, err) + + // Act + _, err = pac.Insert("jumbo", NewSizedValue("v", 1000)) + require.NoError(t, err) + + // Assert + assert.NotNil(t, pac.LookUpWithoutChangingOrder("z_head")) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("jumbo")) + assertAlreadyCompacted(t, pac, probe) + }) + + t.Run("EraseEntriesWithGivenPrefixDrainingToOneSurvivorReclaimsPeakSlack", func(t *testing.T) { + // Arrange + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + for i := range 70 { + key := fmt.Sprintf("batch/k-%03d", i) + _, err := pac.Insert(key, NewSizedValue("v", 10)) + require.NoError(t, err) + } + _, err := pac.Insert("keep/1", NewSizedValue("v", 10)) + require.NoError(t, err) + + // Act + pac.EraseEntriesWithGivenPrefix("batch/") + + // Assert + assert.NotNil(t, pac.LookUpWithoutChangingOrder("keep/1")) + assert.Nil(t, pac.LookUpWithoutChangingOrder("batch/k-000")) + assertAlreadyCompacted(t, pac, probe) + }) + + t.Run("EraseAndUpdateSizeSelfEvictionDrainingToOneSurvivorReclaimPeakSlack", func(t *testing.T) { + // Arrange + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + for i := range 70 { + key := fmt.Sprintf("item-%03d", i) + _, err := pac.Insert(key, NewSizedValue("v", 10)) + require.NoError(t, err) + } + for i := range 68 { + key := fmt.Sprintf("item-%03d", i) + require.NotNil(t, pac.Erase(key)) + } + + // Act + err := pac.UpdateSize("item-069", 1000) + require.NoError(t, err) + + // Assert + assert.NotNil(t, pac.LookUpWithoutChangingOrder("item-068")) + assert.Nil(t, pac.LookUpWithoutChangingOrder("item-069")) + assertAlreadyCompacted(t, pac, probe) + }) + + t.Run("Exact64EntriesDrainedToOneSurvivorAndHighChurnTombstoneDrain", func(t *testing.T) { + // Arrange 1: 64 entries drained to 1 survivor via overwrite. + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + for i := range 64 { + key := string([]byte{byte(i + 1), 'k'}) + _, err := pac.Insert(key, NewSizedValue("v", 10)) + require.NoError(t, err) + } + + firstKey := string([]byte{1, 'k'}) + _, err := pac.Insert(firstKey, NewSizedValue("jumbo", 1000)) + require.NoError(t, err) + + assert.NotNil(t, pac.LookUpWithoutChangingOrder(firstKey)) + assertAlreadyCompacted(t, pac, probe) + + // Arrange 2: High-churn tombstone drain to empty or single survivor. + probeEmpty := newPressureProbe(0.10) + pacEmpty := b.fn(300, WithInvariantChecking(true), probeEmpty.Option()).(PressureAwareCache) + for i := range 100 { + key := string([]byte{byte(i + 1), 'k'}) + _, err = pacEmpty.Insert(key, NewSizedValue("v", 10)) + require.NoError(t, err) + } + + probeOne := newPressureProbe(0.10) + pacOne := b.fn(1000, WithInvariantChecking(true), probeOne.Option()).(PressureAwareCache) + _, err = pacOne.Insert("survivor", NewSizedValue("s", 0)) + require.NoError(t, err) + for i := range 50 { + key := fmt.Sprintf("batch1-%02d", i) + _, err = pacOne.Insert(key, NewSizedValue("v", 10)) + require.NoError(t, err) + } + for i := range 50 { + key := fmt.Sprintf("batch1-%02d", i) + require.NotNil(t, pacOne.Erase(key)) + } + for i := range 50 { + key := fmt.Sprintf("batch2-%02d", i) + _, err = pacOne.Insert(key, NewSizedValue("v", 10)) + require.NoError(t, err) + } + + _, err = pacEmpty.Insert("jumbo", NewSizedValue("v", 300)) + require.NoError(t, err) + + pacOne.EraseEntriesWithGivenPrefix("batch2-") + + assert.NotNil(t, pacEmpty.LookUpWithoutChangingOrder("jumbo")) + assert.NotNil(t, pacOne.LookUpWithoutChangingOrder("survivor")) + assertAlreadyCompacted(t, pacEmpty, probeEmpty) + assertAlreadyCompacted(t, pacOne, probeOne) + }) + }) + } +} + +func TestCompaction_CrossBackendParityOnSingleSurvivorAndShrinkageWatermarks(t *testing.T) { + t.Run("EraseDrainingPeakEntriesToSingleSurvivorAdvancesReclaimEpoch", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + + for i := range 70 { + _, err := pac.Insert(fmt.Sprintf("z-%03d", i), NewSizedValue("v", 0)) + require.NoError(t, err) + } + for i := range 68 { + require.NotNil(t, pac.Erase(fmt.Sprintf("z-%03d", i))) + } + + // Act: Raise pressure to Tier 1 (0.80) and erase "z-068", draining from 70 peak entries to 1 survivor. + probe.Set(0.80) + advanced := probe.ObserveEpochAdvance(t, pac, func() { + require.NotNil(t, pac.Erase("z-068")) + }) + + // Assert + assert.True(t, advanced) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("z-069")) + assertAlreadyCompacted(t, pac, probe) + }) + } + }) + + t.Run("UpdateSizeSelfEvictionDrainingPeakEntriesToSingleSurvivorClampsZeroWatermark", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + // Arrange: Populate 10 zero-size entries, shed 5 under critical pressure (setting watermark to 5), + // then add 65 more entries and drain down to 1 surviving zero-size entry via UpdateSize self-eviction. + probe := newPressureProbe(0.10) + pac := b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + probe.Option(), + ).(PressureAwareCache) + + for i := range 10 { + _, err := pac.Insert(fmt.Sprintf("init-z-%02d", i), NewSizedValue("v", 0)) + require.NoError(t, err) + } + probe.Set(0.95) + require.Len(t, pac.EvaluateMemoryPressure(), 5) + + probe.Set(0.10) + for i := range 64 { + _, err := pac.Insert(fmt.Sprintf("z-%03d", i), NewSizedValue("v", 0)) + require.NoError(t, err) + } + _, err := pac.Insert("p-069", NewSizedValue("v", 10)) + require.NoError(t, err) + + for i := 5; i < 10; i++ { + require.NotNil(t, pac.Erase(fmt.Sprintf("init-z-%02d", i))) + } + for i := range 63 { + require.NotNil(t, pac.Erase(fmt.Sprintf("z-%03d", i))) + } + + // Act: Self-evict "p-069" under Tier 1 pressure, leaving only 1 zero-size entry ("z-063"). + probe.Set(0.80) + advanced := probe.ObserveEpochAdvance(t, pac, func() { + require.NoError(t, pac.UpdateSize("p-069", 1000)) + }) + assert.True(t, advanced) + assertAlreadyCompacted(t, pac, probe) + + // Insert 1 more zero-size entry ("z-064", total 2 zero-size entries) and evaluate under Tier 2 (0.95). + // Because the zero-size watermark was clamped to 1 during single-survivor drain, 1 of the 2 zero-size entries is shed. + _, err = pac.Insert("z-064", NewSizedValue("v", 0)) + require.NoError(t, err) + probe.Set(0.95) + evicted := pac.EvaluateMemoryPressure() + assert.Len(t, evicted, 1) + assert.Nil(t, pac.LookUpWithoutChangingOrder("z-063")) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("z-064")) + }) + } + }) + + t.Run("SmallSheddingPreservesCompactionWatermarksForSubsequentSingleSurvivorDrain", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + probe := newPressureProbe(0.10) + pac := b.fn( + 700, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.90), + probe.Option(), + ).(PressureAwareCache) + + for i := range 64 { + _, err := pac.Insert(fmt.Sprintf("k-%03d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act: Trigger small Tier 2 shedding (evicts 2 of 65 entries, < 25% shrinkage), + // then erase remaining entries down to 1 survivor under Tier 1 (0.80). + probe.Set(0.95) + _, err := pac.Insert("k-064", NewSizedValue("v", 10)) + require.NoError(t, err) + + probe.Set(0.10) + for i := 2; i < 63; i++ { + require.NotNil(t, pac.Erase(fmt.Sprintf("k-%03d", i))) + } + + probe.Set(0.80) + advanced := probe.ObserveEpochAdvance(t, pac, func() { + require.NotNil(t, pac.Erase("k-063")) + }) + + // Assert + assert.True(t, advanced) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("k-064")) + assertAlreadyCompacted(t, pac, probe) + }) + } + }) + + t.Run("ValueBearingInternalNodeShrinkage", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + probe := newPressureProbe(0.10) + pac := b.fn(10000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + + for i := range 20 { + parentKey := fmt.Sprintf("p%02d", i) + childA := fmt.Sprintf("p%02d/a", i) + childB := fmt.Sprintf("p%02d/b", i) + for _, k := range []string{parentKey, childA, childB} { + _, err := pac.Insert(k, NewSizedValue("v", 10)) + require.NoError(t, err) + } + } + + for i := range 20 { + require.NotNil(t, pac.Erase(fmt.Sprintf("p%02d", i))) + } + + // Act: Raise pressure to Tier 1 (0.80) and call EvaluateMemoryPressure(). + probe.Set(0.80) + advanced := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + + // Assert + assert.True(t, advanced) + assertAlreadyCompacted(t, pac, probe) + }) + } + }) + + t.Run("Tier2ShedAndAutoCompactAdvancesReclaimEpochOncePerOperation", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + probe := newPressureProbe(0.10) + pac := b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + probe.Option(), + ).(PressureAwareCache) + + for i := range 20 { + _, err := pac.Insert(fmt.Sprintf("k-%02d", i), NewSizedValue("v", 50)) + require.NoError(t, err) + } + + // Act + probe.Set(0.95) + var evicted []ValueType + advanced := probe.ObserveEpochAdvance(t, pac, func() { + evicted = pac.EvaluateMemoryPressure() + }) + + // Assert: Tier 2 shed+compact evicted 10 entries, compacted slack, and an immediate repeat is a no-op. + require.Len(t, evicted, 10) + assert.True(t, advanced) + assertAlreadyCompacted(t, pac, probe) + + advancedRepeat := probe.ObserveEpochAdvance(t, pac, func() { + assert.Empty(t, pac.EvaluateMemoryPressure()) + }) + assert.False(t, advancedRepeat) + }) + } + }) + + t.Run("Exact25PercentShrinkageAutoCompactionParityAcrossBackends", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + probe := newPressureProbe(0.0) + pac := b.fn( + 1000, + WithInvariantChecking(true), + probe.Option(), + WithCompactionThreshold(0.75), + WithEvictionThreshold(0.90), + ).(PressureAwareCache) + + for i := range 20 { + _, err := pac.Insert(fmt.Sprintf("k%02d", i), StringValue("")) + require.NoError(t, err) + } + for i := range 4 { + require.NotNil(t, pac.Erase(fmt.Sprintf("k%02d", i))) + } + probe.Set(0.80) + + // Act 1: 5th deletion under Tier 1 pressure reaches exact 25% shrinkage from peak (20 -> 15). + advancedFifth := probe.ObserveEpochAdvance(t, pac, func() { + require.NotNil(t, pac.Erase("k04")) + }) + + // Assert 1 + assert.True(t, advancedFifth) + assertAlreadyCompacted(t, pac, probe) + + // Act 2: Erase 1 more zero-byte key ("k05", 15 -> 14, 1/15 < 25% shrinkage) at Tier 1 pressure; + // because watermarks were reset to 15 on the 5th deletion, it does not re-compact. + advancedSixth := probe.ObserveEpochAdvance(t, pac, func() { + require.NotNil(t, pac.Erase("k05")) + }) + assert.False(t, advancedSixth) + }) + } + }) + + t.Run("SmallCacheWithRoutingNodesRespects8EntryHysteresis", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + // Arrange: Insert 6 zero-byte hierarchical keys (6 entries <= 8 hysteresis floor, even though the radix tree + // allocates 10 arena nodes > 8 due to root + 3 intermediate routing nodes "a/", "b/", "c/"). + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + + for _, k := range []string{"a/1", "a/2", "b/1", "b/2", "c/1", "c/2"} { + _, err := pac.Insert(k, NewSizedValue("v", 0)) + require.NoError(t, err) + } + + // Act: Under Tier 1 pressure (0.80), erase 2 zero-byte keys ("a/1", "a/2") while observing epoch advancement. + probe.Set(0.80) + advanced := probe.ObserveEpochAdvance(t, pac, func() { + require.NotNil(t, pac.Erase("a/1")) + require.NotNil(t, pac.Erase("a/2")) + }) + + // Assert: Because peak entry count is 6 <= 8 (below the 8-entry small-cache hysteresis floor) and 0 bytes were freed, + // all backends refrain from auto-compacting or advancing reclamation epoch. + assert.False(t, advanced) + assert.Nil(t, pac.LookUpWithoutChangingOrder("a/1")) + assert.Nil(t, pac.LookUpWithoutChangingOrder("a/2")) + for _, k := range []string{"b/1", "b/2", "c/1", "c/2"} { + assert.NotNil(t, pac.LookUpWithoutChangingOrder(k)) + } + }) + } + }) + + t.Run("Tier1AndTier2AutoCompactionParityAcrossBackends", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("Tier1InlineShrinkageAndExplicitEvaluateAdvanceEpochAndResetSlack", func(t *testing.T) { + probe := newPressureProbe(0.10) + pac := b.fn(10000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + + for i := range 20 { + _, err := pac.Insert(fmt.Sprintf("k/%02d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + for i := range 5 { + require.NotNil(t, pac.Erase(fmt.Sprintf("k/%02d", i))) + } + + // Act 1: Erase the 6th of 20 entries (30% >= 25% shrinkage) under Tier 1 pressure (0.80). + probe.Set(0.80) + advancedInline := probe.ObserveEpochAdvance(t, pac, func() { + require.NotNil(t, pac.Erase("k/05")) + }) + assert.True(t, advancedInline) + assertAlreadyCompacted(t, pac, probe) + + advancedFollowUp := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.False(t, advancedFollowUp) + + // Act 2: Erase 1 more entry at normal pressure (0.10), then invoke EvaluateMemoryPressure() under Tier 1 (0.80). + probe.Set(0.10) + require.NotNil(t, pac.Erase("k/06")) + + probe.Set(0.80) + advancedExplicit := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.True(t, advancedExplicit) + assertAlreadyCompacted(t, pac, probe) + + advancedSecondEval := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.False(t, advancedSecondEval) + }) + + t.Run("Tier2NoShedEvaluateCompactsPendingSlack", func(t *testing.T) { + probe := newPressureProbe(0.10) + pac := b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + probe.Option(), + ).(PressureAwareCache) + + for i := range 20 { + _, err := pac.Insert(fmt.Sprintf("k/%02d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + for i := range 6 { + require.NotNil(t, pac.Erase(fmt.Sprintf("k/%02d", i))) + } + + // Act: Spike pressure to Tier 2 (0.95) and call EvaluateMemoryPressure() while currentSize (140B) <= targetSize (500B). + probe.Set(0.95) + advanced := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.True(t, advanced) + assertAlreadyCompacted(t, pac, probe) + + advancedRepeat := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.False(t, advancedRepeat) + }) + }) + } + }) +} + +func TestCompaction_PreReclaimSlackAndEpochConsistencyAcrossMutations(t *testing.T) { + t.Run("SingleSurvivorDrainDuringInsertDoesNotDoubleAllocateOrRecompactOnEvaluate", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + + for i := range 64 { + _, err := pac.Insert(fmt.Sprintf("k-%03d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act + probe.Set(0.80) + advancedInsert := probe.ObserveEpochAdvance(t, pac, func() { + _, err := pac.Insert("big", NewSizedValue("v", 990)) + require.NoError(t, err) + }) + assert.True(t, advancedInsert) + assertAlreadyCompacted(t, pac, probe) + + advancedEval := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.False(t, advancedEval) + }) + } + }) + + t.Run("EmptyCacheDrainDoesNotSpuriouslyAdvanceEpochOnZeroByteOrNormalPressure", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("ZeroByteLastEntryUnderTier1Pressure", func(t *testing.T) { + probeErase := newPressureProbe(0.80) + probeSelfEvict := newPressureProbe(0.80) + probePrefixEmpty := newPressureProbe(0.80) + cacheErase := b.fn(100, WithInvariantChecking(true), probeErase.Option()).(PressureAwareCache) + cacheSelfEvict := b.fn(100, WithInvariantChecking(true), probeSelfEvict.Option()).(PressureAwareCache) + cachePrefixEmpty := b.fn(100, WithInvariantChecking(true), probePrefixEmpty.Option()).(PressureAwareCache) + + _, err := cacheErase.Insert("z1", NewSizedValue("v", 0)) + require.NoError(t, err) + _, err = cacheSelfEvict.Insert("z1", NewSizedValue("v", 0)) + require.NoError(t, err) + _, err = cachePrefixEmpty.Insert("z1", NewSizedValue("v", 0)) + require.NoError(t, err) + + advancedErase := probeErase.ObserveEpochAdvance(t, cacheErase, func() { + require.NotNil(t, cacheErase.Erase("z1")) + }) + advancedSelfEvict := probeSelfEvict.ObserveEpochAdvance(t, cacheSelfEvict, func() { + require.NoError(t, cacheSelfEvict.UpdateSize("z1", 1000)) + }) + advancedPrefixEmpty := probePrefixEmpty.ObserveEpochAdvance(t, cachePrefixEmpty, func() { + cachePrefixEmpty.EraseEntriesWithGivenPrefix("") + }) + + assert.False(t, advancedErase) + assert.False(t, advancedSelfEvict) + assert.False(t, advancedPrefixEmpty) + }) + + t.Run("NormalPressureLastEntryEraseAndEmptyPrefixClearDoNotAdvanceEpoch", func(t *testing.T) { + var eraseSamples, prefixSamples, emptyPrefixSamples atomic.Int32 + cacheErase := b.fn(100, WithInvariantChecking(true), WithPressureFunc(func() float64 { + eraseSamples.Add(1) + return 0.10 + })).(PressureAwareCache) + cachePrefix := b.fn(100, WithInvariantChecking(true), WithPressureFunc(func() float64 { + prefixSamples.Add(1) + return 0.10 + })).(PressureAwareCache) + cachePrefixEmpty := b.fn(100, WithInvariantChecking(true), WithPressureFunc(func() float64 { + emptyPrefixSamples.Add(1) + return 0.10 + })).(PressureAwareCache) + + _, err := cacheErase.Insert("k1", NewSizedValue("v", 10)) + require.NoError(t, err) + _, err = cachePrefix.Insert("k1", NewSizedValue("v", 10)) + require.NoError(t, err) + _, err = cachePrefixEmpty.Insert("k1", NewSizedValue("v", 10)) + require.NoError(t, err) + + eraseBefore := eraseSamples.Load() + require.NotNil(t, cacheErase.Erase("k1")) + assert.Equal(t, int32(1), eraseSamples.Load()-eraseBefore) + + prefixBefore := prefixSamples.Load() + cachePrefix.EraseEntriesWithGivenPrefix("k") + assert.Equal(t, int32(1), prefixSamples.Load()-prefixBefore) + + emptyBefore := emptyPrefixSamples.Load() + cachePrefixEmpty.EraseEntriesWithGivenPrefix("") + assert.Equal(t, int32(0), emptyPrefixSamples.Load()-emptyBefore) + + assert.Nil(t, cacheErase.LookUpWithoutChangingOrder("k1")) + assert.Nil(t, cachePrefix.LookUpWithoutChangingOrder("k1")) + assert.Nil(t, cachePrefixEmpty.LookUpWithoutChangingOrder("k1")) + assertAlreadyCompacted(t, cacheErase) + assertAlreadyCompacted(t, cachePrefix) + assertAlreadyCompacted(t, cachePrefixEmpty) + }) + + t.Run("FreshCacheAndEmptyPrefixClearAreAlreadyCompactedAndDoNotAdvanceEpoch", func(t *testing.T) { + probe := newPressureProbe(0.80) + pac := b.fn(100, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + + assertAlreadyCompacted(t, pac, probe) + advancedFreshEval := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.False(t, advancedFreshEval) + + // Populate and clear via EraseEntriesWithGivenPrefix(""); resulting empty cache must already be compacted. + probe.Set(0.10) + for i := range 10 { + _, err := pac.Insert(fmt.Sprintf("k-%02d", i), NewSizedValue("v", 5)) + require.NoError(t, err) + } + probe.Set(0.80) + advancedClear := probe.ObserveEpochAdvance(t, pac, func() { + pac.EraseEntriesWithGivenPrefix("") + }) + assert.True(t, advancedClear) + assertAlreadyCompacted(t, pac, probe) + + advancedPostClearEval := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.False(t, advancedPostClearEval) + }) + + t.Run("Draining20ZeroSizeEntriesToEmptyUnderTier1AdvancesEpoch", func(t *testing.T) { + probe := newPressureProbe(0.10) + pac := b.fn(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) + for i := range 20 { + _, err := pac.Insert(fmt.Sprintf("z/%02d", i), NewSizedValue("v", 0)) + require.NoError(t, err) + } + + probe.Set(0.80) + advanced := probe.ObserveEpochAdvance(t, pac, func() { + pac.EraseEntriesWithGivenPrefix("z/") + }) + assert.True(t, advanced) + assert.Nil(t, pac.LookUpWithoutChangingOrder("z/00")) + assertAlreadyCompacted(t, pac, probe) + }) + }) + } + }) + + t.Run("InsertNewKeySingleSurvivorPreCompactionDirtiedByTier2Shed", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + probe := newPressureProbe(0.10) + pac := b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + probe.Option(), + ).(PressureAwareCache) + + for i := range 64 { + _, err := pac.Insert(fmt.Sprintf("e/%02d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + _, err := pac.Insert("survivor", NewSizedValue("s", 5)) + require.NoError(t, err) + + probe.Set(0.95) + advancedInsert := probe.ObserveEpochAdvance(t, pac, func() { + _, err = pac.Insert("jumbo", NewSizedValue("j", 995)) + require.NoError(t, err) + }) + assert.True(t, advancedInsert) + assertAlreadyCompacted(t, pac, probe) + + probe.Set(0.80) + advancedEval := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.False(t, advancedEval) + }) + } + }) + + t.Run("OverwriteAndUpdateSizePreReclaimSlackAndZeroSizeSurvivor", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("OverwriteAndUpdateSizeToSingleSurvivorUnderTier1DoNotDoubleAdvanceEpoch", func(t *testing.T) { + probeOverwrite := newPressureProbe(0.10) + probeUpdate := newPressureProbe(0.10) + cacheOverwrite := b.fn(100, WithInvariantChecking(true), probeOverwrite.Option()).(PressureAwareCache) + cacheUpdateSize := b.fn(100, WithInvariantChecking(true), probeUpdate.Option()).(PressureAwareCache) + + for _, c := range []Cache{cacheOverwrite, cacheUpdateSize} { + _, err := c.Insert("b", NewSizedValue("v", 50)) + require.NoError(t, err) + _, err = c.Insert("a", NewSizedValue("v", 50)) + require.NoError(t, err) + } + + probeOverwrite.Set(0.80) + probeUpdate.Set(0.80) + + advancedOverwrite := probeOverwrite.ObserveEpochAdvance(t, cacheOverwrite, func() { + _, err := cacheOverwrite.Insert("a", NewSizedValue("v2", 90)) + require.NoError(t, err) + }) + advancedUpdate := probeUpdate.ObserveEpochAdvance(t, cacheUpdateSize, func() { + require.NoError(t, cacheUpdateSize.UpdateSize("a", 40)) + }) + assert.True(t, advancedOverwrite) + assert.True(t, advancedUpdate) + assertAlreadyCompacted(t, cacheOverwrite, probeOverwrite) + assertAlreadyCompacted(t, cacheUpdateSize, probeUpdate) + + advancedEvalOverwrite := probeOverwrite.ObserveEpochAdvance(t, cacheOverwrite, func() { + cacheOverwrite.EvaluateMemoryPressure() + }) + advancedEvalUpdate := probeUpdate.ObserveEpochAdvance(t, cacheUpdateSize, func() { + cacheUpdateSize.EvaluateMemoryPressure() + }) + assert.False(t, advancedEvalOverwrite) + assert.False(t, advancedEvalUpdate) + }) + + t.Run("OverwriteAndUpdateSizeWithSurvivingZeroSizeEntryReclaim64DeletedSlack", func(t *testing.T) { + probeOverwrite := newPressureProbe(0.10) + probeUpdate := newPressureProbe(0.10) + cacheOverwrite := b.fn(650, WithInvariantChecking(true), probeOverwrite.Option()).(PressureAwareCache) + cacheUpdateSize := b.fn(650, WithInvariantChecking(true), probeUpdate.Option()).(PressureAwareCache) + + for _, c := range []Cache{cacheOverwrite, cacheUpdateSize} { + for i := range 64 { + _, err := c.Insert(fmt.Sprintf("e/%02d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + _, err := c.Insert("target", NewSizedValue("t", 10)) + require.NoError(t, err) + _, err = c.Insert("zero", NewSizedValue("z", 0)) + require.NoError(t, err) + } + + _, err := cacheOverwrite.Insert("target", NewSizedValue("t2", 650)) + require.NoError(t, err) + require.NoError(t, cacheUpdateSize.UpdateSize("target", 640)) + + for _, item := range []struct { + pac PressureAwareCache + probe *pressureProbe + }{ + {cacheOverwrite, probeOverwrite}, + {cacheUpdateSize, probeUpdate}, + } { + assert.NotNil(t, item.pac.LookUpWithoutChangingOrder("target")) + assert.NotNil(t, item.pac.LookUpWithoutChangingOrder("zero")) + assertAlreadyCompacted(t, item.pac, item.probe) + + item.probe.Set(0.80) + advancedEval := item.probe.ObserveEpochAdvance(t, item.pac, func() { + item.pac.EvaluateMemoryPressure() + }) + assert.False(t, advancedEval) + } + }) + }) + } + }) + + t.Run("InsertNewKeyMassPreInsertEvictionWithTwoZeroSizeSurvivorsCompacts", func(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + probe := newPressureProbe(0.10) + pac := b.fn( + 640, + WithInvariantChecking(true), + probe.Option(), + ).(PressureAwareCache) + for i := range 64 { + _, err := pac.Insert(fmt.Sprintf("p%02d", i), NewSizedValue("val", 10)) + require.NoError(t, err) + } + _, err := pac.Insert("z1", NewSizedValue("zero1", 0)) + require.NoError(t, err) + _, err = pac.Insert("z2", NewSizedValue("zero2", 0)) + require.NoError(t, err) + + evicted, err := pac.Insert("jumbo", NewSizedValue("jumbo-val", 640)) + require.NoError(t, err) + require.Len(t, evicted, 64) + + assert.NotNil(t, pac.LookUpWithoutChangingOrder("z1")) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("z2")) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("jumbo")) + assertAlreadyCompacted(t, pac, probe) + + probe.Set(0.80) + advancedEval := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.False(t, advancedEval) + }) + } + }) +} + +func TestCompaction_NetByteReductionBelowChurnFloor(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("FullDrainBelow64EntriesWithNetByteReductionAdvancesEpochOnceAndCompacts", func(t *testing.T) { + // Arrange: Populate cache (maxSize = 100B) with 10 entries of 10B each (100B total, peakLen = 10 < 64) + // at healthy pressure (0.10). + probe := newPressureProbe(0.10) + pac := b.fn( + 100, + WithInvariantChecking(true), + probe.Option(), + ).(PressureAwareCache) + for i := range 10 { + _, err := pac.Insert(fmt.Sprintf("k-%02d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act: Raise pressure to Tier 1 (0.80) and insert a new 95B entry ("new-large"). + probe.Set(0.80) + var evicted []ValueType + var err error + advancedInsert := probe.ObserveEpochAdvance(t, pac, func() { + evicted, err = pac.Insert("new-large", NewSizedValue("v", 95)) + }) + require.NoError(t, err) + require.Len(t, evicted, 10) + assert.True(t, advancedInsert) + assertAlreadyCompacted(t, pac, probe) + + advancedEval := probe.ObserveEpochAdvance(t, pac, func() { + pac.EvaluateMemoryPressure() + }) + assert.False(t, advancedEval) + }) + + t.Run("PartialDrainToTwoSurvivorsWithNetByteReductionCompactsDirtyStateInInsertAndUpdateSize", func(t *testing.T) { + // Arrange: Populate two caches (maxSize = 100B) with 4 entries: + // "old1" (30B), "old2" (30B), "keep1" (20B), "keep2" (20B) = 100B total (peakLen = 4 <= 8). + probeInsert := newPressureProbe(0.10) + probeUpdate := newPressureProbe(0.10) + cacheInsert := b.fn(100, WithInvariantChecking(true), probeInsert.Option()).(PressureAwareCache) + cacheUpdate := b.fn(100, WithInvariantChecking(true), probeUpdate.Option()).(PressureAwareCache) + + for _, c := range []Cache{cacheInsert, cacheUpdate} { + _, err := c.Insert("old1", NewSizedValue("v", 30)) + require.NoError(t, err) + _, err = c.Insert("old2", NewSizedValue("v", 30)) + require.NoError(t, err) + _, err = c.Insert("keep1", NewSizedValue("v", 20)) + require.NoError(t, err) + _, err = c.Insert("keep2", NewSizedValue("v", 20)) + require.NoError(t, err) + } + + // Act: Under Tier 1 pressure (0.80), achieve net byte reduction (100B -> 90B) via Insert and UpdateSize. + probeInsert.Set(0.80) + probeUpdate.Set(0.80) + + advancedInsert := probeInsert.ObserveEpochAdvance(t, cacheInsert, func() { + _, err := cacheInsert.Insert("new", NewSizedValue("v", 50)) + require.NoError(t, err) + }) + advancedUpdate := probeUpdate.ObserveEpochAdvance(t, cacheUpdate, func() { + require.NoError(t, cacheUpdate.UpdateSize("keep2", 50)) + }) + + // Assert: Both Insert and UpdateSize advanced reclamation epoch AND compacted dirty state. + assert.True(t, advancedInsert) + assert.True(t, advancedUpdate) + assertAlreadyCompacted(t, cacheInsert, probeInsert) + assertAlreadyCompacted(t, cacheUpdate, probeUpdate) + + advancedEvalInsert := probeInsert.ObserveEpochAdvance(t, cacheInsert, func() { + cacheInsert.EvaluateMemoryPressure() + }) + advancedEvalUpdate := probeUpdate.ObserveEpochAdvance(t, cacheUpdate, func() { + cacheUpdate.EvaluateMemoryPressure() + }) + assert.False(t, advancedEvalInsert) + assert.False(t, advancedEvalUpdate) + }) + + t.Run("Tier2SmallCachePartialDrainCompactsDirtyStateWhileLargeCacheSub25PercentEvictionDefersCompaction", func(t *testing.T) { + // Part 1: Small cache (peakEntryLen = 4 <= 8) under Tier 2 (0.95) with RetentionRatio = 0.90 (targetSize = 90B). + // Evicting 2 of 4 entries (50% >= 25% shrinkage) during net-byte-reducing Insert / UpdateSize (100B -> 90B) + // must compact dirty state inline so subsequent Tier 1 EvaluateMemoryPressure() is a no-op. + probeInsert := newPressureProbe(0.10) + probeUpdate := newPressureProbe(0.10) + cacheInsert := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.90), + probeInsert.Option(), + ).(PressureAwareCache) + cacheUpdate := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.90), + probeUpdate.Option(), + ).(PressureAwareCache) + + for _, c := range []Cache{cacheInsert, cacheUpdate} { + _, err := c.Insert("old1", NewSizedValue("v", 30)) + require.NoError(t, err) + _, err = c.Insert("old2", NewSizedValue("v", 30)) + require.NoError(t, err) + _, err = c.Insert("keep1", NewSizedValue("v", 20)) + require.NoError(t, err) + _, err = c.Insert("keep2", NewSizedValue("v", 20)) + require.NoError(t, err) + } + + probeInsert.Set(0.95) + probeUpdate.Set(0.95) + + advancedInsert := probeInsert.ObserveEpochAdvance(t, cacheInsert, func() { + _, err := cacheInsert.Insert("new", NewSizedValue("v", 50)) + require.NoError(t, err) + }) + advancedUpdate := probeUpdate.ObserveEpochAdvance(t, cacheUpdate, func() { + require.NoError(t, cacheUpdate.UpdateSize("keep2", 50)) + }) + assert.True(t, advancedInsert) + assert.True(t, advancedUpdate) + assertAlreadyCompacted(t, cacheInsert, probeInsert) + assertAlreadyCompacted(t, cacheUpdate, probeUpdate) + + probeInsert.Set(0.80) + probeUpdate.Set(0.80) + advancedEvalInsert := probeInsert.ObserveEpochAdvance(t, cacheInsert, func() { + cacheInsert.EvaluateMemoryPressure() + }) + advancedEvalUpdate := probeUpdate.ObserveEpochAdvance(t, cacheUpdate, func() { + cacheUpdate.EvaluateMemoryPressure() + }) + assert.False(t, advancedEvalInsert) + assert.False(t, advancedEvalUpdate) + + // Part 2: Large cache (100 entries: one 10B tail entry + 99 entries of 1B = 109B in 109B maxSize, + // RetentionRatio = 0.99 -> targetSize = 107B) under Tier 2 (0.95). + // Evicting only the single 10B tail entry (1 of 100 entries = 1% < 25% shrinkage) to insert an 8B entry + // reduces live bytes (109B -> 107B) and advances reclaimEpoch, but must NOT thrash O(N) compaction inline. + probeLarge := newPressureProbe(0.10) + cacheLarge := b.fn( + 109, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.99), + probeLarge.Option(), + ).(PressureAwareCache) + + _, err := cacheLarge.Insert("tail-10", NewSizedValue("v", 10)) + require.NoError(t, err) + for i := range 99 { + _, err = cacheLarge.Insert(fmt.Sprintf("k-%03d", i), NewSizedValue("v", 1)) + require.NoError(t, err) + } + + probeLarge.Set(0.95) + advancedLargeInsert := probeLarge.ObserveEpochAdvance(t, cacheLarge, func() { + evicted, err := cacheLarge.Insert("new-8", NewSizedValue("v", 8)) + require.NoError(t, err) + require.Len(t, evicted, 1) + }) + assert.True(t, advancedLargeInsert) + + // Because 1% eviction (< 25%) did not run O(N) compaction inline, an explicit Compact() reclaims the + // 1 deleted slot and advances reclaimEpoch. + probeLarge.Set(0.10) + advancedExplicitCompact := probeLarge.ObserveEpochAdvance(t, cacheLarge, func() { + cacheLarge.Compact() + }) + assert.True(t, advancedExplicitCompact) + }) + + t.Run("InsertSmallerOverwriteInvalidatesConcurrentInFlightSampleAndCompacts", func(t *testing.T) { + // Part 1: Overwriting an existing key with a smaller value under Tier 1 (0.80) or Tier 2 (0.95) + // without evicting any entries (["k1": 30B, "k2": 50B] -> ["k1": 30B, "k2": 10B], 80B -> 40B <= targetSize 50B) + // must advance reclaimEpoch so an in-flight EvaluateMemoryPressure() re-samples pressure. + for _, pressure := range []float64{0.80, 0.95} { + probe := newPressureProbe(0.10) + pac := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + probe.Option(), + ).(PressureAwareCache) + + _, err := pac.Insert("k1", NewSizedValue("v1", 30)) + require.NoError(t, err) + _, err = pac.Insert("k2", NewSizedValue("v2", 50)) + require.NoError(t, err) + + probe.Set(pressure) + advanced := probe.ObserveEpochAdvance(t, pac, func() { + evicted, err := pac.Insert("k2", NewSizedValue("v2-small", 10)) + require.NoError(t, err) + assert.Empty(t, evicted) + }) + assert.True(t, advanced) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, pac.LookUpWithoutChangingOrder("k2")) + } + + // Part 2: When >= 25% churn exists (4 entries inserted, 1 erased -> 3 survivors, deletedSinceCompact*4 >= peakEntryLen) + // under Tier 1 (0.80), overwriting one of the surviving keys with a smaller value compacts dirty state inline. + probeChurn := newPressureProbe(0.10) + cacheChurn := b.fn(1000, WithInvariantChecking(true), probeChurn.Option()).(PressureAwareCache) + for _, k := range []string{"del/00", "keep/01", "keep/02"} { + _, err := cacheChurn.Insert(k, NewSizedValue("v", 10)) + require.NoError(t, err) + } + _, err := cacheChurn.Insert("k1", NewSizedValue("v1", 50)) + require.NoError(t, err) + require.NotNil(t, cacheChurn.Erase("del/00")) + + probeChurn.Set(0.80) + advancedChurn := probeChurn.ObserveEpochAdvance(t, cacheChurn, func() { + evicted, err := cacheChurn.Insert("k1", NewSizedValue("v1-small", 10)) + require.NoError(t, err) + assert.Empty(t, evicted) + }) + assert.True(t, advancedChurn) + assertAlreadyCompacted(t, cacheChurn, probeChurn) + + // Part 3: When a small cache (peakEntryLen = 2 <= 8) has dirty slack down to a sole surviving + // 50B entry "k1" at normal pressure (0.10), subsequently shrinking "k1" in place to 10B via + // Insert("k1", 10B) (with 0 pre-insert evictions) under Tier 1 (0.80) triggers + // shouldReclaimSingleSurvivorOnMutation and compacts the dirty slack inline. + probeSingle := newPressureProbe(0.10) + cacheSingle := b.fn(1000, WithInvariantChecking(true), probeSingle.Option()).(PressureAwareCache) + + _, err = cacheSingle.Insert("del/00", NewSizedValue("v", 10)) + require.NoError(t, err) + _, err = cacheSingle.Insert("k1", NewSizedValue("v1", 50)) + require.NoError(t, err) + require.NotNil(t, cacheSingle.Erase("del/00")) + + probeSingle.Set(0.80) + advancedSingle := probeSingle.ObserveEpochAdvance(t, cacheSingle, func() { + evicted, err := cacheSingle.Insert("k1", NewSizedValue("v1-small", 10)) + require.NoError(t, err) + assert.Empty(t, evicted) + }) + + assert.True(t, advancedSingle) + assertAlreadyCompacted(t, cacheSingle, probeSingle) + }) + }) + } +} + +func TestCompaction_SteadyStateTurnoverAndShrinkageAutoCompaction(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("LowChurnTurnoverDoesNotThrashWhileShrinkageAutoCompacts", func(t *testing.T) { + // Arrange + probe := newPressureProbe(0.80) + c := b.fn( + 100, + WithInvariantChecking(true), + probe.Option(), + WithCompactionThreshold(0.75), + WithEvictionThreshold(0.90), + ).(PressureAwareCache) + + for i := range 10 { + _, err := c.Insert(fmt.Sprintf("init_%02d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act 1: Perform 30 steady-state capacity-turnover inserts (< 64 churn deletions). + advancedTurnover := probe.ObserveEpochAdvance(t, c, func() { + for i := range 30 { + _, err := c.Insert(fmt.Sprintf("turn_%02d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + }) + + // Assert 1: Zero compactions occurred during low-churn capacity turnover. + assert.False(t, advancedTurnover) + + // Act 2: Erase 3 entries so live entry count drops from 10 to 7 (30% shrinkage >= 25%). + require.NotNil(t, c.Erase("turn_27")) + require.NotNil(t, c.Erase("turn_28")) + advancedShrinkage := probe.ObserveEpochAdvance(t, c, func() { + require.NotNil(t, c.Erase("turn_29")) + }) + + // Assert 2: Auto-compaction triggered on >= 25% shrinkage, leaving 7 entries and clean state. + assert.True(t, advancedShrinkage) + assert.NotNil(t, c.LookUpWithoutChangingOrder("turn_20")) + assert.Nil(t, c.LookUpWithoutChangingOrder("turn_29")) + assertAlreadyCompacted(t, c, probe) + }) + + t.Run("HighChurnDeletedSinceCompactTriggersTier1AutoCompaction", func(t *testing.T) { + // Arrange: Populate 100 keys (10B each = 1000B in 1000B cache) at low pressure (0.10), + // then churn 100 new keys so live count stays 100 while accumulating 100 churn deletions (>= 64). + probe := newPressureProbe(0.10) + c := b.fn( + 1000, + WithInvariantChecking(true), + WithCompactionThreshold(0.75), + WithEvictionThreshold(0.90), + probe.Option(), + ).(PressureAwareCache) + + for i := range 100 { + _, err := c.Insert(fmt.Sprintf("init-%03d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + for i := range 100 { + _, err := c.Insert(fmt.Sprintf("churn-%03d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act: Raise pressure to Tier 1 (0.80) and perform a write. + probe.Set(0.80) + var err error + advanced := probe.ObserveEpochAdvance(t, c, func() { + _, err = c.Insert("trigger", NewSizedValue("v", 10)) + }) + + // Assert: High-churn tombstone bloat triggers Tier 1 auto-compaction. + require.NoError(t, err) + assert.True(t, advanced) + assertAlreadyCompacted(t, c, probe) + }) + }) + } +} diff --git a/concurrency_test.go b/concurrency_test.go index e1c0996..cd72d6d 100644 --- a/concurrency_test.go +++ b/concurrency_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus_test +package lru_test import ( "errors" @@ -22,7 +22,7 @@ import ( "sync/atomic" "testing" - lrus "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/google/go-lru" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -38,15 +38,15 @@ func (v concValue) Size() uint64 { func allEngines() []struct { name string - constructor func(maxSize uint64, opts ...lrus.Option) lrus.Cache + constructor func(maxSize uint64, opts ...lru.Option) lru.Cache } { return []struct { name string - constructor func(maxSize uint64, opts ...lrus.Option) lrus.Cache + constructor func(maxSize uint64, opts ...lru.Option) lru.Cache }{ - {"MapCache", lrus.NewMapCache}, - {"RadixCache", lrus.NewRadixCache}, - {"ArenaRadixCache", lrus.NewArenaRadixCache}, + {"MapCache", lru.NewMapCache}, + {"RadixCache", lru.NewRadixCache}, + {"ArenaRadixCache", lru.NewArenaRadixCache}, } } @@ -55,6 +55,7 @@ func allEngines() []struct { func TestConcurrency_MixedOperations(t *testing.T) { for _, eng := range allEngines() { t.Run(eng.name, func(t *testing.T) { + // Arrange const ( numGoroutines = 16 opsPerWorker = 300 @@ -67,11 +68,10 @@ func TestConcurrency_MixedOperations(t *testing.T) { for i := range numKeys { key := fmt.Sprintf("dir_%02d/sub_%02d/file_%03d.txt", i%5, (i/5)%10, i) _, err := cache.Insert(key, concValue{id: key, size: 10}) - if err != nil { - t.Fatalf("unexpected pre-population error: %v", err) - } + require.NoError(t, err) } + // Act var wg sync.WaitGroup for g := range numGoroutines { wg.Add(1) @@ -89,8 +89,8 @@ func TestConcurrency_MixedOperations(t *testing.T) { switch { case op < 30: _, err := cache.Insert(key, concValue{id: key, size: 10}) - if err != nil && !errors.Is(err, lrus.ErrInvalidEntrySize) { - t.Errorf("worker %d: unexpected insert error: %v", workerID, err) + if err != nil { + assert.ErrorIs(t, err, lru.ErrInvalidEntrySize) } case op < 55: _ = cache.LookUp(key) @@ -98,13 +98,13 @@ func TestConcurrency_MixedOperations(t *testing.T) { _ = cache.LookUpWithoutChangingOrder(key) case op < 80: err := cache.UpdateWithoutChangingOrder(key, concValue{id: key + "_upd", size: 10}) - if err != nil && !errors.Is(err, lrus.ErrEntryNotExist) && !errors.Is(err, lrus.ErrInvalidUpdateEntrySize) { - t.Errorf("worker %d: unexpected update error: %v", workerID, err) + if err != nil { + assert.True(t, errors.Is(err, lru.ErrEntryNotExist) || errors.Is(err, lru.ErrInvalidUpdateEntrySize)) } case op < 88: err := cache.UpdateSize(key, 0) - if err != nil && !errors.Is(err, lrus.ErrEntryNotExist) { - t.Errorf("worker %d: unexpected update size error: %v", workerID, err) + if err != nil { + assert.ErrorIs(t, err, lru.ErrEntryNotExist) } case op < 95: _ = cache.Erase(key) @@ -117,6 +117,11 @@ func TestConcurrency_MixedOperations(t *testing.T) { } wg.Wait() + + // Assert + _, err := cache.Insert("post_conc_check", concValue{id: "post_conc_check", size: 10}) + require.NoError(t, err) + assert.NotNil(t, cache.LookUp("post_conc_check")) }) } } @@ -127,6 +132,7 @@ func TestConcurrency_MixedOperations(t *testing.T) { func TestConcurrency_PrefixErasureAtomicity(t *testing.T) { for _, eng := range allEngines() { t.Run(eng.name, func(t *testing.T) { + // Arrange const ( numWriters = 8 opsPerWriter = 100 @@ -134,28 +140,30 @@ func TestConcurrency_PrefixErasureAtomicity(t *testing.T) { cache := eng.constructor(100000) - // Pre-populate target and retained entries. - for i := 0; i < 20; i++ { + for i := range 20 { kTarget := fmt.Sprintf("/target/init_%d", i) - _, _ = cache.Insert(kTarget, concValue{id: kTarget, size: 10}) + _, err := cache.Insert(kTarget, concValue{id: kTarget, size: 10}) + require.NoError(t, err) kKeep := fmt.Sprintf("/keep/init_%d", i) - _, _ = cache.Insert(kKeep, concValue{id: kKeep, size: 10}) + _, err = cache.Insert(kKeep, concValue{id: kKeep, size: 10}) + require.NoError(t, err) } + // Act var writerWg sync.WaitGroup - for w := 0; w < numWriters; w++ { + for w := range numWriters { writerWg.Add(1) go func(workerID int) { defer writerWg.Done() - for op := 0; op < opsPerWriter; op++ { + for op := range opsPerWriter { key := fmt.Sprintf("/target/w%d_%d", workerID, op) - _, _ = cache.Insert(key, concValue{id: key, size: 10}) + _, err := cache.Insert(key, concValue{id: key, size: 10}) + assert.NoError(t, err) } }(w) } - // Concurrent erasures during writes. - for i := 0; i < 10; i++ { + for range 10 { cache.EraseEntriesWithGivenPrefix("/target/") } @@ -164,23 +172,18 @@ func TestConcurrency_PrefixErasureAtomicity(t *testing.T) { // Final prefix erase after writers finish must remove all /target/ keys. cache.EraseEntriesWithGivenPrefix("/target/") - for i := 0; i < 20; i++ { + // Assert + for i := range 20 { kTarget := fmt.Sprintf("/target/init_%d", i) - if val := cache.LookUp(kTarget); val != nil { - t.Fatalf("[%s] pre-existing target key %s survived prefix erasure", eng.name, kTarget) - } + assert.Nil(t, cache.LookUp(kTarget)) kKeep := fmt.Sprintf("/keep/init_%d", i) - if val := cache.LookUp(kKeep); val == nil { - t.Fatalf("[%s] retained key %s was erroneously erased", eng.name, kKeep) - } + assert.NotNil(t, cache.LookUp(kKeep)) } - for w := 0; w < numWriters; w++ { - for op := 0; op < opsPerWriter; op++ { + for w := range numWriters { + for op := range opsPerWriter { key := fmt.Sprintf("/target/w%d_%d", w, op) - if val := cache.LookUp(key); val != nil { - t.Fatalf("[%s] writer target key %s survived final prefix erasure", eng.name, key) - } + assert.Nil(t, cache.LookUp(key)) } } }) @@ -192,6 +195,7 @@ func TestConcurrency_PrefixErasureAtomicity(t *testing.T) { func TestConcurrency_ParallelReadersWithoutChangingOrder(t *testing.T) { for _, eng := range allEngines() { t.Run(eng.name, func(t *testing.T) { + // Arrange const ( totalKeys = 200 numReaders = 16 @@ -201,26 +205,25 @@ func TestConcurrency_ParallelReadersWithoutChangingOrder(t *testing.T) { cache := eng.constructor(capacity) - for i := 0; i < totalKeys; i++ { + for i := range totalKeys { k := fmt.Sprintf("key_%04d", i) _, err := cache.Insert(k, concValue{id: k, size: 10}) - if err != nil { - t.Fatalf("[%s] pre-population failed: %v", eng.name, err) - } + require.NoError(t, err) } + // Act var readerWg sync.WaitGroup - for r := 0; r < numReaders; r++ { + for r := range numReaders { readerWg.Add(1) go func(readerID int) { defer readerWg.Done() - for i := 0; i < readsPerG; i++ { + for i := range readsPerG { targetKey := "key_0000" if i%2 == 1 { targetKey = fmt.Sprintf("key_%04d", (readerID*17+i)%totalKeys) } - if val := cache.LookUpWithoutChangingOrder(targetKey); val == nil { - t.Errorf("[%s] reader %d: unexpected nil lookup for key %s", eng.name, readerID, targetKey) + val := cache.LookUpWithoutChangingOrder(targetKey) + if !assert.NotNil(t, val) { return } } @@ -229,22 +232,19 @@ func TestConcurrency_ParallelReadersWithoutChangingOrder(t *testing.T) { readerWg.Wait() // Fill remaining capacity (50,000 - 2,000 = 48,000 bytes). - for i := 0; i < 48; i++ { + for i := range 48 { k := fmt.Sprintf("filler_%03d", i) _, err := cache.Insert(k, concValue{id: k, size: 1000}) - if err != nil { - t.Fatalf("[%s] failed to insert filler: %v", eng.name, err) - } + require.NoError(t, err) } // Next 10-byte insert must evict key_0000 (the untouched LRU tail). evicted, err := cache.Insert("overflow_trigger", concValue{id: "overflow", size: 10}) - if err != nil { - t.Fatalf("[%s] overflow insert failed: %v", eng.name, err) - } - if len(evicted) != 1 || evicted[0].(concValue).id != "key_0000" { - t.Fatalf("[%s] expected oldest entry key_0000 to be evicted, got %v", eng.name, evicted) - } + + // Assert + require.NoError(t, err) + require.Len(t, evicted, 1) + assert.Equal(t, "key_0000", evicted[0].(concValue).id) }) } } @@ -254,35 +254,38 @@ func TestConcurrency_ParallelReadersWithoutChangingOrder(t *testing.T) { func TestConcurrency_EvictionThrashingWithInvariants(t *testing.T) { for _, eng := range allEngines() { t.Run(eng.name, func(t *testing.T) { + // Arrange const ( numGoroutines = 8 opsPerWorker = 150 capacity = 300 ) - cache := eng.constructor(capacity, lrus.WithInvariantChecking(true)) + cache := eng.constructor(capacity, lru.WithInvariantChecking(true)) var wg sync.WaitGroup var totalEvictions atomic.Int64 - for g := range numGoroutines { + // Act + for range numGoroutines { wg.Add(1) - go func(workerID int) { + go func() { defer wg.Done() for i := range opsPerWorker { key := fmt.Sprintf("inv/p%d/item_%d", i%5, i) evicted, err := cache.Insert(key, concValue{id: key, size: 10}) - if err != nil { - t.Errorf("worker %d: unexpected insert error: %v", workerID, err) - } + assert.NoError(t, err) totalEvictions.Add(int64(len(evicted))) if i%20 == 0 { cache.EraseEntriesWithGivenPrefix(fmt.Sprintf("inv/p%d/", i%5)) } } - }(g) + }() } wg.Wait() + + // Assert + assert.Positive(t, totalEvictions.Load()) }) } } @@ -292,97 +295,101 @@ func TestConcurrency_EvictionThrashingWithInvariants(t *testing.T) { // while memory pressure dynamically oscillates across normal, moderate, and critical tiers // with WithInvariantChecking(true) enabled. func TestConcurrency_MemoryPressureCompactionAndEviction(t *testing.T) { - // Arrange - const ( - numGoroutines = 16 - opsPerWorker = 250 - numKeys = 120 - capacity = 4000 - ) - - var pressureBits atomic.Uint64 - setPressure := func(p float64) { - pressureBits.Store(uint64(p * 1000)) - } - getPressure := func() float64 { - return float64(pressureBits.Load()) / 1000.0 - } - setPressure(0.20) - - cache := lrus.NewArenaRadixCache( - capacity, - lrus.WithInvariantChecking(true), - lrus.WithPressureFunc(getPressure), - lrus.WithCompactionThreshold(0.75), - lrus.WithEvictionThreshold(0.90), - lrus.WithEvictionRetentionRatio(0.50), - ) - - reclaimer, ok := cache.(lrus.PressureAwareCache) - require.True(t, ok, "expected ArenaRadixCache to implement PressureAwareCache") - - // Act - var wg sync.WaitGroup - for g := range numGoroutines { - wg.Add(1) - go func(workerID int) { - defer wg.Done() - r := rand.New(rand.NewSource(int64(workerID*13337 + 99))) - - for step := range opsPerWorker { - // Dynamically oscillate simulated pressure across Normal (0.20), - // Moderate (0.80), and Critical (0.95) tiers. - switch (workerID + step) % 3 { - case 0: - setPressure(0.20) - case 1: - setPressure(0.80) - case 2: - setPressure(0.95) - } + for _, eng := range allEngines() { + t.Run(eng.name, func(t *testing.T) { + // Arrange + const ( + numGoroutines = 16 + opsPerWorker = 250 + numKeys = 120 + capacity = 4000 + ) - op := r.Intn(100) - kIdx := r.Intn(numKeys) - dirIdx := kIdx % 6 - subIdx := (kIdx / 6) % 5 - key := fmt.Sprintf("mp_dir_%02d/sub_%02d/file_%03d.dat", dirIdx, subIdx, kIdx) - - switch { - case op < 30: - _, err := cache.Insert(key, concValue{id: key, size: 10}) - if err != nil { - assert.ErrorIs(t, err, lrus.ErrInvalidEntrySize) - } - case op < 50: - _ = cache.LookUp(key) - case op < 68: - _ = cache.LookUpWithoutChangingOrder(key) - case op < 76: - err := cache.UpdateWithoutChangingOrder(key, concValue{id: key + "_u", size: 10}) - if err != nil { - assert.True(t, errors.Is(err, lrus.ErrEntryNotExist) || errors.Is(err, lrus.ErrInvalidUpdateEntrySize)) - } - case op < 84: - err := cache.UpdateSize(key, 5) - if err != nil { - assert.ErrorIs(t, err, lrus.ErrEntryNotExist) + var pressureBits atomic.Uint64 + setPressure := func(p float64) { + pressureBits.Store(uint64(p * 1000)) + } + getPressure := func() float64 { + return float64(pressureBits.Load()) / 1000.0 + } + setPressure(0.20) + + cache := eng.constructor( + capacity, + lru.WithInvariantChecking(true), + lru.WithPressureFunc(getPressure), + lru.WithCompactionThreshold(0.75), + lru.WithEvictionThreshold(0.90), + lru.WithEvictionRetentionRatio(0.50), + ) + + reclaimer, ok := cache.(lru.PressureAwareCache) + require.True(t, ok, "expected %s to implement PressureAwareCache", eng.name) + + // Act + var wg sync.WaitGroup + for g := range numGoroutines { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + r := rand.New(rand.NewSource(int64(workerID*13337 + 99))) + + for step := range opsPerWorker { + // Dynamically oscillate simulated pressure across Normal (0.20), + // Moderate (0.80), and Critical (0.95) tiers. + switch (workerID + step) % 3 { + case 0: + setPressure(0.20) + case 1: + setPressure(0.80) + case 2: + setPressure(0.95) + } + + op := r.Intn(100) + kIdx := r.Intn(numKeys) + dirIdx := kIdx % 6 + subIdx := (kIdx / 6) % 5 + key := fmt.Sprintf("mp_dir_%02d/sub_%02d/file_%03d.dat", dirIdx, subIdx, kIdx) + + switch { + case op < 30: + _, err := cache.Insert(key, concValue{id: key, size: 10}) + if err != nil { + assert.ErrorIs(t, err, lru.ErrInvalidEntrySize) + } + case op < 50: + _ = cache.LookUp(key) + case op < 68: + _ = cache.LookUpWithoutChangingOrder(key) + case op < 76: + err := cache.UpdateWithoutChangingOrder(key, concValue{id: key + "_u", size: 10}) + if err != nil { + assert.True(t, errors.Is(err, lru.ErrEntryNotExist) || errors.Is(err, lru.ErrInvalidUpdateEntrySize)) + } + case op < 84: + err := cache.UpdateSize(key, 5) + if err != nil { + assert.ErrorIs(t, err, lru.ErrEntryNotExist) + } + case op < 90: + _ = cache.Erase(key) + case op < 95: + prefix := fmt.Sprintf("mp_dir_%02d/", dirIdx) + cache.EraseEntriesWithGivenPrefix(prefix) + case op < 98: + reclaimer.Compact() + default: + _ = reclaimer.EvaluateMemoryPressure() + } } - case op < 90: - _ = cache.Erase(key) - case op < 95: - prefix := fmt.Sprintf("mp_dir_%02d/", dirIdx) - cache.EraseEntriesWithGivenPrefix(prefix) - case op < 98: - reclaimer.Compact() - default: - _ = reclaimer.EvaluateMemoryPressure() - } + }(g) } - }(g) - } - wg.Wait() + wg.Wait() - // Assert: Final compaction and invariant check on quiescent cache succeed cleanly. - reclaimer.Compact() + // Assert: Final compaction and invariant check on quiescent cache succeed cleanly. + reclaimer.Compact() + }) + } } diff --git a/differential_test.go b/differential_test.go index 1703a80..16df9cc 100644 --- a/differential_test.go +++ b/differential_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus_test +package lru_test import ( "fmt" @@ -21,7 +21,9 @@ import ( "strings" "testing" - lrus "github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru" + "github.com/google/go-lru" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type diffValue struct { @@ -46,7 +48,7 @@ func (v *diffValue) String() string { type diffInstance struct { name string invariants bool - cache lrus.Cache + cache lru.Cache } type differentialHarness struct { @@ -55,7 +57,8 @@ type differentialHarness struct { instances []diffInstance } -func newDifferentialHarness(t *testing.T, maxSize uint64) *differentialHarness { +func newDifferentialHarness(t *testing.T, maxSize uint64, opts ...lru.Option) *differentialHarness { + t.Helper() h := &differentialHarness{ t: t, maxSize: maxSize, @@ -63,16 +66,19 @@ func newDifferentialHarness(t *testing.T, maxSize uint64) *differentialHarness { configs := []struct { name string - constructor func(uint64, ...lrus.Option) lrus.Cache + constructor func(uint64, ...lru.Option) lru.Cache }{ - {"MapCache", lrus.NewMapCache}, - {"RadixCache", lrus.NewRadixCache}, - {"ArenaRadixCache", lrus.NewArenaRadixCache}, + {"MapCache", lru.NewMapCache}, + {"RadixCache", lru.NewRadixCache}, + {"ArenaRadixCache", lru.NewArenaRadixCache}, } for _, cfg := range configs { for _, inv := range []bool{false, true} { - c := cfg.constructor(maxSize, lrus.WithInvariantChecking(inv)) + instanceOpts := make([]lru.Option, 0, len(opts)+1) + instanceOpts = append(instanceOpts, lru.WithInvariantChecking(inv)) + instanceOpts = append(instanceOpts, opts...) + c := cfg.constructor(maxSize, instanceOpts...) h.instances = append(h.instances, diffInstance{ name: cfg.name, invariants: inv, @@ -86,45 +92,43 @@ func newDifferentialHarness(t *testing.T, maxSize uint64) *differentialHarness { func (h *differentialHarness) compareErrors(op string, baseErr, targetErr error, instName string, invariants bool) { h.t.Helper() - if (baseErr == nil && targetErr != nil) || (baseErr != nil && targetErr == nil) { - h.t.Fatalf("[%s] error parity mismatch with %s (inv=%v): base err = %v, target err = %v", op, instName, invariants, baseErr, targetErr) - } - if baseErr != nil && targetErr != nil && baseErr.Error() != targetErr.Error() { - h.t.Fatalf("[%s] error message mismatch with %s (inv=%v): base = %q, target = %q", op, instName, invariants, baseErr, targetErr) + if baseErr == nil { + require.NoErrorf(h.t, targetErr, "[%s] error parity mismatch with %s (inv=%v)", op, instName, invariants) + return } + require.Errorf(h.t, targetErr, "[%s] error parity mismatch with %s (inv=%v): base err = %v", op, instName, invariants, baseErr) + require.EqualErrorf(h.t, targetErr, baseErr.Error(), "[%s] error message mismatch with %s (inv=%v)", op, instName, invariants) } -func (h *differentialHarness) compareValues(op string, baseVal, targetVal lrus.ValueType, instName string, invariants bool) { +func (h *differentialHarness) compareValues(op string, baseVal, targetVal lru.ValueType, instName string, invariants bool) { h.t.Helper() - if (baseVal == nil && targetVal != nil) || (baseVal != nil && targetVal == nil) { - h.t.Fatalf("[%s] value nil parity mismatch with %s (inv=%v): base = %v, target = %v", op, instName, invariants, baseVal, targetVal) + if baseVal == nil { + require.Nilf(h.t, targetVal, "[%s] value nil parity mismatch with %s (inv=%v)", op, instName, invariants) + return } - if baseVal != nil && targetVal != nil { - bv, ok1 := baseVal.(*diffValue) - tv, ok2 := targetVal.(*diffValue) - if ok1 && ok2 { - if bv.id != tv.id || bv.size != tv.size { - h.t.Fatalf("[%s] value content mismatch with %s (inv=%v): base = %v, target = %v", op, instName, invariants, bv, tv) - } - } else if baseVal.Size() != targetVal.Size() { - h.t.Fatalf("[%s] value size mismatch with %s (inv=%v): base size = %d, target size = %d", op, instName, invariants, baseVal.Size(), targetVal.Size()) - } + require.NotNilf(h.t, targetVal, "[%s] value nil parity mismatch with %s (inv=%v): base = %v", op, instName, invariants, baseVal) + + bv, ok1 := baseVal.(*diffValue) + tv, ok2 := targetVal.(*diffValue) + if ok1 && ok2 { + require.Equalf(h.t, bv.id, tv.id, "[%s] value id mismatch with %s (inv=%v)", op, instName, invariants) + require.Equalf(h.t, bv.size, tv.size, "[%s] value size mismatch with %s (inv=%v)", op, instName, invariants) + } else { + require.Equalf(h.t, baseVal.Size(), targetVal.Size(), "[%s] value size mismatch with %s (inv=%v)", op, instName, invariants) } } -func (h *differentialHarness) compareEvicted(op string, baseEvicted, targetEvicted []lrus.ValueType, instName string, invariants bool) { +func (h *differentialHarness) compareEvicted(op string, baseEvicted, targetEvicted []lru.ValueType, instName string, invariants bool) { h.t.Helper() - if len(baseEvicted) != len(targetEvicted) { - h.t.Fatalf("[%s] evicted slice length mismatch with %s (inv=%v): base len = %d, target len = %d", op, instName, invariants, len(baseEvicted), len(targetEvicted)) - } + require.Lenf(h.t, targetEvicted, len(baseEvicted), "[%s] evicted slice length mismatch with %s (inv=%v)", op, instName, invariants) for i := range baseEvicted { h.compareValues(fmt.Sprintf("%s.evicted[%d]", op, i), baseEvicted[i], targetEvicted[i], instName, invariants) } } -func (h *differentialHarness) Insert(key string, val lrus.ValueType) []lrus.ValueType { +func (h *differentialHarness) Insert(key string, val lru.ValueType) []lru.ValueType { h.t.Helper() - var baseEvicted []lrus.ValueType + var baseEvicted []lru.ValueType var baseErr error for i, inst := range h.instances { @@ -140,9 +144,9 @@ func (h *differentialHarness) Insert(key string, val lrus.ValueType) []lrus.Valu return baseEvicted } -func (h *differentialHarness) Erase(key string) lrus.ValueType { +func (h *differentialHarness) Erase(key string) lru.ValueType { h.t.Helper() - var baseVal lrus.ValueType + var baseVal lru.ValueType for i, inst := range h.instances { val := inst.cache.Erase(key) @@ -155,9 +159,9 @@ func (h *differentialHarness) Erase(key string) lrus.ValueType { return baseVal } -func (h *differentialHarness) LookUp(key string) lrus.ValueType { +func (h *differentialHarness) LookUp(key string) lru.ValueType { h.t.Helper() - var baseVal lrus.ValueType + var baseVal lru.ValueType for i, inst := range h.instances { val := inst.cache.LookUp(key) @@ -170,9 +174,9 @@ func (h *differentialHarness) LookUp(key string) lrus.ValueType { return baseVal } -func (h *differentialHarness) LookUpWithoutChangingOrder(key string) lrus.ValueType { +func (h *differentialHarness) LookUpWithoutChangingOrder(key string) lru.ValueType { h.t.Helper() - var baseVal lrus.ValueType + var baseVal lru.ValueType for i, inst := range h.instances { val := inst.cache.LookUpWithoutChangingOrder(key) @@ -185,7 +189,7 @@ func (h *differentialHarness) LookUpWithoutChangingOrder(key string) lrus.ValueT return baseVal } -func (h *differentialHarness) UpdateWithoutChangingOrder(key string, val lrus.ValueType) { +func (h *differentialHarness) UpdateWithoutChangingOrder(key string, val lru.ValueType) { h.t.Helper() var baseErr error @@ -204,11 +208,6 @@ func (h *differentialHarness) UpdateSize(key string, delta uint64) { var baseErr error val := h.LookUpWithoutChangingOrder(key) - if val != nil && delta <= math.MaxUint64-val.Size() { - if tv, ok := val.(*diffValue); ok { - tv.size += delta - } - } for i, inst := range h.instances { err := inst.cache.UpdateSize(key, delta) @@ -218,6 +217,12 @@ func (h *differentialHarness) UpdateSize(key string, delta uint64) { h.compareErrors(fmt.Sprintf("UpdateSize(%q, %d)", key, delta), baseErr, err, inst.name, inst.invariants) } } + + if baseErr == nil && val != nil && h.LookUpWithoutChangingOrder(key) != nil { + if tv, ok := val.(*diffValue); ok { + tv.size += delta + } + } } func (h *differentialHarness) EraseEntriesWithGivenPrefix(prefix string) { @@ -237,6 +242,7 @@ func (h *differentialHarness) DrainAndVerifyEvictionOrder(allKnownKeys []string) } func TestDifferential_FlatWorkload(t *testing.T) { + // Arrange r := rand.New(rand.NewSource(1337)) const ( numOps = 5000 @@ -251,6 +257,7 @@ func TestDifferential_FlatWorkload(t *testing.T) { h := newDifferentialHarness(t, cacheCapacity) + // Act for op := range numOps { k := keys[r.Intn(keyPoolSize)] dice := r.Intn(100) @@ -280,10 +287,12 @@ func TestDifferential_FlatWorkload(t *testing.T) { } } + // Assert h.DrainAndVerifyEvictionOrder(keys) } func TestDifferential_HierarchicalDirectoryWorkload(t *testing.T) { + // Arrange r := rand.New(rand.NewSource(4242)) const ( numOps = 5000 @@ -306,6 +315,7 @@ func TestDifferential_HierarchicalDirectoryWorkload(t *testing.T) { h := newDifferentialHarness(t, cacheCapacity) + // Act for op := range numOps { path := paths[r.Intn(len(paths))] dice := r.Intn(100) @@ -333,10 +343,12 @@ func TestDifferential_HierarchicalDirectoryWorkload(t *testing.T) { } } + // Assert h.DrainAndVerifyEvictionOrder(paths) } func TestDifferential_CapacityThrashingAndSizeUpdates(t *testing.T) { + // Arrange r := rand.New(rand.NewSource(7777)) const ( numOps = 5000 @@ -350,7 +362,7 @@ func TestDifferential_CapacityThrashingAndSizeUpdates(t *testing.T) { h := newDifferentialHarness(t, cacheCapacity) - // Explicitly verify uint64 overflow rejection parity across all engines. + // Act: Explicitly verify uint64 overflow rejection parity across all engines. h.Insert("overflow_key", &diffValue{id: "ovf", size: 20}) h.UpdateSize("overflow_key", math.MaxUint64) h.UpdateSize("overflow_key", math.MaxUint64-10) @@ -376,10 +388,12 @@ func TestDifferential_CapacityThrashingAndSizeUpdates(t *testing.T) { } } + // Assert h.DrainAndVerifyEvictionOrder(keys) } func TestDifferential_BoundaryAndEdgeCases(t *testing.T) { + // Arrange r := rand.New(rand.NewSource(12345)) const ( numOps = 3000 @@ -401,6 +415,7 @@ func TestDifferential_BoundaryAndEdgeCases(t *testing.T) { h := newDifferentialHarness(t, cacheCapacity) + // Act for op := range numOps { k := adversarialKeys[r.Intn(len(adversarialKeys))] dice := r.Intn(100) @@ -445,5 +460,211 @@ func TestDifferential_BoundaryAndEdgeCases(t *testing.T) { } } + // Assert h.DrainAndVerifyEvictionOrder(adversarialKeys) } + +func TestDifferential_UpdateSizeSelfEvictionPreservesDiffValueSize(t *testing.T) { + // Arrange + h := newDifferentialHarness(t, 100) + dv := &diffValue{id: "self_evict", size: 50} + h.Insert("k1", dv) + + // Act: Grow k1 by +60 (50 + 60 = 110 > maxSize 100), triggering self-eviction across all backends. + h.UpdateSize("k1", 60) + + // Assert: k1 is evicted and dv.size remains 50 (not mutated to 110). + require.Nil(t, h.LookUpWithoutChangingOrder("k1")) + require.Equal(t, uint64(50), dv.Size()) +} + +func TestDifferential_PressureAwareAndCompactionParity(t *testing.T) { + // Arrange + r := rand.New(rand.NewSource(20260925)) + const ( + numOps = 2000 + cacheCapacity = 1000 + ) + pressure := 0.10 + + h := newDifferentialHarness( + t, + cacheCapacity, + lru.WithPressureFunc(func() float64 { return pressure }), + lru.WithCompactionThreshold(0.75), + lru.WithEvictionThreshold(0.90), + lru.WithEvictionRetentionRatio(0.50), + ) + + keys := make([]string, 80) + for i := range keys { + keys[i] = fmt.Sprintf("bucket_%d/dir_%d/item_%03d", i%4, (i/4)%4, i) + } + + // Act + for op := range numOps { + switch op % 25 { + case 0: + pressure = 0.80 // Tier 1 moderate pressure + case 10: + pressure = 0.95 // Tier 2 critical pressure + case 15: + pressure = 0.10 // Normal pressure + } + + k := keys[r.Intn(len(keys))] + dice := r.Intn(100) + switch { + case dice < 35: + sz := uint64(r.Intn(45) + 5) + h.Insert(k, &diffValue{id: fmt.Sprintf("pv_%d", op), size: sz}) + case dice < 55: + h.LookUp(k) + case dice < 68: + h.LookUpWithoutChangingOrder(k) + case dice < 78: + h.Erase(k) + case dice < 86: + h.UpdateSize(k, uint64(r.Intn(20)+1)) + case dice < 93: + for _, inst := range h.instances { + inst.cache.(lru.PressureAwareCache).Compact() + } + default: + var baseEvicted []lru.ValueType + for i, inst := range h.instances { + ev := inst.cache.(lru.PressureAwareCache).EvaluateMemoryPressure() + if i == 0 { + baseEvicted = ev + } else { + h.compareEvicted("EvaluateMemoryPressure()", baseEvicted, ev, inst.name, inst.invariants) + } + } + } + } + + // Assert + pressure = 0.10 + h.DrainAndVerifyEvictionOrder(keys) +} + +func TestDifferential_UpdateSizeSingleEntryExceedsMaxSizePreservesOtherEntries(t *testing.T) { + // Arrange + h := newDifferentialHarness(t, 100) + h.Insert("k1", &diffValue{id: "v1", size: 20}) + h.Insert("k2", &diffValue{id: "v2", size: 20}) + h.Insert("k3", &diffValue{id: "v3", size: 30}) + + // Act: grow k3 from 30 to 110 (> maxSize 100). Only k3 should be evicted. + h.UpdateSize("k3", 80) + + // Assert + assert.Nil(t, h.LookUpWithoutChangingOrder("k3"), "k3 exceeds maxSize and must be evicted") + assert.NotNil(t, h.LookUpWithoutChangingOrder("k1"), "k1 must not be collateral-evicted") + assert.NotNil(t, h.LookUpWithoutChangingOrder("k2"), "k2 must not be collateral-evicted") +} + +func TestDifferential_InsertNearMaxUint64EvictsWithoutOverflow(t *testing.T) { + // Arrange + const maxCap = uint64(math.MaxUint64) + const halfPlus = uint64(math.MaxUint64/2) + 100 + h := newDifferentialHarness(t, maxCap) + + h.Insert("k1", &diffValue{id: "v1", size: halfPlus}) + + // Act: inserting k2 of size halfPlus would overflow uint64 if added before evicting k1. + evicted := h.Insert("k2", &diffValue{id: "v2", size: halfPlus}) + + // Assert + require.Len(t, evicted, 1) + assert.Nil(t, h.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, h.LookUpWithoutChangingOrder("k2")) + + // Subsequent Erase must not underflow currentSize or panic. + erased := h.Erase("k2") + require.NotNil(t, erased) +} + +func TestDifferential_UpdateSizeMiddleEntrySelfEvictionAndMaxUint64(t *testing.T) { + // Arrange 1: k1 (10B, LRU), k2 (20B, mid), k3 (20B, MRU) in maxSize = 50. + h := newDifferentialHarness(t, 50) + h.Insert("k1", &diffValue{id: "v1", size: 10}) + h.Insert("k2", &diffValue{id: "v2", size: 20}) + h.Insert("k3", &diffValue{id: "v3", size: 20}) + + // Act 1: Grow k2 by +20 (20 -> 40). Since k2 (40) + k3 (20) = 60 > 50, k2 cannot fit alongside newer k3. + h.UpdateSize("k2", 20) + + // Assert 1: k2 is self-evicted while older entry k1 (10B) and newer entry k3 (20B) both survive. + assert.Nil(t, h.LookUpWithoutChangingOrder("k2")) + assert.NotNil(t, h.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, h.LookUpWithoutChangingOrder("k3")) + + // Arrange 2: Near math.MaxUint64, k1 (MaxUint64 - 50) + k2 (50) == MaxUint64. + maxH := newDifferentialHarness(t, math.MaxUint64) + maxH.Insert("k1", &diffValue{id: "v1", size: math.MaxUint64 - 50}) + maxH.Insert("k2", &diffValue{id: "v2", size: 50}) + + // Act 2: Grow k2 by +100. Evicting k1 first prevents uint64 overflow. + maxH.UpdateSize("k2", 100) + + // Assert 2 + assert.Nil(t, maxH.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, maxH.LookUpWithoutChangingOrder("k2")) +} + +func TestDifferential_UpdateSizeLockstepParity(t *testing.T) { + // Updating the MRU head when it requires evicting multiple older entries + h := newDifferentialHarness(t, 100) + h.Insert("k1", &diffValue{id: "v1", size: 10}) + h.Insert("k2", &diffValue{id: "v2", size: 20}) + h.Insert("k3", &diffValue{id: "v3", size: 30}) + h.Insert("k4", &diffValue{id: "v4", size: 30}) + + // cache: k1 (10), k2 (20), k3 (30), k4 (30) + h.UpdateSize("k4", 40) + assert.Nil(t, h.LookUpWithoutChangingOrder("k1")) + assert.Nil(t, h.LookUpWithoutChangingOrder("k2")) + assert.NotNil(t, h.LookUpWithoutChangingOrder("k3")) + assert.NotNil(t, h.LookUpWithoutChangingOrder("k4")) + + // Updating the second-most-recent entry (head.next) when head.size + entry.size + sizeDelta > maxSize (self-evicts) + h2 := newDifferentialHarness(t, 100) + h2.Insert("k1", &diffValue{id: "v1", size: 10}) + h2.Insert("k2", &diffValue{id: "v2", size: 20}) + h2.Insert("k3", &diffValue{id: "v3", size: 30}) // head.next + h2.Insert("k4", &diffValue{id: "v4", size: 40}) // head + h2.UpdateSize("k3", 35) + assert.Nil(t, h2.LookUpWithoutChangingOrder("k3")) // self-evicted + assert.NotNil(t, h2.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, h2.LookUpWithoutChangingOrder("k2")) + assert.NotNil(t, h2.LookUpWithoutChangingOrder("k4")) + + // Updating the second-most-recent entry (head.next) when head.size + entry.size + sizeDelta <= maxSize (evicts older) + h3 := newDifferentialHarness(t, 100) + h3.Insert("k1", &diffValue{id: "v1", size: 10}) + h3.Insert("k2", &diffValue{id: "v2", size: 20}) + h3.Insert("k3", &diffValue{id: "v3", size: 30}) // head.next + h3.Insert("k4", &diffValue{id: "v4", size: 30}) // head + h3.UpdateSize("k3", 30) + assert.Nil(t, h3.LookUpWithoutChangingOrder("k1")) + assert.Nil(t, h3.LookUpWithoutChangingOrder("k2")) + assert.NotNil(t, h3.LookUpWithoutChangingOrder("k3")) + assert.NotNil(t, h3.LookUpWithoutChangingOrder("k4")) + + // Updating the LRU tail entry when it fits (sizeDelta <= maxSize - currentSize) + h4 := newDifferentialHarness(t, 100) + h4.Insert("k1", &diffValue{id: "v1", size: 10}) // tail + h4.Insert("k2", &diffValue{id: "v2", size: 50}) // head + h4.UpdateSize("k1", 30) + assert.NotNil(t, h4.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, h4.LookUpWithoutChangingOrder("k2")) + + // Updating the LRU tail entry when it self-evicts (sizeDelta > maxSize - currentSize) + h5 := newDifferentialHarness(t, 100) + h5.Insert("k1", &diffValue{id: "v1", size: 10}) // tail + h5.Insert("k2", &diffValue{id: "v2", size: 80}) // head + h5.UpdateSize("k1", 20) + assert.Nil(t, h5.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, h5.LookUpWithoutChangingOrder("k2")) +} diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..6ae0ae7 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,79 @@ +# Architecture & Memory-Pressure Reclamation + +This document details the internal data structures, concurrency models, and memory-pressure reclamation mechanics of the three cache engines in `github.com/google/go-lru` (`package lru`). + +--- + +## 1. Architectural Comparison Matrix + +| Dimension | `MapCache` (`NewMapCache`) | `RadixCache` (`NewRadixCache`) | `ArenaRadixCache` (`NewArenaRadixCache`) | +| :--- | :--- | :--- | :--- | +| **Primary Data Structure** | `map[string]*list.Element` + `container/list.List` | Compressed Radix Tree (LCRS) + Intrusive Pointer LRU List | Contiguous Slice Arena `[]arenaRadixNode` + Freelist + Hash Index | +| **Node Representation** | `list.Element` (48 B) + `entry` (40 B) + Map Buckets | `radixNode` struct (80 B heap object) | `arenaRadixNode` struct (64 B contiguous array entry) | +| **Per-Node Heap Allocations** | 2 allocations per entry (0 on update) | 1 allocation per node | **0 allocations** (recycled via intrusive free-list) | +| **Node Pointer Width** | 64-bit pointers | 64-bit pointers (5 pointers / node) | **32-bit indices** (`uint32`, sentinel `nilNode = math.MaxUint32`) | +| **Max Entry Capacity** | Memory / Heap limited | Memory / Heap limited | **`2^32 - 2` total arena nodes** (`0..math.MaxUint32-2`; `nilNode = MaxUint32`, `foregroundNoProtect = MaxUint32-1`) | +| **Point Lookup Latency** | **~50 ns/op** (`O(1)` Hash Map) | ~170 ns/op (`O(K)` Trie Descent) | **~100 ns/op** (`O(1)` FNV-1a Hash Map + Zero-Alloc Path Check) | +| **Sequential Insert Latency** | ~98 ns/op in-place (~383 ns/op turnover) | ~175 ns/op in-place (~472 ns/op turnover) | ~270 ns/op in-place (~515 ns/op turnover, 0 node allocs) | +| **Point Update Latency** | ~82 ns/op (0 allocs) | ~97 ns/op (0 allocs) | ~88 ns/op (0 allocs) | +| **Prefix Erase (100 items)** | ~151.2–219.9 µs (`O(N)` full scan) | **~2.25–2.50 µs** (**65.7x–88.0x faster**) | **~7.3–7.6 µs** (**20.7x–28.9x faster**) | +| **Prefix Erase (50K/100K items)** | ~21.6 ms | **~1.05 ms** (**20.5x faster**) | **~5.1 ms** (**4.2x faster**) | +| **Live Heap Memory** | ~135.9 B/entry at 1M (~163.0 `heap-B/entry` at 100K) | **~94.2 B/entry at 1M** (~96.0 `heap-B/entry` at 100K, **~30.7%–41.1% reduction**) | **~106.9 B/entry at 1M** (~111.2 `heap-B/entry` at 100K, **~21.4%–31.8% reduction**) | +| **GC Pressure & Overhead** | High (millions of distinct heap objects) | Moderate (heap nodes with pointers) | **Ultra-Low** (flat slice; internal tree indices invisible to GC) | +| **Concurrency Lock** | `sync.RWMutex` | `sync.RWMutex` | `sync.RWMutex` | +| **Best Used For** | Flat keys, maximum read/write throughput | File systems, directory trees, prefix purges | Large-scale hierarchical caches (1M+ items) under strict memory limits | + +--- + +## 2. Engine Data Structure Layouts + +### 2.1 `MapCache` (`map_lru.go`) +`MapCache` pairs Go's standard hash map (`map[string]*list.Element`) with a doubly-linked recency list (`container/list.List`) guarded by a `sync.RWMutex`. +- **Operations**: `Insert`, `LookUp`, `LookUpWithoutChangingOrder`, `UpdateWithoutChangingOrder`, `UpdateSize`, and `Erase` execute in `O(1)` time. +- **Prefix Eviction Trade-off**: `EraseEntriesWithGivenPrefix(prefix)` must iterate over all `N` entries in the map (`O(N)`), making it unsuitable for frequent directory-tree purges on large caches. + +### 2.2 `RadixCache` (`radix_lru.go`) +`RadixCache` stores keys in a compressed **Left-Child Right-Sibling (LCRS)** radix tree (`radixNode`, 80 bytes per node). +- **Intrusive Doubly-Linked LRU List**: Each `radixNode` embeds `prev` and `next` pointers alongside tree topology links (`parent`, `child`, `sibling`), avoiding separate `list.Element` wrapper allocations. +- **Subtree Detachment**: `EraseEntriesWithGivenPrefix(prefix)` descends `O(P)` characters to the prefix root, detaches the subtree in `O(1)`, and unlinks the `S` descendant value nodes from the intrusive LRU list in `O(S)` (`O(P + S)` total work). +- **Path Compression**: When an internal node loses children after `Erase` or eviction, `compressPathUpwards` merges single-child routing nodes with their parent to keep tree depth minimal. + +### 2.3 `ArenaRadixCache` (`arena_radix.go`, `arena_radix_lru.go`) +`ArenaRadixCache` eliminates per-node heap objects and pointer-graph scanning by storing all nodes in a contiguous slice `[]arenaRadixNode` (`64 bytes` per node) indexed by 32-bit integers (`uint32`, with `nilNode = math.MaxUint32`). +- **Intrusive O(1) Free-List**: Erased or evicted node slots are pushed onto an intrusive singly-linked free-list (`freeHead`, linked via `next` index with `parent = nilNode`, `child = nilNode`, `sibling = nilNode`, `prev = nilNode`) and recycled on subsequent insertions with zero heap allocations. +- **O(1) 64-Bit FNV-1a Lookup Accelerator**: + - `nodeMap map[uint64]uint32` maps the 64-bit FNV-1a hash of full keys directly to their arena node index. + - `verifyKey(nodeID, key)` walks `parent` indices from `nodeID` up to `root`, verifying byte-for-byte suffix equality against `key` in zero allocations. + - If two distinct keys collide on the same 64-bit FNV-1a hash, `ArenaRadixCache` preserves full correctness by falling back to `O(K)` trie descent (`getNodeKeyWithHash`) and healing `nodeMap` entries on write-locked lookups (`LookUp`, `Insert`, `UpdateWithoutChangingOrder`, `UpdateSize`) or `compactLocked()`. + +--- + +## 3. Two-Tier Memory-Pressure Reclamation (`PressureAwareCache`, `pressure.go`) + +Go's built-in `map` never shrinks its bucket array after deletions, pointer-based trees retain fragmented prefix chains, and `[]arenaRadixNode` retains its peak backing array capacity in the free-list. All three backends (`MapCache`, `RadixCache`, and `ArenaRadixCache`) embed a shared `pressureState` (`pressure.go`) and implement `PressureAwareCache`, providing a unified **two-tier memory-pressure reclamation architecture** configurable via `WithPressureFunc`, `WithMemoryBudget`, `WithCompactionThreshold`, `WithEvictionThreshold`, and `WithEvictionRetentionRatio`. + +### 3.1 Lock-Free Amortized Pressure Sampling +- **Built-In Probe (`DefaultRuntimePressureFunc`)**: Reads `/memory/classes/total:bytes`, `/memory/classes/heap/released:bytes`, `/memory/classes/heap/free:bytes`, and `/memory/classes/heap/objects:bytes` from `runtime/metrics` using a `sync.Pool` of `[4]metrics.Sample` arrays (`metricsSamplePool`, `0 allocs/op`, zero Stop-The-World pauses), with an `O(1)` fast-path when `memoryBudget == 0` that reads `/gc/gomemlimit:bytes` via a pooled `[1]metrics.Sample` array (`gomemlimitSamplePool`) and returns `0.0` immediately when `GOMEMLIMIT` is unbounded without acquiring `worldsema` or sweeping per-P heap stats. +- **Amortized Window (`samplePressureWithEpoch`)**: When using the built-in `DefaultRuntimePressureFunc`, foreground write operations (`Insert`, `UpdateSize`, `Erase`, `EraseEntriesWithGivenPrefix`) sample the runtime metrics probe once every 256 writes (`seq&255 == 1`) or immediately following a reclamation epoch (`pressureNeedsRefresh`), whereas custom `WithPressureFunc` callbacks are sampled on every foreground write and `EvaluateMemoryPressure()` call. +- **Re-Entrancy Guard**: Atomic sampler tracking (`samplingPressure`, `fallbackSampling`, `overflowSamplingGIDs`) ensures that even if a user-supplied `PressureFunc` re-entrantly calls methods on the same `Cache`, it will never deadlock or recurse infinitely. + +### 3.2 Tier 1 — Moderate Pressure (`pressure >= CompactionThreshold`, default `0.75`) +When normalized memory pressure reaches `CompactionThreshold`: +- Triggered automatically on foreground writes whenever backing-structure slack or post-peak entry deletion exceeds 25%, or unconditionally via `Compact()` / `EvaluateMemoryPressure()`. +- `MapCache` reallocates its `map[string]*list.Element` index (`make(map[string]*list.Element, len(c.index))` + `maps.Copy`) when `dirtyIndex` is true to reclaim Go map bucket slack; `RadixCache` eagerly clones prefixes at insertion/split time and zeroes detached nodes upon deletion while resetting compaction watermarks (`onCompacted`); `ArenaRadixCache` allocates a fresh, tightly-sized `newNodes` slice (`len == cap == liveCount`), remaps all 8 `uint32` index fields via `oldToNew`, resets `freeHead = nilNode`, and rebuilds `nodeMap` (`make(map[uint64]uint32, c.len)`) from scratch to release Go hash-map bucket slack. + +### 3.3 Tier 2 — Critical Pressure (`pressure >= EvictionThreshold`, default `0.90`) +When normalized memory pressure reaches `EvictionThreshold`: +- `shedAndCompactLocked()` proactively evicts least-recently-used (`tail`) entries until `currentSize <= maxSize * EvictionRetentionRatio` (default `50%` of `maxSize`), and also proportionally sheds zero-size entries (`size == 0`) down to `EvictionRetentionRatio` (bounded by `targetLen` and `lastReclaimedZeroCount` / `lastReclaimedLen` watermarks so repeated evaluations at sustained critical pressure remain idempotent). +- Foreground insertions pass the MRU head entry as a protected reference so the entry currently being inserted or overwritten is never self-evicted during inline pressure shedding, while foreground size updates (`UpdateSize`) pass an unprotected sentinel (`nil` / `foregroundNoProtect`) and protect the MRU head entry when `tail == head`. +- After shedding LRU entries, `compactDataStructuresLocked()` executes immediately to return both the evicted entries' backing structures and any prior map/tree/arena slack to the Go runtime heap. + +--- + +## 4. O(1)-Space Iterative Invariant Verification + +Enabling `WithInvariantChecking(true)` validates structural integrity across every cache operation: +1. **Capacity Bounds**: `maxSize > 0` and `currentSize <= maxSize`. +2. **Iterative Tree Walk**: Uses parent/sibling pointers (`radixCache`) or `uint32` index links (`ArenaRadixCache`) to traverse the entire LCRS tree in `O(1)` auxiliary space without recursion, verifying parent-child symmetry and absence of cycles. +3. **LRU List Bijection**: Verifies that every value-bearing tree node appears in the doubly-linked `head`/`tail` list with matching forward and reverse traversal counts and exact `currentSize` sum parity. +4. **Free-List & Hash Index Integrity (`ArenaRadixCache`)**: Confirms `len(nodes) - freeCount == treeNodeCount`, verifies zero overlap between free-list slots and live tree nodes, and checks `nodeMap` index consistency. diff --git a/docs/performance.md b/docs/performance.md new file mode 100644 index 0000000..842e843 --- /dev/null +++ b/docs/performance.md @@ -0,0 +1,78 @@ +# Empirical Performance & Heap Footprint + +This document records empirical latency, allocation, subtree prefix eviction, and 1,000,000-key live heap memory footprint benchmarks across `MapCache`, `RadixCache`, and `ArenaRadixCache`. + +All reference benchmarks were collected on an **Intel Xeon CPU @ 2.60GHz (96 vCPUs, `linux/amd64`)** using `go test -bench=. -benchmem`. + +--- + +## 1. Point Operation Latency & Allocations + +| Operation | `MapCache` | `RadixCache` | `ArenaRadixCache` | +| :--- | :--- | :--- | :--- | +| **Lookup (Flat)** | **50.6 ns/op** (0 B, 0 allocs) | 174.0 ns/op (0 B, 0 allocs) | 100.7 ns/op (0 B, 0 allocs) | +| **Lookup (Nested, Depth 2)** | **49.5 ns/op** (0 B, 0 allocs) | 169.6 ns/op (0 B, 0 allocs) | 107.8 ns/op (0 B, 0 allocs) | +| **Lookup (Deeply Nested, Depth 10)** | **64.3 ns/op** (0 B, 0 allocs) | 185.2 ns/op (0 B, 0 allocs) | 194.1 ns/op (0 B, 0 allocs) | +| **Lookup (`LookUpWithoutChangingOrder`)** | **33.2 ns/op** (0 B, 0 allocs) | 163.3 ns/op (0 B, 0 allocs) | 103.8 ns/op (0 B, 0 allocs) | +| **Insert (Existing-Key Overwrite, In-Place)**¹ | **97.7 ns/op** (0 B, 0 allocs in-place) | 178.2 ns/op (0 B, 0 allocs in-place) | 270.6 ns/op (0 B, 0 allocs in-place) | +| **Insert (Nested, Depth 2, In-Place Overwrite)**¹ | **98.1 ns/op** (0 B, 0 allocs in-place) | 172.3 ns/op (0 B, 0 allocs in-place) | 270.9 ns/op (0 B, 0 allocs in-place) | +| **Insert (Deeply Nested, Depth 10, In-Place Overwrite)**¹ | **119.0 ns/op** (0 B, 0 allocs in-place) | 229.9 ns/op (0 B, 0 allocs in-place) | 471.8 ns/op (0 B, 0 allocs in-place) | +| **Insert (`Benchmark_Insert_*`, 50%-Capacity Turnover)**¹ | **383–466 ns/op** (96–144 B, 3 allocs/op) | 472–511 ns/op (132–135 B, 3 allocs/op) | 515–822 ns/op (49–65 B, 2 allocs/op) | +| **Update Value (`UpdateWithoutChangingOrder`)**² | **82.5 ns/op** (0 B internal, 0 allocs) | 97.3 ns/op (0 B, 0 allocs) | 88.6 ns/op (0 B, 0 allocs) | +| **Individual Erase (`Erase`)** | 237.9 ns/op (0 B, 0 allocs) | **147.7 ns/op** (0 B, 0 allocs) | 310.6 ns/op (0 B, 0 allocs) | + +> ¹ **Existing-key overwrite `Insert`** (when the working set fits within `maxSize`) updates the entry in place with **0 B, 0 allocs/op** across `MapCache`, `RadixCache`, and `ArenaRadixCache`. By contrast, `Benchmark_Insert_*` configures `capacity := uint64(len(keys) * 5)` (50,000 B for a 10,000-key × 10 B working set, holding 5,000 of the 10,000 keys), exercising **50%-capacity turnover `Insert`** where every `Insert` after warmup is a new-key insert paired with an LRU tail eviction: `MapCache` and `RadixCache` perform **3 allocs/op** (`strings.Clone` + `*list.Element`/`*entry` or `*radixNode` + evicted `Entry`), whereas `ArenaRadixCache` performs **2 allocs/op** (`strings.Clone` + evicted `Entry`, with **0 node allocations** thanks to intrusive free-list node recycling). +> +> ² `UpdateWithoutChangingOrder` performs **0 internal heap allocations** across all three backends (`MapCache`, `RadixCache`, and `ArenaRadixCache`); any `16 B, 1 alloc/op` reported by `Benchmark_Update_*` when passing `benchValue(10)` by value comes from Go boxing the caller's `benchValue` into the `ValueType` interface parameter inside the benchmark harness. + +### Key Takeaways +- **`MapCache`** achieves the lowest single-key point lookup (~50 ns) and insertion (~98 ns in-place overwrite; ~383–466 ns under 50%-capacity turnover) latency when keys are flat and prefix operations are rare. +- **`ArenaRadixCache`** accelerates radix lookups by **1.6x–1.7x** over `RadixCache` (~100 ns vs ~170 ns) via its 64-bit FNV-1a index and zero-allocation bottom-up key verifier (`verifyKey`). +- All three backends (**`MapCache`**, **`RadixCache`**, and **`ArenaRadixCache`**) perform in-place value updates (`UpdateWithoutChangingOrder` and existing-key `Insert` overwrites) with **0 internal heap allocations per operation**, and `ArenaRadixCache` recycles deleted node indices in steady state via its intrusive free-list (`2 allocs/op` vs `3 allocs/op` during 50%-capacity turnover). + +--- + +## 2. Subtree Prefix Deletion Latency (`EraseEntriesWithGivenPrefix`) + +| Prefix Topology | `MapCache` (`O(N)` Scan) | `RadixCache` (`O(P + S)` Subtree) | `ArenaRadixCache` (`O(P + S)` Subtree) | Speedup vs `MapCache` | +| :--- | :--- | :--- | :--- | :--- | +| **Flat Prefix (100 items)** | 160.6 µs/op | **2.25 µs/op** | ~7.5 µs/op | **71.3x (`Radix`) / ~21.4x (`Arena`) faster** | +| **Nested Prefix (100 items)** | 151.2 µs/op | **2.30 µs/op** | ~7.5 µs/op | **65.7x (`Radix`) / ~20.2x (`Arena`) faster** | +| **Deeply Nested (100 items)** | 219.9 µs/op | **2.50 µs/op** | ~7.6 µs/op | **88.0x (`Radix`) / ~28.9x (`Arena`) faster** | +| **100K Scale (50,000 purged items)** | 21.6 ms/op | **1.05 ms/op** | ~5.1 ms/op | **20.5x (`Radix`) / ~4.2x (`Arena`) faster** | + +--- + +## 3. True Heap Memory Footprint (100K & 1,000,000 Keys) + +Measured via `runtime.ReadMemStats` (`HeapAlloc`) after an isolated garbage collection sweep (`runtime.GC()` + `debug.FreeOSMemory()`): + +| Scale & Key Topology | `MapCache` Heap | `RadixCache` Heap | `ArenaRadixCache` Heap | Memory Reduction vs `MapCache` | +| :--- | :--- | :--- | :--- | :--- | +| **100K Nested (`Benchmark_LargeScale_Insert_100K`)** | ~16.3 MB (~163.0 `heap-B/entry`) | **~9.6 MB** (~96.0 `heap-B/entry`) | ~11.1 MB (~111.2 `heap-B/entry`) | **41.1% less (`Radix`) / 31.8% less (`Arena`)** | +| **1M Flat** (`file_%d.txt`) | 129.6 MB (135.9 B/entry) | **89.8 MB** (94.2 B/entry) | 102.0 MB (106.9 B/entry) | **30.7% less (`Radix`) / 21.4% less (`Arena`)** | +| **1M Nested** (`dir_%04d/file_%04d.txt`) | 129.6 MB (135.9 B/entry) | **90.5 MB** (94.9 B/entry) | 101.9 MB (106.9 B/entry) | **30.2% less (`Radix`) / 21.4% less (`Arena`)** | +| **1M Deeply Nested** (`projects/...`) | 129.6 MB (135.9 B/entry) | **90.9 MB** (95.4 B/entry) | 101.9 MB (106.9 B/entry) | **29.9% less (`Radix`) / 21.4% less (`Arena`)** | + +In addition to reported `b.ReportAllocs()` (`B/op` and `allocs/op`), `Benchmark_LargeScale_Insert_100K` reports `heap-B/entry` (net live heap bytes per entry at 100K scale, where Go map bucket power-of-two sizing accounts for the higher per-entry overhead in `MapCache` relative to 1M scale) and `Benchmark_ArenaRadixCache_Compact` reports `reclaimed-B/op` (live heap bytes reclaimed per compaction pass). + +--- + +## 4. Reproducing Benchmarks & Memory Profiles Locally + +```bash +# Run the complete benchmark suite with memory & custom resource metrics +go test -run=^$ -bench=. -benchmem ./... + +# Run point lookup, insert, update, and erase benchmarks +go test -run=^$ -bench="Benchmark_(Insert|LookUp|Update|Erase)" -benchmem ./... + +# Run multi-core parallel throughput benchmarks (Mixed, ReadHeavy, WriteHeavy) +go test -run=^$ -bench="Benchmark_ParallelThroughput" -benchmem ./... + +# Run 100K-entry scale & ArenaRadixCache compaction/pressure benchmarks +go test -run=^$ -bench="Benchmark_(LargeScale|ArenaRadixCache)" -benchmem ./... + +# Run live heap footprint (heap-B/entry) and compaction reclamation (reclaimed-B/op) benchmarks +go test -run=^$ -bench="Benchmark_(LargeScale_Insert_100K|ArenaRadixCache_Compact)" -benchmem ./... +``` diff --git a/errors.go b/errors.go index 8f9753a..69442af 100644 --- a/errors.go +++ b/errors.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus +package lru import "errors" diff --git a/example_test.go b/example_test.go new file mode 100644 index 0000000..f633341 --- /dev/null +++ b/example_test.go @@ -0,0 +1,119 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lru_test + +import ( + "fmt" + + "github.com/google/go-lru" +) + +// ExampleNew demonstrates creating an LRU cache with the unified constructor +// and caching common types using StringValue, BytesValue, and NewValue. +func ExampleNew() { + cache := lru.New(1024) + + _, _ = cache.Insert("greeting", lru.StringValue("hello, world")) + _, _ = cache.Insert("payload", lru.NewBytesValue([]byte{0xDE, 0xAD, 0xBE, 0xEF})) + _, _ = cache.Insert("inode-42", lru.NewValue(42, 64)) + + if v := cache.LookUp("greeting"); v != nil { + fmt.Printf("greeting=%s (size=%d)\n", v.(lru.StringValue), v.Size()) + } + if v := cache.LookUp("inode-42"); v != nil { + fmt.Printf("inode=%d (size=%d)\n", v.(lru.SizedValue[int]).Value, v.Size()) + } + // Output: + // greeting=hello, world (size=12) + // inode=42 (size=64) +} + +// ExampleNew_eviction demonstrates size-based capacity tracking and LRU eviction order. +func ExampleNew_eviction() { + // Create a cache with a 10-byte capacity. + cache := lru.New(10) + + _, _ = cache.Insert("a", lru.StringValue("1234")) // size 4 (total 4) + _, _ = cache.Insert("b", lru.StringValue("5678")) // size 4 (total 8) + + // Access "a" so "a" becomes MRU and "b" becomes LRU. + _ = cache.LookUp("a") + + // Inserting "c" (size 4) exceeds capacity (8 + 4 > 10), evicting "b". + evicted, err := cache.Insert("c", lru.StringValue("9012")) + if err != nil { + panic(err) + } + + fmt.Printf("evicted count=%d, first=%s\n", len(evicted), evicted[0].(lru.StringValue)) + fmt.Printf("a present=%v, b present=%v, c present=%v\n", + cache.LookUp("a") != nil, + cache.LookUp("b") != nil, + cache.LookUp("c") != nil, + ) + // Output: + // evicted count=1, first=5678 + // a present=true, b present=false, c present=true +} + +// ExampleNew_backendsAndPrefixErase demonstrates selecting the RadixCache backend +// via WithBackend and performing fast hierarchical prefix erasure. +func ExampleNew_backendsAndPrefixErase() { + cache := lru.New(4096, lru.WithBackend(lru.BackendRadix)) + + _, _ = cache.Insert("bucket/dirA/file1.txt", lru.StringValue("data-1")) + _, _ = cache.Insert("bucket/dirA/file2.txt", lru.StringValue("data-2")) + _, _ = cache.Insert("bucket/dirB/file3.txt", lru.StringValue("data-3")) + + // Purge only the "bucket/dirA/" subtree. + cache.EraseEntriesWithGivenPrefix("bucket/dirA/") + + fmt.Printf("dirA/file1=%v, dirB/file3=%v\n", + cache.LookUp("bucket/dirA/file1.txt") != nil, + cache.LookUp("bucket/dirB/file3.txt") != nil, + ) + // Output: + // dirA/file1=false, dirB/file3=true +} + +// ExampleNew_memoryPressure demonstrates configuring the ArenaRadixCache backend +// with a custom memory-pressure callback and triggering two-tier reclamation. +func ExampleNew_memoryPressure() { + pressure := 0.50 // Normal pressure + cache := lru.New( + 100, + lru.WithBackend(lru.BackendArenaRadix), + lru.WithPressureFunc(func() float64 { return pressure }), + lru.WithCompactionThreshold(0.75), + lru.WithEvictionThreshold(0.90), + lru.WithEvictionRetentionRatio(0.50), + ) + + _, _ = cache.Insert("dir/item1", lru.NewValue("v1", 40)) + _, _ = cache.Insert("dir/item2", lru.NewValue("v2", 40)) + + // Simulate Critical Pressure (>= 0.90) to shed LRU entries down to 50% of maxSize (50 bytes). + pressure = 0.95 + paCache := cache.(lru.PressureAwareCache) + shed := paCache.EvaluateMemoryPressure() + + fmt.Printf("shed entries=%d, item1 remaining=%v, item2 remaining=%v\n", + len(shed), + cache.LookUpWithoutChangingOrder("dir/item1") != nil, + cache.LookUpWithoutChangingOrder("dir/item2") != nil, + ) + // Output: + // shed entries=1, item1 remaining=false, item2 remaining=true +} diff --git a/go.mod b/go.mod index 91f06c8..5b5c5b9 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/googlecloudplatform/gcsfuse/v3/internal/cache/lru +module github.com/google/go-lru go 1.26 diff --git a/map_lru.go b/map_lru.go index fe1caf9..22d31d2 100644 --- a/map_lru.go +++ b/map_lru.go @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus +package lru import ( "container/list" "fmt" + "maps" "math" - "reflect" "strings" "sync" ) @@ -55,16 +55,17 @@ type mapCache struct { // Invariant: len(index) == entries.Len() index map[string]*list.Element + // dirtyIndex records whether entries have been deleted since the last index reallocation. + dirtyIndex bool + // mu synchronizes access to internal state. mu sync.RWMutex - // checkInvariantsEnabled indicates whether invariant checks are executed on lock/unlock. - checkInvariantsEnabled bool - - // options holds the parsed cache configuration. - options Options + pressureState } +var foregroundNoProtectElem list.Element + // NewMapCache returns a new map-based LRU Cache initialized with the supplied maxSize. // Optional configuration parameters can be passed via opts (e.g. WithInvariantChecking). // @@ -73,27 +74,21 @@ func NewMapCache(maxSize uint64, opts ...Option) Cache { if maxSize == 0 { panic("maxSize must be greater than zero") } - options := ApplyOptions(opts...) + return newMapCacheWithOptions(maxSize, ApplyOptions(opts...)) +} + +func newMapCacheWithOptions(maxSize uint64, options Options) Cache { c := &mapCache{ - maxSize: maxSize, - index: make(map[string]*list.Element), - checkInvariantsEnabled: options.EnableInvariantChecking, - options: options, + maxSize: maxSize, + index: make(map[string]*list.Element), + pressureState: pressureState{options: options}, } - if c.checkInvariantsEnabled { + if c.options.EnableInvariantChecking { c.checkInvariants() } return c } -// New returns a new Cache instance with the default MapCache implementation. -// It is an alias to NewMapCache. -// -// maxSize must be greater than zero; otherwise New panics. -func New(maxSize uint64, opts ...Option) Cache { - return NewMapCache(maxSize, opts...) -} - // checkInvariants validates internal data structure consistency and panics if any invariant is violated. func (c *mapCache) checkInvariants() { // Invariant 1: maxSize > 0 @@ -106,15 +101,6 @@ func (c *mapCache) checkInvariants() { panic(fmt.Sprintf("CurrentSize %v over maxSize %v", c.currentSize, c.maxSize)) } - // Invariant 3: Element payload type safety - for e := c.entries.Front(); e != nil; e = e.Next() { - switch e.Value.(type) { - case entry: - default: - panic(fmt.Sprintf("Unexpected element type: %v", reflect.TypeOf(e.Value))) - } - } - // Invariant 4: Map-to-list cardinality bijection if c.entries.Len() != len(c.index) { panic(fmt.Sprintf("Length mismatch: %v vs. %v", c.entries.Len(), len(c.index))) @@ -131,10 +117,12 @@ func (c *mapCache) checkInvariants() { } } - // Invariant 6: Bidirectional pointer linkage, map-to-list consistency, and size sum parity + // Invariant 3 & 6: Element payload type safety (*entry), bidirectional pointer linkage, + // map-to-list consistency, and size sum parity. var prevElem *list.Element var sumSize uint64 lruCount := 0 + zeroCount := 0 for e := c.entries.Front(); e != nil; e = e.Next() { lruCount++ @@ -159,17 +147,34 @@ func (c *mapCache) checkInvariants() { } prevElem = e - entryVal := e.Value.(entry) + entryPtr, ok := e.Value.(*entry) + if !ok || entryPtr == nil { + panic(fmt.Sprintf("Unexpected element type: %T", e.Value)) + } + entryVal := *entryPtr + if entryVal.value == nil { + panic(fmt.Sprintf("mapCache invariant violation: unexpected nil value in LRU list for key '%s'", entryVal.key)) + } if c.index[entryVal.key] != e { panic(fmt.Sprintf("Mismatch for key %v", entryVal.key)) } + if math.MaxUint64-sumSize < entryVal.size { + panic("mapCache invariant violation: sumSize uint64 overflow") + } sumSize += entryVal.size + if entryVal.size == 0 { + zeroCount++ + } } if lruCount != c.entries.Len() { panic(fmt.Sprintf("mapCache invariant violation: LRU list count %d does not match entries.Len() %d", lruCount, c.entries.Len())) } + if zeroCount != c.zeroSizeCount { + panic(fmt.Sprintf("mapCache invariant violation: zeroSizeCount %d does not match live zero-size entries %d", c.zeroSizeCount, zeroCount)) + } + // Invariant 7: Sum of entry sizes matches currentSize if sumSize != c.currentSize { panic(fmt.Sprintf("Size sum mismatch: sum of entry sizes %d vs. currentSize %d", sumSize, c.currentSize)) @@ -178,13 +183,10 @@ func (c *mapCache) checkInvariants() { func (c *mapCache) lock() { c.mu.Lock() - if c.checkInvariantsEnabled { - c.checkInvariants() - } } func (c *mapCache) unlock() { - if c.checkInvariantsEnabled { + if c.options.EnableInvariantChecking { c.checkInvariants() } c.mu.Unlock() @@ -192,13 +194,10 @@ func (c *mapCache) unlock() { func (c *mapCache) rLock() { c.mu.RLock() - if c.checkInvariantsEnabled { - c.checkInvariants() - } } func (c *mapCache) rUnlock() { - if c.checkInvariantsEnabled { + if c.options.EnableInvariantChecking { c.checkInvariants() } c.mu.RUnlock() @@ -211,15 +210,45 @@ func (c *mapCache) evictOne() ValueType { if e == nil { return nil } - entryVal := e.Value.(entry) - key := entryVal.key - evictedValue := entryVal.value - c.currentSize -= entryVal.size + entryVal := e.Value.(*entry) + return c.eraseInternal(entryVal.key) +} - c.entries.Remove(e) - delete(c.index, key) +func (c *mapCache) finishDeleteReclaimLocked(sizeBefore, sampledEpoch uint64, pressure float64) { + if c.entries.Len() == 0 { + hadSlack := c.hasEmptyDeleteSlack(false) + c.clearEmptyIndexStateLocked() + if (hadSlack || sizeBefore > 0) && c.reclaimEpoch.Load() == sampledEpoch && c.hasElevatedPressureToInvalidate(pressure) { + c.markReclaimedLocked() + } + return + } + reclaimedSingleSurvivor := false + if c.shouldReclaimSingleSurvivorOnDelete(c.entries.Len(), c.dirtyIndex, false) { + c.compactDataStructuresLocked() + reclaimedSingleSurvivor = true + } + c.maybeReclaimUnderPressureLocked(pressure, &foregroundNoProtectElem) + if (reclaimedSingleSurvivor || c.currentSize < sizeBefore) && c.reclaimEpoch.Load() == sampledEpoch && c.hasElevatedPressureToInvalidate(pressure) { + c.markReclaimedLocked() + } +} - return evictedValue +func (c *mapCache) finishMutationReclaimLocked(evictedValues []ValueType, protectedElem *list.Element, reclaimedPre bool, sizeBefore, sampledEpoch uint64, pressure float64) []ValueType { + evictedByPressure := c.maybeReclaimUnderPressureLocked(pressure, protectedElem) + if len(evictedValues) == 0 { + evictedValues = evictedByPressure + } else if len(evictedByPressure) > 0 { + evictedValues = append(evictedValues, evictedByPressure...) + } + netByteReduced := c.currentSize < sizeBefore + if c.shouldCompactAfterMutation(reclaimedPre, netByteReduced, c.dirtyIndex, pressure) { + c.compactDataStructuresLocked() + } + if c.shouldMarkReclaimedAfterMutation(reclaimedPre, netByteReduced, sampledEpoch, pressure) { + c.markReclaimedLocked() + } + return evictedValues } // Insert inserts or updates the given key and value in the cache. @@ -239,31 +268,58 @@ func (c *mapCache) Insert(key string, value ValueType) ([]ValueType, error) { return nil, ErrInvalidEntrySize } - c.lock() + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) defer c.unlock() + var evictedValues []ValueType + sizeBefore := c.currentSize + reclaimedPreInsert := false + evictedPreInsert := false + e, ok := c.index[key] if ok { - // Update existing entry. - c.currentSize -= e.Value.(entry).size - c.currentSize += valueSize - e.Value = entry{key: key, value: value, size: valueSize} + // Update existing entry in place (0 heap allocations). + entryVal := e.Value.(*entry) + c.onEntrySizeUpdated(entryVal.size, valueSize) c.entries.MoveToFront(e) - } else { - // Add new entry at MRU (front). - e := c.entries.PushFront(entry{key: key, value: value, size: valueSize}) - c.index[key] = e + c.currentSize -= entryVal.size + for valueSize > c.maxSize-c.currentSize && c.entries.Len() > 1 { + evicted := c.evictOne() + if evicted != nil { + evictedValues = append(evictedValues, evicted) + evictedPreInsert = true + } + } + entryVal.value = value + entryVal.size = valueSize c.currentSize += valueSize - } - - var evictedValues []ValueType - for c.currentSize > c.maxSize && c.entries.Len() > 0 { - evicted := c.evictOne() - if evicted != nil { - evictedValues = append(evictedValues, evicted) + if c.shouldReclaimSingleSurvivorOnMutation(c.entries.Len(), c.dirtyIndex, false, evictedPreInsert, c.currentSize, sizeBefore, pressure) { + reclaimedPreInsert = true + } + } else { + // Evict prior to adding new entry if valueSize would exceed remaining capacity (prevents uint64 overflow). + for valueSize > c.maxSize-c.currentSize && c.entries.Len() > 0 { + evicted := c.evictOne() + if evicted != nil { + evictedValues = append(evictedValues, evicted) + evictedPreInsert = true + } + } + if c.shouldReclaimEmptyPreInsert(c.entries.Len(), false, evictedPreInsert, valueSize, sizeBefore, pressure) { + c.clearEmptyIndexStateLocked() + reclaimedPreInsert = true + } else if c.shouldReclaimSingleSurvivorOnMutation(c.entries.Len(), c.dirtyIndex, false, evictedPreInsert, c.currentSize+valueSize, sizeBefore, pressure) { + reclaimedPreInsert = true } + // Clone key to prevent substring keys from pinning large caller backing arrays. + clonedKey := clonePrefix(key) + e = c.entries.PushFront(&entry{key: clonedKey, value: value, size: valueSize}) + c.index[clonedKey] = e + c.onEntryInserted(len(c.index), valueSize) + c.currentSize += valueSize } + evictedValues = c.finishMutationReclaimLocked(evictedValues, e, reclaimedPreInsert, sizeBefore, sampledEpoch, pressure) return evictedValues, nil } @@ -276,23 +332,51 @@ func (c *mapCache) eraseInternal(key string) ValueType { return nil } - entryVal := e.Value.(entry) + entryVal := e.Value.(*entry) deletedEntry := entryVal.value + c.onEntryDeleted(entryVal.size) c.currentSize -= entryVal.size delete(c.index, key) + c.dirtyIndex = true c.entries.Remove(e) + entryVal.key = "" + entryVal.value = nil + entryVal.size = 0 + e.Value = nil return deletedEntry } +func (c *mapCache) clearEmptyIndexStateLocked() { + if c.peakEntryLen > 8 || c.deletedSinceCompact >= 64 { + c.index = make(map[string]*list.Element) + } else { + clear(c.index) + } + c.dirtyIndex = false + c.resetWatermarks() +} + +// resetEmptyIndexLocked releases peak map bucket memory when the cache transitions to empty. +func (c *mapCache) resetEmptyIndexLocked() { + c.clearEmptyIndexStateLocked() + c.markReclaimedLocked() +} + // Erase removes the entry associated with the given key from the cache. // Returns the value of the erased entry, or nil if the key was not found. func (c *mapCache) Erase(key string) ValueType { - c.lock() + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) defer c.unlock() - return c.eraseInternal(key) + sizeBefore := c.currentSize + deleted := c.eraseInternal(key) + if deleted == nil { + return nil + } + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) + return deleted } // LookUp retrieves the value associated with key and updates its position to MRU. @@ -306,7 +390,7 @@ func (c *mapCache) LookUp(key string) ValueType { return nil } c.entries.MoveToFront(e) - return e.Value.(entry).value + return e.Value.(*entry).value } // LookUpWithoutChangingOrder retrieves the value associated with key without altering its LRU position. @@ -319,7 +403,7 @@ func (c *mapCache) LookUpWithoutChangingOrder(key string) ValueType { if !ok { return nil } - return e.Value.(entry).value + return e.Value.(*entry).value } // UpdateWithoutChangingOrder updates the value of an existing key without modifying its LRU position. @@ -342,22 +426,24 @@ func (c *mapCache) UpdateWithoutChangingOrder(key string, value ValueType) error return ErrEntryNotExist } - entryVal := e.Value.(entry) + entryVal := e.Value.(*entry) if valueSize != entryVal.size { return ErrInvalidUpdateEntrySize } - e.Value = entry{key: key, value: value, size: entryVal.size} + entryVal.value = value return nil } // UpdateSize adjusts the size accounting for an existing key by sizeDelta without altering its LRU position. -// If the cache capacity is exceeded as a result of the size adjustment, least recently used entries -// are evicted immediately to ensure the size invariant holds. +// If the entry's updated size exceeds maxSize (or cannot fit alongside entries more recent than key), +// the entry itself is evicted immediately without evicting older entries. Otherwise, least recently used +// entries are evicted to ensure the size invariant holds. // // Returns ErrEntryNotExist if key is not present in the cache. +// Returns ErrInvalidUpdateEntrySize if sizeDelta causes uint64 integer overflow. func (c *mapCache) UpdateSize(key string, sizeDelta uint64) error { - c.lock() + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) defer c.unlock() e, ok := c.index[key] @@ -365,18 +451,80 @@ func (c *mapCache) UpdateSize(key string, sizeDelta uint64) error { return ErrEntryNotExist } - entryVal := e.Value.(entry) - if math.MaxUint64-entryVal.size < sizeDelta || math.MaxUint64-c.currentSize < sizeDelta { + entryVal := e.Value.(*entry) + if math.MaxUint64-entryVal.size < sizeDelta { return ErrInvalidUpdateEntrySize } - entryVal.size += sizeDelta - e.Value = entryVal + if entryVal.size+sizeDelta > c.maxSize { + sizeBefore := c.currentSize + c.eraseInternal(entryVal.key) + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) + return nil + } - c.currentSize += sizeDelta - for c.currentSize > c.maxSize && c.entries.Len() > 0 { + avail := c.maxSize - c.currentSize + if sizeDelta > avail { + maxNewer := c.maxSize - (entryVal.size + sizeDelta) + var newerSize uint64 + headCurr := c.entries.Front() + tailCurr := c.entries.Back() + canFit := false + for { + if headCurr == e { + canFit = newerSize <= maxNewer + break + } + if headCurr != nil { + newerSize += headCurr.Value.(*entry).size + if newerSize > maxNewer { + canFit = false + break + } + headCurr = headCurr.Next() + } + if tailCurr == nil || tailCurr == e { + canFit = sizeDelta <= avail + break + } + avail += tailCurr.Value.(*entry).size + if sizeDelta <= avail { + canFit = true + break + } + tailCurr = tailCurr.Prev() + } + if !canFit { + sizeBefore := c.currentSize + c.eraseInternal(entryVal.key) + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) + return nil + } + } + + sizeBefore := c.currentSize + evictedAny := false + reclaimedPreUpdate := false + for sizeDelta > c.maxSize-c.currentSize && c.entries.Len() > 0 { + if c.entries.Back() == e { + break + } c.evictOne() + evictedAny = true + } + + c.onEntrySizeUpdated(entryVal.size, entryVal.size+sizeDelta) + entryVal.size += sizeDelta + c.currentSize += sizeDelta + if c.shouldReclaimSingleSurvivorOnMutation(c.entries.Len(), c.dirtyIndex, false, evictedAny, c.currentSize, sizeBefore, pressure) { + reclaimedPreUpdate = true + } + + protectedElem := &foregroundNoProtectElem + if e == c.entries.Front() { + protectedElem = e } + c.finishMutationReclaimLocked(nil, protectedElem, reclaimedPreUpdate, sizeBefore, sampledEpoch, pressure) return nil } @@ -384,21 +532,55 @@ func (c *mapCache) UpdateSize(key string, sizeDelta uint64) error { // EraseEntriesWithGivenPrefix removes all entries from the cache whose keys start with prefix. // If prefix is empty (""), all entries in the cache are erased. func (c *mapCache) EraseEntriesWithGivenPrefix(prefix string) { - c.lock() - defer c.unlock() - if prefix == "" { + c.mu.Lock() + defer c.unlock() + + hadEntries := c.entries.Len() > 0 + hadDirtySlack := c.dirtyIndex || c.peakEntryLen > 8 || c.deletedSinceCompact > 0 + if !hadEntries && !hadDirtySlack && c.peakEntryLen == 0 { + return + } + hadReclaimable := c.currentSize > 0 || hadDirtySlack + for e := c.entries.Front(); e != nil; { + next := e.Next() + if entryVal, ok := e.Value.(*entry); ok && entryVal != nil { + entryVal.key = "" + entryVal.value = nil + entryVal.size = 0 + } + c.entries.Remove(e) + e.Value = nil + e = next + } c.entries.Init() - c.index = make(map[string]*list.Element) c.currentSize = 0 + c.clearEmptyIndexStateLocked() + if hadReclaimable && c.hasElevatedPressureToInvalidate(0.0) { + c.markReclaimedLocked() + } return } + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) + defer c.unlock() + + sizeBefore := c.currentSize + erasedAny := false for key := range c.index { if strings.HasPrefix(key, prefix) { c.eraseInternal(key) + erasedAny = true } } + if !erasedAny { + return + } + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) +} + +func (c *mapCache) shouldAutoCompactLocked(protectedElem *list.Element) bool { + return c.shouldAutoCompactEntryCounts(c.dirtyIndex, protectedElem == nil, len(c.index)) } // Compact reallocates the internal hash index to reclaim Go map bucket slack while preserving all live entries. @@ -408,50 +590,109 @@ func (c *mapCache) Compact() { c.compactLocked() } -func (c *mapCache) compactLocked() { - newIndex := make(map[string]*list.Element, len(c.index)) - for k, v := range c.index { - newIndex[k] = v +func (c *mapCache) compactDataStructuresLocked() bool { + if !c.dirtyIndex { + return false } + newIndex := make(map[string]*list.Element, len(c.index)) + maps.Copy(newIndex, c.index) c.index = newIndex + c.dirtyIndex = false + c.onCompacted(len(c.index)) + return true } -// EvaluateMemoryPressure samples the configured memory-pressure probe and executes -// Tier 2 (LRU shedding + map compaction) or Tier 1 (lossless map compaction) if thresholds are met. -func (c *mapCache) EvaluateMemoryPressure() []ValueType { - var pressure float64 - if c.options.PressureFunc != nil { - pressure = c.options.PressureFunc() - if math.IsNaN(pressure) || pressure < 0.0 { - pressure = 0.0 - } +func (c *mapCache) compactLocked() { + if c.compactDataStructuresLocked() { + c.markReclaimedLocked() + } +} + +func (c *mapCache) shedAndCompactLocked(targetSize uint64, retention float64, protectedElem *list.Element) []ValueType { + autoCompactElem := protectedElem + if protectedElem != nil && (protectedElem != c.entries.Front() || protectedElem.Prev() != nil) { + protectedElem = nil } - c.lock() - defer c.unlock() + var protectedSize uint64 + hasProtected := protectedElem != nil + if hasProtected { + protectedSize = protectedElem.Value.(*entry).size + } + effectiveTarget, targetLen, targetZeroCount, unprotectedZeroTarget := c.computeShedTargets(targetSize, retention, c.entries.Len(), hasProtected, protectedSize) - if pressure >= c.options.EvictionThreshold { - retention := c.options.EvictionRetentionRatio - targetSize := computeTargetSize(c.maxSize, retention) - targetLen := 0 - if c.currentSize == 0 && c.entries.Len() > 0 && retention > 0.0 { - targetLen = int(float64(c.entries.Len()) * retention) + needFullFlush := retention == 0.0 + var evicted []ValueType + victim := c.entries.Back() + for victim != nil { + needByteShed := c.currentSize > effectiveTarget || needFullFlush + needZeroShed := !needFullFlush && c.zeroSizeCount > targetZeroCount && c.entries.Len() > targetLen + if !needByteShed && !needZeroShed { + break } - var evicted []ValueType - for c.entries.Len() > 0 { - needByteShed := c.currentSize > targetSize - needZeroSizeShed := c.currentSize == 0 && c.entries.Len() > targetLen - needFullFlush := retention == 0.0 - if !needByteShed && !needZeroSizeShed && !needFullFlush { - break + switch { + case needFullFlush || (needByteShed && c.zeroSizeCount > targetZeroCount): + for victim != nil && victim == protectedElem { + victim = victim.Prev() + } + case needByteShed: + for victim != nil && (victim == protectedElem || victim.Value.(*entry).size == 0) { + victim = victim.Prev() + } + default: + for victim != nil && (victim == protectedElem || victim.Value.(*entry).size > 0) { + victim = victim.Prev() } - evicted = append(evicted, c.evictOne()) } - c.compactLocked() + if victim == nil { + break + } + nextVictim := victim.Prev() + if val := c.eraseInternal(victim.Value.(*entry).key); val != nil { + evicted = append(evicted, val) + } + victim = nextVictim + } + + if len(evicted) > 0 && c.entries.Len() == 0 { + c.resetEmptyIndexLocked() return evicted } - if pressure >= c.options.CompactionThreshold { + + if len(evicted) > 0 { + c.updateZeroWatermarkAfterShed(retention, c.entries.Len(), !hasProtected || effectiveTarget <= targetSize, unprotectedZeroTarget, targetLen) + } + compacted := false + if c.shouldAutoCompactLocked(autoCompactElem) { + compacted = c.compactDataStructuresLocked() + } + if len(evicted) > 0 || compacted { + c.markReclaimedLocked() + } + return evicted +} + +func (c *mapCache) maybeReclaimUnderPressureLocked(pressure float64, protectedElem *list.Element) []ValueType { + if c.isSamplingGoroutine() { + return nil + } + if pressure >= c.options.EvictionThreshold { + retention := c.options.EvictionRetentionRatio + targetSize := computeTargetSize(c.maxSize, retention) + return c.shedAndCompactLocked(targetSize, retention, protectedElem) + } + c.resetZeroWatermarkBelowTier2(pressure) + if pressure >= c.options.CompactionThreshold && c.shouldAutoCompactLocked(protectedElem) { c.compactLocked() } return nil } + +// EvaluateMemoryPressure samples the configured memory-pressure probe and executes +// Tier 2 (LRU shedding + map compaction) or Tier 1 (lossless map compaction) if thresholds are met. +func (c *mapCache) EvaluateMemoryPressure() []ValueType { + _, pressure := c.lockWithPressure(&c.mu, true) + defer c.unlock() + + return c.maybeReclaimUnderPressureLocked(pressure, nil) +} diff --git a/map_lru_test.go b/map_lru_test.go index cb47670..d078ffb 100644 --- a/map_lru_test.go +++ b/map_lru_test.go @@ -12,19 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus +package lru import ( - "errors" "fmt" "testing" -) -const ( - testMaxSize = 50 - testOperationCount = 100 + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +const testMaxSize = 50 + type testData struct { value int64 dataSize uint64 @@ -39,585 +38,707 @@ func setupCacheTest(t *testing.T) Cache { return NewMapCache(testMaxSize, WithInvariantChecking(true)) } -// insertAndAssert inserts key, val into the cache and asserts expected eviction list and error. -func insertAndAssert(t *testing.T, cache Cache, key string, val ValueType, evictedValues []int64, expectedError error) { +func assertEvictedValues(t *testing.T, evicted []ValueType, expectedValues []int64) { t.Helper() - ret, err := cache.Insert(key, val) - - if expectedError != nil { - if !errors.Is(err, expectedError) { - t.Fatalf("expected error %v, got %v", expectedError, err) - } - } else if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if len(evictedValues) != len(ret) { - t.Fatalf("eviction count mismatch: expected %d, got %d", len(evictedValues), len(ret)) + if len(expectedValues) == 0 { + assert.Empty(t, evicted) + return } - for i, evicted := range ret { - td, ok := evicted.(testData) - if !ok { - t.Fatalf("evicted value at index %d is not testData: %T", i, evicted) - } - if td.value != evictedValues[i] { - t.Errorf("evicted value at index %d: expected %d, got %d", i, evictedValues[i], td.value) - } + require.Len(t, evicted, len(expectedValues)) + for i, exp := range expectedValues { + td, ok := evicted[i].(testData) + require.True(t, ok, "evicted value at index %d is not testData: %T", i, evicted[i]) + assert.Equal(t, exp, td.value) } } -func TestLookUpInEmptyCache(t *testing.T) { +func TestMapCache_LookUpInEmptyCache(t *testing.T) { + // Arrange cache := setupCacheTest(t) - if val := cache.LookUp(""); val != nil { - t.Errorf("expected nil, got %v", val) - } - if val := cache.LookUp("taco"); val != nil { - t.Errorf("expected nil, got %v", val) - } + + // Act + valEmpty := cache.LookUp("") + valTaco := cache.LookUp("taco") + + // Assert + assert.Nil(t, valEmpty) + assert.Nil(t, valTaco) } -func TestInsertNilValue(t *testing.T) { +func TestMapCache_InsertNilValue(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "taco", nil, nil, ErrInvalidEntry) + + // Act + evicted, err := cache.Insert("taco", nil) + + // Assert + require.ErrorIs(t, err, ErrInvalidEntry) + assertEvictedValues(t, evicted, nil) } -func TestLookUpUnknownKey(t *testing.T) { +func TestMapCache_LookUpUnknownKey(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "burrito", testData{value: 23, dataSize: 4}, nil, nil) - insertAndAssert(t, cache, "taco", testData{value: 23, dataSize: 8}, nil, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - if val := cache.LookUp(""); val != nil { - t.Errorf("expected nil, got %v", val) - } - if val := cache.LookUp("enchilada"); val != nil { - t.Errorf("expected nil, got %v", val) - } + evicted, err = cache.Insert("taco", testData{value: 23, dataSize: 8}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + + // Act + valEmpty := cache.LookUp("") + valEnchilada := cache.LookUp("enchilada") + + // Assert + assert.Nil(t, valEmpty) + assert.Nil(t, valEnchilada) } -func TestFillUpToCapacity(t *testing.T) { +func TestMapCache_FillUpToCapacity(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "burrito", testData{value: 23, dataSize: 4}, nil, nil) - insertAndAssert(t, cache, "taco", testData{value: 26, dataSize: 20}, nil, nil) - insertAndAssert(t, cache, "enchilada", testData{value: 28, dataSize: 26}, nil, nil) - if val := cache.LookUp("burrito"); val == nil || val.(testData).value != 23 { - t.Errorf("burrito: expected 23, got %v", val) - } - if val := cache.LookUp("taco"); val == nil || val.(testData).value != 26 { - t.Errorf("taco: expected 26, got %v", val) - } - if val := cache.LookUp("enchilada"); val == nil || val.(testData).value != 28 { - t.Errorf("enchilada: expected 28, got %v", val) - } + // Act + evicted1, err1 := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + evicted2, err2 := cache.Insert("taco", testData{value: 26, dataSize: 20}) + evicted3, err3 := cache.Insert("enchilada", testData{value: 28, dataSize: 26}) + + // Assert + require.NoError(t, err1) + assertEvictedValues(t, evicted1, nil) + require.NoError(t, err2) + assertEvictedValues(t, evicted2, nil) + require.NoError(t, err3) + assertEvictedValues(t, evicted3, nil) + + assert.Equal(t, testData{value: 23, dataSize: 4}, cache.LookUp("burrito")) + assert.Equal(t, testData{value: 26, dataSize: 20}, cache.LookUp("taco")) + assert.Equal(t, testData{value: 28, dataSize: 26}, cache.LookUp("enchilada")) } -func TestExpiresLeastRecentlyUsed(t *testing.T) { +func TestMapCache_ExpiresLeastRecentlyUsed(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "burrito", testData{value: 23, dataSize: 4}, nil, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) // Least recent. - insertAndAssert(t, cache, "taco", testData{value: 26, dataSize: 20}, nil, nil) + evicted, err = cache.Insert("taco", testData{value: 26, dataSize: 20}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) // Second most recent. - insertAndAssert(t, cache, "enchilada", testData{value: 28, dataSize: 26}, nil, nil) + evicted, err = cache.Insert("enchilada", testData{value: 28, dataSize: 26}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - if val := cache.LookUp("burrito"); val == nil || val.(testData).value != 23 { - t.Errorf("burrito: expected 23, got %v", val) - } // burrito is now most recent + // Promote burrito to MRU. + assert.Equal(t, testData{value: 23, dataSize: 4}, cache.LookUp("burrito")) - // Insert another, requiring eviction of taco (size 20) to fit queso (size 5). - insertAndAssert(t, cache, "queso", testData{value: 34, dataSize: 5}, []int64{26}, nil) + // Act: Insert another, requiring eviction of taco (size 20) to fit queso (size 5). + evicted, err = cache.Insert("queso", testData{value: 34, dataSize: 5}) - // See what's left. - if val := cache.LookUp("taco"); val != nil { - t.Errorf("expected taco to be evicted, got %v", val) - } - if val := cache.LookUp("burrito"); val == nil || val.(testData).value != 23 { - t.Errorf("burrito: expected 23, got %v", val) - } - if val := cache.LookUp("enchilada"); val == nil || val.(testData).value != 28 { - t.Errorf("enchilada: expected 28, got %v", val) - } - if val := cache.LookUp("queso"); val == nil || val.(testData).value != 34 { - t.Errorf("queso: expected 34, got %v", val) - } + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{26}) + assert.Nil(t, cache.LookUp("taco")) + assert.Equal(t, testData{value: 23, dataSize: 4}, cache.LookUp("burrito")) + assert.Equal(t, testData{value: 28, dataSize: 26}, cache.LookUp("enchilada")) + assert.Equal(t, testData{value: 34, dataSize: 5}, cache.LookUp("queso")) } -func TestOverwrite(t *testing.T) { +func TestMapCache_Overwrite(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "burrito", testData{value: 23, dataSize: 4}, nil, nil) - insertAndAssert(t, cache, "taco", testData{value: 26, dataSize: 20}, nil, nil) - insertAndAssert(t, cache, "enchilada", testData{value: 28, dataSize: 20}, nil, nil) - insertAndAssert(t, cache, "burrito", testData{value: 33, dataSize: 6}, nil, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - // Increase the DataSize while modifying, so eviction of taco should happen. - insertAndAssert(t, cache, "burrito", testData{value: 33, dataSize: 12}, []int64{26}, nil) + evicted, err = cache.Insert("taco", testData{value: 26, dataSize: 20}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - if val := cache.LookUp("taco"); val != nil { - t.Errorf("expected taco to be evicted, got %v", val) - } - if val := cache.LookUp("burrito"); val == nil || val.(testData).value != 33 { - t.Errorf("burrito: expected 33, got %v", val) - } - if val := cache.LookUp("enchilada"); val == nil || val.(testData).value != 28 { - t.Errorf("enchilada: expected 28, got %v", val) - } + evicted, err = cache.Insert("enchilada", testData{value: 28, dataSize: 20}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + + evicted, err = cache.Insert("burrito", testData{value: 33, dataSize: 6}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + + // Act: Increase the DataSize while modifying, so eviction of taco should happen. + evicted, err = cache.Insert("burrito", testData{value: 33, dataSize: 12}) + + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{26}) + assert.Nil(t, cache.LookUp("taco")) + assert.Equal(t, testData{value: 33, dataSize: 12}, cache.LookUp("burrito")) + assert.Equal(t, testData{value: 28, dataSize: 20}, cache.LookUp("enchilada")) } -func TestMultipleEviction(t *testing.T) { +func TestMapCache_MultipleEviction(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "burrito", testData{value: 23, dataSize: 4}, nil, nil) - insertAndAssert(t, cache, "taco", testData{value: 26, dataSize: 20}, nil, nil) - insertAndAssert(t, cache, "enchilada", testData{value: 28, dataSize: 20}, nil, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - // Inserting large entry requires evicting burrito, taco, and enchilada in oldest-first order. - insertAndAssert(t, cache, "large_data", testData{value: 33, dataSize: 45}, []int64{23, 26, 28}, nil) + evicted, err = cache.Insert("taco", testData{value: 26, dataSize: 20}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - if val := cache.LookUp("taco"); val != nil { - t.Errorf("expected taco to be evicted, got %v", val) - } - if val := cache.LookUp("burrito"); val != nil { - t.Errorf("expected burrito to be evicted, got %v", val) - } - if val := cache.LookUp("enchilada"); val != nil { - t.Errorf("expected enchilada to be evicted, got %v", val) - } - if val := cache.LookUp("large_data"); val == nil || val.(testData).value != 33 { - t.Errorf("large_data: expected 33, got %v", val) - } + evicted, err = cache.Insert("enchilada", testData{value: 28, dataSize: 20}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + + // Act: Inserting large entry requires evicting burrito, taco, and enchilada in oldest-first order. + evicted, err = cache.Insert("large_data", testData{value: 33, dataSize: 45}) + + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{23, 26, 28}) + assert.Nil(t, cache.LookUp("taco")) + assert.Nil(t, cache.LookUp("burrito")) + assert.Nil(t, cache.LookUp("enchilada")) + assert.Equal(t, testData{value: 33, dataSize: 45}, cache.LookUp("large_data")) } -func TestWhenEntrySizeMoreThanCacheMaxSize(t *testing.T) { +func TestMapCache_WhenEntrySizeMoreThanCacheMaxSize(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "burrito", testData{value: 23, dataSize: 4}, nil, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - // Insert entry with size greater than maxSize of cache. - insertAndAssert(t, cache, "taco", testData{value: 26, dataSize: testMaxSize + 1}, nil, ErrInvalidEntrySize) + // Act: Insert entry with size greater than maxSize of cache. + evicted, err = cache.Insert("taco", testData{value: 26, dataSize: testMaxSize + 1}) - if val := cache.LookUp("burrito"); val == nil || val.(testData).value != 23 { - t.Errorf("burrito: expected 23, got %v", val) - } + // Assert + require.ErrorIs(t, err, ErrInvalidEntrySize) + assertEvictedValues(t, evicted, nil) + assert.Equal(t, testData{value: 23, dataSize: 4}, cache.LookUp("burrito")) } -func TestEraseWhenKeyPresent(t *testing.T) { +func TestMapCache_EraseWhenKeyPresent(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "burrito", testData{value: 23, dataSize: 4}, nil, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + // Act deletedEntry := cache.Erase("burrito") - if deletedEntry == nil || deletedEntry.(testData).value != 23 { - t.Errorf("expected erased value 23, got %v", deletedEntry) - } - if val := cache.LookUp("burrito"); val != nil { - t.Errorf("expected nil after erase, got %v", val) - } + + // Assert + assert.Equal(t, testData{value: 23, dataSize: 4}, deletedEntry) + assert.Nil(t, cache.LookUp("burrito")) } -func TestEraseCacheWithGivenPrefix(t *testing.T) { +func TestMapCache_EraseCacheWithGivenPrefix(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "a", testData{value: 23, dataSize: 4}, nil, nil) - insertAndAssert(t, cache, "a/b", testData{value: 26, dataSize: 5}, nil, nil) - insertAndAssert(t, cache, "a/b/d", testData{value: 22, dataSize: 6}, nil, nil) - insertAndAssert(t, cache, "a/c", testData{value: 20, dataSize: 6}, nil, nil) - insertAndAssert(t, cache, "b", testData{value: 21, dataSize: 2}, nil, nil) - + _, err := cache.Insert("a", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("a/b/d", testData{value: 22, dataSize: 6}) + require.NoError(t, err) + _, err = cache.Insert("a/c", testData{value: 20, dataSize: 6}) + require.NoError(t, err) + _, err = cache.Insert("b", testData{value: 21, dataSize: 2}) + require.NoError(t, err) + + // Act cache.EraseEntriesWithGivenPrefix("a") - if val := cache.LookUp("a"); val != nil { - t.Errorf("expected nil for a, got %v", val) - } - if val := cache.LookUp("a/b"); val != nil { - t.Errorf("expected nil for a/b, got %v", val) - } - if val := cache.LookUp("a/b/d"); val != nil { - t.Errorf("expected nil for a/b/d, got %v", val) - } - if val := cache.LookUp("a/c"); val != nil { - t.Errorf("expected nil for a/c, got %v", val) - } - if val := cache.LookUp("b"); val == nil || val.Size() != 2 { - t.Errorf("expected b size 2, got %v", val) - } + // Assert + assert.Nil(t, cache.LookUp("a")) + assert.Nil(t, cache.LookUp("a/b")) + assert.Nil(t, cache.LookUp("a/b/d")) + assert.Nil(t, cache.LookUp("a/c")) + valB := cache.LookUp("b") + require.NotNil(t, valB) + assert.Equal(t, uint64(2), valB.Size()) } -func TestEraseCacheWhereNoEntriesExistWithGivenPrefix(t *testing.T) { +func TestMapCache_EraseCacheWhereNoEntriesExistWithGivenPrefix(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "a", testData{value: 23, dataSize: 4}, nil, nil) - insertAndAssert(t, cache, "a/b", testData{value: 26, dataSize: 5}, nil, nil) - insertAndAssert(t, cache, "b", testData{value: 21, dataSize: 2}, nil, nil) - + _, err := cache.Insert("a", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("b", testData{value: 21, dataSize: 2}) + require.NoError(t, err) + + // Act cache.EraseEntriesWithGivenPrefix("c") - if val := cache.LookUp("a"); val == nil || val.Size() != 4 { - t.Errorf("expected a size 4, got %v", val) - } - if val := cache.LookUp("a/b"); val == nil || val.Size() != 5 { - t.Errorf("expected a/b size 5, got %v", val) - } - if val := cache.LookUp("b"); val == nil || val.Size() != 2 { - t.Errorf("expected b size 2, got %v", val) - } + // Assert + valA := cache.LookUp("a") + require.NotNil(t, valA) + assert.Equal(t, uint64(4), valA.Size()) + + valAB := cache.LookUp("a/b") + require.NotNil(t, valAB) + assert.Equal(t, uint64(5), valAB.Size()) + + valB := cache.LookUp("b") + require.NotNil(t, valB) + assert.Equal(t, uint64(2), valB.Size()) } -func TestEraseCacheWithGivenPrefixWithSomeEntriesEvictedDueToCacheSize(t *testing.T) { +func TestMapCache_EraseCacheWithGivenPrefixWithSomeEntriesEvictedDueToCacheSize(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "a", testData{value: 23, dataSize: 20}, nil, nil) - insertAndAssert(t, cache, "a/b", testData{value: 26, dataSize: 10}, nil, nil) - insertAndAssert(t, cache, "a/b/d", testData{value: 22, dataSize: 5}, nil, nil) - insertAndAssert(t, cache, "a/c", testData{value: 20, dataSize: 10}, nil, nil) - insertAndAssert(t, cache, "b", testData{value: 21, dataSize: 15}, []int64{23}, nil) - - // As entry "a" was already evicted by the insertion of "b", only three entries will be removed. + _, err := cache.Insert("a", testData{value: 23, dataSize: 20}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 10}) + require.NoError(t, err) + _, err = cache.Insert("a/b/d", testData{value: 22, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("a/c", testData{value: 20, dataSize: 10}) + require.NoError(t, err) + evicted, err := cache.Insert("b", testData{value: 21, dataSize: 15}) + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{23}) + + // Act: As entry "a" was already evicted by the insertion of "b", only three entries will be removed. cache.EraseEntriesWithGivenPrefix("a") - if val := cache.LookUp("a"); val != nil { - t.Errorf("expected nil for a, got %v", val) - } - if val := cache.LookUp("a/b"); val != nil { - t.Errorf("expected nil for a/b, got %v", val) - } - if val := cache.LookUp("a/b/d"); val != nil { - t.Errorf("expected nil for a/b/d, got %v", val) - } - if val := cache.LookUp("a/c"); val != nil { - t.Errorf("expected nil for a/c, got %v", val) - } - if val := cache.LookUp("b"); val == nil || val.Size() != 15 { - t.Errorf("expected b size 15, got %v", val) - } + // Assert + assert.Nil(t, cache.LookUp("a")) + assert.Nil(t, cache.LookUp("a/b")) + assert.Nil(t, cache.LookUp("a/b/d")) + assert.Nil(t, cache.LookUp("a/c")) + valB := cache.LookUp("b") + require.NotNil(t, valB) + assert.Equal(t, uint64(15), valB.Size()) } -func TestEraseCacheWithEmptyPrefix(t *testing.T) { +func TestMapCache_EraseCacheWithEmptyPrefix(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "a", testData{value: 1, dataSize: 10}, nil, nil) - insertAndAssert(t, cache, "b", testData{value: 2, dataSize: 10}, nil, nil) - insertAndAssert(t, cache, "c", testData{value: 3, dataSize: 10}, nil, nil) - + _, err := cache.Insert("a", testData{value: 1, dataSize: 10}) + require.NoError(t, err) + _, err = cache.Insert("b", testData{value: 2, dataSize: 10}) + require.NoError(t, err) + _, err = cache.Insert("c", testData{value: 3, dataSize: 10}) + require.NoError(t, err) + + // Act cache.EraseEntriesWithGivenPrefix("") - if val := cache.LookUp("a"); val != nil { - t.Errorf("expected nil for a, got %v", val) - } - if val := cache.LookUp("b"); val != nil { - t.Errorf("expected nil for b, got %v", val) - } - if val := cache.LookUp("c"); val != nil { - t.Errorf("expected nil for c, got %v", val) - } + // Assert + assert.Nil(t, cache.LookUp("a")) + assert.Nil(t, cache.LookUp("b")) + assert.Nil(t, cache.LookUp("c")) } -func TestEraseWhenKeyNotPresent(t *testing.T) { +func TestMapCache_EraseWhenKeyNotPresent(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "burrito", testData{value: 23, dataSize: 4}, nil, nil) + _, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + // Act deletedEntry := cache.Erase("taco") - if deletedEntry != nil { - t.Errorf("expected nil when erasing non-existent key, got %v", deletedEntry) - } - if val := cache.LookUp("burrito"); val == nil || val.(testData).value != 23 { - t.Errorf("burrito: expected 23, got %v", val) - } + // Assert + assert.Nil(t, deletedEntry) + assert.Equal(t, testData{value: 23, dataSize: 4}, cache.LookUp("burrito")) } -func TestUpdateWhenKeyPresent(t *testing.T) { +func TestMapCache_UpdateWhenKeyPresent(t *testing.T) { + // Arrange cache := setupCacheTest(t) key := "burrito" data := testData{value: 23, dataSize: 4} - insertAndAssert(t, cache, key, data, nil, nil) + _, err := cache.Insert(key, data) + require.NoError(t, err) newData := testData{value: 2, dataSize: 4} - err := cache.UpdateWithoutChangingOrder(key, newData) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if val := cache.LookUp(key); val == nil || val.(testData).value != 2 { - t.Errorf("expected updated value 2, got %v", val) - } + // Act + err = cache.UpdateWithoutChangingOrder(key, newData) + + // Assert + require.NoError(t, err) + assert.Equal(t, newData, cache.LookUp(key)) } -func TestUpdateWhenKeyNotPresent(t *testing.T) { +func TestMapCache_UpdateWhenKeyNotPresent(t *testing.T) { + // Arrange cache := setupCacheTest(t) key := "burrito" data := testData{value: 23, dataSize: 4} + // Act err := cache.UpdateWithoutChangingOrder(key, data) - if !errors.Is(err, ErrEntryNotExist) { - t.Errorf("expected ErrEntryNotExist, got %v", err) - } + + // Assert + require.ErrorIs(t, err, ErrEntryNotExist) } -func TestUpdateNilValue(t *testing.T) { +func TestMapCache_UpdateNilValue(t *testing.T) { + // Arrange cache := setupCacheTest(t) key := "burrito" data := testData{value: 23, dataSize: 4} - insertAndAssert(t, cache, key, data, nil, nil) + _, err := cache.Insert(key, data) + require.NoError(t, err) - err := cache.UpdateWithoutChangingOrder(key, nil) - if !errors.Is(err, ErrInvalidEntry) { - t.Errorf("expected ErrInvalidEntry, got %v", err) - } + // Act + err = cache.UpdateWithoutChangingOrder(key, nil) + + // Assert + require.ErrorIs(t, err, ErrInvalidEntry) } -func TestUpdateWhenSizeIsDifferent(t *testing.T) { +func TestMapCache_UpdateWhenSizeIsDifferent(t *testing.T) { + // Arrange cache := setupCacheTest(t) key := "burrito" data := testData{value: 23, dataSize: 4} - insertAndAssert(t, cache, key, data, nil, nil) + _, err := cache.Insert(key, data) + require.NoError(t, err) newData := testData{value: 2, dataSize: 3} - err := cache.UpdateWithoutChangingOrder(key, newData) - if !errors.Is(err, ErrInvalidUpdateEntrySize) { - t.Errorf("expected ErrInvalidUpdateEntrySize, got %v", err) - } + // Act + err = cache.UpdateWithoutChangingOrder(key, newData) + + // Assert + require.ErrorIs(t, err, ErrInvalidUpdateEntrySize) } -func TestUpdateNotChangeOrder(t *testing.T) { +func TestMapCache_UpdateNotChangeOrder(t *testing.T) { + // Arrange cache := setupCacheTest(t) key1 := "burrito1" data1 := testData{value: 23, dataSize: 10} - insertAndAssert(t, cache, key1, data1, nil, nil) + _, err := cache.Insert(key1, data1) + require.NoError(t, err) + key2 := "burrito2" data2 := testData{value: 2, dataSize: 40} - insertAndAssert(t, cache, key2, data2, nil, nil) + _, err = cache.Insert(key2, data2) + require.NoError(t, err) + // Act newData := testData{value: 7, dataSize: 10} - err := cache.UpdateWithoutChangingOrder(key1, newData) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + err = cache.UpdateWithoutChangingOrder(key1, newData) + require.NoError(t, err) // Inserting again should evict key1 because key1 was updated without changing order (still LRU). key3 := "burrito3" data3 := testData{value: 3, dataSize: 5} - insertAndAssert(t, cache, key3, data3, []int64{7}, nil) + evicted, err := cache.Insert(key3, data3) + + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{7}) } -func TestLookUpWithoutChangingOrder_WhenKeyPresent(t *testing.T) { +func TestMapCache_LookUpWithoutChangingOrder_WhenKeyPresent(t *testing.T) { + // Arrange cache := setupCacheTest(t) key := "burrito" data := testData{value: 23, dataSize: 4} - insertAndAssert(t, cache, key, data, nil, nil) + _, err := cache.Insert(key, data) + require.NoError(t, err) + // Act value := cache.LookUpWithoutChangingOrder(key) - if value == nil || value.(testData).value != 23 { - t.Errorf("expected 23, got %v", value) - } + + // Assert + assert.Equal(t, data, value) } -func TestLookUpWithoutChangingOrder_WhenKeyNotPresent(t *testing.T) { +func TestMapCache_LookUpWithoutChangingOrder_WhenKeyNotPresent(t *testing.T) { + // Arrange cache := setupCacheTest(t) key := "burrito" + // Act value := cache.LookUpWithoutChangingOrder(key) - if value != nil { - t.Errorf("expected nil, got %v", value) - } + + // Assert + assert.Nil(t, value) } -func TestLookUpWithoutChangingOrder_NotChangeOrder(t *testing.T) { +func TestMapCache_LookUpWithoutChangingOrder_NotChangeOrder(t *testing.T) { + // Arrange cache := setupCacheTest(t) key1 := "burrito1" data1 := testData{value: 23, dataSize: 10} - insertAndAssert(t, cache, key1, data1, nil, nil) + _, err := cache.Insert(key1, data1) + require.NoError(t, err) + key2 := "burrito2" data2 := testData{value: 2, dataSize: 40} - insertAndAssert(t, cache, key2, data2, nil, nil) + _, err = cache.Insert(key2, data2) + require.NoError(t, err) + // Act value := cache.LookUpWithoutChangingOrder(key1) - if value == nil || value.(testData).value != 23 { - t.Errorf("expected 23, got %v", value) - } + assert.Equal(t, data1, value) // Inserting again should evict key1 because key1 was looked up without changing order. key3 := "burrito3" data3 := testData{value: 3, dataSize: 5} - insertAndAssert(t, cache, key3, data3, []int64{23}, nil) + evicted, err := cache.Insert(key3, data3) + + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{23}) } -func TestUpdateSize_Success(t *testing.T) { +func TestMapCache_UpdateSize_Success(t *testing.T) { + // Arrange cache := setupCacheTest(t) - insertAndAssert(t, cache, "file1", testData{value: 10, dataSize: 10}, nil, nil) - insertAndAssert(t, cache, "file2", testData{value: 20, dataSize: 20}, nil, nil) + _, err := cache.Insert("file1", testData{value: 10, dataSize: 10}) + require.NoError(t, err) + _, err = cache.Insert("file2", testData{value: 20, dataSize: 20}) + require.NoError(t, err) - err := cache.UpdateSize("file1", 10) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + // Act + err = cache.UpdateSize("file1", 10) + require.NoError(t, err) // Order should be preserved: file1 is still LRU. // Inserting 20 more units (total size was 10+10+20 = 40, now 40+20 = 60 > 50) evicts file1. - insertAndAssert(t, cache, "file3", testData{value: 30, dataSize: 20}, []int64{10}, nil) + evicted, err := cache.Insert("file3", testData{value: 30, dataSize: 20}) + + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{10}) } -func TestUpdateSize_NonExistentKey(t *testing.T) { +func TestMapCache_UpdateSize_NonExistentKey(t *testing.T) { + // Arrange cache := setupCacheTest(t) + + // Act err := cache.UpdateSize("nonexistent", 10) - if !errors.Is(err, ErrEntryNotExist) { - t.Errorf("expected ErrEntryNotExist, got %v", err) - } + + // Assert + require.ErrorIs(t, err, ErrEntryNotExist) } -func TestUpdateSize_ExceedsMaxSize_Evicts(t *testing.T) { +func TestMapCache_UpdateSize_ExceedsMaxSize_Evicts(t *testing.T) { + // Arrange cache := setupCacheTest(t) // maxSize = 50 - insertAndAssert(t, cache, "key1", testData{value: 1, dataSize: 20}, nil, nil) - insertAndAssert(t, cache, "key2", testData{value: 2, dataSize: 25}, nil, nil) + _, err := cache.Insert("key1", testData{value: 1, dataSize: 20}) + require.NoError(t, err) + _, err = cache.Insert("key2", testData{value: 2, dataSize: 25}) + require.NoError(t, err) - // currentSize was 45. Increasing key2 size by 10 makes currentSize = 55 > 50. + // Act: currentSize was 45. Increasing key2 size by 10 makes currentSize = 55 > 50. // key1 (LRU) must be evicted immediately by UpdateSize to maintain the size invariant. - err := cache.UpdateSize("key2", 10) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if val := cache.LookUp("key1"); val != nil { - t.Errorf("expected key1 to be evicted, got %v", val) - } - if val := cache.LookUp("key2"); val == nil || val.(testData).value != 2 { - t.Errorf("expected key2 to be present with value 2, got %v", val) - } -} - -func TestNewAlias(t *testing.T) { - c := New(100) - if c == nil { - t.Fatal("expected New to return non-nil Cache") - } - _, err := c.Insert("k", testData{value: 1, dataSize: 10}) - if err != nil { - t.Fatalf("unexpected error on Insert: %v", err) - } - if val := c.LookUp("k"); val == nil || val.(testData).value != 1 { - t.Errorf("expected value 1, got %v", val) - } -} - -func TestCheckInvariants_InvalidMaxSizePanic(t *testing.T) { - t.Run("DefaultOptions", func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on zero maxSize with default options") - } - }() - _ = NewMapCache(0) - }) - - t.Run("InvariantsEnabled", func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on zero maxSize with invariant checking enabled") - } - }() - _ = NewMapCache(0, WithInvariantChecking(true)) - }) - - t.Run("InvariantsDisabled", func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on zero maxSize with invariant checking disabled") - } - }() - _ = NewMapCache(0, WithInvariantChecking(false)) - }) + err = cache.UpdateSize("key2", 10) - t.Run("NewAliasDefaultOptions", func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on New(0) with default options") - } - }() - _ = New(0) - }) + // Assert + require.NoError(t, err) + assert.Nil(t, cache.LookUp("key1")) + assert.Equal(t, testData{value: 2, dataSize: 25}, cache.LookUp("key2")) } -func TestCheckInvariants_PanicOnCorruption(t *testing.T) { +// TestMapCache_CheckInvariants_PanicOnCorruption verifies that mapCache.checkInvariants() detects and panics +// on internal data structure corruption across all checked invariants. +// +// White-box testing rationale: +// All public Cache methods maintain internal size, cardinality, pointer, and type invariants. +// Exercising the panic branches inside checkInvariants() therefore requires directly mutating +// unexported mapCache fields (currentSize, index, entries, zeroSizeCount) to inject synthetic +// corruption. Without direct white-box testing of checkInvariants(), a defect in the invariant +// validator itself could mask silent state corruption during development and fuzzing. +func TestMapCache_CheckInvariants_PanicOnCorruption(t *testing.T) { t.Run("CurrentSizeExceedsMaxSize", func(t *testing.T) { + // Arrange c := NewMapCache(10).(*mapCache) c.currentSize = 20 - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on currentSize > maxSize") - } - }() - c.checkInvariants() + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) t.Run("LengthMismatch", func(t *testing.T) { + // Arrange c := NewMapCache(10).(*mapCache) c.index["dummy"] = nil - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on length mismatch") - } - }() - c.checkInvariants() + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) t.Run("KeyMismatch", func(t *testing.T) { + // Arrange c := NewMapCache(50).(*mapCache) - e := c.entries.PushFront(entry{key: "correctKey", value: testData{1, 5}}) + e := c.entries.PushFront(&entry{key: "correctKey", value: testData{1, 5}, size: 5}) c.index["wrongKey"] = e - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on key mismatch") - } - }() - c.checkInvariants() + c.currentSize = 5 + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) t.Run("InvalidElementType", func(t *testing.T) { + // Arrange c := NewMapCache(50).(*mapCache) e := c.entries.PushFront("not-an-entry-struct") c.index["someKey"] = e - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on invalid element type") - } - }() - c.checkInvariants() + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) + }) + + t.Run("ValueTypeEntryInsteadOfPointer", func(t *testing.T) { + // Arrange + c := NewMapCache(100, WithInvariantChecking(true)).(*mapCache) + el := c.entries.PushFront(entry{ + key: "bad_value_type", + value: NewSizedValue("v", 5), + size: 5, + }) + c.index["bad_value_type"] = el + c.currentSize = 5 + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) + }) + + t.Run("ZeroSizeCountMismatch", func(t *testing.T) { + // Arrange + c := NewMapCache(100).(*mapCache) + _, err := c.Insert("z", NewStringValue("")) + require.NoError(t, err) + c.zeroSizeCount = 0 + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) t.Run("SizeSumMismatch", func(t *testing.T) { + // Arrange c := NewMapCache(500).(*mapCache) - _, _ = c.Insert("k1", testData{value: 1, dataSize: 10}) - c.currentSize += 1 - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on size sum mismatch") - } - }() - c.checkInvariants() + _, err := c.Insert("k1", testData{value: 1, dataSize: 10}) + require.NoError(t, err) + c.currentSize++ + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) } -func TestMapCache_EraseEntriesWithGivenPrefix_EmptyPrefixFastPath(t *testing.T) { - c := NewMapCache(1000, WithInvariantChecking(true)).(*mapCache) - +func TestMapCache_Compact(t *testing.T) { + // Arrange + probe := newPressureProbe(0.10) + c := NewMapCache(1000, WithInvariantChecking(true), probe.Option()).(PressureAwareCache) for i := 0; i < 20; i++ { - key := fmt.Sprintf("entry_%d", i) - _, err := c.Insert(key, testData{value: int64(i), dataSize: 10}) - if err != nil { - t.Fatalf("Insert failed: %v", err) - } + _, err := c.Insert(fmt.Sprintf("entry_%d", i), testData{value: int64(i), dataSize: 10}) + require.NoError(t, err) } - if c.currentSize != 200 { - t.Fatalf("expected currentSize 200, got %d", c.currentSize) + + // Act & Assert 1: Clean Compact is a zero-allocation no-op + assertAlreadyCompacted(t, c, probe) + + // Arrange 2: Delete half the entries to dirty the index + for i := 0; i < 10; i++ { + erased := c.Erase(fmt.Sprintf("entry_%d", i)) + require.NotNil(t, erased) } - c.EraseEntriesWithGivenPrefix("") + // Act 2: Compact dirty index (re-allocates map once and advances reclaimEpoch, then subsequent Compact is zero-alloc) + assert.True(t, probe.ObserveEpochAdvance(t, c, c.Compact)) - if c.currentSize != 0 { - t.Errorf("expected currentSize 0 after empty prefix erase, got %d", c.currentSize) + // Assert 2 + assertAlreadyCompacted(t, c, probe) + for i := 0; i < 10; i++ { + assert.Nil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("entry_%d", i))) } - if c.entries.Len() != 0 { - t.Errorf("expected entries.Len() 0, got %d", c.entries.Len()) + for i := 10; i < 20; i++ { + assert.Equal(t, testData{value: int64(i), dataSize: 10}, c.LookUp(fmt.Sprintf("entry_%d", i))) + } +} + +func TestMapCache_EvaluateMemoryPressure(t *testing.T) { + // Arrange + probe := newPressureProbe(0.10) + c := NewMapCache( + 100, + WithInvariantChecking(true), + probe.Option(), + WithCompactionThreshold(0.75), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + for i := 0; i < 10; i++ { + _, err := c.Insert(fmt.Sprintf("k%d", i), testData{value: int64(i), dataSize: 10}) + require.NoError(t, err) } - if len(c.index) != 0 { - t.Errorf("expected len(index) 0, got %d", len(c.index)) + require.NotNil(t, c.Erase("k0")) + require.NotNil(t, c.Erase("k1")) + + // Act & Assert 1: Below CompactionThreshold (0.50 < 0.75) does nothing + probe.Set(0.50) + var evicted []ValueType + advancedBelow := probe.ObserveEpochAdvance(t, c, func() { + evicted = c.EvaluateMemoryPressure() + }) + assert.Empty(t, evicted) + assert.False(t, advancedBelow) + + // Act & Assert 2: Tier 1 CompactionThreshold (0.80 in [0.75, 0.90)) compacts without evicting + probe.Set(0.80) + advancedTier1 := probe.ObserveEpochAdvance(t, c, func() { + evicted = c.EvaluateMemoryPressure() + }) + assert.Empty(t, evicted) + assert.True(t, advancedTier1) + assertAlreadyCompacted(t, c, probe) + for i := 2; i < 10; i++ { + assert.NotNil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k%d", i))) } - // Erasing empty prefix on an already empty cache must be a safe no-op. - c.EraseEntriesWithGivenPrefix("") - if c.currentSize != 0 || c.entries.Len() != 0 || len(c.index) != 0 { - t.Errorf("expected empty cache state after erasing empty cache") + // Act & Assert 3: Tier 2 EvictionThreshold (0.95 >= 0.90) sheds down to targetSize = 50 and compacts + probe.Set(0.95) + advancedTier2 := probe.ObserveEpochAdvance(t, c, func() { + evicted = c.EvaluateMemoryPressure() + }) + assertEvictedValues(t, evicted, []int64{2, 3, 4}) + assert.True(t, advancedTier2) + assertAlreadyCompacted(t, c, probe) + for i := 2; i < 5; i++ { + assert.Nil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k%d", i))) + } + for i := 5; i < 10; i++ { + assert.NotNil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("k%d", i))) } + + // Act & Assert 4: Repeated EvaluateMemoryPressure when clean and at targetSize is a no-op + advancedRepeat := probe.ObserveEpochAdvance(t, c, func() { + evicted = c.EvaluateMemoryPressure() + }) + assert.Empty(t, evicted) + assert.False(t, advancedRepeat) + assertAlreadyCompacted(t, c, probe) } diff --git a/options.go b/options.go index 73787b6..e8192ea 100644 --- a/options.go +++ b/options.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus +package lru import ( "math" @@ -20,6 +20,37 @@ import ( "sync" ) +// Backend identifies the underlying cache data structure engine constructed by New. +type Backend uint8 + +const ( + // BackendMap selects the hash-map + doubly-linked list LRU engine (MapCache). + // This is the default backend when WithBackend is not specified. + BackendMap Backend = iota + + // BackendRadix selects the pointer-based Left-Child Right-Sibling (LCRS) radix tree + // LRU engine (RadixCache), optimized for hierarchical keys and fast prefix eviction. + BackendRadix + + // BackendArenaRadix selects the contiguous-slice 32-bit index arena-backed radix tree + // LRU engine (ArenaRadixCache) with two-tier memory-pressure reclamation. + BackendArenaRadix +) + +// String returns the human-readable name of the cache backend. +func (b Backend) String() string { + switch b { + case BackendMap: + return "MapCache" + case BackendRadix: + return "RadixCache" + case BackendArenaRadix: + return "ArenaRadixCache" + default: + return "UnknownBackend" + } +} + // Default thresholds for two-tier memory-pressure reclamation. const ( // DefaultCompactionThreshold is the normalized memory pressure [0.0, 1.0+] @@ -40,24 +71,40 @@ const ( // indicate moderate pressure, and values >= EvictionThreshold indicate critical pressure. type PressureFunc func() float64 -// metricsSamplePool pools 5-element runtime/metrics sample arrays so DefaultRuntimePressureFunc +// gomemlimitSamplePool pools 1-element runtime/metrics sample arrays so DefaultRuntimePressureFunc(0) +// can check /gc/gomemlimit:bytes without acquiring runtime.metricsLock or sweeping per-P heap stats +// when GOMEMLIMIT is unbounded. +var gomemlimitSamplePool = sync.Pool{ + New: func() any { + return &[1]metrics.Sample{ + {Name: "/gc/gomemlimit:bytes"}, + } + }, +} + +// metricsSamplePool pools 4-element runtime/metrics sample arrays so DefaultRuntimePressureFunc // executes with 0 heap allocations per call (avoiding slice escape to heap in metrics.Read). var metricsSamplePool = sync.Pool{ New: func() any { - return &[5]metrics.Sample{ + return &[4]metrics.Sample{ {Name: "/memory/classes/total:bytes"}, {Name: "/memory/classes/heap/released:bytes"}, {Name: "/memory/classes/heap/free:bytes"}, {Name: "/memory/classes/heap/objects:bytes"}, - {Name: "/gc/gomemlimit:bytes"}, } }, } // Options contains configuration parameters for Cache instances. -// Note: Memory-pressure reclamation options (PressureFunc, MemoryBudget, CompactionThreshold, -// EvictionThreshold, EvictionRetentionRatio) are used by ArenaRadixCache. +// Memory-pressure reclamation options (PressureFunc, MemoryBudget, CompactionThreshold, +// EvictionThreshold, EvictionRetentionRatio) configure both automatic amortized foreground +// reclamation (on Insert, Erase, UpdateSize, and EraseEntriesWithGivenPrefix) and explicit +// EvaluateMemoryPressure() / Compact() calls across ArenaRadixCache, MapCache, and RadixCache. type Options struct { + // Backend selects the underlying cache engine when calling New. + // Defaults to BackendMap. + Backend Backend + // EnableInvariantChecking enables internal data structure integrity and invariant validation. // When enabled, cache operations execute comprehensive validation checks (e.g. bidirectional pointer // consistency, tree structure validity, size accounting parity) and panic if corruption is detected. @@ -82,11 +129,13 @@ type Options struct { // If CompactionThreshold > EvictionThreshold, thresholds are reconciled to preserve ordering. CompactionThreshold float64 hasCustomCompactionThreshold bool + customCompactionBits uint64 // EvictionThreshold specifies the normalized pressure threshold for Tier 2 LRU shedding + compaction. // Defaults to DefaultEvictionThreshold (0.90) if <= 0, NaN, or Inf. EvictionThreshold float64 hasCustomEvictionThreshold bool + customEvictionBits uint64 // EvictionRetentionRatio specifies the fraction [0.0, 1.0] of cache maxSize to retain // when Critical Pressure (EvictionThreshold) is reached. @@ -97,6 +146,14 @@ type Options struct { // Option is a functional option for configuring a Cache instance. type Option func(*Options) +// WithBackend configures the cache engine backend constructed by New. +// Supported backends: BackendMap (default), BackendRadix, BackendArenaRadix. +func WithBackend(backend Backend) Option { + return func(o *Options) { + o.Backend = backend + } +} + // WithInvariantChecking returns an Option that enables or disables internal invariant checking. func WithInvariantChecking(enabled bool) Option { return func(o *Options) { @@ -121,18 +178,34 @@ func WithMemoryBudget(bytes uint64) Option { } // WithCompactionThreshold configures the Moderate Pressure threshold for lossless arena/map compaction. +// Passing a non-positive, NaN, or infinite value resets CompactionThreshold to DefaultCompactionThreshold. func WithCompactionThreshold(threshold float64) Option { return func(o *Options) { + if math.IsNaN(threshold) || math.IsInf(threshold, 0) || threshold <= 0 { + o.CompactionThreshold = DefaultCompactionThreshold + o.hasCustomCompactionThreshold = false + o.customCompactionBits = 0 + return + } o.CompactionThreshold = threshold o.hasCustomCompactionThreshold = true + o.customCompactionBits = math.Float64bits(threshold) } } // WithEvictionThreshold configures the Critical Pressure threshold for proactive LRU shedding. +// Passing a non-positive, NaN, or infinite value resets EvictionThreshold to DefaultEvictionThreshold. func WithEvictionThreshold(threshold float64) Option { return func(o *Options) { + if math.IsNaN(threshold) || math.IsInf(threshold, 0) || threshold <= 0 { + o.EvictionThreshold = DefaultEvictionThreshold + o.hasCustomEvictionThreshold = false + o.customEvictionBits = 0 + return + } o.EvictionThreshold = threshold o.hasCustomEvictionThreshold = true + o.customEvictionBits = math.Float64bits(threshold) } } @@ -149,24 +222,27 @@ func WithEvictionRetentionRatio(ratio float64) Option { // against memoryBudget (if > 0) or /gc/gomemlimit:bytes (if configured < math.MaxInt64). func DefaultRuntimePressureFunc(memoryBudget uint64) PressureFunc { return func() float64 { - samples := metricsSamplePool.Get().(*[5]metrics.Sample) - metrics.Read(samples[:]) - - totalBytes := samples[0].Value.Uint64() - releasedBytes := samples[1].Value.Uint64() - heapFreeBytes := samples[2].Value.Uint64() - heapObjectsBytes := samples[3].Value.Uint64() - gomemlimit := samples[4].Value.Uint64() - metricsSamplePool.Put(samples) - limitBytes := memoryBudget if limitBytes == 0 { + limitSample := gomemlimitSamplePool.Get().(*[1]metrics.Sample) + metrics.Read(limitSample[:]) + gomemlimit := limitSample[0].Value.Uint64() + gomemlimitSamplePool.Put(limitSample) if gomemlimit == 0 || gomemlimit >= uint64(math.MaxInt64) { return 0.0 } limitBytes = gomemlimit } + samples := metricsSamplePool.Get().(*[4]metrics.Sample) + metrics.Read(samples[:]) + + totalBytes := samples[0].Value.Uint64() + releasedBytes := samples[1].Value.Uint64() + heapFreeBytes := samples[2].Value.Uint64() + heapObjectsBytes := samples[3].Value.Uint64() + metricsSamplePool.Put(samples) + var usedBytes uint64 reclaimedOrFree := releasedBytes + heapFreeBytes if totalBytes > reclaimedOrFree { @@ -182,15 +258,26 @@ func DefaultRuntimePressureFunc(memoryBudget uint64) PressureFunc { // ApplyOptions parses and applies the provided slice of Option functions onto a default Options configuration. func ApplyOptions(opts ...Option) Options { options := Options{ + Backend: BackendMap, CompactionThreshold: DefaultCompactionThreshold, EvictionThreshold: DefaultEvictionThreshold, EvictionRetentionRatio: DefaultEvictionRetentionRatio, } + for _, opt := range opts { if opt != nil { opt(&options) } } + if options.Backend != BackendMap && options.Backend != BackendRadix && options.Backend != BackendArenaRadix { + options.Backend = BackendMap + } + if options.hasCustomCompactionThreshold && math.Float64bits(options.CompactionThreshold) != options.customCompactionBits { + options.hasCustomCompactionThreshold = false + } + if options.hasCustomEvictionThreshold && math.Float64bits(options.EvictionThreshold) != options.customEvictionBits { + options.hasCustomEvictionThreshold = false + } if math.IsNaN(options.CompactionThreshold) || math.IsInf(options.CompactionThreshold, 0) || options.CompactionThreshold <= 0 { options.CompactionThreshold = DefaultCompactionThreshold options.hasCustomCompactionThreshold = false @@ -199,27 +286,40 @@ func ApplyOptions(opts ...Option) Options { options.EvictionThreshold = DefaultEvictionThreshold options.hasCustomEvictionThreshold = false } - if options.CompactionThreshold > options.EvictionThreshold { - switch { - case options.hasCustomCompactionThreshold && !options.hasCustomEvictionThreshold: - // Caller raised CompactionThreshold above default EvictionThreshold; advance EvictionThreshold + if options.CompactionThreshold > options.EvictionThreshold || + (options.CompactionThreshold == options.EvictionThreshold && options.hasCustomCompactionThreshold != options.hasCustomEvictionThreshold) { + if options.hasCustomCompactionThreshold && !options.hasCustomEvictionThreshold && options.EvictionThreshold == DefaultEvictionThreshold { + // Caller raised CompactionThreshold at or above default EvictionThreshold; advance EvictionThreshold // to preserve the Tier 1 compaction window. - options.EvictionThreshold = math.Min(1.0, options.CompactionThreshold+(DefaultEvictionThreshold-DefaultCompactionThreshold)) - if options.EvictionThreshold < options.CompactionThreshold { - options.EvictionThreshold = options.CompactionThreshold + options.EvictionThreshold = options.CompactionThreshold + (DefaultEvictionThreshold - DefaultCompactionThreshold) + if math.IsInf(options.EvictionThreshold, 1) { + options.EvictionThreshold = math.MaxFloat64 + } + if options.EvictionThreshold <= options.CompactionThreshold { + if options.CompactionThreshold < math.MaxFloat64 { + options.EvictionThreshold = math.Nextafter(options.CompactionThreshold, math.MaxFloat64) + } else { + options.EvictionThreshold = options.CompactionThreshold + } } - case !options.hasCustomCompactionThreshold && options.hasCustomEvictionThreshold: - // Caller lowered EvictionThreshold below default CompactionThreshold; scale CompactionThreshold - // proportionally to preserve the Tier 1 compaction window. + } else { + // Scale CompactionThreshold proportionally below EvictionThreshold to preserve a non-empty Tier 1 window. options.CompactionThreshold = options.EvictionThreshold * (DefaultCompactionThreshold / DefaultEvictionThreshold) - default: - options.CompactionThreshold = options.EvictionThreshold + if options.CompactionThreshold == 0 { + options.CompactionThreshold = math.SmallestNonzeroFloat64 + } + if options.CompactionThreshold >= options.EvictionThreshold && options.EvictionThreshold > math.SmallestNonzeroFloat64 { + options.CompactionThreshold = math.Nextafter(options.EvictionThreshold, 0) + } } } - if math.IsNaN(options.EvictionRetentionRatio) || math.IsInf(options.EvictionRetentionRatio, -1) || options.EvictionRetentionRatio < 0 { + switch { + case math.IsNaN(options.EvictionRetentionRatio) || math.IsInf(options.EvictionRetentionRatio, -1) || options.EvictionRetentionRatio < 0: options.EvictionRetentionRatio = DefaultEvictionRetentionRatio - } else if math.IsInf(options.EvictionRetentionRatio, 1) || options.EvictionRetentionRatio > 1.0 { + case math.IsInf(options.EvictionRetentionRatio, 1) || options.EvictionRetentionRatio > 1.0: options.EvictionRetentionRatio = 1.0 + case options.EvictionRetentionRatio == 0: + options.EvictionRetentionRatio = 0.0 } if options.PressureFunc == nil { options.PressureFunc = DefaultRuntimePressureFunc(options.MemoryBudget) diff --git a/options_test.go b/options_test.go index 733782a..ee73347 100644 --- a/options_test.go +++ b/options_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus +package lru import ( "math" @@ -33,38 +33,38 @@ func (v testValue) Size() uint64 { } func TestValueTypeInterface(t *testing.T) { + // Arrange var val ValueType = testValue{size: 42} - if val.Size() != 42 { - t.Errorf("expected size 42, got %d", val.Size()) - } + + // Act + size := val.Size() + + // Assert + assert.Equal(t, uint64(42), size) } func TestOptions_Default(t *testing.T) { + // Arrange & Act opts := ApplyOptions() - if opts.EnableInvariantChecking { - t.Errorf("expected EnableInvariantChecking to be false by default, got true") - } + + // Assert + assert.False(t, opts.EnableInvariantChecking) } func TestOptions_WithInvariantChecking(t *testing.T) { + // Arrange & Act optsTrue := ApplyOptions(WithInvariantChecking(true)) - if !optsTrue.EnableInvariantChecking { - t.Errorf("expected EnableInvariantChecking to be true, got false") - } - optsFalse := ApplyOptions(WithInvariantChecking(false)) - if optsFalse.EnableInvariantChecking { - t.Errorf("expected EnableInvariantChecking to be false, got true") - } - - // Chained options and nil resilience. optsChained := ApplyOptions(nil, WithInvariantChecking(false), nil, WithInvariantChecking(true)) - if !optsChained.EnableInvariantChecking { - t.Errorf("expected EnableInvariantChecking to be true after chained options, got false") - } + + // Assert + assert.True(t, optsTrue.EnableInvariantChecking) + assert.False(t, optsFalse.EnableInvariantChecking) + assert.True(t, optsChained.EnableInvariantChecking) } func TestSentinelErrors(t *testing.T) { + // Arrange tests := []struct { err error expected string @@ -87,62 +87,51 @@ func TestSentinelErrors(t *testing.T) { }, } + // Act & Assert for _, tc := range tests { - if tc.err == nil { - t.Errorf("expected non-nil error") - } else if tc.err.Error() != tc.expected { - t.Errorf("expected error message %q, got %q", tc.expected, tc.err.Error()) - } + require.Error(t, tc.err) + assert.EqualError(t, tc.err, tc.expected) } } func TestConstructors_Validation(t *testing.T) { - constructors := []struct { + // Arrange + constructors := append(allBackends(), struct { name string fn func(uint64, ...Option) Cache - }{ - {"NewMapCache", NewMapCache}, - {"New", New}, - {"NewRadixCache", NewRadixCache}, - {"NewArenaRadixCache", NewArenaRadixCache}, - } + }{"New", New}) + // Act & Assert for _, c := range constructors { - t.Run(c.name+"/ZeroMaxSize_DefaultOptions", func(t *testing.T) { - defer func() { - r := recover() - if r == nil { - t.Fatalf("[%s] expected panic on maxSize == 0 with default options, got none", c.name) - } - }() - _ = c.fn(0) - }) - - t.Run(c.name+"/ZeroMaxSize_InvariantsDisabled", func(t *testing.T) { - defer func() { - r := recover() - if r == nil { - t.Fatalf("[%s] expected panic on maxSize == 0 with invariants disabled, got none", c.name) - } - }() - _ = c.fn(0, WithInvariantChecking(false)) - }) - - t.Run(c.name+"/ZeroMaxSize_InvariantsEnabled", func(t *testing.T) { - defer func() { - r := recover() - if r == nil { - t.Fatalf("[%s] expected panic on maxSize == 0 with invariants enabled, got none", c.name) - } - }() - _ = c.fn(0, WithInvariantChecking(true)) - }) - - t.Run(c.name+"/PositiveBoundary_MaxSize1", func(t *testing.T) { - cache := c.fn(1) - if cache == nil { - t.Fatalf("[%s] expected non-nil cache for maxSize == 1", c.name) - } + t.Run(c.name, func(t *testing.T) { + t.Run("ZeroMaxSize_DefaultOptions", func(t *testing.T) { + // Arrange, Act & Assert + assert.Panics(t, func() { + _ = c.fn(0) + }) + }) + + t.Run("ZeroMaxSize_InvariantsDisabled", func(t *testing.T) { + // Arrange, Act & Assert + assert.Panics(t, func() { + _ = c.fn(0, WithInvariantChecking(false)) + }) + }) + + t.Run("ZeroMaxSize_InvariantsEnabled", func(t *testing.T) { + // Arrange, Act & Assert + assert.Panics(t, func() { + _ = c.fn(0, WithInvariantChecking(true)) + }) + }) + + t.Run("PositiveBoundary_MaxSize1", func(t *testing.T) { + // Arrange & Act + cache := c.fn(1) + + // Assert + require.NotNil(t, cache) + }) }) } } @@ -152,19 +141,23 @@ func TestOptions_MemoryPressureDefaults(t *testing.T) { opts := ApplyOptions() // Assert - assert.Equal(t, 0.75, DefaultCompactionThreshold) - assert.Equal(t, 0.90, DefaultEvictionThreshold) - assert.Equal(t, 0.50, DefaultEvictionRetentionRatio) - assert.Equal(t, DefaultCompactionThreshold, opts.CompactionThreshold) - assert.Equal(t, DefaultEvictionThreshold, opts.EvictionThreshold) - assert.Equal(t, DefaultEvictionRetentionRatio, opts.EvictionRetentionRatio) + assert.InDelta(t, 0.75, DefaultCompactionThreshold, 1e-9) + assert.InDelta(t, 0.90, DefaultEvictionThreshold, 1e-9) + assert.InDelta(t, 0.50, DefaultEvictionRetentionRatio, 1e-9) + assert.InDelta(t, DefaultCompactionThreshold, opts.CompactionThreshold, 1e-9) + assert.InDelta(t, DefaultEvictionThreshold, opts.EvictionThreshold, 1e-9) + assert.InDelta(t, DefaultEvictionRetentionRatio, opts.EvictionRetentionRatio, 1e-9) require.NotNil(t, opts.PressureFunc) - assert.False(t, opts.hasCustomPressureFunc) + assert.GreaterOrEqual(t, opts.PressureFunc(), 0.0) } func TestOptions_MemoryPressureCustomAndValidation(t *testing.T) { // Arrange - customFn := func() float64 { return 0.88 } + var customCalls int + customFn := func() float64 { + customCalls++ + return 0.88 + } // Act opts := ApplyOptions( @@ -181,53 +174,41 @@ func TestOptions_MemoryPressureCustomAndValidation(t *testing.T) { WithEvictionRetentionRatio(1.5), ) optsNegativeRetention := ApplyOptions(WithEvictionRetentionRatio(-0.25)) - optsInverted := ApplyOptions( - WithCompactionThreshold(0.85), - WithEvictionThreshold(0.70), - ) optsNaNAndInf := ApplyOptions( WithCompactionThreshold(math.NaN()), WithEvictionThreshold(math.Inf(1)), WithEvictionRetentionRatio(math.NaN()), ) + optsDirectStructPressure := ApplyOptions(func(o *Options) { + o.PressureFunc = customFn + }) // Assert assert.Equal(t, uint64(256*1024*1024), opts.MemoryBudget) - assert.Equal(t, 0.60, opts.CompactionThreshold) - assert.Equal(t, 0.85, opts.EvictionThreshold) - assert.Equal(t, 0.35, opts.EvictionRetentionRatio) + assert.InDelta(t, 0.60, opts.CompactionThreshold, 1e-9) + assert.InDelta(t, 0.85, opts.EvictionThreshold, 1e-9) + assert.InDelta(t, 0.35, opts.EvictionRetentionRatio, 1e-9) require.NotNil(t, opts.PressureFunc) - assert.True(t, opts.hasCustomPressureFunc) - assert.Equal(t, 0.88, opts.PressureFunc()) - - assert.Equal(t, 0.0, optsZeroRetention.EvictionRetentionRatio) + assert.InDelta(t, 0.88, opts.PressureFunc(), 1e-9) - assert.Equal(t, DefaultCompactionThreshold, optsClamped.CompactionThreshold) - assert.Equal(t, DefaultEvictionThreshold, optsClamped.EvictionThreshold) - assert.Equal(t, 1.0, optsClamped.EvictionRetentionRatio) + assert.InDelta(t, 0.0, optsZeroRetention.EvictionRetentionRatio, 1e-9) - assert.Equal(t, DefaultEvictionRetentionRatio, optsNegativeRetention.EvictionRetentionRatio) + assert.InDelta(t, DefaultCompactionThreshold, optsClamped.CompactionThreshold, 1e-9) + assert.InDelta(t, DefaultEvictionThreshold, optsClamped.EvictionThreshold, 1e-9) + assert.InDelta(t, 1.0, optsClamped.EvictionRetentionRatio, 1e-9) - assert.Equal(t, 0.70, optsInverted.CompactionThreshold) - assert.Equal(t, 0.70, optsInverted.EvictionThreshold) + assert.InDelta(t, DefaultEvictionRetentionRatio, optsNegativeRetention.EvictionRetentionRatio, 1e-9) - assert.Equal(t, DefaultCompactionThreshold, optsNaNAndInf.CompactionThreshold) - assert.Equal(t, DefaultEvictionThreshold, optsNaNAndInf.EvictionThreshold) - assert.Equal(t, DefaultEvictionRetentionRatio, optsNaNAndInf.EvictionRetentionRatio) + assert.InDelta(t, DefaultCompactionThreshold, optsNaNAndInf.CompactionThreshold, 1e-9) + assert.InDelta(t, DefaultEvictionThreshold, optsNaNAndInf.EvictionThreshold, 1e-9) + assert.InDelta(t, DefaultEvictionRetentionRatio, optsNaNAndInf.EvictionRetentionRatio, 1e-9) - // F8: Single threshold customization preserves Tier 1 compaction window. - optsHighCompactionOnly := ApplyOptions(WithCompactionThreshold(0.92)) - assert.Equal(t, 0.92, optsHighCompactionOnly.CompactionThreshold) - assert.Greater(t, optsHighCompactionOnly.EvictionThreshold, optsHighCompactionOnly.CompactionThreshold) - - optsLowEvictionOnly := ApplyOptions(WithEvictionThreshold(0.60)) - assert.Equal(t, 0.60, optsLowEvictionOnly.EvictionThreshold) - assert.Less(t, optsLowEvictionOnly.CompactionThreshold, optsLowEvictionOnly.EvictionThreshold) - - optsDirectStructPressure := ApplyOptions(func(o *Options) { - o.PressureFunc = customFn - }) - assert.True(t, optsDirectStructPressure.hasCustomPressureFunc) + require.NotNil(t, optsDirectStructPressure.PressureFunc) + callsBefore := customCalls + c := NewMapCache(100, func(o *Options) { o.PressureFunc = customFn }) + _, err := c.Insert("k", NewSizedValue("v", 10)) + require.NoError(t, err) + assert.Greater(t, customCalls, callsBefore) } func TestDefaultRuntimePressureFunc(t *testing.T) { @@ -235,10 +216,13 @@ func TestDefaultRuntimePressureFunc(t *testing.T) { prevLimit := debug.SetMemoryLimit(-1) defer debug.SetMemoryLimit(prevLimit) - // Act & Assert 1: Unbounded GOMEMLIMIT (math.MaxInt64) with MemoryBudget == 0 returns 0.0. + // Act & Assert 1: Unbounded GOMEMLIMIT (math.MaxInt64) with MemoryBudget == 0 returns 0.0 in 0 allocs. debug.SetMemoryLimit(math.MaxInt64) probeUnbounded := DefaultRuntimePressureFunc(0) - assert.Equal(t, 0.0, probeUnbounded()) + assert.InDelta(t, 0.0, probeUnbounded(), 1e-9) + assert.InDelta(t, 0.0, testing.AllocsPerRun(50, func() { + _ = probeUnbounded() + }), 1e-9) // Act & Assert 2: Configured 1 GiB GOMEMLIMIT returns positive normalized pressure in (0.0, 1.0). const oneGiB = int64(1 << 30) @@ -259,7 +243,7 @@ func TestDefaultRuntimePressureFunc(t *testing.T) { allocsPerRun := testing.AllocsPerRun(50, func() { _ = probeBudget512MB() }) - assert.Equal(t, 0.0, allocsPerRun) + assert.InDelta(t, 0.0, allocsPerRun, 1e-9) // Act & Assert 4: Concurrent race-free reads across 16 goroutines. var wg sync.WaitGroup @@ -274,3 +258,331 @@ func TestDefaultRuntimePressureFunc(t *testing.T) { } wg.Wait() } + +func TestOptions_ThresholdReconciliationAndEdgeCases(t *testing.T) { + subnormal2 := math.Float64frombits(2) // 2 * math.SmallestNonzeroFloat64 + compactionOneULPBelowEviction := math.Nextafter(DefaultEvictionThreshold, 0) + compactionOneULPBelowDefault := math.Nextafter(DefaultCompactionThreshold, 0) + + tests := []struct { + name string + opts []Option + wantCompaction float64 + wantEviction float64 + exactBits bool + wantStrictLess bool + }{ + { + name: "SmallestNonzeroFloat64SingleEvictionThreshold", + opts: []Option{WithEvictionThreshold(math.SmallestNonzeroFloat64)}, + wantCompaction: math.SmallestNonzeroFloat64, + wantEviction: math.SmallestNonzeroFloat64, + exactBits: true, + }, + { + name: "SmallestNonzeroFloat64InvertedWithCompactionThreshold", + opts: []Option{WithCompactionThreshold(0.50), WithEvictionThreshold(math.SmallestNonzeroFloat64)}, + wantCompaction: math.SmallestNonzeroFloat64, + wantEviction: math.SmallestNonzeroFloat64, + exactBits: true, + }, + { + name: "SmallestNonzeroFloat64CompactionThresholdOnly", + opts: []Option{WithCompactionThreshold(math.SmallestNonzeroFloat64)}, + wantCompaction: math.SmallestNonzeroFloat64, + wantEviction: DefaultEvictionThreshold, + exactBits: true, + }, + { + name: "MaxFloat64CompactionThresholdClampsEvictionThresholdWithoutInf", + opts: []Option{WithCompactionThreshold(math.MaxFloat64)}, + wantCompaction: math.MaxFloat64, + wantEviction: math.MaxFloat64, + exactBits: true, + }, + { + name: "SubnormalEvictionThresholdSingleOptionMaintainsStrictInequality", + opts: []Option{WithEvictionThreshold(subnormal2)}, + wantCompaction: math.SmallestNonzeroFloat64, + wantEviction: subnormal2, + exactBits: true, + wantStrictLess: true, + }, + { + name: "SubnormalEvictionThresholdInvertedWithCompactionMaintainsStrictInequality", + opts: []Option{WithCompactionThreshold(0.50), WithEvictionThreshold(subnormal2)}, + wantCompaction: math.SmallestNonzeroFloat64, + wantEviction: subnormal2, + exactBits: true, + wantStrictLess: true, + }, + { + name: "OneULPBelowDefaultEvictionThresholdPreservedBitForBit", + opts: []Option{WithCompactionThreshold(compactionOneULPBelowEviction)}, + wantCompaction: compactionOneULPBelowEviction, + wantEviction: DefaultEvictionThreshold, + exactBits: true, + wantStrictLess: true, + }, + { + name: "OneULPBelowDefaultCompactionThresholdPreservedBitForBit", + opts: []Option{WithCompactionThreshold(compactionOneULPBelowDefault)}, + wantCompaction: compactionOneULPBelowDefault, + wantEviction: DefaultEvictionThreshold, + exactBits: true, + wantStrictLess: true, + }, + { + name: "WithCompactionThreshold95AdvancesEvictionThresholdTo110", + opts: []Option{WithCompactionThreshold(0.95)}, + wantCompaction: 0.95, + wantEviction: 1.10, + }, + { + name: "WithCompactionThreshold100AdvancesEvictionThresholdTo115", + opts: []Option{WithCompactionThreshold(1.0)}, + wantCompaction: 1.0, + wantEviction: 1.15, + }, + { + name: "WithLowEvictionThreshold60ScalesCompactionBelow60", + opts: []Option{WithEvictionThreshold(0.60)}, + wantCompaction: 0.60 * (DefaultCompactionThreshold / DefaultEvictionThreshold), + wantEviction: 0.60, + wantStrictLess: true, + }, + { + name: "InvertedCompaction85AndEviction70ScalesCompactionBelow70", + opts: []Option{ + WithCompactionThreshold(0.85), + WithEvictionThreshold(0.70), + }, + wantCompaction: 0.70 * (DefaultCompactionThreshold / DefaultEvictionThreshold), + wantEviction: 0.70, + wantStrictLess: true, + }, + { + name: "ExplicitEqualThresholdsAt90PreservesEquality", + opts: []Option{ + WithCompactionThreshold(DefaultEvictionThreshold), + WithEvictionThreshold(DefaultEvictionThreshold), + }, + wantCompaction: 0.90, + wantEviction: 0.90, + }, + { + name: "ExplicitEqualThresholdsAt75PreservesEquality", + opts: []Option{ + WithCompactionThreshold(DefaultCompactionThreshold), + WithEvictionThreshold(DefaultCompactionThreshold), + }, + wantCompaction: 0.75, + wantEviction: 0.75, + }, + { + name: "ResetCompactionThresholdToZeroBeforeEviction75ScalesDefaultCompaction", + opts: []Option{ + WithCompactionThreshold(0.80), + WithCompactionThreshold(0), + WithEvictionThreshold(0.75), + }, + wantCompaction: 0.625, + wantEviction: 0.75, + }, + { + name: "DirectStructCompactionMutationDoesNotPinAndScalesBackToDefault", + opts: []Option{func(o *Options) { o.CompactionThreshold = 0.95 }}, + wantCompaction: DefaultCompactionThreshold, + wantEviction: DefaultEvictionThreshold, + }, + { + name: "WithEvictionPinAndDirectCompactionStructMutationScalesBelowPinnedEviction", + opts: []Option{ + WithEvictionThreshold(DefaultEvictionThreshold), + func(o *Options) { o.CompactionThreshold = 0.95 }, + }, + wantCompaction: DefaultCompactionThreshold, + wantEviction: DefaultEvictionThreshold, + }, + { + name: "MultiOptionExplicitDefaultEvictionBeforeCompaction95ScalesToDefault", + opts: []Option{ + WithEvictionThreshold(DefaultEvictionThreshold), + WithCompactionThreshold(0.95), + }, + wantCompaction: 0.75, + wantEviction: 0.90, + }, + { + name: "MultiOptionExplicitDefaultEvictionAfterCompaction95ScalesToDefault", + opts: []Option{ + WithCompactionThreshold(0.95), + WithEvictionThreshold(DefaultEvictionThreshold), + }, + wantCompaction: 0.75, + wantEviction: 0.90, + }, + { + name: "MultiOptionEviction90OverriddenBy80WithCompaction95ScalesBelow80", + opts: []Option{ + WithEvictionThreshold(DefaultEvictionThreshold), + WithEvictionThreshold(0.80), + WithCompactionThreshold(0.95), + }, + wantCompaction: 0.80 * (DefaultCompactionThreshold / DefaultEvictionThreshold), + wantEviction: 0.80, + }, + { + name: "MultiOptionEviction80OverriddenBy90WithCompaction95ScalesBelow90", + opts: []Option{ + WithEvictionThreshold(0.80), + WithEvictionThreshold(DefaultEvictionThreshold), + WithCompactionThreshold(0.95), + }, + wantCompaction: 0.75, + wantEviction: 0.90, + }, + { + name: "MultiStepOptionToDefaultCompactionPreservesEqualityWithEviction75", + opts: []Option{ + WithCompactionThreshold(0.60), + WithEvictionThreshold(DefaultCompactionThreshold), + WithCompactionThreshold(DefaultCompactionThreshold), + }, + wantCompaction: 0.75, + wantEviction: 0.75, + }, + { + name: "EqualThresholdsAfterEarlierEvictionResetPreservesEquality", + opts: []Option{ + WithEvictionThreshold(0.80), + WithEvictionThreshold(0), + func(o *Options) { + WithCompactionThreshold(0.90)(o) + WithEvictionThreshold(0.90)(o) + }, + }, + wantCompaction: 0.90, + wantEviction: 0.90, + }, + { + name: "WithCompaction80FollowedByDirectEvictionMutation60ScalesCompactionBelow60", + opts: []Option{ + WithCompactionThreshold(0.80), + func(o *Options) { o.EvictionThreshold = 0.60 }, + }, + wantCompaction: 0.60 * (DefaultCompactionThreshold / DefaultEvictionThreshold), + wantEviction: 0.60, + }, + { + name: "WithCompaction80FollowedByDirectCompactionMutation95ScalesBelowDefaultEviction", + opts: []Option{ + WithCompactionThreshold(0.80), + func(o *Options) { o.CompactionThreshold = 0.95 }, + }, + wantCompaction: DefaultCompactionThreshold, + wantEviction: DefaultEvictionThreshold, + }, + { + name: "WithEviction80FollowedByDirectEvictionMutationToDefaultAdvancesWithCompaction95", + opts: []Option{ + WithEvictionThreshold(0.80), + func(o *Options) { o.EvictionThreshold = DefaultEvictionThreshold }, + WithCompactionThreshold(0.95), + }, + wantCompaction: 0.95, + wantEviction: 1.10, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := ApplyOptions(tc.opts...) + assert.Greater(t, got.CompactionThreshold, 0.0) + assert.False(t, math.IsInf(got.EvictionThreshold, 0)) + if tc.exactBits { + assert.Equal(t, math.Float64bits(tc.wantCompaction), math.Float64bits(got.CompactionThreshold)) + assert.Equal(t, math.Float64bits(tc.wantEviction), math.Float64bits(got.EvictionThreshold)) + } else { + assert.InDelta(t, tc.wantCompaction, got.CompactionThreshold, 1e-9) + assert.InDelta(t, tc.wantEviction, got.EvictionThreshold, 1e-9) + } + if tc.wantStrictLess { + assert.Less(t, got.CompactionThreshold, got.EvictionThreshold) + } + }) + } + + t.Run("NegativeZeroRetentionRatioNormalizedToPositiveZero", func(t *testing.T) { + optsNegZero := ApplyOptions(WithEvictionRetentionRatio(math.Copysign(0.0, -1.0))) + assert.False(t, math.Signbit(optsNegZero.EvictionRetentionRatio)) + assert.Equal(t, uint64(0), math.Float64bits(optsNegZero.EvictionRetentionRatio)) + }) +} + +func TestOptions_CustomAndConditionalOptionClosures(t *testing.T) { + t.Run("StatefulClosuresExecutedExactlyOnceAndObserveExactDefaults", func(t *testing.T) { + // Arrange + invocations := 0 + var observedCompaction, observedEviction float64 + statefulOpt := func(o *Options) { + invocations++ + observedCompaction = o.CompactionThreshold + observedEviction = o.EvictionThreshold + WithCompactionThreshold(o.CompactionThreshold + 0.20)(o) + } + + // Act + got := ApplyOptions(statefulOpt) + + // Assert + assert.Equal(t, 1, invocations) + assert.Equal(t, math.Float64bits(DefaultCompactionThreshold), math.Float64bits(observedCompaction)) + assert.Equal(t, math.Float64bits(DefaultEvictionThreshold), math.Float64bits(observedEviction)) + assert.InDelta(t, 0.95, got.CompactionThreshold, 1e-9) + assert.InDelta(t, 1.10, got.EvictionThreshold, 1e-9) + }) + + t.Run("RelativeOptionSettingCompactionEqualToEvictionPreservesEquality", func(t *testing.T) { + // Arrange + matchEvictionOpt := func(o *Options) { + WithEvictionThreshold(DefaultEvictionThreshold)(o) + WithCompactionThreshold(o.EvictionThreshold)(o) + } + + // Act + got := ApplyOptions(matchEvictionOpt) + + // Assert + assert.InDelta(t, 0.90, got.CompactionThreshold, 1e-9) + assert.InDelta(t, 0.90, got.EvictionThreshold, 1e-9) + }) + + t.Run("TakenAndUntakenConditionalBranchesPinOnlyWhenExecuted", func(t *testing.T) { + // Arrange + conditionalPinEviction := func(o *Options) { + if o.EnableInvariantChecking { + WithEvictionThreshold(DefaultEvictionThreshold)(o) + } + } + combinedUntakenBranch := func(o *Options) { + WithCompactionThreshold(0.95)(o) + if o.MemoryBudget > 0 { + WithEvictionThreshold(DefaultEvictionThreshold)(o) + } + } + + // Act + gotUntaken := ApplyOptions(conditionalPinEviction, WithCompactionThreshold(0.95)) + gotTaken := ApplyOptions(WithInvariantChecking(true), conditionalPinEviction, WithCompactionThreshold(0.95)) + gotCombinedUntaken := ApplyOptions(combinedUntakenBranch) + + // Assert + assert.InDelta(t, 0.95, gotUntaken.CompactionThreshold, 1e-9) + assert.InDelta(t, 1.10, gotUntaken.EvictionThreshold, 1e-9) + assert.InDelta(t, 0.75, gotTaken.CompactionThreshold, 1e-9) + assert.InDelta(t, 0.90, gotTaken.EvictionThreshold, 1e-9) + assert.InDelta(t, 0.95, gotCombinedUntaken.CompactionThreshold, 1e-9) + assert.InDelta(t, 1.10, gotCombinedUntaken.EvictionThreshold, 1e-9) + }) +} diff --git a/pressure.go b/pressure.go new file mode 100644 index 0000000..d9dcc27 --- /dev/null +++ b/pressure.go @@ -0,0 +1,550 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lru + +import ( + "math" + "runtime" + "strings" + "sync" + "sync/atomic" +) + +var goroutineIDBufPool = sync.Pool{ + New: func() any { + return new([32]byte) + }, +} + +func currentGoroutineID() uint64 { + bufPtr := goroutineIDBufPool.Get().(*[32]byte) + n := runtime.Stack(bufPtr[:32], false) + const prefix = "goroutine " + var id uint64 + if n > len(prefix) { + for i := len(prefix); i < n; i++ { + b := bufPtr[i] + if b < '0' || b > '9' { + break + } + id = id*10 + uint64(b-'0') + } + } + goroutineIDBufPool.Put(bufPtr) + return id +} + +var byteStrings [256]string + +func init() { + for i := range 256 { + byteStrings[i] = string(byte(i)) + } +} + +// clonePrefix returns an independent copy of s that never shares s's underlying backing array, +// using preallocated 1-byte strings for single-character prefixes to avoid heap allocations. +func clonePrefix(s string) string { + switch len(s) { + case 0: + return "" + case 1: + return byteStrings[s[0]] + default: + return strings.Clone(s) + } +} + +// longestCommonPrefix finds the length of the longest common prefix of a and b. +func longestCommonPrefix(a, b string) int { + i := 0 + for i < len(a) && i < len(b) && a[i] == b[i] { + i++ + } + return i +} + +// computeTargetSize safely computes uint64(float64(maxSize) * retention) without +// float64-to-uint64 overflow at math.MaxUint64 or zero-truncation when maxSize == 1 and retention > 0. +func computeTargetSize(maxSize uint64, retention float64) uint64 { + if retention <= 0.0 { + return 0 + } + if retention >= 1.0 { + return maxSize + } + f := float64(maxSize) * retention + if f >= float64(maxSize) || f >= float64(math.MaxUint64) { + return maxSize + } + target := uint64(f) + if target == 0 && maxSize > 0 { + return 1 + } + return target +} + +// pressureState encapsulates lock-free memory-pressure sampling, reclamation epoch tracking, +// and shared compaction/zero-size entry watermarks embedded across all three cache backends. +type pressureState struct { + options Options + + peakEntryLen int + deletedSinceCompact int + zeroSizeCount int + lastReclaimedZeroCount int + lastReclaimedLen int + hasValidSample bool + + pressureWriteMu sync.Mutex + samplingPressure atomic.Bool + fallbackSampling atomic.Bool + pressureInitialized atomic.Bool + pressureNeedsRefresh atomic.Bool + overflowSamplingCount atomic.Int32 + samplingGID atomic.Uint64 + fallbackGID atomic.Uint64 + samplingReentrantReclaim atomic.Bool + fallbackReentrantReclaim atomic.Bool + overflowSamplingGIDs sync.Map + reentrantReclaimGIDs sync.Map + pressureSampleSeq atomic.Uint64 + pressureInvokeSeq atomic.Uint64 + pressureMaxStoredSeq atomic.Uint64 + cachedPressureBits atomic.Uint64 + cachedPressureEpoch atomic.Uint64 + reclaimEpoch atomic.Uint64 + externalReclaimEpoch atomic.Uint64 +} + +const ( + samplerSlotPrimary = iota + samplerSlotFallback + samplerSlotOverflow +) + +func (p *pressureState) hasOverflowSamplingGID(gid uint64) bool { + if gid == 0 || p.overflowSamplingCount.Load() <= 0 { + return false + } + _, ok := p.overflowSamplingGIDs.Load(gid) + return ok +} + +func (p *pressureState) isCurrentGoroutineSampling(gid uint64) bool { + if gid == 0 { + return false + } + return p.samplingGID.Load() == gid || + p.fallbackGID.Load() == gid || + p.hasOverflowSamplingGID(gid) +} + +func (p *pressureState) checkSamplingGoroutine() (bool, uint64) { + if !p.options.hasCustomPressureFunc { + return false, 0 + } + sGID := p.samplingGID.Load() + fGID := p.fallbackGID.Load() + oCount := p.overflowSamplingCount.Load() + if sGID == 0 && fGID == 0 && oCount <= 0 { + return false, 0 + } + gid := currentGoroutineID() + return p.isCurrentGoroutineSampling(gid), gid +} + +func (p *pressureState) isSamplingGoroutine() bool { + sampling, _ := p.checkSamplingGoroutine() + return sampling +} + +func (p *pressureState) recordReentrantReclaimGID(gid uint64) { + if gid == 0 { + return + } + switch { + case p.samplingGID.Load() == gid: + p.samplingReentrantReclaim.Store(true) + case p.fallbackGID.Load() == gid: + p.fallbackReentrantReclaim.Store(true) + case p.hasOverflowSamplingGID(gid): + p.reentrantReclaimGIDs.Store(gid, struct{}{}) + } +} + +// markReclaimedLocked invalidates cached pre-reclamation pressure and increments reclaimEpoch. +// Caller MUST hold the cache's write lock. +func (p *pressureState) markReclaimedLocked() { + p.pressureWriteMu.Lock() + defer p.pressureWriteMu.Unlock() + sampling, gid := p.checkSamplingGoroutine() + if sampling { + p.recordReentrantReclaimGID(gid) + } else { + p.samplingReentrantReclaim.Store(false) + p.fallbackReentrantReclaim.Store(false) + p.reentrantReclaimGIDs.Clear() + } + p.pressureNeedsRefresh.Store(true) + p.pressureInitialized.Store(false) + p.cachedPressureBits.Store(0) + p.reclaimEpoch.Add(1) + if !sampling { + p.externalReclaimEpoch.Add(1) + } +} + +func (p *pressureState) hasElevatedPressureToInvalidate(pressure float64) bool { + thresh := p.options.CompactionThreshold + if pressure >= thresh || + math.Float64frombits(p.cachedPressureBits.Load()) >= thresh || + p.pressureNeedsRefresh.Load() { + return true + } + if !p.samplingPressure.Load() && + !p.fallbackSampling.Load() && + p.overflowSamplingCount.Load() <= 0 { + return false + } + return !p.isSamplingGoroutine() +} + +func (p *pressureState) resolvePostRetryPressureLocked(sampledEpoch uint64, pressure float64, validSample bool) (uint64, float64) { + currentEpoch := p.reclaimEpoch.Load() + var resolvedEpoch uint64 + var resolvedPressure float64 + switch { + case sampledEpoch == currentEpoch: + p.hasValidSample = validSample + resolvedEpoch = sampledEpoch + resolvedPressure = pressure + case p.pressureInitialized.Load() && !p.pressureNeedsRefresh.Load() && p.cachedPressureEpoch.Load() == currentEpoch: + p.hasValidSample = true + resolvedEpoch = currentEpoch + resolvedPressure = math.Float64frombits(p.cachedPressureBits.Load()) + default: + p.hasValidSample = false + resolvedEpoch = currentEpoch + resolvedPressure = 0.0 + } + p.resetZeroWatermarkBelowTier2(resolvedPressure) + return resolvedEpoch, resolvedPressure +} + +func (p *pressureState) lockWithPressure(mu *sync.RWMutex, fresh bool) (uint64, float64) { + var sampledEpoch uint64 + var pressure float64 + var validSample bool + if fresh { + sampledEpoch, pressure, validSample = p.samplePressureFreshWithEpoch() + } else { + sampledEpoch, pressure, validSample = p.samplePressureWithEpoch() + } + mu.Lock() + for retries := 0; sampledEpoch != p.reclaimEpoch.Load() && retries < 2; retries++ { + mu.Unlock() + if fresh { + sampledEpoch, pressure, validSample = p.samplePressureFreshWithEpoch() + } else { + sampledEpoch, pressure, validSample = p.samplePressureWithEpoch() + } + mu.Lock() + } + return p.resolvePostRetryPressureLocked(sampledEpoch, pressure, validSample) +} + +func (p *pressureState) storeSampledPressureWithSeq(epoch, extEpoch, invokeSeq uint64, val float64, gid uint64, slot int) (uint64, float64, bool) { + p.pressureWriteMu.Lock() + defer p.pressureWriteMu.Unlock() + + var stored bool + if p.reclaimEpoch.Load() == epoch { + if invokeSeq != 0 && invokeSeq < p.pressureMaxStoredSeq.Load() { + if p.pressureInitialized.Load() && !p.pressureNeedsRefresh.Load() && p.cachedPressureEpoch.Load() == epoch { + val = math.Float64frombits(p.cachedPressureBits.Load()) + } + } else { + if invokeSeq > p.pressureMaxStoredSeq.Load() { + p.pressureMaxStoredSeq.Store(invokeSeq) + } + bits := math.Float64bits(val) + p.pressureInitialized.Store(false) + p.cachedPressureEpoch.Store(epoch) + p.cachedPressureBits.Store(bits) + p.pressureInitialized.Store(true) + p.pressureNeedsRefresh.Store(false) + stored = true + } + } else if !p.pressureInitialized.Load() || p.cachedPressureEpoch.Load() != p.reclaimEpoch.Load() { + p.pressureNeedsRefresh.Store(true) + } + + if gid != 0 { + var reentrant bool + switch slot { + case samplerSlotPrimary: + reentrant = p.samplingReentrantReclaim.Swap(false) + case samplerSlotFallback: + reentrant = p.fallbackReentrantReclaim.Swap(false) + default: + _, reentrant = p.reentrantReclaimGIDs.LoadAndDelete(gid) + } + if reentrant && !stored && p.externalReclaimEpoch.Load() == extEpoch { + epoch = p.reclaimEpoch.Load() + } + } + + return epoch, val, stored +} + +func (p *pressureState) invokeAndStorePressure(gid uint64, slot int) (uint64, float64, bool) { + if p.options.hasCustomPressureFunc && gid != 0 { + switch slot { + case samplerSlotPrimary: + p.samplingReentrantReclaim.Store(false) + case samplerSlotFallback: + p.fallbackReentrantReclaim.Store(false) + default: + p.reentrantReclaimGIDs.Delete(gid) + } + } + extEpoch := p.externalReclaimEpoch.Load() + epoch := p.reclaimEpoch.Load() + invokeSeq := p.pressureInvokeSeq.Add(1) + val := p.options.PressureFunc() + if math.IsNaN(val) || val < 0.0 { + val = 0.0 + } + epoch, val, _ = p.storeSampledPressureWithSeq(epoch, extEpoch, invokeSeq, val, gid, slot) + return epoch, val, true +} + +// samplePressureFreshWithEpoch evaluates p.options.PressureFunc lock-free outside the cache lock +// with a goroutine-aware re-entrancy guard and cold-start fallback. +func (p *pressureState) samplePressureFreshWithEpoch() (uint64, float64, bool) { + if p.options.PressureFunc == nil { + return p.reclaimEpoch.Load(), 0.0, true + } + sampling, gid := p.checkSamplingGoroutine() + if sampling { + return p.reclaimEpoch.Load(), 0.0, false + } + if p.samplingPressure.CompareAndSwap(false, true) { + if p.options.hasCustomPressureFunc { + if gid == 0 { + gid = currentGoroutineID() + } + p.samplingGID.Store(gid) + defer func() { + p.samplingGID.Store(0) + p.samplingPressure.Store(false) + }() + } else { + defer p.samplingPressure.Store(false) + } + return p.invokeAndStorePressure(gid, samplerSlotPrimary) + } + if p.options.hasCustomPressureFunc { + if gid == 0 { + gid = currentGoroutineID() + } + if p.isCurrentGoroutineSampling(gid) { + return p.reclaimEpoch.Load(), 0.0, false + } + } + if p.fallbackSampling.CompareAndSwap(false, true) { + if p.options.hasCustomPressureFunc { + p.fallbackGID.Store(gid) + defer func() { + p.fallbackGID.Store(0) + p.fallbackSampling.Store(false) + }() + } else { + defer p.fallbackSampling.Store(false) + } + return p.invokeAndStorePressure(gid, samplerSlotFallback) + } + if p.options.hasCustomPressureFunc { + p.overflowSamplingGIDs.Store(gid, struct{}{}) + } + p.overflowSamplingCount.Add(1) + defer func() { + if p.options.hasCustomPressureFunc { + p.overflowSamplingGIDs.Delete(gid) + } + p.overflowSamplingCount.Add(-1) + }() + return p.invokeAndStorePressure(gid, samplerSlotOverflow) +} + +// samplePressureWithEpoch returns the current memory pressure and epoch for foreground cache operations. +// Custom PressureFunc callbacks are invoked on every call; the default runtime/metrics probe is amortized +// across a 256-operation window (and immediately refreshed after any reclamation/compaction cycle). +func (p *pressureState) samplePressureWithEpoch() (uint64, float64, bool) { + if p.options.hasCustomPressureFunc { + return p.samplePressureFreshWithEpoch() + } + epoch := p.reclaimEpoch.Load() + seq := p.pressureSampleSeq.Add(1) + if (seq&255) == 1 || !p.pressureInitialized.Load() || p.pressureNeedsRefresh.Load() || p.cachedPressureEpoch.Load() != epoch { + return p.samplePressureFreshWithEpoch() + } + bits := p.cachedPressureBits.Load() + if !p.pressureInitialized.Load() || p.pressureNeedsRefresh.Load() || p.cachedPressureEpoch.Load() != epoch || p.reclaimEpoch.Load() != epoch { + return p.samplePressureFreshWithEpoch() + } + return epoch, math.Float64frombits(bits), true +} + +func (p *pressureState) onEntryInserted(newLen int, size uint64) { + if newLen > p.peakEntryLen { + p.peakEntryLen = newLen + } + if size == 0 { + p.zeroSizeCount++ + } +} + +func (p *pressureState) onEntrySizeUpdated(oldSize, newSize uint64) { + if oldSize == 0 && newSize > 0 && p.zeroSizeCount > 0 { + p.zeroSizeCount-- + if p.zeroSizeCount < p.lastReclaimedZeroCount { + p.lastReclaimedZeroCount = p.zeroSizeCount + } + } else if oldSize > 0 && newSize == 0 { + p.zeroSizeCount++ + p.lastReclaimedLen = 0 + } +} + +func (p *pressureState) onEntryDeleted(size uint64) { + p.deletedSinceCompact++ + if size == 0 && p.zeroSizeCount > 0 { + p.zeroSizeCount-- + if p.zeroSizeCount < p.lastReclaimedZeroCount { + p.lastReclaimedZeroCount = p.zeroSizeCount + } + } else if size > 0 { + p.lastReclaimedLen = 0 + } +} + +func (p *pressureState) resetWatermarks() { + p.peakEntryLen = 0 + p.deletedSinceCompact = 0 + p.zeroSizeCount = 0 + p.lastReclaimedZeroCount = 0 + p.lastReclaimedLen = 0 +} + +func (p *pressureState) onCompacted(newLen int) { + p.deletedSinceCompact = 0 + p.peakEntryLen = newLen + if p.zeroSizeCount < p.lastReclaimedZeroCount { + p.lastReclaimedZeroCount = p.zeroSizeCount + } + if newLen < p.lastReclaimedLen { + p.lastReclaimedLen = newLen + } +} + +func (p *pressureState) shouldAutoCompactEntryCounts(isDirty, isBackground bool, currentLen int) bool { + if isBackground { + return isDirty + } + if !isDirty { + return false + } + if p.peakEntryLen > 8 && p.peakEntryLen > currentLen && p.peakEntryLen-currentLen >= 2 && uint64(p.peakEntryLen-currentLen)*4 >= uint64(p.peakEntryLen) { + return true + } + return p.deletedSinceCompact >= 64 && p.deletedSinceCompact >= currentLen +} + +func (p *pressureState) hasEmptyDeleteSlack(extraSlack bool) bool { + return extraSlack || p.peakEntryLen > 8 || p.deletedSinceCompact >= 64 +} + +func (p *pressureState) shouldReclaimSingleSurvivorOnDelete(currentLen int, isDirty, extraChurn bool) bool { + return currentLen == 1 && isDirty && (extraChurn || p.peakEntryLen >= 64 || p.deletedSinceCompact >= 64) +} + +func (p *pressureState) shouldReclaimEmptyPreInsert(currentLen int, extraChurn, evictedPre bool, newSize, sizeBefore uint64, pressure float64) bool { + return currentLen == 0 && (extraChurn || p.peakEntryLen >= 64 || p.deletedSinceCompact >= 64 || (evictedPre && newSize < sizeBefore && p.hasElevatedPressureToInvalidate(pressure))) +} + +func (p *pressureState) shouldReclaimSingleSurvivorOnMutation(currentLen int, isDirty, extraChurn, evictedPre bool, postSize, sizeBefore uint64, pressure float64) bool { + return (currentLen == 1 || (evictedPre && currentLen-p.zeroSizeCount <= 1)) && isDirty && + (extraChurn || p.peakEntryLen >= 64 || p.deletedSinceCompact >= 64 || (currentLen == 1 && postSize < sizeBefore && p.hasElevatedPressureToInvalidate(pressure))) +} + +func (p *pressureState) shouldCompactAfterMutation(reclaimedPre, netByteReduced, isDirty bool, pressure float64) bool { + return isDirty && (reclaimedPre || (netByteReduced && pressure >= p.options.CompactionThreshold && uint64(p.deletedSinceCompact)*4 >= uint64(p.peakEntryLen))) +} + +func (p *pressureState) shouldMarkReclaimedAfterMutation(reclaimedPre, netByteReduced bool, sampledEpoch uint64, pressure float64) bool { + return (reclaimedPre || netByteReduced) && p.reclaimEpoch.Load() == sampledEpoch && p.hasElevatedPressureToInvalidate(pressure) +} + +func (p *pressureState) computeShedTargets(targetSize uint64, retention float64, currentLen int, hasProtected bool, protectedSize uint64) (effectiveTarget uint64, targetLen, targetZeroCount, unprotectedZeroTarget int) { + effectiveTarget = targetSize + if hasProtected && retention > 0.0 && protectedSize > effectiveTarget { + effectiveTarget = protectedSize + } + if retention > 0.0 { + if currentLen > 0 { + targetLen = int(float64(currentLen) * retention) + if (!hasProtected || protectedSize <= targetSize) && p.lastReclaimedLen > targetLen { + targetLen = p.lastReclaimedLen + } + if hasProtected && targetLen < 1 { + targetLen = 1 + } + } + if p.zeroSizeCount > 0 { + unprotectedZeroTarget = max(int(float64(p.zeroSizeCount)*retention), p.lastReclaimedZeroCount) + targetZeroCount = unprotectedZeroTarget + if hasProtected && protectedSize == 0 && targetZeroCount < 1 { + targetZeroCount = 1 + } + } + } + return effectiveTarget, targetLen, targetZeroCount, unprotectedZeroTarget +} + +func (p *pressureState) updateZeroWatermarkAfterShed(retention float64, currentLen int, unprotectedLenAllowed bool, unprotectedZeroTarget, targetLen int) { + if retention > 0.0 && p.zeroSizeCount > 0 { + p.lastReclaimedZeroCount = min(p.zeroSizeCount, unprotectedZeroTarget) + if unprotectedLenAllowed { + p.lastReclaimedLen = min(currentLen, targetLen) + } else { + p.lastReclaimedLen = 0 + } + } else { + p.lastReclaimedZeroCount = 0 + p.lastReclaimedLen = 0 + } +} + +func (p *pressureState) resetZeroWatermarkBelowTier2(pressure float64) { + if p.lastReclaimedZeroCount == 0 && p.lastReclaimedLen == 0 { + return + } + if p.options.PressureFunc == nil || (p.hasValidSample && pressure < p.options.EvictionThreshold && !p.isSamplingGoroutine()) { + p.lastReclaimedZeroCount = 0 + p.lastReclaimedLen = 0 + } +} diff --git a/pressure_test.go b/pressure_test.go new file mode 100644 index 0000000..2814ef7 --- /dev/null +++ b/pressure_test.go @@ -0,0 +1,2888 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lru + +import ( + "fmt" + "math" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPressure_CriticalPressureSheddingAndZeroSizeRetention(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("PreservesZeroSizeEntriesAfterByteTargetMet", func(t *testing.T) { + // Arrange + pressure := 0.0 + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + // Insert 1 positive-size entry (60B) at LRU tail, followed by 50 zero-size entries toward MRU. + _, err := cache.Insert("lru_60", NewSizedValue("payload", 60)) + require.NoError(t, err) + for i := range 50 { + _, err = cache.Insert("zero_"+string(rune('A'+i)), NewStringValue("")) + require.NoError(t, err) + } + + // Act: trigger critical-pressure shedding (targetSize = 50, targetZeroCount = 25). + pressure = 0.95 + evicted := cache.EvaluateMemoryPressure() + + // Assert: "lru_60" plus the 25 oldest zero-size entries are shed (26 total), + // while the 25 newest zero-size entries remain in the cache. + require.Len(t, evicted, 26) + assert.Nil(t, cache.LookUpWithoutChangingOrder("lru_60")) + assert.Nil(t, cache.LookUpWithoutChangingOrder("zero_A")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("zero_"+string(rune('A'+49)))) + }) + + t.Run("ManyZeroPlusOneByteShedsZeroSizeEntries", func(t *testing.T) { + // Arrange + pressure := 0.0 + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + for i := range 100 { + _, err := cache.Insert(fmt.Sprintf("z-%03d", i), NewStringValue("")) + require.NoError(t, err) + } + _, err := cache.Insert("one_byte", NewStringValue("x")) + require.NoError(t, err) + + // Act: Critical pressure must shed ~50 zero-size entries even though currentSize == 1 <= targetSize (50). + pressure = 0.95 + evicted := cache.EvaluateMemoryPressure() + + // Assert + assert.GreaterOrEqual(t, len(evicted), 50) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("one_byte")) + }) + + t.Run("ZeroAtTailPlusLargeMRU", func(t *testing.T) { + // Arrange + pressure := 0.0 + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + for i := range 10 { + _, err := cache.Insert(fmt.Sprintf("z-%02d", i), NewStringValue("")) + require.NoError(t, err) + } + _, err := cache.Insert("mru_60", NewSizedValue("large", 60)) + require.NoError(t, err) + + // Act: Critical pressure (targetSize = 50, targetLen = 5) must shed mru_60 and retain 5 zero-size entries. + pressure = 0.95 + evicted := cache.EvaluateMemoryPressure() + + // Assert + assert.Len(t, evicted, 6) + assert.Nil(t, cache.LookUpWithoutChangingOrder("mru_60")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z-09")) + }) + + t.Run("ForegroundZeroSizeShedding", func(t *testing.T) { + // Arrange: Configure critical pressure (0.95) and 50% retention. + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return 0.95 }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ) + + // Act: Insert 100 zero-size entries via foreground Insert under critical pressure. + totalEvicted := 0 + for i := range 100 { + evicted, err := cache.Insert(fmt.Sprintf("fg_zero_%03d", i), NewStringValue("")) + require.NoError(t, err) + totalEvicted += len(evicted) + } + + // Assert: Foreground writes shed 99 older zero-size entries, retaining only the latest MRU entry. + assert.Equal(t, 99, totalEvicted) + assert.Nil(t, cache.LookUpWithoutChangingOrder("fg_zero_000")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("fg_zero_099")) + }) + + t.Run("PositiveTailPlusZeroSizeEntriesShedTogether", func(t *testing.T) { + // Arrange: Insert 1 entry of 51B at tail (targetSize = 50B) followed by 100 zero-size entries. + pressure := 0.0 + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + _, err := cache.Insert("tail_51", NewSizedValue("v", 51)) + require.NoError(t, err) + for i := range 100 { + _, err = cache.Insert(fmt.Sprintf("z_%03d", i), NewStringValue("")) + require.NoError(t, err) + } + + // Act: EvaluateMemoryPressure under 0.95 must shed both "tail_51" and 50 zero-size entries (51 total). + pressure = 0.95 + evicted := cache.EvaluateMemoryPressure() + + // Assert + assert.Len(t, evicted, 51) + assert.Nil(t, cache.LookUpWithoutChangingOrder("tail_51")) + assert.Nil(t, cache.LookUpWithoutChangingOrder("z_000")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z_099")) + }) + + t.Run("PreservesPositiveEntriesBelowTargetSizeAndIsIdempotent", func(t *testing.T) { + // Arrange: 50 positive-size 10B entries (500B == targetSize) + 2 zero-size entries at MRU. + pressure := 0.0 + cache := b.fn( + 1000, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + for i := range 50 { + _, err := cache.Insert(fmt.Sprintf("pos_%02d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + _, err := cache.Insert("zero_1", NewStringValue("")) + require.NoError(t, err) + _, err = cache.Insert("zero_2", NewStringValue("")) + require.NoError(t, err) + + // Act: Invoke EvaluateMemoryPressure() 6 times under sustained critical pressure (0.95). + pressure = 0.95 + evicted1 := cache.EvaluateMemoryPressure() + var subsequentEvicted int + for range 5 { + subsequentEvicted += len(cache.EvaluateMemoryPressure()) + } + + // Assert: Call 1 evicts only 1 zero-size entry (zero_1); Calls 2..6 evict 0 entries; + // all 50 positive-size entries (500B) and zero_2 remain intact. + assert.Len(t, evicted1, 1) + assert.Zero(t, subsequentEvicted) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("pos_00")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("pos_49")) + assert.Nil(t, cache.LookUpWithoutChangingOrder("zero_1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("zero_2")) + }) + + t.Run("SinglePassCursorSkipsRetainedZeroSizeTailEntries", func(t *testing.T) { + // Arrange: Populate 1,000 zero-size entries at the LRU tail followed by 1x1B entry ("p_seed") + // and 500x1B entries ("p_0000".."p_0499") toward MRU (501B total > targetSize 500B). + pressure := 0.0 + cache := b.fn( + 1000, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + for i := range 1000 { + _, err := cache.Insert(fmt.Sprintf("z_%04d", i), NewStringValue("")) + require.NoError(t, err) + } + _, err := cache.Insert("p_seed", NewSizedValue("x", 1)) + require.NoError(t, err) + for i := range 500 { + _, err = cache.Insert(fmt.Sprintf("p_%04d", i), NewSizedValue("x", 1)) + require.NoError(t, err) + } + + // First critical-pressure evaluation sheds "p_seed" (1B) + 500 oldest zero-size entries ("z_0000".."z_0499"), + // naturally establishing the retained zero-size watermark at 500 while keeping "z_0500".."z_0999" at the LRU tail. + pressure = 0.95 + evictedInitial := cache.EvaluateMemoryPressure() + require.Len(t, evictedInitial, 501) + + // Grow the MRU entry by +100B under critical pressure (bringing currentSize to 600B > targetSize 500B), + // which sheds 100x1B entries ("p_0000".."p_0099") past the 500 retained zero-size tail entries in a single pass. + err = cache.UpdateSize("p_0499", 100) + require.NoError(t, err) + + // Assert: All 500 retained zero-size tail entries survive while the 100 oldest 1B entries were evicted. + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z_0500")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z_0999")) + assert.Nil(t, cache.LookUpWithoutChangingOrder("p_0000")) + assert.Nil(t, cache.LookUpWithoutChangingOrder("p_0099")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("p_0100")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("p_0499")) + }) + }) + } +} + +func TestPressure_ForegroundMutations(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("ForegroundInsertShedsOldestEntriesAcrossAllBackends", func(t *testing.T) { + // Arrange: Configure critical pressure (0.95) and 50% retention (targetSize = 50B). + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return 0.95 }), + WithEvictionRetentionRatio(0.50), + ) + + // Act: Insert 4 entries of 20B each (80B total > 50B targetSize). + _, err := cache.Insert("k1", NewSizedValue("v1", 20)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 20)) + require.NoError(t, err) + _, err = cache.Insert("k3", NewSizedValue("v3", 20)) + require.NoError(t, err) + _, err = cache.Insert("k4", NewSizedValue("v4", 20)) + require.NoError(t, err) + + // Assert: Foreground pressure reclamation sheds oldest entries (k1, k2) across all 3 backends. + assert.Nil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.Nil(t, cache.LookUpWithoutChangingOrder("k2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k3")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k4")) + }) + + t.Run("ForegroundEraseTriggersTier2SheddingOfRemainingExcess", func(t *testing.T) { + // Arrange: Insert 4 entries of 20B (80B total) at 0.0 pressure, then raise pressure to 0.95 (targetSize = 50B). + pressure := 0.0 + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ) + + _, err := cache.Insert("p/1", NewSizedValue("v1", 20)) + require.NoError(t, err) + _, err = cache.Insert("p/2", NewSizedValue("v2", 20)) + require.NoError(t, err) + _, err = cache.Insert("other/3", NewSizedValue("v3", 20)) + require.NoError(t, err) + _, err = cache.Insert("other/4", NewSizedValue("v4", 20)) + require.NoError(t, err) + + // Act: Erase "other/4" (leaving 60B > targetSize 50B) under 0.95 critical pressure. + pressure = 0.95 + erased := cache.Erase("other/4") + + // Assert: Foreground Erase also triggers Tier 2 shedding of "p/1" (oldest 20B) so currentSize drops to 40B <= 50B. + require.NotNil(t, erased) + assert.Nil(t, cache.LookUpWithoutChangingOrder("p/1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("p/2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("other/3")) + }) + + t.Run("UpdateSizeStrictLRUEvictsOldestTailEntry", func(t *testing.T) { + // Arrange: k1 (20B) at LRU tail, k2 (30B) at MRU head, maxSize = 100, targetSize = 50. + pressure := 0.10 + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ) + + _, err := cache.Insert("k1", NewSizedValue("v1", 20)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 30)) + require.NoError(t, err) + + // Act: Raise pressure to 0.95 and grow LRU tail entry k1 by +10 (20 -> 30, currentSize = 60 > 50). + pressure = 0.95 + err = cache.UpdateSize("k1", 10) + + // Assert: Strict LRU eviction order must evict k1 (oldest at LRU tail) and retain k2 (newest at MRU head). + require.NoError(t, err) + assert.Nil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + + t.Run("ProtectedMRUEntryAboveTargetSizeDoesNotSpuriouslyAdvanceEpoch", func(t *testing.T) { + // Arrange: Empty cache with maxSize=1000, retention=0.50 (targetSize=500), and critical pressure 0.95. + probe := newPressureProbe(0.95) + cache := b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + probe.Option(), + ).(PressureAwareCache) + + // Act: Insert a single 800B entry (> targetSize 500B). Because it is the protected MRU head + // and there are no older entries or dirty index buckets, 0 evictions and 0 compactions occur. + var evicted []ValueType + var err error + advanced := probe.ObserveEpochAdvance(t, cache, func() { + evicted, err = cache.Insert("protected_jumbo", NewSizedValue("v", 800)) + }) + + // Assert: The reclamation epoch must not advance because no memory was evicted or compacted. + require.NoError(t, err) + assert.Empty(t, evicted) + assert.False(t, advanced) + }) + + t.Run("UpdateSizeWithZeroDeltaProtectsMRUHeadUnderCriticalPressure", func(t *testing.T) { + // Arrange: Insert "head" (800B > targetSize 500B) under critical pressure (0.95). + cache := b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { return 0.95 }), + ) + _, err := cache.Insert("head", NewSizedValue("v", 800)) + require.NoError(t, err) + require.NotNil(t, cache.LookUpWithoutChangingOrder("head")) + + // Act: Call UpdateSize("head", 0) on the MRU head entry. + err = cache.UpdateSize("head", 0) + + // Assert: Updating the MRU head with sizeDelta == 0 must protect the MRU head just like sizeDelta > 0. + require.NoError(t, err) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("head")) + }) + + t.Run("ZeroByteErasureAndSelfEvictionDoNotSpuriouslyAdvanceReclaimEpoch", func(t *testing.T) { + // Arrange: Populate 100B cache with 3 zero-size tail entries and 1 100B MRU entry under Tier 1 pressure (0.80). + probe := newPressureProbe(0.80) + cache := b.fn( + 100, + WithInvariantChecking(true), + probe.Option(), + ).(PressureAwareCache) + _, err := cache.Insert("z_erase", NewSizedValue("z", 0)) + require.NoError(t, err) + _, err = cache.Insert("z_prefix/1", NewSizedValue("z", 0)) + require.NoError(t, err) + _, err = cache.Insert("z_self_evict", NewSizedValue("z", 0)) + require.NoError(t, err) + _, err = cache.Insert("mru_full", NewSizedValue("full", 100)) + require.NoError(t, err) + + // Act: Erase a 0B entry, prefix-erase a 0B entry, and self-evict a 0B entry via UpdateSize (+10B when avail == 0). + advanced := probe.ObserveEpochAdvance(t, cache, func() { + _ = cache.Erase("z_erase") + cache.EraseEntriesWithGivenPrefix("z_prefix/") + err = cache.UpdateSize("z_self_evict", 10) + }) + require.NoError(t, err) + + // Assert: Because currentSize remained 100B (0 bytes freed) and no compaction ran, the reclamation epoch must not advance. + assert.False(t, advanced) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("mru_full")) + }) + + t.Run("UpdateSizeNetByteGrowthUnderTier1PressureDoesNotAdvanceReclaimEpoch", func(t *testing.T) { + // Arrange: Insert "small" (10B) and "target" (20B) in a 100B cache under Tier 1 pressure (0.80). + probe := newPressureProbe(0.80) + cache := b.fn( + 100, + WithInvariantChecking(true), + probe.Option(), + ).(PressureAwareCache) + + _, err := cache.Insert("small", NewSizedValue("v", 10)) + require.NoError(t, err) + _, err = cache.Insert("target", NewSizedValue("v", 20)) + require.NoError(t, err) + + // Act: Grow "target" by +80B (to 100B). This evicts "small" (10B), so net currentSize increases from 30B to 100B. + advanced := probe.ObserveEpochAdvance(t, cache, func() { + err = cache.UpdateSize("target", 80) + }) + + // Assert: Because net cache memory grew from 30B to 100B and no compaction occurred, the reclamation epoch must not advance. + require.NoError(t, err) + assert.False(t, advanced) + assert.Nil(t, cache.LookUpWithoutChangingOrder("small")) + require.NotNil(t, cache.LookUpWithoutChangingOrder("target")) + assert.NoError(t, cache.UpdateWithoutChangingOrder("target", NewSizedValue("v2", 100))) + }) + }) + } +} + +func TestPressure_KeyAndPrefixMissesDoNotEvictLiveEntriesUnderCriticalPressure(t *testing.T) { + missOps := []struct { + name string + act func(t *testing.T, c Cache) + }{ + { + name: "EraseMiss", + act: func(t *testing.T, c Cache) { + t.Helper() + deleted := c.Erase("absent_key") + assert.Nil(t, deleted) + }, + }, + { + name: "UpdateSizeMiss", + act: func(t *testing.T, c Cache) { + t.Helper() + err := c.UpdateSize("absent_key", 10) + assert.ErrorIs(t, err, ErrEntryNotExist) + }, + }, + { + name: "EraseEntriesWithGivenPrefixMiss", + act: func(t *testing.T, c Cache) { + t.Helper() + c.EraseEntriesWithGivenPrefix("absent_prefix/") + }, + }, + } + + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + for _, op := range missOps { + t.Run(op.name, func(t *testing.T) { + // Arrange: Populate cache with "victim" (300B) and "protected_head" (500B) at low pressure (0.10) + // so currentSize is 800B > targetSize (500B), then raise pressure to 0.95 before the miss operation. + pressure := 0.10 + cache := b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { return pressure }), + ) + + _, err := cache.Insert("victim", NewSizedValue("v_victim", 300)) + require.NoError(t, err) + + _, err = cache.Insert("protected_head", NewSizedValue("v_head", 500)) + require.NoError(t, err) + require.NotNil(t, cache.LookUpWithoutChangingOrder("victim")) + require.NotNil(t, cache.LookUpWithoutChangingOrder("protected_head")) + + // Act: Perform a key/prefix miss under critical pressure. + pressure = 0.95 + op.act(t, cache) + + // Assert: A key or prefix miss must be a state no-op and must not evict live entries. + assert.NotNil(t, cache.LookUpWithoutChangingOrder("victim")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("protected_head")) + }) + } + }) + } +} + +func TestPressure_ShedAndCompactZeroSizeWatermarkAccuracyWhenTargetLenStopsShedding(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("ProtectedMRUHeadInsertThenEvaluate", func(t *testing.T) { + // Arrange: Populate 100B cache at 0.10 pressure with 5x8B positive-size entries (40B total) + // followed by 4 zero-size entries. + pressure := 0.10 + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { return pressure }), + ).(PressureAwareCache) + + for i := range 5 { + _, err := cache.Insert(fmt.Sprintf("p8_%d", i), NewSizedValue("v", 8)) + require.NoError(t, err) + } + for i := range 4 { + _, err := cache.Insert(fmt.Sprintf("z_%d", i), NewSizedValue("z", 0)) + require.NoError(t, err) + } + + // Act: Under critical pressure (0.95), insert "mru_60" (60B), then call EvaluateMemoryPressure(). + pressure = 0.95 + evicted1, err := cache.Insert("mru_60", NewSizedValue("v", 60)) + require.NoError(t, err) + require.Len(t, evicted1, 5) + + evicted2 := cache.EvaluateMemoryPressure() + + // Assert: EvaluateMemoryPressure must evict "mru_60" plus 2 zero-size entries (3 total), + // leaving 2 zero-size entries in the cache. + assert.Len(t, evicted2, 3) + assert.Nil(t, cache.LookUpWithoutChangingOrder("z_0")) + assert.Nil(t, cache.LookUpWithoutChangingOrder("z_1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z_2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z_3")) + }) + }) + } +} + +func TestPressure_EraseEmptyPrefixInvalidationAndEmptyNoOp(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("NonEmptyCacheEraseEmptyPrefixInvalidatesStaleCachedPressure", func(t *testing.T) { + // Arrange: Populate cache and seed critical pressure (0.95) before resetting via EraseEntriesWithGivenPrefix(""). + probe := newPressureProbe(0.10) + c := b.fn( + 200, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + probe.Option(), + ).(PressureAwareCache) + + _, err := c.Insert("pre_1", NewSizedValue("v1", 40)) + require.NoError(t, err) + + probe.Set(0.95) + _ = c.EvaluateMemoryPressure() + probe.Set(0.10) + + // Act: EraseEntriesWithGivenPrefix("") on a non-empty cache must advance the reclamation epoch and invalidate cached pressure. + advanced := probe.ObserveEpochAdvance(t, c, func() { + c.EraseEntriesWithGivenPrefix("") + }) + + // Assert + assert.True(t, advanced) + _, err = c.Insert("post_1", NewSizedValue("v1", 60)) + require.NoError(t, err) + _, err = c.Insert("post_2", NewSizedValue("v2", 60)) + require.NoError(t, err) + assert.NotNil(t, c.LookUpWithoutChangingOrder("post_1")) + assert.NotNil(t, c.LookUpWithoutChangingOrder("post_2")) + }) + + t.Run("InFlightSampleAcrossReclamationDoesNotClearPressureNeedsRefresh", func(t *testing.T) { + // Arrange: Goroutine 1 starts sampling 0.95 before EraseEntriesWithGivenPrefix("") reclaims the cache + // and pressure returns to 0.10. + var blockSample atomic.Bool + var currentPressure atomic.Uint64 + currentPressure.Store(math.Float64bits(0.10)) + inSample := make(chan struct{}) + releaseSample := make(chan struct{}) + + c := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if blockSample.CompareAndSwap(true, false) { + close(inSample) + <-releaseSample + return 0.95 + } + return math.Float64frombits(currentPressure.Load()) + }), + ).(PressureAwareCache) + + _, err := c.Insert("seed", NewSizedValue("v", 20)) + require.NoError(t, err) + + blockSample.Store(true) + done := make(chan struct{}) + go func() { + defer close(done) + _ = c.EvaluateMemoryPressure() + }() + <-inSample + + // Act: Reclaim all entries while the stale 0.95 sample is in-flight, then release the sampler. + c.EraseEntriesWithGivenPrefix("") + close(releaseSample) + <-done + + // Assert: Stale pre-reclamation 0.95 sample was rejected; subsequent inserts above targetSize (50B) at 0.10 survive. + _, err = c.Insert("k1", NewSizedValue("v1", 40)) + require.NoError(t, err) + _, err = c.Insert("k2", NewSizedValue("v2", 40)) + require.NoError(t, err) + assert.NotNil(t, c.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, c.LookUpWithoutChangingOrder("k2")) + }) + }) + } +} + +func TestPressure_EmptyPrefixOnAlreadyEmptyCacheDoesNotInvalidate(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("ZeroAllocsOnEmptyCache", func(t *testing.T) { + // Arrange: Create a fresh empty cache. + cache := b.fn(1000, WithInvariantChecking(true)) + + // Act & Assert: Calling EraseEntriesWithGivenPrefix("") on an already-empty cache must allocate 0 objects. + allocs := testing.AllocsPerRun(20, func() { + cache.EraseEntriesWithGivenPrefix("") + }) + assert.Zero(t, allocs, + "%s.EraseEntriesWithGivenPrefix(\"\") on an already-empty cache allocated memory on every call", b.name) + }) + + t.Run("DoesNotForceConcurrentWriterToResamplePressureOnEmptyCache", func(t *testing.T) { + // Arrange: Empty cache where Goroutine A samples healthy 0.10 pressure for Insert, + // and while Goroutine A is about to return from PressureFunc, Goroutine B calls + // EraseEntriesWithGivenPrefix("") on the already-empty cache. + var cache Cache + var writerSamples atomic.Int32 + var interceptWriter atomic.Bool + writerInSample := make(chan struct{}) + releaseWriter := make(chan struct{}) + + cache = b.fn( + 1000, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if interceptWriter.CompareAndSwap(true, false) { + writerSamples.Add(1) + close(writerInSample) + <-releaseWriter + return 0.10 + } + if writerSamples.Load() > 0 { + writerSamples.Add(1) + } + return 0.10 + }), + ) + + interceptWriter.Store(true) + insertDone := make(chan error, 1) + go func() { + _, err := cache.Insert("k1", NewSizedValue("v1", 10)) + insertDone <- err + }() + <-writerInSample + + // Act: Call EraseEntriesWithGivenPrefix("") on the already-empty cache while Goroutine A is in flight. + cache.EraseEntriesWithGivenPrefix("") + close(releaseWriter) + require.NoError(t, <-insertDone) + + // Assert: Because the cache was already empty and clean, EraseEntriesWithGivenPrefix("") must not + // advance the reclamation epoch or force Goroutine A to re-run PressureFunc. + assert.Equal(t, int32(1), writerSamples.Load(), + "%s.EraseEntriesWithGivenPrefix(\"\") on an already-empty cache advanced the reclamation epoch and forced concurrent Insert to re-sample PressureFunc", b.name) + }) + }) + } +} + +func TestPressure_ConcurrentColdStartZeroPressureAndPostReclamationSampling(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("ColdStartConcurrentCriticalPressure", func(t *testing.T) { + // Arrange + var blockFirst atomic.Bool + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.0)) + entered := make(chan struct{}) + release := make(chan struct{}) + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if blockFirst.CompareAndSwap(true, false) { + close(entered) + <-release + } + return math.Float64frombits(pressureBits.Load()) + }), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + _, err := cache.Insert("k1", NewSizedValue("v1", 40)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 40)) + require.NoError(t, err) + _, err = cache.Insert("tmp", NewSizedValue("vt", 10)) + require.NoError(t, err) + _ = cache.Erase("tmp") + + // Invalidate cached pressure via Compact() to simulate cold uninitialized pressure before spike. + cache.Compact() + pressureBits.Store(math.Float64bits(0.95)) + blockFirst.Store(true) + + // Act: Goroutine 1 blocks inside PressureFunc() while Goroutine 2 calls EvaluateMemoryPressure(). + done1 := make(chan []ValueType, 1) + go func() { + done1 <- cache.EvaluateMemoryPressure() + }() + <-entered + + evicted2 := cache.EvaluateMemoryPressure() + close(release) + evicted1 := <-done1 + + // Assert: Goroutine 2 itself sheds k1 (40B) down to targetSize (50B) while Goroutine 1 is blocked. + assert.Len(t, evicted2, 1) + assert.Empty(t, evicted1) + assert.Nil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + + t.Run("ZeroPressureConcurrentNoFalseEviction", func(t *testing.T) { + // Arrange + var blockFirst atomic.Bool + entered := make(chan struct{}) + release := make(chan struct{}) + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if blockFirst.CompareAndSwap(true, false) { + close(entered) + <-release + } + return 0.0 + }), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + _, err := cache.Insert("k0", NewSizedValue("v0", 40)) + require.NoError(t, err) + _, err = cache.Insert("k1", NewSizedValue("v1", 40)) + require.NoError(t, err) + + blockFirst.Store(true) + + // Act: Goroutine 1 blocks inside PressureFunc() returning 0.0 while Goroutine 2 evaluates pressure. + done1 := make(chan []ValueType, 1) + go func() { + done1 <- cache.EvaluateMemoryPressure() + }() + <-entered + + evicted2 := cache.EvaluateMemoryPressure() + close(release) + evicted1 := <-done1 + + // Assert: Zero pressure (0.0) must never fabricate EvictionThreshold or shed entries. + assert.Empty(t, evicted2) + assert.Empty(t, evicted1) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k0")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k1")) + }) + + t.Run("ReentrantZeroPressureNoFalseEviction", func(t *testing.T) { + // Arrange + var cache PressureAwareCache + reentrant := false + cache = b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if reentrant && cache != nil { + _ = cache.EvaluateMemoryPressure() + } + return 0.0 + }), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + _, err := cache.Insert("k0", NewSizedValue("v0", 40)) + require.NoError(t, err) + _, err = cache.Insert("k1", NewSizedValue("v1", 40)) + require.NoError(t, err) + + // Act: Enable re-entrant EvaluateMemoryPressure() inside PressureFunc() returning 0.0. + reentrant = true + evicted := cache.EvaluateMemoryPressure() + + // Assert: Neither re-entrant nor outer call evicts entries when pressure is 0.0. + assert.Empty(t, evicted) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k0")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k1")) + }) + + t.Run("PostReclamationClearsLastSampledPressure", func(t *testing.T) { + // Arrange + var blockFirst atomic.Bool + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.95)) + entered := make(chan struct{}) + release := make(chan struct{}) + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if blockFirst.CompareAndSwap(true, false) { + close(entered) + <-release + } + return math.Float64frombits(pressureBits.Load()) + }), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + // Seed high pressure (0.95) in cachedPressureBits, then reclaim and restore healthy pressure (0.10). + _ = cache.EvaluateMemoryPressure() + cache.EraseEntriesWithGivenPrefix("") + pressureBits.Store(math.Float64bits(0.10)) + + _, err := cache.Insert("k1", NewSizedValue("v1", 40)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 40)) + require.NoError(t, err) + _, err = cache.Insert("tmp", NewSizedValue("vt", 10)) + require.NoError(t, err) + _ = cache.Erase("tmp") + cache.Compact() + + blockFirst.Store(true) + + // Act: Goroutine 1 blocks sampling 0.10 while Goroutine 2 evaluates pressure after reclamation. + done1 := make(chan []ValueType, 1) + go func() { + done1 <- cache.EvaluateMemoryPressure() + }() + <-entered + + evicted2 := cache.EvaluateMemoryPressure() + close(release) + evicted1 := <-done1 + + // Assert: Post-reclamation writer must not read stale pre-reclamation 0.95 pressure and evict k1. + assert.Empty(t, evicted2) + assert.Empty(t, evicted1) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + }) + } +} + +func TestPressure_EpochInvalidationAndResamplingSynchronization(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("NoPostReclamationRepoisoningOfLastSampled", func(t *testing.T) { + // Arrange + var blockNext atomic.Bool + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + entered := make(chan struct{}) + release := make(chan struct{}) + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if blockNext.CompareAndSwap(true, false) { + close(entered) + <-release + } + return math.Float64frombits(pressureBits.Load()) + }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + _, err := cache.Insert("k1", NewSizedValue("v1", 40)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 40)) + require.NoError(t, err) + + pressureBits.Store(math.Float64bits(0.95)) + _, err = cache.Insert("k4", NewSizedValue("v4", 30)) + require.NoError(t, err) + require.Nil(t, cache.LookUpWithoutChangingOrder("k1")) + require.Nil(t, cache.LookUpWithoutChangingOrder("k2")) + require.NotNil(t, cache.LookUpWithoutChangingOrder("k4")) + + // Restore healthy pressure (0.10) and block Goroutine 1 inside fresh pressure sampling. + pressureBits.Store(math.Float64bits(0.10)) + blockNext.Store(true) + done1 := make(chan []ValueType, 1) + go func() { + done1 <- cache.EvaluateMemoryPressure() + }() + <-entered + + // Act: While Goroutine 1 is in flight, Goroutine 2 inserts k5 and calls EvaluateMemoryPressure(). + evictedK5, err := cache.Insert("k5", NewSizedValue("v5", 30)) + require.NoError(t, err) + evicted2 := cache.EvaluateMemoryPressure() + close(release) + evicted1 := <-done1 + + // Assert + assert.Empty(t, evictedK5) + assert.Empty(t, evicted2) + assert.Empty(t, evicted1) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k4")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k5")) + }) + + t.Run("SampledEpochMismatchResamplesFreshPressureOutsideLock", func(t *testing.T) { + // Arrange: Create real reclaimable slack so a concurrent Compact() bumps the reclamation epoch while Insert samples. + inFirstSample := make(chan struct{}) + proceedSample := make(chan struct{}) + var armed atomic.Bool + var trapFirst atomic.Bool + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if !armed.Load() { + return 0.10 + } + if trapFirst.CompareAndSwap(true, false) { + close(inFirstSample) + <-proceedSample + } + return 0.95 + }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + _, err := cache.Insert("keep_zero", NewSizedValue("", 0)) + require.NoError(t, err) + _, err = cache.Insert("scratch", NewSizedValue("s", 10)) + require.NoError(t, err) + require.NotNil(t, cache.Erase("scratch")) + trapFirst.Store(true) + armed.Store(true) + + doneInsert := make(chan error, 1) + go func() { + _, err := cache.Insert("k1", NewSizedValue("v1", 60)) + doneInsert <- err + }() + <-inFirstSample + + // Bump the reclamation epoch via real compaction while Insert's initial pressure sample is in flight. + cache.Compact() + close(proceedSample) + + // Act + err = <-doneInsert + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 20)) + require.NoError(t, err) + + // Assert: Insert re-sampled 0.95 outside lock instead of reading 0.0 from the invalidated pressure cache. + assert.Nil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + + t.Run("NormalPressureErasePreservesReclaimEpoch", func(t *testing.T) { + // Arrange: Insert 3 entries at normal 0.10 pressure (< CompactionThreshold 0.75). + var samples atomic.Int32 + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + samples.Add(1) + return 0.10 + }), + ) + _, err := cache.Insert("k1", NewSizedValue("v1", 20)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 40)) + require.NoError(t, err) + _, err = cache.Insert("k3", NewSizedValue("v3", 40)) + require.NoError(t, err) + samples.Store(0) + + // Act: Erase k1 at normal pressure while k2 and k3 (80B > targetSize 50B) remain in the cache. + erased := cache.Erase("k1") + + // Assert: Normal-pressure Erase samples once, does not retry or shed remaining entries. + require.NotNil(t, erased) + assert.Equal(t, int32(1), samples.Load()) + assert.Nil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k3")) + }) + + t.Run("EvaluateMemoryPressureResamplesOnEpochAdvanceBeforeLock", func(t *testing.T) { + // Arrange + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + var blockEval atomic.Bool + evalSampled := make(chan struct{}) + releaseEval := make(chan struct{}) + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + p := math.Float64frombits(pressureBits.Load()) + if blockEval.CompareAndSwap(true, false) { + close(evalSampled) + <-releaseEval + } + return p + }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + _, err := cache.Insert("seed_1", NewSizedValue("v", 5)) + require.NoError(t, err) + _, err = cache.Insert("seed_2", NewSizedValue("v", 5)) + require.NoError(t, err) + _ = cache.Erase("seed_1") + + pressureBits.Store(math.Float64bits(0.95)) + blockEval.Store(true) + + doneEval := make(chan []ValueType, 1) + go func() { + doneEval <- cache.EvaluateMemoryPressure() + }() + <-evalSampled + + pressureBits.Store(math.Float64bits(0.10)) + cache.Compact() + _, err = cache.Insert("live_1", NewSizedValue("v1", 40)) + require.NoError(t, err) + _, err = cache.Insert("live_2", NewSizedValue("v2", 40)) + require.NoError(t, err) + + // Act: Release Goroutine A so it acquires c.mu.Lock() in EvaluateMemoryPressure(). + close(releaseEval) + evicted := <-doneEval + + // Assert + assert.Empty(t, evicted) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("live_1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("live_2")) + }) + + t.Run("EraseDuringInFlightSamplerInvalidatesReclaimEpoch", func(t *testing.T) { + // Arrange + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + var blockSampler atomic.Bool + samplerInFlight := make(chan struct{}) + releaseSampler := make(chan struct{}) + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if blockSampler.CompareAndSwap(true, false) { + close(samplerInFlight) + <-releaseSampler + return 0.95 + } + return math.Float64frombits(pressureBits.Load()) + }), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + ) + + _, err := cache.Insert("k1", NewSizedValue("v1", 40)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 40)) + require.NoError(t, err) + _, err = cache.Insert("k3", NewSizedValue("v3", 10)) + require.NoError(t, err) + + blockSampler.Store(true) + doneInsert := make(chan []ValueType, 1) + go func() { + ev, _ := cache.Insert("k4", NewSizedValue("v4", 20)) + doneInsert <- ev + }() + <-samplerInFlight + + // Act + erased := cache.Erase("k1") + require.NotNil(t, erased) + + close(releaseSampler) + evicted := <-doneInsert + + // Assert + assert.Empty(t, evicted) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k3")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k4")) + }) + + t.Run("PanicDuringEpochResampleDoesNotDoubleUnlockOrRunCheckInvariantsUnlocked", func(t *testing.T) { + // Arrange + var armed atomic.Bool + var sampleCount atomic.Int32 + firstSampleReady := make(chan struct{}) + epochBumped := make(chan struct{}) + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if !armed.Load() { + return 0.10 + } + n := sampleCount.Add(1) + if n == 1 { + close(firstSampleReady) + <-epochBumped + return 0.10 + } + panic("synthetic PressureFunc failure during resample") + }), + ).(PressureAwareCache) + + _, err := cache.Insert("k1", NewSizedValue("v1", 20)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 20)) + require.NoError(t, err) + _ = cache.Erase("k1") + armed.Store(true) + + panicObserved := make(chan any, 1) + go func() { + defer func() { + panicObserved <- recover() + }() + _ = cache.EvaluateMemoryPressure() + }() + + <-firstSampleReady + cache.Compact() + close(epochBumped) + + // Act + recovered := <-panicObserved + + // Assert + require.Equal(t, "synthetic PressureFunc failure during resample", recovered) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + + t.Run("ChildGoroutineReclaimingInsidePressureFuncTerminatesInBoundedRetries", func(t *testing.T) { + // Arrange + var cacheRef Cache + var inChild atomic.Bool + var parentSampleCalls atomic.Int32 + + cache := b.fn( + 500, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if cacheRef != nil && inChild.CompareAndSwap(false, true) { + parentSampleCalls.Add(1) + done := make(chan struct{}) + go func() { + defer func() { + inChild.Store(false) + close(done) + }() + _, _ = cacheRef.Insert("tmp_item", NewSizedValue("v", 1)) + cacheRef.EraseEntriesWithGivenPrefix("tmp_") + }() + <-done + } + return 0.95 + }), + WithEvictionRetentionRatio(0.50), + ) + cacheRef = cache + + // Act + parentSampleCalls.Store(0) + _, err := cache.Insert("target_key", NewSizedValue("val", 10)) + _ = cache.(PressureAwareCache).EvaluateMemoryPressure() + + // Assert: Both Insert and EvaluateMemoryPressure terminate in bounded retries (3 parent samples each = 6 total). + require.NoError(t, err) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("target_key")) + assert.Equal(t, int32(6), parentSampleCalls.Load()) + }) + + t.Run("ExhaustedEpochRetriesUseLatestEpochPressure", func(t *testing.T) { + // Arrange + var cache PressureAwareCache + var currentPressure atomic.Uint64 + currentPressure.Store(math.Float64bits(0.10)) + + var interceptCaller atomic.Bool + sampleEntered := make(chan struct{}, 4) + sampleProceed := make(chan struct{}, 4) + + cache = b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if interceptCaller.Load() { + sampleEntered <- struct{}{} + <-sampleProceed + return 0.95 + } + return math.Float64frombits(currentPressure.Load()) + }), + ).(PressureAwareCache) + + _, err := cache.Insert("survivor", NewSizedValue("v", 600)) + require.NoError(t, err) + + interceptCaller.Store(true) + done := make(chan []ValueType, 1) + go func() { + done <- cache.EvaluateMemoryPressure() + }() + + for attempt := range 3 { + <-sampleEntered + interceptCaller.Store(false) + if attempt == 2 { + currentPressure.Store(math.Float64bits(0.10)) + } + _, _ = cache.Insert("tmp", NewSizedValue("t", 10)) + _ = cache.Erase("tmp") + cache.Compact() + if attempt == 2 { + _ = cache.EvaluateMemoryPressure() + } + if attempt < 2 { + interceptCaller.Store(true) + } + sampleProceed <- struct{}{} + } + + // Act + evicted := <-done + + // Assert + assert.Empty(t, evicted) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("survivor")) + }) + }) + } +} + +func TestPressure_ReentrancyAndOverflowSamplerCoordination(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("ReentrantPressureFuncDoesNotStackOverflow", func(t *testing.T) { + // Arrange + var pac PressureAwareCache + reentered := false + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if !reentered && pac != nil { + reentered = true + _ = pac.EvaluateMemoryPressure() + } + return 0.95 + }), + ) + pac = cache.(PressureAwareCache) + _, err := cache.Insert("k1", NewSizedValue("v1", 80)) + require.NoError(t, err) + + // Act & Assert: EvaluateMemoryPressure must not infinitely recurse. + evicted := pac.EvaluateMemoryPressure() + assert.True(t, reentered) + assert.Len(t, evicted, 1) + }) + + t.Run("ReentrantCompactFollowedByEvaluateDoesNotReinvokePressureFunc", func(t *testing.T) { + // Arrange + var pac PressureAwareCache + reentrantCall := false + invocations := 0 + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + invocations++ + if reentrantCall && pac != nil { + reentrantCall = false + pac.Compact() + _ = pac.EvaluateMemoryPressure() + } + return 0.50 + }), + ) + pac = cache.(PressureAwareCache) + + _, err := cache.Insert("seed", NewSizedValue("v", 10)) + require.NoError(t, err) + invocations = 0 + reentrantCall = true + + // Act + _ = pac.EvaluateMemoryPressure() + + // Assert + assert.Equal(t, 1, invocations) + }) + + t.Run("ThreePlusConcurrentCallersReentrantPressureFuncNoStackOverflow", func(t *testing.T) { + // Arrange: 4 concurrent callers enter fresh pressure sampling during cold start in deterministic order + // so G1 holds the primary sampler slot (order == 1), G2 holds the fallback sampler slot (order == 2), and G3 + G4 + // execute the 3rd+ overflow path (order == 3, 4). + const numCallers = 4 + var pac PressureAwareCache + var cache Cache + var activeDepth sync.Map + var maxDepth atomic.Int32 + var enteredCount atomic.Int32 + var overflowDone atomic.Int32 + firstEntered := make(chan struct{}) + secondEntered := make(chan struct{}) + allEntered := make(chan struct{}) + releaseHolders := make(chan struct{}) + + cache = b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + gid := testGoroutineID() + val, _ := activeDepth.LoadOrStore(gid, new(atomic.Int32)) + depthPtr := val.(*atomic.Int32) + d := depthPtr.Add(1) + defer depthPtr.Add(-1) + + for { + prev := maxDepth.Load() + if d <= prev || maxDepth.CompareAndSwap(prev, d) { + break + } + } + if d > 1 { + return 0.10 + } + + order := enteredCount.Add(1) + switch order { + case 1: + close(firstEntered) + case 2: + close(secondEntered) + case numCallers: + close(allEntered) + } + <-allEntered + + // G3 and G4 (3rd+ overflow callers) make re-entrant cache calls while G1 and G2 are still inside PressureFunc. + if (order == 3 || order == 4) && pac != nil { + pac.Compact() + _ = pac.EvaluateMemoryPressure() + _, _ = cache.Insert(fmt.Sprintf("reentrant_%d", order), NewSizedValue("v", 10)) + _ = cache.Erase(fmt.Sprintf("reentrant_%d", order)) + if overflowDone.Add(1) == numCallers-2 { + close(releaseHolders) + } + } + if order <= 2 { + <-releaseHolders + } + return 0.10 + }), + ) + pac = cache.(PressureAwareCache) + + // Act: Launch G1 first (primary), then G2 (fallback), then G3 and G4 (overflow). + var wg sync.WaitGroup + wg.Add(numCallers) + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + <-firstEntered + + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + <-secondEntered + + for range numCallers - 2 { + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + } + wg.Wait() + + // Assert: Every goroutine's PressureFunc recursion depth stayed strictly 1 (no stack overflow) + // and none of the re-entrant calls inside G3/G4 bumped the reclamation epoch (enteredCount == 4 with zero retries). + assert.Equal(t, int32(1), maxDepth.Load()) + assert.Equal(t, int32(numCallers), enteredCount.Load()) + assert.Nil(t, cache.LookUpWithoutChangingOrder("reentrant_3")) + assert.Nil(t, cache.LookUpWithoutChangingOrder("reentrant_4")) + }) + + t.Run("ReentrantReclamationInsidePressureFuncNotOverwrittenByStoreSampledPressure", func(t *testing.T) { + // Arrange: PressureFunc performs a re-entrant Compact() (which marks the cache reclaimed) + // and returns the pre-reclamation reading 0.95 on that call while subsequent calls return 0.10. + var pac PressureAwareCache + var reentrantCompact atomic.Bool + var currentPressure atomic.Uint64 + currentPressure.Store(math.Float64bits(0.10)) + + cache := b.fn( + 200, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if reentrantCompact.CompareAndSwap(true, false) && pac != nil { + pac.Compact() + return 0.95 + } + return math.Float64frombits(currentPressure.Load()) + }), + ) + pac = cache.(PressureAwareCache) + + for i := range 5 { + _, err := cache.Insert(fmt.Sprintf("k-%d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + _ = cache.Erase("k-0") + + // Act: Trigger fresh pressure sampling with reentrantCompact enabled, then insert 80B at 0.10 pressure + // (bringing total occupancy to 130B > targetSize 100B). + reentrantCompact.Store(true) + _, err := cache.Insert("k-after", NewSizedValue("v", 10)) + require.NoError(t, err) + evicted, err := cache.Insert("k-large", NewSizedValue("v", 80)) + require.NoError(t, err) + + // Assert: Because Compact() reclaimed memory during PressureFunc, the pre-reclamation 0.95 reading + // did not overwrite the post-reclamation pressure state, so k-large and all surviving keys remain intact. + assert.Empty(t, evicted) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k-1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k-after")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k-large")) + }) + + t.Run("SamplePressureFreshRetainsPostReclamationDirectReturnValue", func(t *testing.T) { + // Arrange: Populate cache with 2 entries of 40B (80B > targetSize 50B) at healthy pressure (0.10) + // and create slack via a temporary entry so Compact() reclaims memory across all backends. + var pac PressureAwareCache + reentrantCompact := false + pressure := 0.10 + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if reentrantCompact && pac != nil { + reentrantCompact = false + pac.Compact() + } + return pressure + }), + ) + pac = cache.(PressureAwareCache) + + _, err := cache.Insert("k1", NewSizedValue("v1", 40)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 40)) + require.NoError(t, err) + _, err = cache.Insert("tmp", NewSizedValue("vt", 10)) + require.NoError(t, err) + require.NotNil(t, cache.Erase("tmp")) + + // Act: Evaluate memory pressure when PressureFunc performs re-entrant Compact() before returning critical pressure (0.95). + pressure = 0.95 + reentrantCompact = true + evicted := pac.EvaluateMemoryPressure() + + // Assert: Because lossless Compact() does not reduce live byte occupancy (80B > 50B targetSize), + // the 0.95 return value from PressureFunc must not be discarded as 0.0 and must shed LRU tail "k1". + assert.Len(t, evicted, 1) + assert.Nil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + + t.Run("ForegroundReclaimDuringOverflowSamplingAdvancesReclaimEpoch", func(t *testing.T) { + // Arrange: Prepare a cache with two 40B live entries and dirty/fragmented state so Compact() reclaims. + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + var trapActive atomic.Bool + enteredCh := make(chan struct{}, 3) + releaseCh := make(chan struct{}) + + cache := b.fn(100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + p := math.Float64frombits(pressureBits.Load()) + if trapActive.Load() { + enteredCh <- struct{}{} + <-releaseCh + } + return p + }), + ) + pac, ok := cache.(PressureAwareCache) + require.True(t, ok) + + _, err := cache.Insert("k1", NewBytesValue([]byte(strings.Repeat("a", 40)))) + require.NoError(t, err) + _, err = cache.Insert("k2", NewBytesValue([]byte(strings.Repeat("b", 40)))) + require.NoError(t, err) + _, err = cache.Insert("scratch", NewBytesValue([]byte("x"))) + require.NoError(t, err) + require.NotNil(t, cache.Erase("scratch")) + + pressureBits.Store(math.Float64bits(0.95)) + trapActive.Store(true) + + // Act: 3 concurrent samplers enter PressureFunc (primary + fallback + 1 overflow sampler). + var wg sync.WaitGroup + for range 3 { + wg.Add(1) + go func() { + defer wg.Done() + pac.EvaluateMemoryPressure() + }() + } + for range 3 { + <-enteredCh + } + trapActive.Store(false) + pressureBits.Store(math.Float64bits(0.10)) + pac.Compact() // Foreground reclamation while an overflow sampler is in flight + close(releaseCh) + wg.Wait() + + // Assert: Stale 0.95 pre-Compact samples must be invalidated by Compact(), preserving both k1 and k2. + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + + t.Run("ReentrantInsertPreservesLastSampledPressureAndZeroWatermark", func(t *testing.T) { + // Arrange: Populate 4 zero-size entries at 0.0 pressure, then shed 2 of them at 0.95 pressure + // so 2 zero-size entries ("z-2", "z-3") remain with a zero-size watermark of 2. + var cache Cache + var reentrantInsert atomic.Bool + var pressure atomic.Uint64 + pressure.Store(math.Float64bits(0.0)) + + cache = b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if reentrantInsert.CompareAndSwap(true, false) && cache != nil { + _, _ = cache.Insert("reentrant_key", NewSizedValue("v", 10)) + } + return math.Float64frombits(pressure.Load()) + }), + ) + pac := cache.(PressureAwareCache) + + for i := range 4 { + _, err := cache.Insert(fmt.Sprintf("z-%d", i), NewSizedValue("v", 0)) + require.NoError(t, err) + } + pressure.Store(math.Float64bits(0.95)) + evicted := pac.EvaluateMemoryPressure() + require.Len(t, evicted, 2) + require.NotNil(t, cache.LookUpWithoutChangingOrder("z-2")) + require.NotNil(t, cache.LookUpWithoutChangingOrder("z-3")) + + // Act: Insert "trigger" under 0.95 while PressureFunc performs a re-entrant 10B Insert. + reentrantInsert.Store(true) + _, err := cache.Insert("trigger", NewSizedValue("v", 10)) + require.NoError(t, err) + + // Assert: Re-entrant Insert at synthetic 0.0 pressure did not reset the zero-size watermark to 0, + // so the retained zero-size entries ("z-2", "z-3") were not re-shed. + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z-2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z-3")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("reentrant_key")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("trigger")) + }) + }) + } +} + +func TestPressure_ConcurrentReentrantSamplersAndOverflowEpochInvalidation(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("PrimaryAndFallbackReentrantCompactDoNotForceResample", func(t *testing.T) { + // Arrange: Primary sampler G1 and fallback sampler G2 both call Compact() re-entrantly + // inside PressureFunc. Each slot's self-reclamation must be isolated so neither G1 nor G2 + // is forced to retry PressureFunc in lockWithPressure. + var pac PressureAwareCache + var cache Cache + var armed atomic.Bool + var calls atomic.Int32 + primaryEntered := make(chan struct{}) + fallbackEntered := make(chan struct{}) + releasePrimary := make(chan struct{}) + releaseFallback := make(chan struct{}) + primaryCompacted := make(chan struct{}) + + cache = b.fn( + 1000, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { + if !armed.Load() { + return 0.10 + } + n := calls.Add(1) + switch n { + case 1: + // Primary sampler G1 compacts initial dirty slack from "k1". + close(primaryEntered) + <-releasePrimary + pac.Compact() + close(primaryCompacted) + <-releaseFallback + return 0.10 + case 2: + // Fallback sampler G2 creates dirty slack and compacts it re-entrantly. + close(fallbackEntered) + <-primaryCompacted + _, _ = cache.Insert("k2", NewSizedValue("v2", 10)) + _ = cache.Erase("k2") + pac.Compact() + close(releaseFallback) + return 0.10 + default: + return 0.10 + } + }), + ) + pac = cache.(PressureAwareCache) + + _, err := cache.Insert("keep", NewSizedValue("vk", 10)) + require.NoError(t, err) + _, err = cache.Insert("k1", NewSizedValue("v1", 10)) + require.NoError(t, err) + require.NotNil(t, cache.Erase("k1")) // Leaves non-empty dirty slack so G1's Compact() reclaims + armed.Store(true) + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + <-primaryEntered + + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + <-fallbackEntered + + // Act: Release G1 to run re-entrant Compact(), then G2 runs re-entrant Compact(), and both return. + close(releasePrimary) + wg.Wait() + + // Assert: Neither G1 nor G2 was forced to re-sample in lockWithPressure (exactly 2 PressureFunc calls). + assert.Equal(t, int32(2), calls.Load()) + }) + + t.Run("PrimaryReentrantCompactWhileOverflowActiveInvalidatesStaleSamples", func(t *testing.T) { + // Arrange: Primary sampler G1, fallback sampler G2, and overflow sampler G3 are all in-flight. + // When G1 performs a re-entrant Compact() on real dirty slack while G3 is in overflow, + // the reclamation epoch must advance so G2 and G3's stale 0.95 readings from before G1's Compact() are discarded. + var pac PressureAwareCache + var cache Cache + var armed atomic.Bool + var orderCounter atomic.Int32 + g1Ready := make(chan struct{}) + g2Ready := make(chan struct{}) + g3Ready := make(chan struct{}) + releaseG1 := make(chan struct{}) + releaseG2AndG3 := make(chan struct{}) + g1Compacted := make(chan struct{}) + + cache = b.fn( + 200, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if !armed.Load() { + return 0.10 + } + order := orderCounter.Add(1) + switch order { + case 1: + // Primary sampler G1 + close(g1Ready) + <-releaseG1 + pac.Compact() + close(g1Compacted) + return 0.10 + case 2: + // Fallback sampler G2 + close(g2Ready) + <-releaseG2AndG3 + return 0.95 + case 3: + // Overflow sampler G3 + close(g3Ready) + <-releaseG2AndG3 + return 0.95 + default: + return 0.10 + } + }), + ) + pac = cache.(PressureAwareCache) + + // Populate two 60B live entries (120B > targetSize 100B) plus a deleted "tmp" entry at 0.10 pressure + // so G1's re-entrant Compact() performs a real compaction (compactDataStructuresLocked() == true). + _, err := cache.Insert("k1", NewSizedValue("v1", 60)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 60)) + require.NoError(t, err) + _, err = cache.Insert("tmp", NewSizedValue("vt", 10)) + require.NoError(t, err) + require.NotNil(t, cache.Erase("tmp")) + armed.Store(true) + + // Act: Launch G1 first (primary), then G2 (fallback), then G3 (overflow) deterministically. + var wg sync.WaitGroup + wg.Add(3) + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + <-g1Ready + + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + <-g2Ready + + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + <-g3Ready + + // Release G1 first so G1 calls pac.Compact() re-entrantly while G3 is still in overflow, + // then release G2 and G3 which return stale 0.95 from the pre-Compact epoch. + close(releaseG1) + <-g1Compacted + close(releaseG2AndG3) + wg.Wait() + + // Assert: Because G1's re-entrant Compact() advanced the reclamation epoch even while G3 was in overflow, + // G2 and G3's stale 0.95 readings were invalidated and neither k1 nor k2 was evicted. + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + t.Run("ExternalReclamationBeforeReentrantCompactInvalidatesEpoch", func(t *testing.T) { + // Arrange: Sampler G1 is in flight. An external goroutine G2 reclaims. + // Then G1 performs a re-entrant Compact() and returns a stale 0.95 reading. + // The sampler must detect the external reclamation, retry PressureFunc, + // get 0.10, and NOT evict entries. + var pac PressureAwareCache + var cache Cache + var calls atomic.Int32 + var armed atomic.Bool + g1Entered := make(chan struct{}) + releaseG1Compact := make(chan struct{}) + releaseG1Finish := make(chan struct{}) + + cache = b.fn( + 200, + WithInvariantChecking(true), + WithCompactionThreshold(0.90), + WithEvictionThreshold(0.90), + WithPressureFunc(func() float64 { + if !armed.Load() { + return 0.10 + } + n := calls.Add(1) + if n == 1 { + // G1 in-flight + close(g1Entered) + // Wait for G2 external reclamation + <-releaseG1Compact + // Perform re-entrant Compact + _, _ = cache.Insert("k_scratch", NewSizedValue("v_scratch", 10)) + _ = cache.Erase("k_scratch") + pac.Compact() + <-releaseG1Finish + return 0.95 // Stale pre-reclamation reading + } + // Retry should occur, returning 0.10 + return 0.10 + }), + ) + pac = cache.(PressureAwareCache) + + _, err := cache.Insert("k1", NewSizedValue("v1", 100)) + require.NoError(t, err) + + armed.Store(true) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + + <-g1Entered + + // External reclamation G2 + _, _ = cache.Insert("k_ext", NewSizedValue("v_ext", 10)) + _ = cache.Erase("k_ext") + pac.Compact() // This increments externalReclaimEpoch + + close(releaseG1Compact) + close(releaseG1Finish) + wg.Wait() + + // Assert: The sampler detected the external reclamation across its own reentrant compaction, + // retried, got 0.10, and did not shed k1. + assert.GreaterOrEqual(t, calls.Load(), int32(2), "Should retry PressureFunc due to intervening external reclamation") + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k1"), "k1 should not be shed because pressure dropped to 0.10") + }) + }) + } +} + +func TestPressure_PreSampleEpochLoadAndConcurrentSamplerOrdering(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("PreSampleEpochLoadUsesActualSampledEpoch", func(t *testing.T) { + // Arrange: Populate two 40B entries (80B > targetSize 50B) plus a deleted scratch entry + // so Compact() can advance the reclamation epoch while G1 holds the primary sampler slot. + var armed atomic.Bool + var blockG1 atomic.Bool + var g2SampleCalls atomic.Int32 + g1InSample := make(chan struct{}) + releaseG1 := make(chan struct{}) + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if !armed.Load() { + return 0.10 + } + if blockG1.CompareAndSwap(true, false) { + close(g1InSample) + <-releaseG1 + return 0.10 + } + g2SampleCalls.Add(1) + return 0.95 + }), + ) + pac := cache.(PressureAwareCache) + + _, err := cache.Insert("k1", NewSizedValue("v1", 40)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 40)) + require.NoError(t, err) + _, err = cache.Insert("scratch", NewSizedValue("vs", 10)) + require.NoError(t, err) + require.NotNil(t, cache.Erase("scratch")) + + // G1 enters PressureFunc at epoch 0 and holds the primary sampler slot. + armed.Store(true) + blockG1.Store(true) + g1Done := make(chan struct{}) + go func() { + defer close(g1Done) + _ = pac.EvaluateMemoryPressure() + }() + <-g1InSample + + // Advance the reclamation epoch to 1 via Compact() while G1 still holds the primary sampler slot from epoch 0. + pac.Compact() + + // Act: G2 inserts k3 (10B), spins against G1's primary sampler slot, and samples 0.95 via the fallback slot in epoch 1. + evicted, err := cache.Insert("k3", NewSizedValue("v3", 10)) + g2Calls := g2SampleCalls.Load() + close(releaseG1) + <-g1Done + + // Assert: G2 used the actual epoch (1) at which its fallback sample was taken (1 sample call, no false retry) + // and shed k1 under 0.95 pressure. + require.NoError(t, err) + assert.Equal(t, int32(1), g2Calls) + assert.Len(t, evicted, 1) + assert.Nil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k3")) + }) + + t.Run("OlderInFlightSamplerDoesNotOverwriteNewerCompletedSample", func(t *testing.T) { + // Arrange + var mode atomic.Int32 + g1Entered := make(chan struct{}) + releaseG1 := make(chan struct{}) + + cache := b.fn(1000, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + m := mode.Load() + if m == 0 { + return 0.10 + } + if m == 1 && mode.CompareAndSwap(1, 2) { + close(g1Entered) + <-releaseG1 + return 0.10 + } + return 0.95 + }), + ) + + for i := range 80 { + _, err := cache.Insert(fmt.Sprintf("k/%02d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act + mode.Store(1) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + _ = cache.Erase("nonexistent-1") + }() + <-g1Entered + + _ = cache.Erase("nonexistent-2") + close(releaseG1) + wg.Wait() + + _, err := cache.Insert("post-spike", NewSizedValue("v", 10)) + require.NoError(t, err) + + // Assert: Compute surviving byte total solely via the public LookUpWithoutChangingOrder API. + var survivingBytes uint64 + for i := range 80 { + if v := cache.LookUpWithoutChangingOrder(fmt.Sprintf("k/%02d", i)); v != nil { + survivingBytes += v.Size() + } + } + if v := cache.LookUpWithoutChangingOrder("post-spike"); v != nil { + survivingBytes += v.Size() + } + assert.LessOrEqual(t, survivingBytes, uint64(500)) + assert.Nil(t, cache.LookUpWithoutChangingOrder("k/00")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("post-spike")) + }) + + t.Run("ReentrantReclamationInsidePressureFuncInvalidatesConcurrentCallerEpoch", func(t *testing.T) { + // Arrange + var phase atomic.Int32 + g2Sampled := make(chan struct{}) + g1Reclaimed := make(chan struct{}) + + var pac PressureAwareCache + cache := b.fn(1000, + WithInvariantChecking(true), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + switch phase.Load() { + case 1: + if phase.CompareAndSwap(1, 2) { + close(g2Sampled) + <-g1Reclaimed + return 0.95 + } + phase.Store(3) + pac.Compact() + return 0.10 + case 2: + phase.Store(3) + pac.Compact() + return 0.10 + default: + return 0.10 + } + }), + ) + pac = cache.(PressureAwareCache) + + for i := range 85 { + _, err := cache.Insert(fmt.Sprintf("k/%02d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + _, err := cache.Insert("scratch", NewSizedValue("v", 10)) + require.NoError(t, err) + require.NotNil(t, cache.Erase("scratch")) + + // Act + phase.Store(1) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + _, err := cache.Insert("g2-key", NewSizedValue("v", 10)) + assert.NoError(t, err) + }() + + <-g2Sampled + pac.EvaluateMemoryPressure() + close(g1Reclaimed) + wg.Wait() + + // Assert: Verify all 85 original entries plus "g2-key" (860B total) survived via public API. + var survivingBytes uint64 + for i := range 85 { + v := cache.LookUpWithoutChangingOrder(fmt.Sprintf("k/%02d", i)) + if assert.NotNil(t, v) { + survivingBytes += v.Size() + } + } + vG2 := cache.LookUpWithoutChangingOrder("g2-key") + if assert.NotNil(t, vG2) { + survivingBytes += vG2.Size() + } + assert.Equal(t, uint64(860), survivingBytes) + }) + + t.Run("OverflowSamplerReentrantCompactAdvancesEpochAndInvalidatesPrimaryAndFallback", func(t *testing.T) { + // Arrange: Primary sampler G1, fallback sampler G2, and overflow sampler G3 all enter PressureFunc. + // When the 3rd (overflow) sampler G3 calls pac.Compact() re-entrantly on dirty slack, + // markReclaimedLocked() must unconditionally advance reclaimEpoch so G1 and G2's stale 0.95 + // readings are invalidated and resampled as 0.10. + var pac PressureAwareCache + var cache Cache + var armed atomic.Bool + var orderCounter atomic.Int32 + g1Ready := make(chan struct{}) + g2Ready := make(chan struct{}) + g3Ready := make(chan struct{}) + releaseG3 := make(chan struct{}) + g3Compacted := make(chan struct{}) + releaseG1AndG2 := make(chan struct{}) + + cache = b.fn( + 200, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if !armed.Load() { + return 0.10 + } + switch orderCounter.Add(1) { + case 1: + // Primary sampler G1 + close(g1Ready) + <-releaseG1AndG2 + return 0.95 + case 2: + // Fallback sampler G2 + close(g2Ready) + <-releaseG1AndG2 + return 0.95 + case 3: + // Overflow sampler G3 performs re-entrant Compact() + close(g3Ready) + <-releaseG3 + pac.Compact() + close(g3Compacted) + return 0.10 + default: + return 0.10 + } + }), + ) + pac = cache.(PressureAwareCache) + + _, err := cache.Insert("k1", NewSizedValue("v1", 60)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 60)) + require.NoError(t, err) + _, err = cache.Insert("tmp", NewSizedValue("vt", 10)) + require.NoError(t, err) + require.NotNil(t, cache.Erase("tmp")) + armed.Store(true) + + var wg sync.WaitGroup + wg.Add(3) + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + <-g1Ready + + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + <-g2Ready + + go func() { + defer wg.Done() + _ = pac.EvaluateMemoryPressure() + }() + <-g3Ready + + // Act: Release overflow sampler G3 to run re-entrant Compact(), then release G1 and G2. + close(releaseG3) + <-g3Compacted + close(releaseG1AndG2) + wg.Wait() + + // Assert: Neither k1 nor k2 was shed because G3's overflow re-entrant Compact() invalidated G1 and G2's epochs. + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + + t.Run("ExternalReclamationAfterSamplerReentrantCompactClearsExemptionAndForcesResample", func(t *testing.T) { + // Arrange: Primary sampler G1 performs a re-entrant Compact() inside PressureFunc, and before G1 returns 0.95, + // an external caller creates and compacts new dirty slack. The external reclamation must clear G1's + // re-entrant exemption so G1 is forced to resample (0.10) in lockWithPressure. + var pac PressureAwareCache + var cache Cache + var armed atomic.Bool + var calls atomic.Int32 + g1AfterReentrantCompact := make(chan struct{}) + releaseG1 := make(chan struct{}) + + cache = b.fn( + 200, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if !armed.Load() { + return 0.10 + } + if calls.Add(1) == 1 { + pac.Compact() // G1's own re-entrant Compact() + close(g1AfterReentrantCompact) + <-releaseG1 + return 0.95 + } + return 0.10 + }), + ) + pac = cache.(PressureAwareCache) + + _, err := cache.Insert("k1", NewSizedValue("v1", 60)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 60)) + require.NoError(t, err) + _, err = cache.Insert("tmp1", NewSizedValue("vt", 10)) + require.NoError(t, err) + require.NotNil(t, cache.Erase("tmp1")) + armed.Store(true) + + g1Done := make(chan struct{}) + go func() { + defer close(g1Done) + _ = pac.EvaluateMemoryPressure() + }() + <-g1AfterReentrantCompact + + // External caller creates dirty slack and compacts while G1 is still inside PressureFunc. + armed.Store(false) + _, err = cache.Insert("tmp2", NewSizedValue("vt", 10)) + require.NoError(t, err) + require.NotNil(t, cache.Erase("tmp2")) + armed.Store(true) + pac.Compact() + + // Act: Release G1 to return stale 0.95; lockWithPressure must reject the cleared exemption and resample 0.10. + close(releaseG1) + <-g1Done + + // Assert: G1 resampled (calls >= 2) and both k1 and k2 survived. + assert.GreaterOrEqual(t, calls.Load(), int32(2)) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + + t.Run("ContendedEvaluateMemoryPressureDoesNotHijackOlderPreSpikeSample", func(t *testing.T) { + // Arrange: G1 holds the primary sampler slot prepared to return a pre-spike 0.10 reading, + // while G2 completes a fallback 0.95 sample. When G3 subsequently calls EvaluateMemoryPressure() + // while G1 is still holding the primary slot, G3 must invoke PressureFunc itself (0.95) and shed k1 + // rather than hijacking G1's stale 0.10 sample. + var armed atomic.Bool + var calls atomic.Int32 + g1Entered := make(chan struct{}) + releaseG1 := make(chan struct{}) + + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if !armed.Load() { + return 0.10 + } + if calls.Add(1) == 1 { + close(g1Entered) + <-releaseG1 + return 0.10 + } + return 0.95 + }), + ) + pac := cache.(PressureAwareCache) + + _, err := cache.Insert("k1", NewSizedValue("v1", 40)) + require.NoError(t, err) + _, err = cache.Insert("k2", NewSizedValue("v2", 40)) + require.NoError(t, err) + armed.Store(true) + + g1Done := make(chan struct{}) + go func() { + defer close(g1Done) + _ = cache.Erase("miss-1") + }() + <-g1Entered + + // G2 completes a 0.95 sample via the fallback slot while G1 is blocked in the primary slot. + _ = cache.Erase("miss-2") + + // Act: G3 calls EvaluateMemoryPressure() while G1 is still in the primary slot. + evicted := pac.EvaluateMemoryPressure() + callsWhileG1Blocked := calls.Load() + close(releaseG1) + <-g1Done + + // Assert: G3 sampled 0.95 while G1 was still blocked (3 calls before releasing G1) and shed k1. + assert.Equal(t, int32(3), callsWhileG1Blocked) + assert.Len(t, evicted, 1) + assert.Nil(t, cache.LookUpWithoutChangingOrder("k1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("k2")) + }) + }) + } +} + +func TestPressure_ZeroSizeWatermarksAndIdempotency(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("ProtectedZeroSizeInsertDoesNotLeakFloorOneIntoSubsequentEvaluateMemoryPressure", func(t *testing.T) { + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.0), + WithPressureFunc(func() float64 { + return math.Float64frombits(pressureBits.Load()) + }), + ) + pac := cache.(PressureAwareCache) + + for _, k := range []string{"z1", "z2", "z3"} { + _, err := cache.Insert(k, NewSizedValue("v", 0)) + require.NoError(t, err) + } + + // Act 1: Under Tier 2 (0.95) with RetentionRatio = 0.0, inserting "z4" (0B) protects the newly inserted + // MRU entry ("z4") while shedding "z1", "z2", "z3". + pressureBits.Store(math.Float64bits(0.95)) + evictedInsert, err := cache.Insert("z4", NewSizedValue("v", 0)) + require.NoError(t, err) + require.Len(t, evictedInsert, 3) + require.NotNil(t, cache.LookUpWithoutChangingOrder("z4")) + + // Act 2: Immediately call unprotected EvaluateMemoryPressure(). Because Insert did not leak its + // temporary floor of 1 into lastReclaimedZeroCount, "z4" must now be shed (RetentionRatio = 0.0 -> targetZero = 0). + evictedEval := pac.EvaluateMemoryPressure() + assert.Len(t, evictedEval, 1) + assert.Nil(t, cache.LookUpWithoutChangingOrder("z4")) + }) + + t.Run("TargetLenBoundedZeroSheddingIsIdempotentUntilPositiveSurvivorErased", func(t *testing.T) { + // Arrange: 2 positive-size entries at LRU tail (p1, p2: 30B each) + 2 zero-size entries in middle (z1, z2: 0B) + // + 1 positive-size entry at MRU head (p3: 40B). Total = 5 entries, 100B in 100B maxSize, RetentionRatio = 0.60. + // Under Tier 2 (0.95), targetSize = 60B, targetLen = int(5 * 0.60) = 3, targetZero = int(2 * 0.60) = 1. + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.60), + WithPressureFunc(func() float64 { + return math.Float64frombits(pressureBits.Load()) + }), + ) + pac := cache.(PressureAwareCache) + + for _, k := range []string{"p1", "p2"} { + _, err := cache.Insert(k, NewSizedValue("v", 30)) + require.NoError(t, err) + } + for _, k := range []string{"z1", "z2"} { + _, err := cache.Insert(k, NewSizedValue("v", 0)) + require.NoError(t, err) + } + _, err := cache.Insert("p3", NewSizedValue("v", 40)) + require.NoError(t, err) + + // Act 1: First EvaluateMemoryPressure() at 0.95 sheds p1 and p2 (60B) to reach 40B <= targetSize (60B), + // stopping at c.len == 3 == targetLen with z1, z2, p3 surviving (zeroCount = 2 > targetZero = 1). + pressureBits.Store(math.Float64bits(0.95)) + evictedFirst := pac.EvaluateMemoryPressure() + require.Len(t, evictedFirst, 2) + require.NotNil(t, cache.LookUpWithoutChangingOrder("z1")) + require.NotNil(t, cache.LookUpWithoutChangingOrder("z2")) + require.NotNil(t, cache.LookUpWithoutChangingOrder("p3")) + + // Act 2: Immediate second EvaluateMemoryPressure() at 0.95 must be 100% idempotent (0 evictions). + evictedSecond := pac.EvaluateMemoryPressure() + assert.Empty(t, evictedSecond) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("p3")) + + // Act 3: Erase positive-size survivor "p3". Deferred zero-size entry "z1" is now shed, leaving "z2". + require.NotNil(t, cache.Erase("p3")) + _ = pac.EvaluateMemoryPressure() + assert.Nil(t, cache.LookUpWithoutChangingOrder("z1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z2")) + }) + + t.Run("InterleavedZeroSizeAtTailShedsInTrueLRUOrderDuringByteShedding", func(t *testing.T) { + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + return math.Float64frombits(pressureBits.Load()) + }), + ) + pac := cache.(PressureAwareCache) + + // Populate in LRU-to-MRU order: z1 (0B), p1 (80B), p2 (20B). + _, err := cache.Insert("z1", NewSizedValue("vz", 0)) + require.NoError(t, err) + _, err = cache.Insert("p1", NewSizedValue("vp1", 80)) + require.NoError(t, err) + _, err = cache.Insert("p2", NewSizedValue("vp2", 20)) + require.NoError(t, err) + + // Act: EvaluateMemoryPressure() at 0.95 (targetSize = 50B) must evict z1 and p1 in True LRU order. + pressureBits.Store(math.Float64bits(0.95)) + evicted := pac.EvaluateMemoryPressure() + require.Len(t, evicted, 2) + assert.Nil(t, cache.LookUpWithoutChangingOrder("z1")) + assert.Nil(t, cache.LookUpWithoutChangingOrder("p1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("p2")) + }) + + t.Run("InterleavedZeroSizeDuringByteSheddingPreservesLRUOrderWithoutOverEviction", func(t *testing.T) { + // Arrange: Populate in LRU-to-MRU order with maxSize = 100, retention = 0.60: + // p1=5B, p2=5B, p3=5B, p4=5B, z1=0B, p5=25B, z2=0B, z3=0B, p6=55B (total 100B, 9 entries, 3 zero-size). + // Under Tier 2 (0.95): targetSize = 60B, targetLen = int(9*0.60) = 5, targetZeroCount = int(3*0.60) = 1. + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.60), + WithPressureFunc(func() float64 { + return math.Float64frombits(pressureBits.Load()) + }), + ) + pac := cache.(PressureAwareCache) + + entries := []struct { + key string + val int64 + size uint64 + }{ + {"p1", 1, 5}, + {"p2", 2, 5}, + {"p3", 3, 5}, + {"p4", 4, 5}, + {"z1", 5, 0}, + {"p5", 6, 25}, + {"z2", 7, 0}, + {"z3", 8, 0}, + {"p6", 9, 55}, + } + for _, e := range entries { + _, err := cache.Insert(e.key, testData{value: e.val, dataSize: e.size}) + require.NoError(t, err) + } + + // Act: EvaluateMemoryPressure() at 0.95 must evict [p1, p2, p3, p4, z1, p5] in strict LRU order, + // without evicting p5 before z1 and without over-evicting z2 after byte shedding reaches 55B <= 60B. + pressureBits.Store(math.Float64bits(0.95)) + evicted := pac.EvaluateMemoryPressure() + + // Assert + assertEvictedValues(t, evicted, []int64{1, 2, 3, 4, 5, 6}) + for _, k := range []string{"p1", "p2", "p3", "p4", "z1", "p5"} { + assert.Nil(t, cache.LookUpWithoutChangingOrder(k)) + } + for _, k := range []string{"z2", "z3", "p6"} { + assert.NotNil(t, cache.LookUpWithoutChangingOrder(k)) + } + }) + + t.Run("NoOpUpdateSizeAfterEvaluateDoesNotRatchetZeroEntries", func(t *testing.T) { + // Arrange: Populate [p1: 30B, p2: 30B, z1: 0B, z2: 0B, p3: 40B] (100B, 5 entries, retention = 0.60). + // First EvaluateMemoryPressure() at 0.95 sheds p1, p2 and stops at targetLen = 3 with [z1, z2, p3] (40B <= 60B). + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.60), + WithPressureFunc(func() float64 { + return math.Float64frombits(pressureBits.Load()) + }), + ) + pac := cache.(PressureAwareCache) + + for _, k := range []string{"p1", "p2"} { + _, err := cache.Insert(k, NewSizedValue("v", 30)) + require.NoError(t, err) + } + for _, k := range []string{"z1", "z2"} { + _, err := cache.Insert(k, NewSizedValue("v", 0)) + require.NoError(t, err) + } + _, err := cache.Insert("p3", NewSizedValue("v", 40)) + require.NoError(t, err) + + pressureBits.Store(math.Float64bits(0.95)) + require.Len(t, pac.EvaluateMemoryPressure(), 2) + + // Act: Call UpdateSize("p3", 0) (no-op delta) and UpdateSize("p3", 10) (grows p3 to 50B <= targetSize 60B) under 0.95. + require.NoError(t, cache.UpdateSize("p3", 0)) + require.NoError(t, cache.UpdateSize("p3", 10)) + + // Assert: Protected mutation with protectedSize <= targetSize must not ratchet-evict z1 or z2. + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z1")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("p3")) + }) + + t.Run("PositiveToZeroOverwriteResetsLastReclaimedLen", func(t *testing.T) { + // Arrange: Populate [p1: 30B, p2: 30B, z1: 0B, z2: 0B, p3: 40B] (100B, 5 entries, retention = 0.60). + // First EvaluateMemoryPressure() at 0.95 sheds p1, p2 and stops at targetLen = 3 with [z1, z2, p3] + // (lastReclaimedLen = 3, lastReclaimedZeroCount = 1). + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + cache := b.fn( + 100, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.60), + WithPressureFunc(func() float64 { + return math.Float64frombits(pressureBits.Load()) + }), + ) + pac := cache.(PressureAwareCache) + + for _, k := range []string{"p1", "p2"} { + _, err := cache.Insert(k, NewSizedValue("v", 30)) + require.NoError(t, err) + } + for _, k := range []string{"z1", "z2"} { + _, err := cache.Insert(k, NewSizedValue("v", 0)) + require.NoError(t, err) + } + _, err := cache.Insert("p3", NewSizedValue("v", 40)) + require.NoError(t, err) + + pressureBits.Store(math.Float64bits(0.95)) + evictedFirst := pac.EvaluateMemoryPressure() + require.Len(t, evictedFirst, 2) + + // Act: Convert positive survivor p3 (40B) into a 0B entry via overwrite under 0.95, + // then verify EvaluateMemoryPressure() is idempotent. + evictedOverwrite, err := cache.Insert("p3", NewSizedValue("v0", 0)) + require.NoError(t, err) + evictedSecond := pac.EvaluateMemoryPressure() + + // Assert: Overwriting p3 from 40B to 0B reset lastReclaimedLen from 3 to 0, allowing + // the excess zero-size entries (z1, z2) to be shed down to targetZeroCount = 1 ("p3"). + assert.Len(t, evictedOverwrite, 2) + assert.Empty(t, evictedSecond) + assert.Nil(t, cache.LookUpWithoutChangingOrder("z1")) + assert.Nil(t, cache.LookUpWithoutChangingOrder("z2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("p3")) + }) + + t.Run("ValidZeroPressureSampleWhileRefreshFlagSetResetsZeroWatermark", func(t *testing.T) { + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + cache := b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + return math.Float64frombits(pressureBits.Load()) + }), + ) + pac := cache.(PressureAwareCache) + + for i := range 4 { + _, err := cache.Insert(fmt.Sprintf("z-%d", i), NewSizedValue("v", 0)) + require.NoError(t, err) + } + + // Shed 2 of 4 zero-size entries at 0.95 (sets lastReclaimedZeroCount = 2 and pressureNeedsRefresh = true). + pressureBits.Store(math.Float64bits(0.95)) + require.Len(t, pac.EvaluateMemoryPressure(), 2) + + // Act: Drop pressure to exact 0.0 while pressureNeedsRefresh is set, and call EvaluateMemoryPressure(). + // Because 0.0 is a genuine sample (hasValidSample == true), lastReclaimedZeroCount must reset to 0. + pressureBits.Store(math.Float64bits(0.0)) + assert.Empty(t, pac.EvaluateMemoryPressure()) + + // Spike pressure back to 0.95: 1 of the 2 remaining zero-size entries ("z-2") must be shed. + pressureBits.Store(math.Float64bits(0.95)) + evicted := pac.EvaluateMemoryPressure() + assert.Len(t, evicted, 1) + assert.Nil(t, cache.LookUpWithoutChangingOrder("z-2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z-3")) + }) + }) + } +} + +type deadlockProbeValue struct { + size uint64 + onSize func() +} + +func (v deadlockProbeValue) Size() uint64 { + if v.onSize != nil { + v.onSize() + } + return v.size +} + +func TestPressure_TargetSizeBoundsAndSafeSizeCallbacks(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("UpdateWithoutChangingOrderSamplesValueSizeOutsideLock", func(t *testing.T) { + // Arrange + c := b.fn(100, WithInvariantChecking(true)) + _, err := c.Insert("k1", NewSizedValue("v1", 10)) + require.NoError(t, err) + + readSucceededInsideSize := false + probe := deadlockProbeValue{ + size: 10, + onSize: func() { + if c.LookUpWithoutChangingOrder("k1") != nil && c.LookUp("k1") != nil { + readSucceededInsideSize = true + } + }, + } + + // Act + err = c.UpdateWithoutChangingOrder("k1", probe) + + // Assert + require.NoError(t, err) + assert.True(t, readSucceededInsideSize) + }) + + t.Run("SafeSizeCallbackAndPressureAwareCache", func(t *testing.T) { + // Arrange + pressure := 0.10 + cache := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionRetentionRatio(0.50), + ) + for i := range 10 { + _, err := cache.Insert(fmt.Sprintf("k-%d", i), NewSizedValue("v", 10)) + require.NoError(t, err) + } + + // Act: Re-entrant LookUpWithoutChangingOrder inside ValueType.Size() and PressureAwareCache evaluation. + err := cache.UpdateWithoutChangingOrder("k-0", deadlockProbeValue{ + size: 10, + onSize: func() { + _ = cache.LookUpWithoutChangingOrder("k-0") + }, + }) + pac, ok := cache.(PressureAwareCache) + require.True(t, ok) + pac.Compact() + pressure = 0.95 + evicted := pac.EvaluateMemoryPressure() + + // Assert + require.NoError(t, err) + assert.Len(t, evicted, 5) + }) + + t.Run("ComputeTargetSizeBoundsAndZeroSizeEntryShedding", func(t *testing.T) { + // Arrange & Act 1: Zero-size entry shedding at 50% retention. + pressure := 0.10 + c := b.fn( + 100, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + for i := range 10 { + _, err := c.Insert(fmt.Sprintf("zero-%d", i), NewSizedValue("v", 0)) + require.NoError(t, err) + } + pressure = 0.95 + evicted := c.EvaluateMemoryPressure() + + // Assert 1 + assert.Len(t, evicted, 5) + for i := range 5 { + assert.Nil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("zero-%d", i))) + } + for i := 5; i < 10; i++ { + assert.NotNil(t, c.LookUpWithoutChangingOrder(fmt.Sprintf("zero-%d", i))) + } + + // Arrange & Act 2: maxSize == 1 with positive retention (0.50) clamps targetSize to 1 + // instead of truncating to 0, so a 1-byte entry survives EvaluateMemoryPressure(). + minCache := b.fn( + 1, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return 0.95 }), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + _, err := minCache.Insert("one-byte", NewSizedValue("x", 1)) + require.NoError(t, err) + assert.Empty(t, minCache.EvaluateMemoryPressure()) + assert.NotNil(t, minCache.LookUpWithoutChangingOrder("one-byte")) + + // Arrange & Act 3: maxSize == math.MaxUint64 with retention 1.0, 0.50, and Nextafter(1.0, 0.0) + // avoids float64-to-uint64 overflow in targetSize calculation. + for _, retention := range []float64{1.0, 0.50, math.Nextafter(1.0, 0.0)} { + maxCache := b.fn( + math.MaxUint64, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return 0.95 }), + WithEvictionRetentionRatio(retention), + ).(PressureAwareCache) + _, err := maxCache.Insert("large", NewSizedValue("v", 1<<62)) + require.NoError(t, err) + assert.Empty(t, maxCache.EvaluateMemoryPressure()) + assert.NotNil(t, maxCache.LookUpWithoutChangingOrder("large")) + } + }) + + t.Run("EvaluateMemoryPressureWithZeroRetentionFlushesAllEntries", func(t *testing.T) { + // Arrange + pressure := 0.10 + c := b.fn( + 1000, + WithInvariantChecking(true), + WithPressureFunc(func() float64 { return pressure }), + WithEvictionRetentionRatio(0.0), + ).(PressureAwareCache) + _, err := c.Insert("only-key", NewSizedValue("v", 100)) + require.NoError(t, err) + + // Act: EvaluateMemoryPressure passes the unprotected sentinel so all entries are shed when retention == 0.0. + pressure = 0.95 + evicted := c.EvaluateMemoryPressure() + + // Assert + assert.Len(t, evicted, 1) + assert.Nil(t, c.LookUpWithoutChangingOrder("only-key")) + }) + }) + } +} + +func TestPressure_BelowTier2ResetsZeroWatermarks(t *testing.T) { + for _, b := range allBackends() { + t.Run(b.name, func(t *testing.T) { + t.Run("SingleSurvivorDeletionResetsWatermarks", func(t *testing.T) { + probe := newPressureProbe(0.10) + cache := b.fn( + 100, + WithInvariantChecking(true), + probe.Option(), + WithCompactionThreshold(0.70), + WithEvictionThreshold(0.80), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + for i := 0; i < 127; i++ { + _, err := cache.Insert(fmt.Sprintf("p-%03d", i), NewSizedValue("z", 0)) + require.NoError(t, err) + } + _, err := cache.Insert("survivor", NewSizedValue("z", 0)) + require.NoError(t, err) + + // At pressure 0.80 (retention 0.50), 64 of 128 zero-size entries survive, + // setting lastReclaimedZeroCount = 64, lastReclaimedLen = 64, peakEntryLen = 64. + probe.Set(0.80) + evicted := cache.EvaluateMemoryPressure() + require.Len(t, evicted, 64) + require.NotNil(t, cache.LookUpWithoutChangingOrder("survivor")) + + // Drop below Tier 2 and delete all 63 remaining "p-" entries in one operation, + // triggering shouldReclaimSingleSurvivorOnDelete == true with "survivor" as the sole entry. + probe.Set(0.10) + cache.EraseEntriesWithGivenPrefix("p-") + require.NotNil(t, cache.LookUpWithoutChangingOrder("survivor")) + + // Return to Tier 2 (retention 0.75 => int(1 * 0.75) == 0). + // Because the single-survivor deletion below Tier 2 reset lastReclaimedZeroCount to 0, + // "survivor" must now be shed. + probe.Set(0.875) + evictedAfter := cache.EvaluateMemoryPressure() + assert.Len(t, evictedAfter, 1) + assert.Nil(t, cache.LookUpWithoutChangingOrder("survivor")) + }) + + t.Run("MissedMutationsBelowTier2ResetsWatermarks", func(t *testing.T) { + ops := []struct { + name string + act func(c Cache) + }{ + {"EraseMiss", func(c Cache) { c.Erase("non-existent-key") }}, + {"ErasePrefixMiss", func(c Cache) { c.EraseEntriesWithGivenPrefix("non-existent-prefix") }}, + {"UpdateSizeMiss", func(c Cache) { _ = c.UpdateSize("non-existent-key", 1) }}, + } + + for _, op := range ops { + t.Run(op.name, func(t *testing.T) { + probe := newPressureProbe(0.10) + cache := b.fn( + 100, + WithInvariantChecking(true), + probe.Option(), + WithCompactionThreshold(0.70), + WithEvictionThreshold(0.80), + WithEvictionRetentionRatio(0.50), + ).(PressureAwareCache) + + _, err := cache.Insert("z1", NewSizedValue("z", 0)) + require.NoError(t, err) + _, err = cache.Insert("z2", NewSizedValue("z", 0)) + require.NoError(t, err) + + probe.Set(0.875) + evicted := cache.EvaluateMemoryPressure() + require.Len(t, evicted, 1) + require.NotNil(t, cache.LookUpWithoutChangingOrder("z2")) + + probe.Set(0.10) + op.act(cache) + + probe.Set(0.875) + evictedAfter := cache.EvaluateMemoryPressure() + assert.Len(t, evictedAfter, 1) + assert.Nil(t, cache.LookUpWithoutChangingOrder("z2")) + }) + } + }) + + t.Run("PostRetryPressureResolutionAndSamplingGoroutineWatermarkGuard", func(t *testing.T) { + // Arrange: Seed 4 zero-size entries and shed 2 under Tier 2 (0.95) so lastReclaimedZeroCount == 2. + // Then exercise post-retry pressure resolution when a re-entrant operation inside PressureFunc + // runs while watermarks are active, followed by a non-sampling below-Tier-2 post-retry resolution. + var cache Cache + var pac PressureAwareCache + var pressureBits atomic.Uint64 + pressureBits.Store(math.Float64bits(0.10)) + var reentrantMiss atomic.Bool + + cache = b.fn( + 1000, + WithInvariantChecking(true), + WithEvictionThreshold(0.90), + WithEvictionRetentionRatio(0.50), + WithPressureFunc(func() float64 { + if reentrantMiss.CompareAndSwap(true, false) && cache != nil { + _ = cache.Erase("absent-during-sampling") + } + return math.Float64frombits(pressureBits.Load()) + }), + ) + pac = cache.(PressureAwareCache) + + for i := range 4 { + _, err := cache.Insert(fmt.Sprintf("z-%d", i), NewSizedValue("z", 0)) + require.NoError(t, err) + } + + pressureBits.Store(math.Float64bits(0.95)) + require.Len(t, pac.EvaluateMemoryPressure(), 2) + + // Act 1: Re-entrant Erase("absent-during-sampling") inside PressureFunc while outer pressure is 0.95 + // must not clear lastReclaimedZeroCount, so EvaluateMemoryPressure() remains idempotent (0 evictions). + reentrantMiss.Store(true) + assert.Empty(t, pac.EvaluateMemoryPressure()) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z-2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z-3")) + + // Act 2: Non-sampling operation below Tier 2 (0.10) resets the watermark so a subsequent 0.95 evaluation sheds z-2. + pressureBits.Store(math.Float64bits(0.10)) + _ = cache.Erase("absent-below-tier2") + pressureBits.Store(math.Float64bits(0.95)) + evictedAfterReset := pac.EvaluateMemoryPressure() + assert.Len(t, evictedAfterReset, 1) + assert.Nil(t, cache.LookUpWithoutChangingOrder("z-2")) + assert.NotNil(t, cache.LookUpWithoutChangingOrder("z-3")) + }) + }) + } +} diff --git a/radix_lru.go b/radix_lru.go index 27b65b7..4b743ca 100644 --- a/radix_lru.go +++ b/radix_lru.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus +package lru import ( "fmt" @@ -59,24 +59,28 @@ type radixCache struct { // mu synchronizes concurrent access to all cache data structures. mu sync.RWMutex - // opts contains configuration options such as invariant checking. - opts Options + pressureState } +var foregroundNoProtectNode radixNode + // NewRadixCache creates a new RadixCache instance bounded by maxSize (in bytes). // maxSize must be greater than zero; otherwise NewRadixCache panics. func NewRadixCache(maxSize uint64, opts ...Option) Cache { if maxSize == 0 { panic("maxSize must be greater than zero") } - options := ApplyOptions(opts...) + return newRadixCacheWithOptions(maxSize, ApplyOptions(opts...)) +} + +func newRadixCacheWithOptions(maxSize uint64, options Options) Cache { c := &radixCache{ - maxSize: maxSize, - root: &radixNode{}, - opts: options, + maxSize: maxSize, + root: &radixNode{}, + pressureState: pressureState{options: options}, } - if c.opts.EnableInvariantChecking { + if c.options.EnableInvariantChecking { c.checkInvariants() } @@ -96,12 +100,19 @@ func (c *radixCache) checkInvariants() { // INVARIANT 3: LRU list validation lruCount := 0 + zeroCount := 0 var sumSize uint64 var prevNode *radixNode for curr := c.head; curr != nil; curr = curr.next { lruCount++ + if math.MaxUint64-sumSize < curr.size { + panic("radixCache invariant violation: sumSize uint64 overflow") + } sumSize += curr.size + if curr.size == 0 { + zeroCount++ + } if curr.value == nil { panic(fmt.Sprintf("radixCache invariant violation: unexpected nil value in LRU list for prefix '%s'", curr.prefix)) } @@ -133,6 +144,10 @@ func (c *radixCache) checkInvariants() { panic(fmt.Sprintf("radixCache invariant violation: LRU list count %d does not match tracked len %d", lruCount, c.len)) } + if zeroCount != c.zeroSizeCount { + panic(fmt.Sprintf("radixCache invariant violation: zeroSizeCount %d does not match live zero-size entries %d", c.zeroSizeCount, zeroCount)) + } + if sumSize != c.currentSize { panic(fmt.Sprintf("radixCache: currentSize drift: currentSize=%d sumSize=%d", c.currentSize, sumSize)) } @@ -151,6 +166,9 @@ func (c *radixCache) checkInvariants() { if c.root == nil { panic("radixCache invariant violation: root node is nil") } + if c.root.prefix != "" { + panic("radixCache invariant violation: root node must have empty prefix") + } if c.root.parent != nil { panic("radixCache invariant violation: root node must not have a parent") } @@ -167,12 +185,22 @@ func (c *radixCache) checkInvariants() { for curr != nil { if curr.value != nil { treeCount++ + if math.MaxUint64-treeSumSize < curr.size { + panic("radixCache invariant violation: treeSumSize uint64 overflow") + } treeSumSize += curr.size - // A node is verifiably in the LRU list if it is the head or has a non-nil prev pointer. - inLRU := c.head == curr || curr.prev != nil + // A node is verifiably in the LRU list iff it is the head (with nil prev) or its prev's next points back to it. + inLRU := (c.head == curr && curr.prev == nil) || (curr.prev != nil && curr.prev.next == curr) if !inLRU { panic(fmt.Sprintf("radixCache invariant violation: node with prefix '%s' has value but is missing from LRU list", curr.prefix)) } + } else { + if curr.size != 0 { + panic(fmt.Sprintf("radixCache invariant violation: routing node with prefix '%s' has non-zero size %d", curr.prefix, curr.size)) + } + if curr.prev != nil || curr.next != nil || c.head == curr || c.tail == curr { + panic(fmt.Sprintf("radixCache invariant violation: routing node with prefix '%s' has non-nil LRU pointers", curr.prefix)) + } } // Validate child pointers and sibling ordering @@ -222,15 +250,6 @@ func (c *radixCache) checkInvariants() { } } -// longestCommonPrefix finds the length of the longest common prefix of a and b. -func longestCommonPrefix(a, b string) int { - i := 0 - for i < len(a) && i < len(b) && a[i] == b[i] { - i++ - } - return i -} - // getChild finds a child node whose prefix starts with byte b. // Takes advantage of sorted sibling order for early-exit termination. func (n *radixNode) getChild(b byte) *radixNode { @@ -267,6 +286,7 @@ func (n *radixNode) removeChild(childToRemove *radixNode) { return } } + panic("removeChild: requested child not found in sibling list") } // replaceChild finds oldChild in the sibling linked list and substitutes it with newChild, @@ -274,6 +294,7 @@ func (n *radixNode) removeChild(childToRemove *radixNode) { func (n *radixNode) replaceChild(oldChild, newChild *radixNode) { for pcurr := &n.child; *pcurr != nil; pcurr = &(*pcurr).sibling { if *pcurr == oldChild { + newChild.parent = n newChild.sibling = oldChild.sibling *pcurr = newChild @@ -282,12 +303,13 @@ func (n *radixNode) replaceChild(oldChild, newChild *radixNode) { return } } + panic("replaceChild: requested child not found in sibling list") } -// insertNode inserts a new key into the radix tree and returns the leaf node and previous value (if any). -func (c *radixCache) insertNode(key string, value ValueType) (*radixNode, ValueType) { +// insertNode inserts a new key into the radix tree and returns the leaf node. +func (c *radixCache) insertNode(key string, value ValueType) *radixNode { if value == nil { - return nil, nil + return nil } node := c.root @@ -295,20 +317,19 @@ func (c *radixCache) insertNode(key string, value ValueType) (*radixNode, ValueT for { if len(search) == 0 { - oldValue := node.value node.value = value - return node, oldValue + return node } child := node.getChild(search[0]) if child == nil { // Clone the substring to prevent memory leaks from sliced string headers pinning large backing arrays newLeaf := &radixNode{ - prefix: strings.Clone(search), + prefix: clonePrefix(search), value: value, } node.addChild(newLeaf) - return newLeaf, nil + return newLeaf } lcp := longestCommonPrefix(search, child.prefix) @@ -319,29 +340,31 @@ func (c *radixCache) insertNode(key string, value ValueType) (*radixNode, ValueT continue } + // Clone both split prefix halves so surviving intermediate routing nodes never pin + // the underlying backing arrays of large evicted leaf keys. + oldPrefix := child.prefix splitNode := &radixNode{ - prefix: strings.Clone(child.prefix[:lcp]), + prefix: clonePrefix(oldPrefix[:lcp]), parent: node, } node.replaceChild(child, splitNode) - child.prefix = strings.Clone(child.prefix[lcp:]) + child.prefix = clonePrefix(oldPrefix[lcp:]) child.sibling = nil splitNode.addChild(child) if lcp == len(search) { - oldValue := splitNode.value splitNode.value = value - return splitNode, oldValue + return splitNode } newLeaf := &radixNode{ - prefix: strings.Clone(search[lcp:]), + prefix: clonePrefix(search[lcp:]), value: value, } splitNode.addChild(newLeaf) - return newLeaf, nil + return newLeaf } } @@ -393,6 +416,7 @@ func (c *radixCache) compressPathUpwards(curr *radixNode) { if curr.child == nil { parent := curr.parent parent.removeChild(curr) + curr.prefix = "" curr = parent continue } @@ -402,11 +426,11 @@ func (c *radixCache) compressPathUpwards(curr *radixNode) { onlyChild.prefix = curr.prefix + onlyChild.prefix onlyChild.parent = curr.parent - parent := curr.parent curr.parent.replaceChild(curr, onlyChild) + curr.child = nil + curr.prefix = "" - curr = parent - continue + return } break @@ -475,6 +499,9 @@ func (c *radixCache) remove(node *radixNode) { // It returns the deleted ValueType. func (c *radixCache) eraseInternal(node *radixNode) ValueType { deletedEntry := node.value + if deletedEntry != nil { + c.onEntryDeleted(node.size) + } c.currentSize -= node.size node.size = 0 @@ -484,6 +511,15 @@ func (c *radixCache) eraseInternal(node *radixNode) ValueType { return deletedEntry } +func (c *radixCache) clearEmptyTreeStateLocked() { + c.resetWatermarks() +} + +func (c *radixCache) resetEmptyTreeLocked() { + c.clearEmptyTreeStateLocked() + c.markReclaimedLocked() +} + // evictOne removes and returns the least recently used entry (c.tail). func (c *radixCache) evictOne() ValueType { node := c.tail @@ -498,6 +534,7 @@ func (c *radixCache) sweepAndUnlink(node *radixNode) { curr := node for curr != nil { if curr.value != nil { + c.onEntryDeleted(curr.size) c.currentSize -= curr.size curr.size = 0 c.remove(curr) @@ -510,12 +547,25 @@ func (c *radixCache) sweepAndUnlink(node *radixNode) { } for curr != node && curr.sibling == nil { - curr = curr.parent + parent := curr.parent + curr.prefix = "" + curr.parent = nil + curr.child = nil + curr = parent } if curr == node { + curr.prefix = "" + curr.parent = nil + curr.child = nil + curr.sibling = nil return } - curr = curr.sibling + next := curr.sibling + curr.prefix = "" + curr.parent = nil + curr.child = nil + curr.sibling = nil + curr = next } } @@ -523,6 +573,57 @@ func (c *radixCache) sweepAndUnlink(node *radixNode) { // Cache Interface Implementation // ============================================================================ +func (c *radixCache) unlock() { + if c.options.EnableInvariantChecking { + c.checkInvariants() + } + c.mu.Unlock() +} + +func (c *radixCache) rUnlock() { + if c.options.EnableInvariantChecking { + c.checkInvariants() + } + c.mu.RUnlock() +} + +func (c *radixCache) finishDeleteReclaimLocked(sizeBefore, sampledEpoch uint64, pressure float64) { + if c.len == 0 { + hadSlack := c.hasEmptyDeleteSlack(false) + c.clearEmptyTreeStateLocked() + if (hadSlack || sizeBefore > 0) && c.reclaimEpoch.Load() == sampledEpoch && c.hasElevatedPressureToInvalidate(pressure) { + c.markReclaimedLocked() + } + return + } + reclaimedSingleSurvivor := false + if c.shouldReclaimSingleSurvivorOnDelete(c.len, c.deletedSinceCompact > 0, false) { + c.compactDataStructuresLocked() + reclaimedSingleSurvivor = true + } + c.maybeReclaimUnderPressureLocked(pressure, &foregroundNoProtectNode) + if (reclaimedSingleSurvivor || c.currentSize < sizeBefore) && c.reclaimEpoch.Load() == sampledEpoch && c.hasElevatedPressureToInvalidate(pressure) { + c.markReclaimedLocked() + } +} + +func (c *radixCache) finishMutationReclaimLocked(evictedValues []ValueType, protectedNode *radixNode, reclaimedPre bool, sizeBefore, sampledEpoch uint64, pressure float64) []ValueType { + evictedByPressure := c.maybeReclaimUnderPressureLocked(pressure, protectedNode) + if len(evictedValues) == 0 { + evictedValues = evictedByPressure + } else if len(evictedByPressure) > 0 { + evictedValues = append(evictedValues, evictedByPressure...) + } + netByteReduced := c.currentSize < sizeBefore + if c.shouldCompactAfterMutation(reclaimedPre, netByteReduced, c.deletedSinceCompact > 0, pressure) { + c.compactDataStructuresLocked() + } + if c.shouldMarkReclaimedAfterMutation(reclaimedPre, netByteReduced, sampledEpoch, pressure) { + c.markReclaimedLocked() + } + return evictedValues +} + // Insert inserts or updates a key-value entry in the cache. // If the key exists, its value is updated and moved to MRU. // If capacity is exceeded, excess LRU entries are evicted and returned. @@ -539,61 +640,75 @@ func (c *radixCache) Insert(key string, value ValueType) ([]ValueType, error) { return nil, ErrInvalidEntrySize } - c.mu.Lock() - defer func() { - if c.opts.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) + defer c.unlock() - if node, oldValue := c.insertNode(key, value); oldValue != nil { + var evictedValues []ValueType + sizeBefore := c.currentSize + reclaimedPreInsert := false + evictedPreInsert := false + + node, exists := c.getNode(key) + if exists { + c.onEntrySizeUpdated(node.size, valueSize) + c.moveToFront(node) c.currentSize -= node.size - c.currentSize += valueSize + for valueSize > c.maxSize-c.currentSize && c.tail != nil && c.tail != node { + if evicted := c.evictOne(); evicted != nil { + evictedValues = append(evictedValues, evicted) + evictedPreInsert = true + } + } + node.value = value node.size = valueSize - c.moveToFront(node) + c.currentSize += valueSize + reclaimedPreInsert = c.shouldReclaimSingleSurvivorOnMutation(c.len, c.deletedSinceCompact > 0, false, evictedPreInsert, c.currentSize, sizeBefore, pressure) } else { + // Evict from the LRU tail before inserting into the trie to avoid redundant node splits and merges. + for valueSize > c.maxSize-c.currentSize && c.tail != nil { + if evicted := c.evictOne(); evicted != nil { + evictedValues = append(evictedValues, evicted) + evictedPreInsert = true + } + } + if c.shouldReclaimEmptyPreInsert(c.len, false, evictedPreInsert, valueSize, sizeBefore, pressure) { + c.clearEmptyTreeStateLocked() + reclaimedPreInsert = true + } else if c.shouldReclaimSingleSurvivorOnMutation(c.len, c.deletedSinceCompact > 0, false, evictedPreInsert, c.currentSize+valueSize, sizeBefore, pressure) { + reclaimedPreInsert = true + } + node = c.insertNode(key, value) node.size = valueSize c.pushFront(node) + c.onEntryInserted(c.len, valueSize) c.currentSize += valueSize } - var evictedValues []ValueType - for c.currentSize > c.maxSize && c.tail != nil { - evictedValues = append(evictedValues, c.evictOne()) - } - + evictedValues = c.finishMutationReclaimLocked(evictedValues, node, reclaimedPreInsert, sizeBefore, sampledEpoch, pressure) return evictedValues, nil } // Erase removes the entry associated with key, returning its value (or nil if not found). func (c *radixCache) Erase(key string) (value ValueType) { - c.mu.Lock() - defer func() { - if c.opts.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) + defer c.unlock() node, ok := c.getNode(key) if !ok { return nil } - return c.eraseInternal(node) + sizeBefore := c.currentSize + deleted := c.eraseInternal(node) + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) + return deleted } // LookUp retrieves the value for key and promotes it to the MRU position. // Returns nil if key is not found. func (c *radixCache) LookUp(key string) (value ValueType) { c.mu.Lock() - defer func() { - if c.opts.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + defer c.unlock() node, ok := c.getNode(key) if !ok { @@ -608,12 +723,7 @@ func (c *radixCache) LookUp(key string) (value ValueType) { // Returns nil if key is not found. func (c *radixCache) LookUpWithoutChangingOrder(key string) (value ValueType) { c.mu.RLock() - defer func() { - if c.opts.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.RUnlock() - }() + defer c.rUnlock() node, ok := c.getNode(key) if !ok { @@ -636,12 +746,7 @@ func (c *radixCache) UpdateWithoutChangingOrder(key string, value ValueType) err valueSize := value.Size() c.mu.Lock() - defer func() { - if c.opts.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + defer c.unlock() node, ok := c.getNode(key) if !ok { @@ -657,32 +762,91 @@ func (c *radixCache) UpdateWithoutChangingOrder(key string, value ValueType) err } // UpdateSize updates the size accounting for an existing key by sizeDelta and evicts excess entries if needed. +// If node.size + sizeDelta exceeds maxSize (or cannot fit alongside entries more recent than node), +// only node itself is evicted without evicting older entries. // // Returns ErrEntryNotExist if key does not exist. +// Returns ErrInvalidUpdateEntrySize if sizeDelta causes uint64 integer overflow. func (c *radixCache) UpdateSize(key string, sizeDelta uint64) error { - c.mu.Lock() - defer func() { - if c.opts.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) + defer c.unlock() node, ok := c.getNode(key) if !ok { return ErrEntryNotExist } - if math.MaxUint64-node.size < sizeDelta || math.MaxUint64-c.currentSize < sizeDelta { + if math.MaxUint64-node.size < sizeDelta { return ErrInvalidUpdateEntrySize } + if node.size+sizeDelta > c.maxSize { + sizeBefore := c.currentSize + c.eraseInternal(node) + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) + return nil + } + + avail := c.maxSize - c.currentSize + if sizeDelta > avail { + maxNewer := c.maxSize - (node.size + sizeDelta) + var newerSize uint64 + headCurr := c.head + tailCurr := c.tail + canFit := false + for { + if headCurr == node { + canFit = newerSize <= maxNewer + break + } + if headCurr != nil { + newerSize += headCurr.size + if newerSize > maxNewer { + canFit = false + break + } + headCurr = headCurr.next + } + if tailCurr == nil || tailCurr == node { + canFit = sizeDelta <= avail + break + } + avail += tailCurr.size + if sizeDelta <= avail { + canFit = true + break + } + tailCurr = tailCurr.prev + } + if !canFit { + sizeBefore := c.currentSize + c.eraseInternal(node) + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) + return nil + } + } + + sizeBefore := c.currentSize + evictedAny := false + reclaimedPreUpdate := false + for sizeDelta > c.maxSize-c.currentSize && c.tail != nil { + if c.tail == node { + break + } + c.evictOne() + evictedAny = true + } + + c.onEntrySizeUpdated(node.size, node.size+sizeDelta) node.size += sizeDelta c.currentSize += sizeDelta + reclaimedPreUpdate = c.shouldReclaimSingleSurvivorOnMutation(c.len, c.deletedSinceCompact > 0, false, evictedAny, c.currentSize, sizeBefore, pressure) - for c.currentSize > c.maxSize && c.tail != nil { - c.evictOne() + protectedNode := &foregroundNoProtectNode + if node == c.head { + protectedNode = node } + c.finishMutationReclaimLocked(nil, protectedNode, reclaimedPreUpdate, sizeBefore, sampledEpoch, pressure) return nil } @@ -690,23 +854,35 @@ func (c *radixCache) UpdateSize(key string, sizeDelta uint64) error { // EraseEntriesWithGivenPrefix deletes all entries whose keys start with prefix. // Prunes subtrees in O(prefix_length + subtree_size) time and sweeps detached nodes. func (c *radixCache) EraseEntriesWithGivenPrefix(prefix string) { - c.mu.Lock() - defer func() { - if c.opts.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() - if prefix == "" { - c.root = &radixNode{} + c.mu.Lock() + defer c.unlock() + + hadEntries := c.len > 0 || (c.root != nil && (c.root.value != nil || c.root.child != nil)) + hadDirtySlack := c.deletedSinceCompact > 0 || c.peakEntryLen > 8 + if !hadEntries && !hadDirtySlack && c.peakEntryLen == 0 { + return + } + hadReclaimable := c.currentSize > 0 || hadDirtySlack + if c.root != nil { + c.sweepAndUnlink(c.root) + } else { + c.root = &radixNode{} + } c.head = nil c.tail = nil c.currentSize = 0 c.len = 0 + c.clearEmptyTreeStateLocked() + if hadReclaimable && c.hasElevatedPressureToInvalidate(0.0) { + c.markReclaimedLocked() + } return } + sampledEpoch, pressure := c.lockWithPressure(&c.mu, false) + defer c.unlock() + node := c.root search := prefix @@ -719,9 +895,11 @@ func (c *radixCache) EraseEntriesWithGivenPrefix(prefix string) { lcp := longestCommonPrefix(search, child.prefix) if lcp == len(search) { + sizeBefore := c.currentSize node.removeChild(child) c.sweepAndUnlink(child) c.compressPathUpwards(node) + c.finishDeleteReclaimLocked(sizeBefore, sampledEpoch, pressure) return } @@ -738,53 +916,117 @@ func (c *radixCache) EraseEntriesWithGivenPrefix(prefix string) { // Compact satisfies PressureAwareCache on radixCache (pointer-based nodes are reclaimed directly by Go GC upon deletion). func (c *radixCache) Compact() { c.mu.Lock() - defer func() { - if c.opts.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() + defer c.unlock() + c.compactLocked() } -// EvaluateMemoryPressure samples the configured memory-pressure probe and sheds LRU tail entries -// down to maxSize * EvictionRetentionRatio if critical pressure is reached. -func (c *radixCache) EvaluateMemoryPressure() []ValueType { - var pressure float64 - if c.opts.PressureFunc != nil { - pressure = c.opts.PressureFunc() - if math.IsNaN(pressure) || pressure < 0.0 { - pressure = 0.0 - } +func (c *radixCache) compactDataStructuresLocked() bool { + if c.deletedSinceCompact == 0 && c.peakEntryLen <= c.len { + return false } + c.onCompacted(c.len) + return true +} - c.mu.Lock() - defer func() { - if c.opts.EnableInvariantChecking { - c.checkInvariants() - } - c.mu.Unlock() - }() +func (c *radixCache) compactLocked() { + if c.compactDataStructuresLocked() { + c.markReclaimedLocked() + } +} - if pressure >= c.opts.EvictionThreshold { - retention := c.opts.EvictionRetentionRatio - targetSize := computeTargetSize(c.maxSize, retention) - targetLen := 0 - if c.currentSize == 0 && c.len > 0 && retention > 0.0 { - targetLen = int(float64(c.len) * retention) - } - var evicted []ValueType - for c.tail != nil { - needByteShed := c.currentSize > targetSize - needZeroSizeShed := c.currentSize == 0 && c.len > targetLen - needFullFlush := retention == 0.0 - if !needByteShed && !needZeroSizeShed && !needFullFlush { - break +func (c *radixCache) shouldAutoCompactLocked(protectedNode *radixNode) bool { + return c.shouldAutoCompactEntryCounts(c.deletedSinceCompact > 0, protectedNode == nil, c.len) +} + +func (c *radixCache) shedAndCompactLocked(targetSize uint64, retention float64, protectedNode *radixNode) []ValueType { + autoCompactNode := protectedNode + if protectedNode != nil && (protectedNode != c.head || protectedNode.prev != nil) { + protectedNode = nil + } + + var protectedSize uint64 + hasProtected := protectedNode != nil + if hasProtected { + protectedSize = protectedNode.size + } + effectiveTarget, targetLen, targetZeroCount, unprotectedZeroTarget := c.computeShedTargets(targetSize, retention, c.len, hasProtected, protectedSize) + + needFullFlush := retention == 0.0 + var evicted []ValueType + victim := c.tail + for victim != nil { + needByteShed := c.currentSize > effectiveTarget || needFullFlush + needZeroShed := !needFullFlush && c.zeroSizeCount > targetZeroCount && c.len > targetLen + if !needByteShed && !needZeroShed { + break + } + switch { + case needFullFlush || (needByteShed && c.zeroSizeCount > targetZeroCount): + for victim != nil && victim == protectedNode { + victim = victim.prev + } + case needByteShed: + for victim != nil && (victim == protectedNode || victim.size == 0) { + victim = victim.prev } - if val := c.evictOne(); val != nil { - evicted = append(evicted, val) + default: + for victim != nil && (victim == protectedNode || victim.size > 0) { + victim = victim.prev } } + if victim == nil { + break + } + nextVictim := victim.prev + if val := c.eraseInternal(victim); val != nil { + evicted = append(evicted, val) + } + victim = nextVictim + } + + if len(evicted) > 0 && c.len == 0 { + c.resetEmptyTreeLocked() return evicted } - return nil + + if len(evicted) > 0 { + c.updateZeroWatermarkAfterShed(retention, c.len, !hasProtected || effectiveTarget <= targetSize, unprotectedZeroTarget, targetLen) + } + compacted := false + if c.shouldAutoCompactLocked(autoCompactNode) { + compacted = c.compactDataStructuresLocked() + } + if len(evicted) > 0 || compacted { + c.markReclaimedLocked() + } + return evicted +} + +func (c *radixCache) maybeReclaimUnderPressureLocked(pressure float64, protectedNode *radixNode) []ValueType { + if c.isSamplingGoroutine() { + return nil + } + var evicted []ValueType + if pressure >= c.options.EvictionThreshold { + retention := c.options.EvictionRetentionRatio + targetSize := computeTargetSize(c.maxSize, retention) + evicted = c.shedAndCompactLocked(targetSize, retention, protectedNode) + } else { + c.resetZeroWatermarkBelowTier2(pressure) + if pressure >= c.options.CompactionThreshold { + if c.shouldAutoCompactLocked(protectedNode) { + c.compactLocked() + } + } + } + return evicted +} + +// EvaluateMemoryPressure samples the configured memory-pressure probe and sheds LRU tail entries +// down to maxSize * EvictionRetentionRatio if critical pressure is reached. +func (c *radixCache) EvaluateMemoryPressure() []ValueType { + _, pressure := c.lockWithPressure(&c.mu, true) + defer c.unlock() + + return c.maybeReclaimUnderPressureLocked(pressure, nil) } diff --git a/radix_lru_test.go b/radix_lru_test.go index ff145d9..98ec007 100644 --- a/radix_lru_test.go +++ b/radix_lru_test.go @@ -12,558 +12,536 @@ // See the License for the specific language governing permissions and // limitations under the License. -package lrus +package lru import ( - "errors" + "strings" "testing" -) + "unsafe" -const ( - radixTestMaxSize = 50 - radixTestOperationCount = 100 + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -type radixTestData struct { - value int64 - dataSize uint64 -} - -func (d radixTestData) Size() uint64 { - return d.dataSize -} - func setupRadixCacheTest(t *testing.T) Cache { t.Helper() - return NewRadixCache(radixTestMaxSize, WithInvariantChecking(true)) -} - -func insertAndAssertRadix(t *testing.T, cache Cache, key string, val ValueType, expectedEvictedValues []int64, expectedErr error) { - t.Helper() - evicted, err := cache.Insert(key, val) - if expectedErr != nil { - if err == nil { - t.Fatalf("expected error %v, got nil", expectedErr) - } - if !errors.Is(err, expectedErr) && err.Error() != expectedErr.Error() { - t.Fatalf("expected error %v, got %v", expectedErr, err) - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if len(expectedEvictedValues) == 0 { - if len(evicted) != 0 { - t.Fatalf("expected no evicted values, got %d", len(evicted)) - } - return - } - - if len(evicted) != len(expectedEvictedValues) { - t.Fatalf("expected %d evicted values, got %d", len(expectedEvictedValues), len(evicted)) - } - - for i, exp := range expectedEvictedValues { - actualData, ok := evicted[i].(radixTestData) - if !ok { - // Also support pointer if used - ptrData, ptrOk := evicted[i].(*radixTestData) - if ptrOk { - actualData = *ptrData - } else { - t.Fatalf("evicted value %d is not of type radixTestData: %T", i, evicted[i]) - } - } - if actualData.value != exp { - t.Fatalf("evicted[%d] expected value %d, got %d", i, exp, actualData.value) - } - } + return NewRadixCache(testMaxSize, WithInvariantChecking(true)) } func TestRadixCache_LookUpInEmptyCache(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - if v := cache.LookUp(""); v != nil { - t.Fatalf("expected nil for empty key in empty cache, got %v", v) - } - if v := cache.LookUp("taco"); v != nil { - t.Fatalf("expected nil for non-existent key, got %v", v) - } + + // Act + valEmpty := cache.LookUp("") + valTaco := cache.LookUp("taco") + + // Assert + assert.Nil(t, valEmpty) + assert.Nil(t, valTaco) } func TestRadixCache_InsertNilValue(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "taco", nil, []int64{}, ErrInvalidEntry) + + // Act + evicted, err := cache.Insert("taco", nil) + + // Assert + require.ErrorIs(t, err, ErrInvalidEntry) + assertEvictedValues(t, evicted, nil) } func TestRadixCache_InsertEmptyKey(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "", radixTestData{value: 42, dataSize: 10}, []int64{}, nil) + // Act + evicted, err := cache.Insert("", testData{value: 42, dataSize: 10}) - lookupVal := cache.LookUp("") - if lookupVal == nil { - t.Fatalf("expected value for empty key, got nil") - } - if lookupVal.(radixTestData).value != 42 { - t.Fatalf("expected 42, got %d", lookupVal.(radixTestData).value) - } - if cache.LookUp("taco") != nil { - t.Fatalf("expected nil for taco") - } + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + assert.Equal(t, testData{value: 42, dataSize: 10}, cache.LookUp("")) + assert.Nil(t, cache.LookUp("taco")) } func TestRadixCache_LookUpUnknownKey(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "burrito", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) - insertAndAssertRadix(t, cache, "taco", radixTestData{value: 23, dataSize: 8}, []int64{}, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - if cache.LookUp("") != nil { - t.Fatalf("expected nil for empty key") - } - if cache.LookUp("enchilada") != nil { - t.Fatalf("expected nil for enchilada") - } + evicted, err = cache.Insert("taco", testData{value: 23, dataSize: 8}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + + // Act + valEmpty := cache.LookUp("") + valEnchilada := cache.LookUp("enchilada") + + // Assert + assert.Nil(t, valEmpty) + assert.Nil(t, valEnchilada) } func TestRadixCache_FillUpToCapacity(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "burrito", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) - insertAndAssertRadix(t, cache, "taco", radixTestData{value: 26, dataSize: 20}, []int64{}, nil) - insertAndAssertRadix(t, cache, "enchilada", radixTestData{value: 28, dataSize: 26}, []int64{}, nil) - if v := cache.LookUp("burrito"); v == nil || v.(radixTestData).value != 23 { - t.Fatalf("expected 23 for burrito, got %v", v) - } - if v := cache.LookUp("taco"); v == nil || v.(radixTestData).value != 26 { - t.Fatalf("expected 26 for taco, got %v", v) - } - if v := cache.LookUp("enchilada"); v == nil || v.(radixTestData).value != 28 { - t.Fatalf("expected 28 for enchilada, got %v", v) - } + // Act + evicted1, err1 := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + evicted2, err2 := cache.Insert("taco", testData{value: 26, dataSize: 20}) + evicted3, err3 := cache.Insert("enchilada", testData{value: 28, dataSize: 26}) + + // Assert + require.NoError(t, err1) + assertEvictedValues(t, evicted1, nil) + require.NoError(t, err2) + assertEvictedValues(t, evicted2, nil) + require.NoError(t, err3) + assertEvictedValues(t, evicted3, nil) + + assert.Equal(t, testData{value: 23, dataSize: 4}, cache.LookUp("burrito")) + assert.Equal(t, testData{value: 26, dataSize: 20}, cache.LookUp("taco")) + assert.Equal(t, testData{value: 28, dataSize: 26}, cache.LookUp("enchilada")) } func TestRadixCache_ExpiresLeastRecentlyUsed(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "burrito", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) // Least recent. - insertAndAssertRadix(t, cache, "taco", radixTestData{value: 26, dataSize: 20}, []int64{}, nil) + evicted, err = cache.Insert("taco", testData{value: 26, dataSize: 20}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) // Second most recent. - insertAndAssertRadix(t, cache, "enchilada", radixTestData{value: 28, dataSize: 26}, []int64{}, nil) + evicted, err = cache.Insert("enchilada", testData{value: 28, dataSize: 26}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - if v := cache.LookUp("burrito"); v == nil || v.(radixTestData).value != 23 { - t.Fatalf("expected 23 for burrito, got %v", v) - } + assert.Equal(t, testData{value: 23, dataSize: 4}, cache.LookUp("burrito")) - // Insert another, should evict taco (value 26). - insertAndAssertRadix(t, cache, "queso", radixTestData{value: 34, dataSize: 5}, []int64{26}, nil) + // Act: Insert another, should evict taco (value 26). + evicted, err = cache.Insert("queso", testData{value: 34, dataSize: 5}) - // See what's left. - if cache.LookUp("taco") != nil { - t.Fatalf("expected taco to be evicted") - } - if v := cache.LookUp("burrito"); v == nil || v.(radixTestData).value != 23 { - t.Fatalf("expected 23 for burrito, got %v", v) - } - if v := cache.LookUp("enchilada"); v == nil || v.(radixTestData).value != 28 { - t.Fatalf("expected 28 for enchilada, got %v", v) - } - if v := cache.LookUp("queso"); v == nil || v.(radixTestData).value != 34 { - t.Fatalf("expected 34 for queso, got %v", v) - } + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{26}) + assert.Nil(t, cache.LookUp("taco")) + assert.Equal(t, testData{value: 23, dataSize: 4}, cache.LookUp("burrito")) + assert.Equal(t, testData{value: 28, dataSize: 26}, cache.LookUp("enchilada")) + assert.Equal(t, testData{value: 34, dataSize: 5}, cache.LookUp("queso")) } func TestRadixCache_Overwrite(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "burrito", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) - insertAndAssertRadix(t, cache, "taco", radixTestData{value: 26, dataSize: 20}, []int64{}, nil) - insertAndAssertRadix(t, cache, "enchilada", radixTestData{value: 28, dataSize: 20}, []int64{}, nil) - insertAndAssertRadix(t, cache, "burrito", radixTestData{value: 33, dataSize: 6}, []int64{}, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - // Increase dataSize while modifying, so eviction should happen (taco evicted) - insertAndAssertRadix(t, cache, "burrito", radixTestData{value: 33, dataSize: 12}, []int64{26}, nil) + evicted, err = cache.Insert("taco", testData{value: 26, dataSize: 20}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - if cache.LookUp("taco") != nil { - t.Fatalf("expected taco to be evicted") - } - if v := cache.LookUp("burrito"); v == nil || v.(radixTestData).value != 33 { - t.Fatalf("expected 33 for burrito, got %v", v) - } - if v := cache.LookUp("enchilada"); v == nil || v.(radixTestData).value != 28 { - t.Fatalf("expected 28 for enchilada, got %v", v) - } + evicted, err = cache.Insert("enchilada", testData{value: 28, dataSize: 20}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + + evicted, err = cache.Insert("burrito", testData{value: 33, dataSize: 6}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + + // Act: Increase dataSize while modifying, so eviction should happen (taco evicted) + evicted, err = cache.Insert("burrito", testData{value: 33, dataSize: 12}) + + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{26}) + assert.Nil(t, cache.LookUp("taco")) + assert.Equal(t, testData{value: 33, dataSize: 12}, cache.LookUp("burrito")) + assert.Equal(t, testData{value: 28, dataSize: 20}, cache.LookUp("enchilada")) } func TestRadixCache_MultipleEviction(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "burrito", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) - insertAndAssertRadix(t, cache, "taco", radixTestData{value: 26, dataSize: 20}, []int64{}, nil) - insertAndAssertRadix(t, cache, "enchilada", radixTestData{value: 28, dataSize: 20}, []int64{}, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - // Inserting large data evicts all three existing items - insertAndAssertRadix(t, cache, "large_data", radixTestData{value: 33, dataSize: 45}, []int64{23, 26, 28}, nil) + evicted, err = cache.Insert("taco", testData{value: 26, dataSize: 20}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - if cache.LookUp("taco") != nil { - t.Fatalf("expected taco to be evicted") - } - if cache.LookUp("burrito") != nil { - t.Fatalf("expected burrito to be evicted") - } - if cache.LookUp("enchilada") != nil { - t.Fatalf("expected enchilada to be evicted") - } - if v := cache.LookUp("large_data"); v == nil || v.(radixTestData).value != 33 { - t.Fatalf("expected 33 for large_data, got %v", v) - } + evicted, err = cache.Insert("enchilada", testData{value: 28, dataSize: 20}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + + // Act: Inserting large data evicts all three existing items + evicted, err = cache.Insert("large_data", testData{value: 33, dataSize: 45}) + + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{23, 26, 28}) + assert.Nil(t, cache.LookUp("taco")) + assert.Nil(t, cache.LookUp("burrito")) + assert.Nil(t, cache.LookUp("enchilada")) + assert.Equal(t, testData{value: 33, dataSize: 45}, cache.LookUp("large_data")) } func TestRadixCache_WhenEntrySizeMoreThanCacheMaxSize(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "burrito", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) - // Insert entry with size greater than maxSize of cache. - insertAndAssertRadix(t, cache, "taco", radixTestData{value: 26, dataSize: radixTestMaxSize + 1}, []int64{}, ErrInvalidEntrySize) + // Act: Insert entry with size greater than maxSize of cache. + evicted, err = cache.Insert("taco", testData{value: 26, dataSize: testMaxSize + 1}) - if v := cache.LookUp("burrito"); v == nil || v.(radixTestData).value != 23 { - t.Fatalf("expected 23 for burrito, got %v", v) - } + // Assert + require.ErrorIs(t, err, ErrInvalidEntrySize) + assertEvictedValues(t, evicted, nil) + assert.Equal(t, testData{value: 23, dataSize: 4}, cache.LookUp("burrito")) } func TestRadixCache_EraseWhenKeyPresent(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "burrito", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) + evicted, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + assertEvictedValues(t, evicted, nil) + // Act deletedEntry := cache.Erase("burrito") - if deletedEntry == nil || deletedEntry.(radixTestData).value != 23 { - t.Fatalf("expected erased value 23, got %v", deletedEntry) - } - if cache.LookUp("burrito") != nil { - t.Fatalf("expected burrito to be gone") - } + + // Assert + assert.Equal(t, testData{value: 23, dataSize: 4}, deletedEntry) + assert.Nil(t, cache.LookUp("burrito")) } func TestRadixCache_EraseCacheWithGivenPrefix(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "a", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) - insertAndAssertRadix(t, cache, "a/b", radixTestData{value: 26, dataSize: 5}, []int64{}, nil) - insertAndAssertRadix(t, cache, "a/b/d", radixTestData{value: 22, dataSize: 6}, []int64{}, nil) - insertAndAssertRadix(t, cache, "a/c", radixTestData{value: 20, dataSize: 6}, []int64{}, nil) - insertAndAssertRadix(t, cache, "b", radixTestData{value: 21, dataSize: 2}, []int64{}, nil) - + _, err := cache.Insert("a", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("a/b/d", testData{value: 22, dataSize: 6}) + require.NoError(t, err) + _, err = cache.Insert("a/c", testData{value: 20, dataSize: 6}) + require.NoError(t, err) + _, err = cache.Insert("b", testData{value: 21, dataSize: 2}) + require.NoError(t, err) + + // Act cache.EraseEntriesWithGivenPrefix("a") - if cache.LookUp("a") != nil { - t.Fatalf("expected 'a' to be erased") - } - if cache.LookUp("a/b") != nil { - t.Fatalf("expected 'a/b' to be erased") - } - if cache.LookUp("a/b/d") != nil { - t.Fatalf("expected 'a/b/d' to be erased") - } - if cache.LookUp("a/c") != nil { - t.Fatalf("expected 'a/c' to be erased") - } - if v := cache.LookUp("b"); v == nil || v.Size() != 2 { - t.Fatalf("expected 'b' to remain with size 2, got %v", v) - } + // Assert + assert.Nil(t, cache.LookUp("a")) + assert.Nil(t, cache.LookUp("a/b")) + assert.Nil(t, cache.LookUp("a/b/d")) + assert.Nil(t, cache.LookUp("a/c")) + valB := cache.LookUp("b") + require.NotNil(t, valB) + assert.Equal(t, uint64(2), valB.Size()) } func TestRadixCache_EraseCacheWithEmptyPrefix(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - - insertAndAssertRadix(t, cache, "a", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) - insertAndAssertRadix(t, cache, "a/b", radixTestData{value: 26, dataSize: 5}, []int64{}, nil) - insertAndAssertRadix(t, cache, "b", radixTestData{value: 21, dataSize: 2}, []int64{}, nil) - + _, err := cache.Insert("a", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("b", testData{value: 21, dataSize: 2}) + require.NoError(t, err) + + // Act cache.EraseEntriesWithGivenPrefix("") - if cache.LookUp("a") != nil { - t.Fatalf("expected 'a' to be erased") - } - if cache.LookUp("a/b") != nil { - t.Fatalf("expected 'a/b' to be erased") - } - if cache.LookUp("b") != nil { - t.Fatalf("expected 'b' to be erased") - } + // Assert + assert.Nil(t, cache.LookUp("a")) + assert.Nil(t, cache.LookUp("a/b")) + assert.Nil(t, cache.LookUp("b")) } func TestRadixCache_EraseCacheWhereNoEntriesExistWithGivenPrefix(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "a", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) - insertAndAssertRadix(t, cache, "a/b", radixTestData{value: 26, dataSize: 5}, []int64{}, nil) - insertAndAssertRadix(t, cache, "b", radixTestData{value: 21, dataSize: 2}, []int64{}, nil) - + _, err := cache.Insert("a", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("b", testData{value: 21, dataSize: 2}) + require.NoError(t, err) + + // Act cache.EraseEntriesWithGivenPrefix("c") - if v := cache.LookUp("a"); v == nil || v.Size() != 4 { - t.Fatalf("expected 'a' to have size 4, got %v", v) - } - if v := cache.LookUp("a/b"); v == nil || v.Size() != 5 { - t.Fatalf("expected 'a/b' to have size 5, got %v", v) - } - if v := cache.LookUp("b"); v == nil || v.Size() != 2 { - t.Fatalf("expected 'b' to have size 2, got %v", v) - } + // Assert + valA := cache.LookUp("a") + require.NotNil(t, valA) + assert.Equal(t, uint64(4), valA.Size()) + + valAB := cache.LookUp("a/b") + require.NotNil(t, valAB) + assert.Equal(t, uint64(5), valAB.Size()) + + valB := cache.LookUp("b") + require.NotNil(t, valB) + assert.Equal(t, uint64(2), valB.Size()) } func TestRadixCache_EraseCacheWithGivenPrefixWithSomeEntriesEvictedDueToCacheSize(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "a", radixTestData{value: 23, dataSize: 20}, []int64{}, nil) - insertAndAssertRadix(t, cache, "a/b", radixTestData{value: 26, dataSize: 10}, []int64{}, nil) - insertAndAssertRadix(t, cache, "a/b/d", radixTestData{value: 22, dataSize: 5}, []int64{}, nil) - insertAndAssertRadix(t, cache, "a/c", radixTestData{value: 20, dataSize: 10}, []int64{}, nil) - insertAndAssertRadix(t, cache, "b", radixTestData{value: 21, dataSize: 15}, []int64{23}, nil) - - // Entry "a" was already evicted by insertion of "b". Erasing prefix "a" cleans remaining descendants. + _, err := cache.Insert("a", testData{value: 23, dataSize: 20}) + require.NoError(t, err) + _, err = cache.Insert("a/b", testData{value: 26, dataSize: 10}) + require.NoError(t, err) + _, err = cache.Insert("a/b/d", testData{value: 22, dataSize: 5}) + require.NoError(t, err) + _, err = cache.Insert("a/c", testData{value: 20, dataSize: 10}) + require.NoError(t, err) + evicted, err := cache.Insert("b", testData{value: 21, dataSize: 15}) + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{23}) + + // Act: Entry "a" was already evicted by insertion of "b". Erasing prefix "a" cleans remaining descendants. cache.EraseEntriesWithGivenPrefix("a") - if cache.LookUp("a") != nil { - t.Fatalf("expected 'a' to be nil") - } - if cache.LookUp("a/b") != nil { - t.Fatalf("expected 'a/b' to be nil") - } - if cache.LookUp("a/b/d") != nil { - t.Fatalf("expected 'a/b/d' to be nil") - } - if cache.LookUp("a/c") != nil { - t.Fatalf("expected 'a/c' to be nil") - } - if v := cache.LookUp("b"); v == nil || v.Size() != 15 { - t.Fatalf("expected 'b' to remain with size 15, got %v", v) - } + // Assert + assert.Nil(t, cache.LookUp("a")) + assert.Nil(t, cache.LookUp("a/b")) + assert.Nil(t, cache.LookUp("a/b/d")) + assert.Nil(t, cache.LookUp("a/c")) + valB := cache.LookUp("b") + require.NotNil(t, valB) + assert.Equal(t, uint64(15), valB.Size()) } func TestRadixCache_EraseWhenKeyNotPresent(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) - insertAndAssertRadix(t, cache, "burrito", radixTestData{value: 23, dataSize: 4}, []int64{}, nil) + _, err := cache.Insert("burrito", testData{value: 23, dataSize: 4}) + require.NoError(t, err) + // Act deletedEntry := cache.Erase("taco") - if deletedEntry != nil { - t.Fatalf("expected nil for non-existent key, got %v", deletedEntry) - } - if v := cache.LookUp("burrito"); v == nil || v.(radixTestData).value != 23 { - t.Fatalf("expected burrito to remain 23, got %v", v) - } + + // Assert + assert.Nil(t, deletedEntry) + assert.Equal(t, testData{value: 23, dataSize: 4}, cache.LookUp("burrito")) } func TestRadixCache_UpdateSize(t *testing.T) { t.Run("NonExistentKey", func(t *testing.T) { + // Arrange cache := NewRadixCache(100, WithInvariantChecking(true)) + // Act err := cache.UpdateSize("key1", 20) - if !errors.Is(err, ErrEntryNotExist) { - t.Fatalf("expected ErrEntryNotExist, got %v", err) - } + + // Assert + require.ErrorIs(t, err, ErrEntryNotExist) }) - t.Run("Immediate Eviction", func(t *testing.T) { + t.Run("ImmediateEviction", func(t *testing.T) { + // Arrange cache := NewRadixCache(100, WithInvariantChecking(true)) - data1 := radixTestData{value: 1, dataSize: 10} - data2 := radixTestData{value: 2, dataSize: 70} - _, _ = cache.Insert("key1", data1) - _, _ = cache.Insert("key2", data2) - + data1 := testData{value: 1, dataSize: 10} + data2 := testData{value: 2, dataSize: 70} + _, err := cache.Insert("key1", data1) + require.NoError(t, err) + _, err = cache.Insert("key2", data2) + require.NoError(t, err) + + // Act errUpdate := cache.UpdateSize("key1", 30) - if errUpdate != nil { - t.Fatalf("unexpected error: %v", errUpdate) - } - if cache.LookUp("key1") != nil { - t.Fatalf("expected key1 to be evicted") - } - if cache.LookUp("key2") == nil { - t.Fatalf("expected key2 to be present") - } + + // Assert + require.NoError(t, errUpdate) + assert.Nil(t, cache.LookUp("key1")) + assert.NotNil(t, cache.LookUp("key2")) }) } func TestRadixCache_UpdateSize_ExceedsMaxSize(t *testing.T) { + // Arrange cache := NewRadixCache(100, WithInvariantChecking(true)) - data := &radixTestData{value: 1, dataSize: 50} + data := testData{value: 1, dataSize: 50} _, err := cache.Insert("file.txt", data) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + require.NoError(t, err) - data.dataSize = 150 + // Act err = cache.UpdateSize("file.txt", 100) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if cache.LookUp("file.txt") != nil { - t.Fatalf("expected file.txt to be evicted") - } + // Assert + require.NoError(t, err) + assert.Nil(t, cache.LookUp("file.txt")) } func TestRadixCache_UpdateWhenKeyPresent(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) key := "burrito" - data := radixTestData{value: 23, dataSize: 4} - insertAndAssertRadix(t, cache, key, data, []int64{}, nil) - newData := radixTestData{value: 2, dataSize: 4} + data := testData{value: 23, dataSize: 4} + _, err := cache.Insert(key, data) + require.NoError(t, err) + newData := testData{value: 2, dataSize: 4} - err := cache.UpdateWithoutChangingOrder(key, newData) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if v := cache.LookUp(key); v == nil || v.(radixTestData).value != 2 { - t.Fatalf("expected value 2, got %v", v) - } + // Act + err = cache.UpdateWithoutChangingOrder(key, newData) + + // Assert + require.NoError(t, err) + assert.Equal(t, newData, cache.LookUp(key)) } func TestRadixCache_UpdateWhenKeyNotPresent(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) key := "burrito" - data := radixTestData{value: 23, dataSize: 4} + data := testData{value: 23, dataSize: 4} + // Act err := cache.UpdateWithoutChangingOrder(key, data) - if !errors.Is(err, ErrEntryNotExist) { - t.Fatalf("expected ErrEntryNotExist, got %v", err) - } + + // Assert + require.ErrorIs(t, err, ErrEntryNotExist) } func TestRadixCache_UpdateWhenSizeIsDifferent(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) key := "burrito" - data := radixTestData{value: 23, dataSize: 4} - insertAndAssertRadix(t, cache, key, data, []int64{}, nil) - newData := radixTestData{value: 2, dataSize: 3} + data := testData{value: 23, dataSize: 4} + _, err := cache.Insert(key, data) + require.NoError(t, err) + newData := testData{value: 2, dataSize: 3} - err := cache.UpdateWithoutChangingOrder(key, newData) - if !errors.Is(err, ErrInvalidUpdateEntrySize) { - t.Fatalf("expected ErrInvalidUpdateEntrySize, got %v", err) - } + // Act + err = cache.UpdateWithoutChangingOrder(key, newData) + + // Assert + require.ErrorIs(t, err, ErrInvalidUpdateEntrySize) } func TestRadixCache_UpdateNotChangeOrder(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) key1 := "burrito1" - data1 := radixTestData{value: 23, dataSize: 10} - insertAndAssertRadix(t, cache, key1, data1, []int64{}, nil) + data1 := testData{value: 23, dataSize: 10} + _, err := cache.Insert(key1, data1) + require.NoError(t, err) + key2 := "burrito2" - data2 := radixTestData{value: 2, dataSize: 40} - insertAndAssertRadix(t, cache, key2, data2, []int64{}, nil) + data2 := testData{value: 2, dataSize: 40} + _, err = cache.Insert(key2, data2) + require.NoError(t, err) - newData := radixTestData{value: 7, dataSize: 10} - err := cache.UpdateWithoutChangingOrder(key1, newData) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + // Act + newData := testData{value: 7, dataSize: 10} + err = cache.UpdateWithoutChangingOrder(key1, newData) + require.NoError(t, err) // Inserting again should evict key1 because key1 was updated without changing order (still at tail) key3 := "burrito3" - data3 := radixTestData{value: 3, dataSize: 5} - insertAndAssertRadix(t, cache, key3, data3, []int64{7}, nil) + data3 := testData{value: 3, dataSize: 5} + evicted, err := cache.Insert(key3, data3) + + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{7}) } func TestRadixCache_UpdateSize_DoubleCountingDivergence(t *testing.T) { + // Arrange const maxSize = 100 const initialSize = 50 const sizeDelta = 50 // New total size will be 50 + 50 = 100 (exactly at maxSize) radixCache := NewRadixCache(maxSize, WithInvariantChecking(true)) - radixVal := &radixTestData{value: 2, dataSize: initialSize} - - // 1. Insert initial entry - _, err := radixCache.Insert("file.txt", radixVal) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + _, err := radixCache.Insert("file.txt", testData{value: 1, dataSize: initialSize}) + require.NoError(t, err) - // 2. Simulate incremental file growth in memory - radixVal.dataSize += sizeDelta - - // 3. Notify radixCache of the growth + // Act: Grow tracked size via UpdateSize first, then update value payload at the new size err = radixCache.UpdateSize("file.txt", sizeDelta) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + require.NoError(t, err) - if v := radixCache.LookUp("file.txt"); v == nil { - t.Fatalf("expected file.txt to remain in cache") - } + err = radixCache.UpdateWithoutChangingOrder("file.txt", testData{value: 2, dataSize: initialSize + sizeDelta}) + + // Assert + require.NoError(t, err) + assert.Equal(t, testData{value: 2, dataSize: initialSize + sizeDelta}, radixCache.LookUp("file.txt")) } func TestRadixCache_LookUpWithoutChangingOrder_WhenKeyPresent(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) key := "burrito" - data := radixTestData{value: 23, dataSize: 4} - insertAndAssertRadix(t, cache, key, data, []int64{}, nil) + data := testData{value: 23, dataSize: 4} + _, err := cache.Insert(key, data) + require.NoError(t, err) + // Act value := cache.LookUpWithoutChangingOrder(key) - if value == nil || value.(radixTestData).value != 23 { - t.Fatalf("expected 23, got %v", value) - } + + // Assert + assert.Equal(t, data, value) } func TestRadixCache_LookUpWithoutChangingOrder_WhenKeyNotPresent(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) key := "burrito" + // Act value := cache.LookUpWithoutChangingOrder(key) - if value != nil { - t.Fatalf("expected nil for non-existent key, got %v", value) - } + + // Assert + assert.Nil(t, value) } func TestRadixCache_LookUpWithoutChangingOrder_NotChangeOrder(t *testing.T) { + // Arrange cache := setupRadixCacheTest(t) key1 := "burrito1" - data1 := radixTestData{value: 23, dataSize: 10} - insertAndAssertRadix(t, cache, key1, data1, []int64{}, nil) + data1 := testData{value: 23, dataSize: 10} + _, err := cache.Insert(key1, data1) + require.NoError(t, err) + key2 := "burrito2" - data2 := radixTestData{value: 2, dataSize: 40} - insertAndAssertRadix(t, cache, key2, data2, []int64{}, nil) + data2 := testData{value: 2, dataSize: 40} + _, err = cache.Insert(key2, data2) + require.NoError(t, err) + // Act value := cache.LookUpWithoutChangingOrder(key1) - if value == nil || value.(radixTestData).value != 23 { - t.Fatalf("expected 23, got %v", value) - } + assert.Equal(t, data1, value) // Inserting again should evict key1 because key1 was looked up without changing order key3 := "burrito3" - data3 := radixTestData{value: 3, dataSize: 5} - insertAndAssertRadix(t, cache, key3, data3, []int64{23}, nil) -} - -func TestRadixCache_NewRadixCache_InvalidMaxSize(t *testing.T) { - t.Run("DefaultOptions", func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Fatalf("expected panic on maxSize == 0 with default options, got none") - } - }() - _ = NewRadixCache(0) - }) - - t.Run("InvariantsEnabled", func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Fatalf("expected panic on maxSize == 0 with invariant checking enabled, got none") - } - }() - _ = NewRadixCache(0, WithInvariantChecking(true)) - }) + data3 := testData{value: 3, dataSize: 5} + evicted, err := cache.Insert(key3, data3) - t.Run("InvariantsDisabled", func(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Fatalf("expected panic on maxSize == 0 with invariant checking disabled, got none") - } - }() - _ = NewRadixCache(0, WithInvariantChecking(false)) - }) + // Assert + require.NoError(t, err) + assertEvictedValues(t, evicted, []int64{23}) } func TestRadixCache_ComplexEdgeSplitsAndMerges(t *testing.T) { + // Arrange cache := NewRadixCache(1000, WithInvariantChecking(true)) - keys := []string{ "car", "cart", @@ -579,57 +557,43 @@ func TestRadixCache_ComplexEdgeSplitsAndMerges(t *testing.T) { } for i, k := range keys { - _, err := cache.Insert(k, radixTestData{value: int64(i + 1), dataSize: 10}) - if err != nil { - t.Fatalf("failed to insert %q: %v", k, err) - } + _, err := cache.Insert(k, testData{value: int64(i + 1), dataSize: 10}) + require.NoError(t, err) } - // Verify all keys are present for i, k := range keys { - v := cache.LookUp(k) - if v == nil || v.(radixTestData).value != int64(i+1) { - t.Fatalf("expected %d for %q, got %v", i+1, k, v) - } + assert.Equal(t, testData{value: int64(i + 1), dataSize: 10}, cache.LookUp(k)) } - // Erase prefix "car" -> should remove "car", "cart", "card", "carpet", "care", "careful" + // Act 1: Erase prefix "car" -> should remove "car", "cart", "card", "carpet", "care", "careful" cache.EraseEntriesWithGivenPrefix("car") + // Assert 1 carKeys := []string{"car", "cart", "card", "carpet", "care", "careful"} for _, k := range carKeys { - if v := cache.LookUp(k); v != nil { - t.Fatalf("expected %q to be erased, got %v", k, v) - } + assert.Nil(t, cache.LookUp(k)) } - // Remaining keys should still be present nonCarKeys := []string{"cat", "catch", "dog", "door", "dorm"} for _, k := range nonCarKeys { - if v := cache.LookUp(k); v == nil { - t.Fatalf("expected %q to remain in cache", k) - } + assert.NotNil(t, cache.LookUp(k)) } - // Erase remaining keys one by one to verify compressPathUpwards in reverse + // Act 2: Erase remaining keys one by one to verify compressPathUpwards in reverse for _, k := range nonCarKeys { v := cache.Erase(k) - if v == nil { - t.Fatalf("expected non-nil erased value for %q", k) - } + require.NotNil(t, v) } - // Cache should now be completely empty + // Assert 2: Cache should now be completely empty for _, k := range keys { - if cache.LookUp(k) != nil { - t.Fatalf("expected cache to be empty, found %q", k) - } + assert.Nil(t, cache.LookUp(k)) } } func TestRadixCache_BinaryAndUnicodeKeys(t *testing.T) { + // Arrange cache := NewRadixCache(1000, WithInvariantChecking(true)) - unicodeKeys := []string{ "日本語/ディレクトリ/ファイル1", "日本語/ディレクトリ/ファイル2", @@ -642,112 +606,276 @@ func TestRadixCache_BinaryAndUnicodeKeys(t *testing.T) { } for i, k := range unicodeKeys { - _, err := cache.Insert(k, radixTestData{value: int64(i + 1), dataSize: 10}) - if err != nil { - t.Fatalf("failed to insert key %q: %v", k, err) - } + _, err := cache.Insert(k, testData{value: int64(i + 1), dataSize: 10}) + require.NoError(t, err) } for i, k := range unicodeKeys { - v := cache.LookUp(k) - if v == nil || v.(radixTestData).value != int64(i+1) { - t.Fatalf("expected value %d for %q, got %v", i+1, k, v) - } + assert.Equal(t, testData{value: int64(i + 1), dataSize: 10}, cache.LookUp(k)) } - // Erase prefix "日本語/" + // Act: Erase prefix "日本語/" cache.EraseEntriesWithGivenPrefix("日本語/") - if cache.LookUp("日本語/ディレクトリ/ファイル1") != nil { - t.Fatalf("expected unicode key to be erased") - } - if cache.LookUp("日本語/ディレクトリ/ファイル2") != nil { - t.Fatalf("expected unicode key to be erased") - } - if cache.LookUp("日本語/別のディレクトリ/ファイル3") != nil { - t.Fatalf("expected unicode key to be erased") - } - if cache.LookUp("🚀/rocket/one") == nil { - t.Fatalf("expected rocket key to remain") - } + + // Assert + assert.Nil(t, cache.LookUp("日本語/ディレクトリ/ファイル1")) + assert.Nil(t, cache.LookUp("日本語/ディレクトリ/ファイル2")) + assert.Nil(t, cache.LookUp("日本語/別のディレクトリ/ファイル3")) + assert.NotNil(t, cache.LookUp("🚀/rocket/one")) } func TestRadixCache_UpdateWithoutChangingOrder_Errors(t *testing.T) { + // Arrange cache := NewRadixCache(100, WithInvariantChecking(true)) - // Nil value returns ErrInvalidEntry + // Act err := cache.UpdateWithoutChangingOrder("any", nil) - if !errors.Is(err, ErrInvalidEntry) { - t.Fatalf("expected ErrInvalidEntry, got %v", err) - } + + // Assert + require.ErrorIs(t, err, ErrInvalidEntry) } func TestRadixCache_OptionsToggle(t *testing.T) { - // Without invariant checking (production mode) + // Arrange cacheProd := NewRadixCache(100, WithInvariantChecking(false)) - _, err := cacheProd.Insert("k1", radixTestData{value: 1, dataSize: 10}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // With invariant checking (debug/test mode) cacheDebug := NewRadixCache(100, WithInvariantChecking(true)) - _, err = cacheDebug.Insert("k1", radixTestData{value: 1, dataSize: 10}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } + + // Act + _, errProd := cacheProd.Insert("k1", testData{value: 1, dataSize: 10}) + _, errDebug := cacheDebug.Insert("k1", testData{value: 1, dataSize: 10}) + + // Assert + require.NoError(t, errProd) + require.NoError(t, errDebug) } +// TestRadixCache_CheckInvariants_PanicScenarios verifies that radixCache.checkInvariants() detects +// and panics on internal tree, LRU list, and size accounting corruption. +// +// White-box testing rationale: +// All public Cache operations preserve radix tree and doubly-linked LRU invariants. Triggering +// the panic branches inside checkInvariants() requires directly mutating unexported radixCache +// and radixNode fields (currentSize, head.next.prev, root.child.parent, zeroSizeCount) to inject +// synthetic corruption. Without direct white-box verification, bugs in the invariant checker +// itself would go undetected. func TestRadixCache_CheckInvariants_PanicScenarios(t *testing.T) { t.Run("CurrentSizeExceedsMaxSize", func(t *testing.T) { + // Arrange c := NewRadixCache(10).(*radixCache) c.currentSize = 20 - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on currentSize > maxSize") - } - }() - c.checkInvariants() + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) t.Run("CorruptLRULinks", func(t *testing.T) { + // Arrange c := NewRadixCache(50).(*radixCache) - _, _ = c.Insert("k1", radixTestData{value: 1, dataSize: 10}) - _, _ = c.Insert("k2", radixTestData{value: 2, dataSize: 10}) - // Corrupt prev pointer + _, err := c.Insert("k1", testData{value: 1, dataSize: 10}) + require.NoError(t, err) + _, err = c.Insert("k2", testData{value: 2, dataSize: 10}) + require.NoError(t, err) c.head.next.prev = nil - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on corrupt prev pointer") - } - }() - c.checkInvariants() + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) t.Run("CorruptTreeParent", func(t *testing.T) { + // Arrange c := NewRadixCache(50).(*radixCache) - _, _ = c.Insert("a/b", radixTestData{value: 1, dataSize: 10}) - // Corrupt child's parent pointer + _, err := c.Insert("a/b", testData{value: 1, dataSize: 10}) + require.NoError(t, err) if c.root.child != nil { c.root.child.parent = nil } - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on corrupt parent pointer") - } - }() - c.checkInvariants() + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) + }) + + t.Run("ZeroSizeCountMismatch", func(t *testing.T) { + // Arrange + c := NewRadixCache(100).(*radixCache) + _, err := c.Insert("z", NewStringValue("")) + require.NoError(t, err) + c.zeroSizeCount = 0 + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) }) t.Run("SizeSumMismatch", func(t *testing.T) { + // Arrange c := NewRadixCache(50).(*radixCache) - _, _ = c.Insert("k1", radixTestData{value: 1, dataSize: 10}) - // Corrupt currentSize by 1 so sumSize != currentSize - c.currentSize += 1 - defer func() { - if r := recover(); r == nil { - t.Errorf("expected panic on size sum mismatch") - } - }() - c.checkInvariants() + _, err := c.Insert("k1", testData{value: 1, dataSize: 10}) + require.NoError(t, err) + c.currentSize++ + + // Act & Assert + assert.Panics(t, func() { + c.checkInvariants() + }) + }) +} + +// TestRadixCache_RoutingPrefixDoesNotPinLargeKeyBackingArray verifies that splitting an edge +// created by a 64 KiB key clones the internal routing node's prefix string so erasing the 64 KiB +// key does not pin the caller's 64 KiB backing array in memory. +// +// White-box testing rationale: +// Public Cache lookups only inspect entry values and cannot observe whether an internal non-value +// routing node's prefix string header aliases the backing byte array of an erased key. Inspecting +// unsafe.StringData(c.root.child.prefix) deterministically verifies zero substring backing-array retention. +func TestRadixCache_RoutingPrefixDoesNotPinLargeKeyBackingArray(t *testing.T) { + // Arrange + largeKey := strings.Clone("dir/" + strings.Repeat("X", 1<<16)) + largeStart := uintptr(unsafe.Pointer(unsafe.StringData(largeKey))) + largeEnd := largeStart + uintptr(len(largeKey)) + + c := NewRadixCache(1<<20, WithInvariantChecking(true)).(*radixCache) + _, err := c.Insert(largeKey, NewStringValue("v")) + require.NoError(t, err) + + // Act: Split "dir/..." at "dir/" and erase the large key. + _, err = c.Insert("dir/a", NewStringValue("va")) + require.NoError(t, err) + _, err = c.Insert("dir/b", NewStringValue("vb")) + require.NoError(t, err) + _ = c.Erase(largeKey) + + // Assert: Routing node prefix "dir/" does not alias largeKey's backing array. + routingNode := c.root.child + require.NotNil(t, routingNode) + assert.Equal(t, "dir/", routingNode.prefix) + prefixPtr := uintptr(unsafe.Pointer(unsafe.StringData(routingNode.prefix))) + assert.True(t, prefixPtr < largeStart || prefixPtr >= largeEnd, "routing node prefix must not pin largeKey backing array") +} + +func TestRadixCache_PreInsertEvictionAvoidsRedundantSplitAndMergeAllocations(t *testing.T) { + // Arrange + c := NewRadixCache(50) + alphaKey := "prefix/alpha" + betaKey := "prefix/beta" + v1 := ValueType(NewSizedValue("v1", 40)) + v2 := ValueType(NewSizedValue("v2", 40)) + + _, err := c.Insert(alphaKey, v1) + require.NoError(t, err) + toggle := false + + // Act: Alternate inserting "prefix/beta" and "prefix/alpha" (40B each in a 50B cache), + // forcing eviction of the sibling before insertNode. + allocs := testing.AllocsPerRun(20, func() { + if toggle { + _, _ = c.Insert(alphaKey, v1) + } else { + _, _ = c.Insert(betaKey, v2) + } + toggle = !toggle + }) + + // Assert: Pre-eviction avoids splitNode + prefix split + compressPathUpwards string re-concatenation. + assert.LessOrEqual(t, allocs, 3.0) + assert.Nil(t, c.LookUpWithoutChangingOrder(alphaKey)) + assert.NotNil(t, c.LookUpWithoutChangingOrder(betaKey)) +} + +// TestRadixCache_DetachedNodesClearPrefixesAndTreePointers verifies that detached and merged +// radixNode structs have their prefix strings, values, tree pointers, and LRU links cleared upon removal. +// +// White-box testing rationale: +// Public Cache methods return nil for erased keys regardless of whether detached radixNode structs +// retain non-nil parent, child, sibling, prev, next, or prefix references. Inspecting the detached +// radixNode fields directly is necessary to deterministically verify that no pointer chains remain +// to pin adjacent live subtrees or key buffers in the Go heap. +func TestRadixCache_DetachedNodesClearPrefixesAndTreePointers(t *testing.T) { + t.Run("CompressPathUpwardsClearsDetachedLeafAndMergedRoutingNodeReferences", func(t *testing.T) { + // Arrange + rc := NewRadixCache(1000, WithInvariantChecking(true)).(*radixCache) + _, err := rc.Insert("group/sub/item1", NewSizedValue("v1", 10)) + require.NoError(t, err) + _, err = rc.Insert("group/sub/item2", NewSizedValue("v2", 10)) + require.NoError(t, err) + + routingNode := rc.root.getChild('g') + require.NotNil(t, routingNode) + leaf1 := routingNode.getChild('1') + require.NotNil(t, leaf1) + + // Act: Erase "group/sub/item1", pruning leaf1 and merging routingNode into leaf2. + erased := rc.Erase("group/sub/item1") + + // Assert: Both pruned leaf1 and merged routingNode must clear prefix string headers and tree pointers. + require.NotNil(t, erased) + assert.Empty(t, leaf1.prefix) + assert.Nil(t, leaf1.parent) + assert.Empty(t, routingNode.prefix) + assert.Nil(t, routingNode.child) + assert.Nil(t, routingNode.parent) + }) + + t.Run("EraseEntriesWithGivenPrefixClearsDetachedSubtreePrefixesAndPointers", func(t *testing.T) { + // Arrange + rc := NewRadixCache(1000, WithInvariantChecking(true)).(*radixCache) + _, err := rc.Insert("dir/sub/a", NewSizedValue("va", 10)) + require.NoError(t, err) + _, err = rc.Insert("dir/sub/b", NewSizedValue("vb", 10)) + require.NoError(t, err) + + subtreeRoot := rc.root.getChild('d') + require.NotNil(t, subtreeRoot) + leafA := subtreeRoot.getChild('a') + require.NotNil(t, leafA) + leafB := subtreeRoot.getChild('b') + require.NotNil(t, leafB) + + // Act + rc.EraseEntriesWithGivenPrefix("dir/") + + // Assert: Every node in the detached subtree must clear its prefix string header and tree pointers. + assert.Empty(t, subtreeRoot.prefix) + assert.Nil(t, subtreeRoot.child) + assert.Nil(t, subtreeRoot.parent) + assert.Empty(t, leafA.prefix) + assert.Nil(t, leafA.parent) + assert.Nil(t, leafA.sibling) + assert.Empty(t, leafB.prefix) + assert.Nil(t, leafB.parent) + }) + + t.Run("EraseEmptyPrefixSweepsAndZeroesDetachedTrieNodes", func(t *testing.T) { + // Arrange + rc := NewRadixCache(1000, WithInvariantChecking(true)).(*radixCache) + _, err := rc.Insert("dir/sub/a", NewSizedValue("va", 10)) + require.NoError(t, err) + _, err = rc.Insert("dir/sub/b", NewSizedValue("vb", 10)) + require.NoError(t, err) + + routingNode := rc.root.getChild('d') + require.NotNil(t, routingNode) + leafA := routingNode.getChild('a') + require.NotNil(t, leafA) + + // Act + rc.EraseEntriesWithGivenPrefix("") + + // Assert: Detached radixNode structs must have prefix, value, tree pointers, and LRU pointers cleared. + assert.Empty(t, routingNode.prefix) + assert.Nil(t, routingNode.child) + assert.Nil(t, routingNode.parent) + assert.Empty(t, leafA.prefix) + assert.Nil(t, leafA.value) + assert.Nil(t, leafA.parent) + assert.Nil(t, leafA.prev) + assert.Nil(t, leafA.next) }) } diff --git a/values.go b/values.go new file mode 100644 index 0000000..b0c0052 --- /dev/null +++ b/values.go @@ -0,0 +1,101 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lru + +import "strings" + +// StringValue is a ValueType wrapper for a standard Go string whose Size() is its byte length (len(s)). +type StringValue string + +// NewStringValue clones s and wraps it as a StringValue implementing ValueType. +// Cloning prevents substring values from pinning large underlying caller backing arrays in memory. +func NewStringValue(s string) StringValue { + return StringValue(strings.Clone(s)) +} + +// Size returns the byte length of the string (uint64(len(s))). +func (s StringValue) Size() uint64 { + return uint64(len(s)) +} + +// String returns the underlying string. +func (s StringValue) String() string { + return string(s) +} + +// BytesValue is an immutable, comparable ValueType wrapper for a byte slice whose Size() is its byte length (len(b)). +type BytesValue struct { + data string + nonNil bool +} + +// NewBytesValue clones b and wraps it as an immutable BytesValue implementing ValueType. +// Cloning prevents caller mutations from racing with cached readers and prevents sub-slices +// from pinning large underlying backing arrays in memory. +func NewBytesValue(b []byte) BytesValue { + if b == nil { + return BytesValue{} + } + return BytesValue{ + data: string(b), + nonNil: true, + } +} + +// Size returns the byte length of the slice (uint64(len(b))). +func (b BytesValue) Size() uint64 { + return uint64(len(b.data)) +} + +// Bytes returns a defensive copy of the underlying []byte slice. +func (b BytesValue) Bytes() []byte { + if len(b.data) == 0 { + if !b.nonNil { + return nil + } + return []byte{} + } + return []byte(b.data) +} + +// SizedValue wraps an arbitrary value of type T with an explicit logical or byte size, +// allowing any type to be cached without defining a custom struct implementing ValueType. +type SizedValue[T any] struct { + Value T + ByteSize uint64 +} + +// NewSizedValue wraps val with the specified logical or byte size as a SizedValue[T]. +func NewSizedValue[T any](val T, size uint64) SizedValue[T] { + return SizedValue[T]{ + Value: val, + ByteSize: size, + } +} + +// NewValue is a shorthand constructor alias for NewSizedValue. +func NewValue[T any](val T, size uint64) SizedValue[T] { + return NewSizedValue(val, size) +} + +// Size returns the logical or byte size configured for this entry. +func (v SizedValue[T]) Size() uint64 { + return v.ByteSize +} + +// Unwrap returns the underlying value of type T. +func (v SizedValue[T]) Unwrap() T { + return v.Value +} diff --git a/values_test.go b/values_test.go new file mode 100644 index 0000000..e6b5e27 --- /dev/null +++ b/values_test.go @@ -0,0 +1,139 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package lru + +import ( + "strings" + "testing" + "unsafe" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestBackendSelectionAndUnifiedNew verifies that New() dispatches to the expected +// concrete backend implementation (*mapCache, *radixCache, *arenaRadix) for each Backend option. +// +// White-box testing rationale: +// All three backends implement the identical public Cache and PressureAwareCache interfaces, so +// black-box behavioral assertions cannot distinguish which concrete engine New() constructed. +// Type-asserting the unexported concrete pointer types (*mapCache, *radixCache, *arenaRadix) is +// required to verify that WithBackend() selects the intended underlying data structure. +func TestBackendSelectionAndUnifiedNew(t *testing.T) { + // Arrange & Act + defaultCache := New(1024) + mapCacheInstance := New(1024, WithBackend(BackendMap)) + radixCacheInstance := New(1024, WithBackend(BackendRadix)) + arenaCacheInstance := New(1024, WithBackend(BackendArenaRadix)) + unknownBackendCache := New(1024, WithBackend(Backend(99))) + + // Assert concrete backend types & String() representations + assert.Equal(t, "MapCache", BackendMap.String()) + assert.Equal(t, "RadixCache", BackendRadix.String()) + assert.Equal(t, "ArenaRadixCache", BackendArenaRadix.String()) + assert.Equal(t, "UnknownBackend", Backend(99).String()) + + _, isDefaultMap := defaultCache.(*mapCache) + assert.True(t, isDefaultMap, "default New() should construct *mapCache") + + _, isMap := mapCacheInstance.(*mapCache) + assert.True(t, isMap, "WithBackend(BackendMap) should construct *mapCache") + _, mapImplementsPressureAware := mapCacheInstance.(PressureAwareCache) + assert.True(t, mapImplementsPressureAware, "MapCache returned by New should implement PressureAwareCache") + + _, isRadix := radixCacheInstance.(*radixCache) + assert.True(t, isRadix, "WithBackend(BackendRadix) should construct *radixCache") + _, radixImplementsPressureAware := radixCacheInstance.(PressureAwareCache) + assert.True(t, radixImplementsPressureAware, "RadixCache returned by New should implement PressureAwareCache") + + _, isArena := arenaCacheInstance.(*arenaRadix) + assert.True(t, isArena, "WithBackend(BackendArenaRadix) should construct *arenaRadix") + _, implementsPressureAware := arenaCacheInstance.(PressureAwareCache) + assert.True(t, implementsPressureAware, "ArenaRadixCache returned by New should implement PressureAwareCache") + + _, isUnknownNormalizedToMap := unknownBackendCache.(*mapCache) + assert.True(t, isUnknownNormalizedToMap, "unrecognized Backend value should normalize to BackendMap") +} + +func TestValueWrappers(t *testing.T) { + // Arrange + sv := NewStringValue("hello-lru") + bv := NewBytesValue([]byte{0x01, 0x02, 0x03, 0x04}) + type customPayload struct { + ID int + Name string + } + sized := NewSizedValue(customPayload{ID: 7, Name: "node"}, 128) + shorthand := NewValue("inline-payload", 64) + + // Assert ValueType interface compliance & helper methods + assert.Equal(t, uint64(9), sv.Size()) + assert.Equal(t, "hello-lru", sv.String()) + + assert.Equal(t, uint64(4), bv.Size()) + assert.Equal(t, []byte{0x01, 0x02, 0x03, 0x04}, bv.Bytes()) + + var v1, v2, v3 ValueType = NewBytesValue([]byte("alpha")), NewBytesValue([]byte("alpha")), NewBytesValue([]byte("beta")) + areEqual := v1 == v2 + areDiff := v1 == v3 + assert.Equal(t, v1, v2) + assert.True(t, areEqual) + assert.False(t, areDiff) + + // Verify NewBytesValue clones input slice and Bytes() clones output slice + raw := []byte("immutable-payload") + immutableBV := NewBytesValue(raw) + raw[0] = 'X' + out := immutableBV.Bytes() + out[1] = 'Y' + assert.Equal(t, []byte("immutable-payload"), immutableBV.Bytes()) + assert.Nil(t, NewBytesValue(nil).Bytes()) + emptyBytes := NewBytesValue([]byte{}).Bytes() + assert.NotNil(t, emptyBytes) + assert.Empty(t, emptyBytes) + + // Verify NewStringValue clones substring backing buffer so large parent string is not pinned + largeBuffer := strings.Repeat("X", 64*1024) + substring := largeBuffer[100:116] + clonedSV := NewStringValue(substring) + assert.Equal(t, StringValue(substring), clonedSV) + assert.NotSame(t, unsafe.StringData(substring), unsafe.StringData(string(clonedSV))) + + assert.Equal(t, uint64(128), sized.Size()) + assert.Equal(t, customPayload{ID: 7, Name: "node"}, sized.Unwrap()) + + assert.Equal(t, uint64(64), shorthand.Size()) + assert.Equal(t, "inline-payload", shorthand.Unwrap()) + + // Act & Assert: Verify caching across all three backends + for _, b := range []Backend{BackendMap, BackendRadix, BackendArenaRadix} { + cache := New(512, WithBackend(b), WithInvariantChecking(true)) + evicted, err := cache.Insert("k1", sv) + require.NoError(t, err) + assert.Empty(t, evicted) + + evicted, err = cache.Insert("k2", bv) + require.NoError(t, err) + assert.Empty(t, evicted) + + evicted, err = cache.Insert("k3", sized) + require.NoError(t, err) + assert.Empty(t, evicted) + + assert.Equal(t, sv, cache.LookUp("k1")) + assert.Equal(t, bv, cache.LookUp("k2")) + assert.Equal(t, sized, cache.LookUp("k3")) + } +}