diff --git a/.gitignore b/.gitignore index 6d1e18b..f16400b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ coverage/ .pnpm-store/ test/perf/last-results.json test/perf/last-profile.json +test/perf/last-memory-results.json +test/perf/last-memory-report.md +test/perf/last-memory-profile.json test/perf/*-results.json profiles/ package-lock.json diff --git a/AGENTS.md b/AGENTS.md index 189353c..63fc963 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,8 @@ This is `couch-auth-proxy`: a TypeScript (Hono) reverse proxy that enforces per- - `pnpm lint`, `pnpm fmt:check`, `pnpm typecheck`, `pnpm test` (unit) need no running services. - `pnpm test:integration` requires the docker stack up first (`docker compose up -d --build`); it hits the proxy at `http://127.0.0.1:8000`. - `pnpm test:perf` is the ACL load harness (multi-client Pouch sync + HTTP r/w ops/sec). Prefer the dev overlay so direct Couch is on `5985` for overhead compare: `docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build`. Not part of CI. -- Request phase profiling: set `PROFILE=true` (compose overlay `docker-compose.profile.yml`, or `pnpm docker:up:profile`). Scrapes `GET /_couch-auth-proxy/profile` (auth / acl / aclMiss / upstream / filter). Convenience: `pnpm test:perf:profile`. Host CPU profiles: build then `pnpm start:profile` (writes under `profiles/`). +- Request phase profiling: set `PROFILE=true` (compose overlay `docker-compose.profile.yml`, or `pnpm docker:up:profile`). Scrapes `GET /_couch-auth-proxy/profile` (auth / acl / aclMiss / upstream / filter + process memory / resource sizes). Convenience: `pnpm test:perf:profile`. Host CPU profiles: build then `pnpm start:profile` (writes under `profiles/`). +- Memory stability soak (opt-in): `pnpm test:perf:memory` brings up the profile overlay (`PROFILE=true`, `NODE_OPTIONS=--expose-gc`), runs a multi-minute steady-state ACL load while scraping heap/rss trends, and writes `test/perf/last-memory-report.md`. Tunable via `PERF_MEMORY_DURATION_SEC` (default 300). ### Gotchas diff --git a/README.md b/README.md index da97786..7fff817 100644 --- a/README.md +++ b/README.md @@ -118,27 +118,27 @@ Unmapped endpoints return **404** for non-admins (default-deny). `_list`, `_show ## Ops -| Variable | Purpose | -| ---------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `COUCH_URL` | Upstream Couch (no creds) | -| `COUCH_ADMIN_USER` / `COUCH_ADMIN_PASSWORD` or `COUCH_ADMIN_URL` | Admin for ACL maintenance + `_changes` follow | -| `COUCH_PRELOAD_DBS` | Comma-separated DBs to warm on boot | -| `ACL_AUTO_INSTALL` | Auto-PUT `_design/acl` when missing on app DBs (default `true`). Never installs into `_users` / `_replicator` / `_global_changes`. Prefer `false` in production when ddocs are provisioned out-of-band. | -| `ACL_DB_INCLUDE` / `ACL_DB_EXCLUDE` | Opt-in database allow/deny lists (CSV). Entries are exact names or `/regex/flags`. Empty = historical behaviour. Exclude wins. Non-admins only; hidden DBs are omitted from `/_all_dbs` and return **404**. Example: `ACL_DB_INCLUDE=/^data-/`. | -| `ACL_ROUTE_INCLUDE` / `ACL_ROUTE_EXCLUDE` | Opt-in API surface allow/deny lists (CSV). Entries are feature/bundle names (`pouch-sync`, `session`, `changes`, …), `METHOD /restmap-path` templates, or `/regex/flags` over `METHOD pathname`. Empty = all restmap routes. Exclude wins. Non-admins get **403**. | -| `AUTH_RESOLVE_VIA_COUCH_SESSION` | Default `true` | -| `JWT_LOCAL_VERIFY` / `JWT_HMAC_SECRET` | Optional local Bearer JWT verification; required together when Couch session resolution is disabled | -| `JWT_ROLES_CLAIM_PATH` / `JWT_REQUIRED_CLAIMS` | Local JWT role claim path and comma-separated required claims | -| `COUCH_MAX_ID_LENGTH` | Maximum accepted document-id length (default `200`) | -| `CORS_ORIGINS` | Comma allowlist (**required for browser CORS**; empty = no Origin reflection) | -| `TRUST_PROXY_HOPS` | Trusted reverse-proxy hops for client IP (default `0` = ignore `X-Forwarded-For`) | -| `SESSION_CACHE_TTL_MS` / `SESSION_CACHE_MAX` | Session principal cache TTL (default `5000`) + LRU size. Cuts sequential `/_session` cost under sync/HTTP load; role/`_admin` revocation from Couch can lag by up to the TTL. Set `0` for immediate re-resolve on every request. Concurrent identical credentials also coalesce in-flight (no extra stale window). | -| `RATE_LIMIT_*` | Global + per-IP limits | -| `MAX_BODY_BYTES` | Request body ceiling (Content-Length + streamed bodies) | -| `SHUTDOWN_TIMEOUT_MS` | Drain timeout before force-exit | -| `PORT` / `HOST` | Listen address | -| `LOG_LEVEL` | Minimum log level: `verbose`, `debug`, `info`, `warn`, `error` (aliases: `trace`→`verbose`, `warning`→`warn`). Default `debug` outside production, `info` in production. Use `verbose` to trace ACL allow/deny decisions (actors, resolvers, filters, session tokens). | -| `PROFILE` | Opt-in request phase profiling (`auth` / `acl` / `aclMiss` / `upstream` / `filter`). Adds phase ms to access logs and exposes `GET/POST /_couch-auth-proxy/profile[/reset]` for the perf harness. Default off — leave disabled in production. | +| Variable | Purpose | +| ---------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `COUCH_URL` | Upstream Couch (no creds) | +| `COUCH_ADMIN_USER` / `COUCH_ADMIN_PASSWORD` or `COUCH_ADMIN_URL` | Admin for ACL maintenance + `_changes` follow | +| `COUCH_PRELOAD_DBS` | Comma-separated DBs to warm on boot | +| `ACL_AUTO_INSTALL` | Auto-PUT `_design/acl` when missing on app DBs (default `true`). Never installs into `_users` / `_replicator` / `_global_changes`. Prefer `false` in production when ddocs are provisioned out-of-band. | +| `ACL_DB_INCLUDE` / `ACL_DB_EXCLUDE` | Opt-in database allow/deny lists (CSV). Entries are exact names or `/regex/flags`. Empty = historical behaviour. Exclude wins. Non-admins only; hidden DBs are omitted from `/_all_dbs` and return **404**. Example: `ACL_DB_INCLUDE=/^data-/`. | +| `ACL_ROUTE_INCLUDE` / `ACL_ROUTE_EXCLUDE` | Opt-in API surface allow/deny lists (CSV). Entries are feature/bundle names (`pouch-sync`, `session`, `changes`, …), `METHOD /restmap-path` templates, or `/regex/flags` over `METHOD pathname`. Empty = all restmap routes. Exclude wins. Non-admins get **403**. | +| `AUTH_RESOLVE_VIA_COUCH_SESSION` | Default `true` | +| `JWT_LOCAL_VERIFY` / `JWT_HMAC_SECRET` | Optional local Bearer JWT verification; required together when Couch session resolution is disabled | +| `JWT_ROLES_CLAIM_PATH` / `JWT_REQUIRED_CLAIMS` | Local JWT role claim path and comma-separated required claims | +| `COUCH_MAX_ID_LENGTH` | Maximum accepted document-id length (default `200`) | +| `CORS_ORIGINS` | Comma allowlist (**required for browser CORS**; empty = no Origin reflection) | +| `TRUST_PROXY_HOPS` | Trusted reverse-proxy hops for client IP (default `0` = ignore `X-Forwarded-For`) | +| `SESSION_CACHE_TTL_MS` / `SESSION_CACHE_MAX` | Session principal cache TTL (default `5000`) + LRU size. Cuts sequential `/_session` cost under sync/HTTP load; role/`_admin` revocation from Couch can lag by up to the TTL. Set `0` for immediate re-resolve on every request. Concurrent identical credentials also coalesce in-flight (no extra stale window). | +| `RATE_LIMIT_*` | Global + per-IP limits | +| `MAX_BODY_BYTES` | Request body ceiling (Content-Length + streamed bodies) | +| `SHUTDOWN_TIMEOUT_MS` | Drain timeout before force-exit | +| `PORT` / `HOST` | Listen address | +| `LOG_LEVEL` | Minimum log level: `verbose`, `debug`, `info`, `warn`, `error` (aliases: `trace`→`verbose`, `warning`→`warn`). Default `debug` outside production, `info` in production. Use `verbose` to trace ACL allow/deny decisions (actors, resolvers, filters, session tokens). | +| `PROFILE` | Opt-in request phase profiling (`auth` / `acl` / `aclMiss` / `upstream` / `filter`) plus scrapeable process memory / ACL+session resource sizes on `GET /_couch-auth-proxy/profile`. Also exposes `POST .../profile/reset` and optional `POST .../profile/gc` (needs `--expose-gc`). Default off — leave disabled in production. | Structured JSON logs go to stdout/stderr (`ts`, `level`, `component`, `msg`, …). Secret-looking fields (`authorization`, `cookie`, `password`, `token`, `secret`, …) are redacted. @@ -217,6 +217,10 @@ pnpm test:perf # writes test/perf/last-results.json; not in CI pnpm test:perf:profile # compose profile overlay + scrape /_couch-auth-proxy/profile # Host CPU profile (after pnpm build; Couch on :5985 via docker:up:dev): # PROFILE=true pnpm start:profile # writes CPU profiles under ./profiles/ + +# Long-running memory stability soak (opt-in PROFILE memory probe; default 5 min) +pnpm test:perf:memory # writes test/perf/last-memory-{results.json,report.md} +# PERF_MEMORY_DURATION_SEC=600 pnpm test:perf:memory ``` ### Performance notes diff --git a/docker-compose.profile.yml b/docker-compose.profile.yml index 0b80029..cb7face 100644 --- a/docker-compose.profile.yml +++ b/docker-compose.profile.yml @@ -1,11 +1,15 @@ -# Profiling overlay: enables request phase timers on the proxy. +# Profiling overlay: enables request phase timers + memory probe on the proxy. # Usage: # docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.profile.yml up -d --build # pnpm test:perf:profile +# pnpm test:perf:memory services: couch-auth-proxy: environment: PROFILE: "true" + # Expose V8 GC for optional POST /_couch-auth-proxy/profile/gc samples used + # by the memory-stability soak. Harmless when unused; still opt-in via this overlay. + NODE_OPTIONS: "--expose-gc" # Session principal cache defaults to 5000ms in the app. Override here to # attribute auth vs ACL under an alternate TTL (0 = re-resolve every request). # SESSION_CACHE_TTL_MS: "0" diff --git a/docs/memory-stability.md b/docs/memory-stability.md new file mode 100644 index 0000000..4ee1438 --- /dev/null +++ b/docs/memory-stability.md @@ -0,0 +1,70 @@ +# Memory stability report + +Generated: 2026-07-24T22:24:16.881Z +Branch / harness: `feat/memory-stability-perf-38c4` via `pnpm test:perf:memory` +Proxy: compose profile overlay (`PROFILE=true`, `NODE_OPTIONS=--expose-gc`) + +## Verdict + +**STABLE — no evidence of a memory leak** + +- V8 `heapUsed` was flat-to-down over a 5-minute steady-state ACL load (net **−1.55 MiB**). +- In-process ACL map size plateaued (`acl_rows` **504 → 504**), so cache growth does not explain any residual process size change. +- Linear heap slope after warmup was **~0.45 KiB/s** with weak correlation (**r = 0.094**) — consistent with GC sawtooth, not a leak. +- Median heap in the last third of the steady window was only **~200 KiB** above the first third. +- RSS rose modestly (**+9.6 MiB**, ~48 KiB/s, r = 0.988) while heap declined. That pattern matches native allocator retention / freelist growth under sustained HTTP, not retained JavaScript objects. Slope remained under the soak threshold (128 KiB/s). + +## Load (steady-state) + +| Metric | Value | +| ----------------------- | ------------------------------------------ | +| duration | 302.8 s | +| clients | 6 | +| seed docs | 300 | +| samples | 150 (2 s interval; 25% warmup discarded) | +| ops | 185,926 (613.9 ops/s) | +| docs read / written | 600,998 / 37,188 | +| error rate | 0.00% | +| latency p50 / p95 / p99 | 7.0 / 24.8 / 31.2 ms | +| forced GC samples | yes (`POST /_couch-auth-proxy/profile/gc`) | + +Workload reuses rotating document slots and a fixed mixed-ACL corpus so the ACL row count plateaus while still exercising auth, ACL lookup, `_bulk_get` filtering, and writes. + +## Memory trend (steady state) + +| Signal | Value | +| --------------------------- | --------------------------------- | +| heap_used first → last | 16.48 → 14.93 MiB (Δ −1.55 MiB) | +| heap_used slope | 0.45 KiB/s (r = 0.094) | +| heap median 1st → 3rd third | 16.27 → 16.47 MiB | +| rss first → last | 217.10 → 226.73 MiB (Δ +9.63 MiB) | +| rss slope | 47.88 KiB/s (r = 0.988) | +| acl_rows first → last | 504 → 504 (Δ 0) | + +## Method + +1. Seed a fixed mixed-ACL corpus through the proxy. +2. Run concurrent HTTP readers/writers that reuse rotating document slots so the in-memory ACL map plateaus. +3. While load runs, scrape `GET /_couch-auth-proxy/profile` (opt-in `PROFILE=true`) for `process.memoryUsage()` plus ACL/session resource sizes. +4. Optionally `POST /_couch-auth-proxy/profile/gc` each sample when the proxy was started with `--expose-gc` (profile compose overlay). +5. Discard the leading warmup fraction, fit heap/rss vs time, and compare median heap in the first vs last third of the steady window. Expected ACL-row growth is budgeted; unexplained growth fails the assessment. + +## Thresholds used + +| Threshold | Value | +| --------------------------------- | ----------- | +| max heap slope | 64.0 KiB/s | +| max rss slope | 128.0 KiB/s | +| max unexplained heap median shift | 48.0 MiB | +| heap budget per new ACL row | 2048 B | +| min steady samples | 8 | + +## How to reproduce + +```bash +pnpm test:perf:memory +# longer soak: +PERF_MEMORY_DURATION_SEC=600 pnpm test:perf:memory +``` + +Artifacts (gitignored): `test/perf/last-memory-results.json`, `test/perf/last-memory-report.md`, `test/perf/last-memory-profile.json`. diff --git a/package.json b/package.json index 94a64a5..bb0fc2e 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "test:integration": "vitest run --config vitest.integration.config.ts", "test:perf": "vitest run --config vitest.perf.config.ts", "test:perf:profile": "bash scripts/perf-profile.sh", + "test:perf:memory": "bash scripts/perf-memory.sh", "docker:up": "docker compose up -d --build", "docker:up:dev": "docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build", "docker:up:profile": "docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.profile.yml up -d --build", diff --git a/scripts/perf-memory.sh b/scripts/perf-memory.sh new file mode 100755 index 0000000..b32d7ea --- /dev/null +++ b/scripts/perf-memory.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Bring up the compose stack with PROFILE=true (+ optional --expose-gc), run the +# long-running memory stability soak, and write JSON + Markdown reports. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +COMPOSE=(docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.profile.yml) +PROXY_URL="${COUCH_AUTH_PROXY_URL:-http://127.0.0.1:8000}" +RESULTS_PATH="${PERF_MEMORY_RESULTS_PATH:-test/perf/last-memory-results.json}" +REPORT_PATH="${PERF_MEMORY_REPORT_PATH:-test/perf/last-memory-report.md}" +# Default 5-minute soak; override with PERF_MEMORY_DURATION_SEC. +export PERF_MEMORY_DURATION_SEC="${PERF_MEMORY_DURATION_SEC:-300}" + +die() { + echo "error: $*" >&2 + exit 1 +} + +echo "==> starting stack with PROFILE=true (memory probe + optional expose-gc)" +"${COMPOSE[@]}" up -d --build + +echo "==> waiting for ready" +for i in $(seq 1 90); do + if curl -sf "${PROXY_URL}/_couch-auth-proxy/ready" >/dev/null; then + break + fi + if [[ "$i" -eq 90 ]]; then + "${COMPOSE[@]}" logs couch-auth-proxy couchdb || true + die "proxy not ready at ${PROXY_URL}" + fi + sleep 2 +done + +echo "==> checking PROFILE memory probe" +curl -sf "${PROXY_URL}/_couch-auth-proxy/profile" >/dev/null || + die "profile endpoint unavailable — is PROFILE=true on the proxy?" + +echo "==> running memory soak (${PERF_MEMORY_DURATION_SEC}s)" +PERF_MEMORY_RESULTS_PATH="${RESULTS_PATH}" \ + PERF_MEMORY_REPORT_PATH="${REPORT_PATH}" \ + PERF_MEMORY_REQUIRED=1 \ + pnpm exec vitest run --config vitest.perf.memory.config.ts + +echo "==> final profile snapshot" +curl -sf "${PROXY_URL}/_couch-auth-proxy/profile" | tee test/perf/last-memory-profile.json >/dev/null +echo +echo "Wrote ${RESULTS_PATH}" +echo "Wrote ${REPORT_PATH}" +echo "Wrote test/perf/last-memory-profile.json" +if [[ -f "${REPORT_PATH}" ]]; then + echo + sed -n '1,80p' "${REPORT_PATH}" +fi diff --git a/src/acl/cache.ts b/src/acl/cache.ts index 1cc94da..850ec8b 100644 --- a/src/acl/cache.ts +++ b/src/acl/cache.ts @@ -110,6 +110,32 @@ export class AclCache { return this.dbs.values(); } + /** + * Compact size counters for opt-in PROFILE memory scrapes. + * Does not walk row contents — O(dbs). + */ + resourceStats(): { + aclDbs: number; + aclRows: number; + aclTombstones: number; + aclInflightEnsures: number; + aclInflightRefreshes: number; + } { + let aclRows = 0; + let aclTombstones = 0; + for (const state of this.dbs.values()) { + aclRows += state.acl.size; + aclTombstones += state.tombstones?.size ?? 0; + } + return { + aclDbs: this.dbs.size, + aclRows, + aclTombstones, + aclInflightEnsures: this.inflight.size, + aclInflightRefreshes: this.refreshInflight.size, + }; + } + /** * Read only the bucket policy needed by `/_all_dbs`. * diff --git a/src/app.ts b/src/app.ts index 53520d1..a7677e1 100644 --- a/src/app.ts +++ b/src/app.ts @@ -20,6 +20,7 @@ import { bodyLimit } from "./middleware/bodyLimit.js"; import { registerRoutes } from "./routes/register.js"; import { jsonResponse } from "./proxy/forward.js"; import { createLogger } from "./util/log.js"; +import { captureProcessMemory, tryForceGc, type ResourceStats } from "./util/memory.js"; import { ProfileAggregator } from "./util/profile.js"; const log = createLogger("app"); @@ -154,7 +155,7 @@ export function createApp(services: AppServices): Hono { }); /** - * Scrapeable phase-timing snapshot when `PROFILE=true`. + * Scrapeable phase-timing + memory snapshot when `PROFILE=true`. * Returns 404 when profiling is off so probes stay non-sensitive by default. */ app.get("/_couch-auth-proxy/profile", (c) => { @@ -162,7 +163,14 @@ export function createApp(services: AppServices): Hono { if (!c.get("config").server.profile || !agg) { return jsonResponse({ error: "not_found", reason: "Profiling disabled" }, 404); } - return jsonResponse(agg.snapshot()); + const acl = c.get("aclCache").resourceStats(); + const session = c.get("sessions").resourceStats(); + const resources: ResourceStats = { ...acl, ...session }; + return jsonResponse({ + ...agg.snapshot(), + memory: captureProcessMemory(), + resources, + }); }); /** Reset aggregated profile counters (load harness between phases). */ @@ -175,6 +183,22 @@ export function createApp(services: AppServices): Hono { return jsonResponse({ ok: true }); }); + /** + * Best-effort V8 GC when the process was started with `--expose-gc`. + * Opt-in only (`PROFILE=true`); used by the memory-stability harness between samples. + */ + app.post("/_couch-auth-proxy/profile/gc", (c) => { + if (!c.get("config").server.profile || !c.get("profileAggregator")) { + return jsonResponse({ error: "not_found", reason: "Profiling disabled" }, 404); + } + const ran = tryForceGc(); + return jsonResponse({ + ok: true, + gc: ran, + memory: captureProcessMemory(), + }); + }); + registerRoutes(app, services.accessPolicy); // Keyed ACL view failures may happen after the DB gate. Keep their response diff --git a/src/auth/session.ts b/src/auth/session.ts index 86f4191..7fd07de 100644 --- a/src/auth/session.ts +++ b/src/auth/session.ts @@ -38,6 +38,17 @@ export class SessionResolver { this.cache = new LruMap(config.couch.sessionCacheMaxEntries); } + /** + * Compact size counters for opt-in PROFILE memory scrapes. + * Session cache is LRU-bounded (`SESSION_CACHE_MAX`); inflight should stay near 0 at rest. + */ + resourceStats(): { sessionCacheEntries: number; sessionInflight: number } { + return { + sessionCacheEntries: this.cache.size, + sessionInflight: this.inflight.size, + }; + } + /** * Resolve identity from incoming request headers. * Missing credentials → anonymous. Couch 401 → anonymous (upstream may still reject). diff --git a/src/util/memory.ts b/src/util/memory.ts new file mode 100644 index 0000000..865ad97 --- /dev/null +++ b/src/util/memory.ts @@ -0,0 +1,88 @@ +/** + * Opt-in process memory snapshots for `/_couch-auth-proxy/profile`. + * + * Only consulted when `PROFILE=true`. Cheap (`process.memoryUsage()`); no + * allocators or heap dumps — meant for long-running load harness scrapes. + */ + +/** Bytes from `process.memoryUsage()`. */ +export type ProcessMemorySnapshot = { + rss: number; + heapTotal: number; + heapUsed: number; + external: number; + arrayBuffers: number; +}; + +/** Bounded in-process structure sizes correlated with expected heap growth. */ +export type ResourceStats = { + aclDbs: number; + aclRows: number; + aclTombstones: number; + aclInflightEnsures: number; + aclInflightRefreshes: number; + sessionCacheEntries: number; + sessionInflight: number; +}; + +/** Capture current process memory counters. */ +export function captureProcessMemory(): ProcessMemorySnapshot { + const m = process.memoryUsage(); + return { + rss: m.rss, + heapTotal: m.heapTotal, + heapUsed: m.heapUsed, + external: m.external, + arrayBuffers: m.arrayBuffers, + }; +} + +/** Format bytes for harness / console output. */ +export function formatBytes(bytes: number): string { + const sign = bytes < 0 ? "-" : ""; + const abs = Math.abs(bytes); + if (abs < 1024) { + const whole = Number.isInteger(abs) ? String(abs) : abs.toFixed(1); + return `${sign}${whole}B`; + } + if (abs < 1024 * 1024) return `${sign}${(abs / 1024).toFixed(1)}KiB`; + if (abs < 1024 * 1024 * 1024) return `${sign}${(abs / (1024 * 1024)).toFixed(2)}MiB`; + return `${sign}${(abs / (1024 * 1024 * 1024)).toFixed(2)}GiB`; +} + +/** Human-readable memory + resource lines for harness logs. */ +export function formatMemorySnapshot( + memory: ProcessMemorySnapshot, + resources?: ResourceStats, + label = "process memory", +): string { + const lines = [ + `=== ${label} ===`, + `rss: ${formatBytes(memory.rss)}`, + `heap_used: ${formatBytes(memory.heapUsed)}`, + `heap_total: ${formatBytes(memory.heapTotal)}`, + `external: ${formatBytes(memory.external)}`, + `array_buffers: ${formatBytes(memory.arrayBuffers)}`, + ]; + if (resources) { + lines.push( + `acl_dbs: ${resources.aclDbs}`, + `acl_rows: ${resources.aclRows}`, + `acl_tombstones:${resources.aclTombstones}`, + `session_cache: ${resources.sessionCacheEntries}`, + `session_inflight:${resources.sessionInflight}`, + ); + } + return lines.join("\n"); +} + +/** + * Request a V8 GC when the process was started with `--expose-gc`. + * Returns false when GC is unavailable (typical production / default PROFILE). + */ +export function tryForceGc(): boolean { + const gc = (globalThis as { gc?: () => void }).gc; + if (typeof gc !== "function") return false; + gc(); + return true; +} diff --git a/src/util/profile.ts b/src/util/profile.ts index f2e52b2..1c13a0e 100644 --- a/src/util/profile.ts +++ b/src/util/profile.ts @@ -4,11 +4,14 @@ * When enabled, middleware installs an AsyncLocalStorage request profile and * hot-path helpers accumulate wall time for auth / ACL / upstream / filter. * Aggregated stats are exposed via `/_couch-auth-proxy/profile` for the perf - * harness; per-request phase ms are also attached to structured access logs. + * harness (including process memory + resource sizes); per-request phase ms + * are also attached to structured access logs. * * Disabled by default — zero ALS / timer cost on the hot path when off. */ import { AsyncLocalStorage } from "node:async_hooks"; +import type { ProcessMemorySnapshot, ResourceStats } from "./memory.js"; +export type { ProcessMemorySnapshot, ResourceStats } from "./memory.js"; /** Timed phases on the ACL proxy hot path. */ export const PROFILE_PHASES = ["auth", "acl", "aclMiss", "upstream", "filter"] as const; @@ -45,6 +48,12 @@ export type ProfileSnapshot = { * overlap or when other work is unattributed — use as a relative signal). */ phaseShareOfMean: Record; + /** + * Present on scrape responses from the HTTP probe (not on bare aggregator + * snapshots). Process `memoryUsage()` + in-process cache sizes. + */ + memory?: ProcessMemorySnapshot; + resources?: ResourceStats; }; type PhaseAccum = { diff --git a/test/perf/acl-sync-load.test.ts b/test/perf/acl-sync-load.test.ts index 9f84265..992b3c2 100644 --- a/test/perf/acl-sync-load.test.ts +++ b/test/perf/acl-sync-load.test.ts @@ -31,6 +31,7 @@ * docker compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.profile.yml up -d --build * pnpm test:perf:profile * When available, each phase scrapes `/_couch-auth-proxy/profile` (auth/acl/aclMiss/upstream/filter). + * Long-running memory stability soak (separate config): `pnpm test:perf:memory`. */ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; diff --git a/test/perf/memory-stability.test.ts b/test/perf/memory-stability.test.ts new file mode 100644 index 0000000..02b5bc1 --- /dev/null +++ b/test/perf/memory-stability.test.ts @@ -0,0 +1,470 @@ +/** + * Long-running memory stability assessment (docker compose + PROFILE=true). + * + * Sustains ACL-challenging HTTP traffic against a fixed corpus while scraping + * `/_couch-auth-proxy/profile` memory + resource counters. After a warmup + * window, heap/rss trends are fit with linear regression and checked against + * leak thresholds (ACL-row growth is budgeted as expected cache cost). + * + * Opt-in only — requires the profile overlay (or `PROFILE=true` on the proxy): + * pnpm test:perf:memory + * + * Tunables (env): + * PERF_MEMORY_DURATION_SEC soak length (default 300) + * PERF_MEMORY_SAMPLE_MS sample interval (default 2000) + * PERF_MEMORY_CLIENTS concurrent workers (default 6) + * PERF_MEMORY_SEED_DOCS fixed corpus size (default 300) + * PERF_MEMORY_WARMUP_FRAC discarded leading fraction (default 0.25) + * PERF_MEMORY_FORCE_GC POST /profile/gc each sample when available (default 1) + * PERF_MEMORY_RESULTS_PATH JSON output (default test/perf/last-memory-results.json) + * PERF_MEMORY_REPORT_PATH Markdown report (default test/perf/last-memory-report.md) + */ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + PROXY, + adminHeaders, + authHeaders, + createUserIfMissing, + ensureDbOpenForDemoUsers, + mintJwt, + sleep, + waitForReady, + waitUntil, +} from "../integration/helpers.js"; +import { LatencyTracker, OpCounter, formatRate, rateReport, timed } from "./metrics.js"; +import { + DEFAULT_STABILITY_THRESHOLDS, + analyzeMemoryTrend, + formatTrendReport, + memoryEndpointReady, + requestServerGc, + scrapeMemorySample, + type MemorySample, + type MemoryTrendReport, + type StabilityThresholds, +} from "./memory.js"; +import { resetServerProfile } from "./profile.js"; + +type Principal = { + name: string; + pass: string; + roles: string[]; + jwt: string; +}; + +const suiteId = Date.now().toString(36); +const DB = `perfmem-${suiteId}`; + +const DURATION_SEC = Math.max(30, Number(process.env.PERF_MEMORY_DURATION_SEC ?? 300)); +const SAMPLE_MS = Math.max(250, Number(process.env.PERF_MEMORY_SAMPLE_MS ?? 2000)); +const CLIENTS = Math.max(1, Number(process.env.PERF_MEMORY_CLIENTS ?? 6)); +const SEED_DOCS = Math.max(40, Number(process.env.PERF_MEMORY_SEED_DOCS ?? 300)); +const WARMUP_FRAC = Math.min( + 0.6, + Math.max(0.05, Number(process.env.PERF_MEMORY_WARMUP_FRAC ?? 0.25)), +); +const FORCE_GC = !["0", "false", "no"].includes( + String(process.env.PERF_MEMORY_FORCE_GC ?? "1").toLowerCase(), +); +const RESULTS_PATH = + process.env.PERF_MEMORY_RESULTS_PATH ?? + path.join(path.dirname(fileURLToPath(import.meta.url)), "last-memory-results.json"); +const REPORT_PATH = + process.env.PERF_MEMORY_REPORT_PATH ?? + path.join(path.dirname(fileURLToPath(import.meta.url)), "last-memory-report.md"); + +const thresholds: StabilityThresholds = { + ...DEFAULT_STABILITY_THRESHOLDS, + warmupFraction: WARMUP_FRAC, + maxHeapSlopeBytesPerSec: Number( + process.env.PERF_MEMORY_MAX_HEAP_SLOPE_BPS ?? + DEFAULT_STABILITY_THRESHOLDS.maxHeapSlopeBytesPerSec, + ), + maxRssSlopeBytesPerSec: Number( + process.env.PERF_MEMORY_MAX_RSS_SLOPE_BPS ?? + DEFAULT_STABILITY_THRESHOLDS.maxRssSlopeBytesPerSec, + ), + maxHeapThirdGrowthBytes: Number( + process.env.PERF_MEMORY_MAX_HEAP_THIRD_GROWTH ?? + DEFAULT_STABILITY_THRESHOLDS.maxHeapThirdGrowthBytes, + ), +}; + +const principals: Principal[] = []; +const samples: MemorySample[] = []; +let seedIds: string[] = []; +let trend: MemoryTrendReport | null = null; +let loadReport: ReturnType | null = null; +let gcAvailable = false; + +function aclPattern(i: number, owner: Principal): Record { + switch (i % 5) { + case 0: + return { creator: owner.name, kind: "private" }; + case 1: + return { + creator: owner.name, + acl: ["u-alice", "u-bob", "u-carol", "u-dave"].filter((u) => u !== `u-${owner.name}`), + kind: "shared-readers", + }; + case 2: + return { creator: owner.name, acl: ["r-readers"], kind: "role-readers" }; + case 3: + return { creator: owner.name, owners: ["u-bob", "u-carol"], kind: "owners" }; + default: + return { kind: "open", body: "any-member" }; + } +} + +async function seedCorpus(): Promise { + const ids: string[] = []; + const batchSize = 50; + for (let start = 0; start < SEED_DOCS; start += batchSize) { + const end = Math.min(SEED_DOCS, start + batchSize); + const docs = []; + for (let i = start; i < end; i++) { + const owner = principals[i % principals.length]!; + const id = `mem-seed-${suiteId}-${i}`; + ids.push(id); + docs.push({ + _id: id, + ...aclPattern(i, owner), + n: i, + body: `seed-${i}`, + }); + } + const res = await fetch(`${PROXY}/${DB}/_bulk_docs`, { + method: "POST", + headers: { ...adminHeaders(), "Content-Type": "application/json" }, + body: JSON.stringify({ docs }), + }); + if (!res.ok) throw new Error(`seed bulk: ${res.status} ${await res.text()}`); + } + + const openId = ids.find((_, i) => i % 5 === 4) ?? ids[0]!; + await waitUntil( + `mem seed readable ${openId}`, + async () => { + const res = await fetch(`${PROXY}/${DB}/${encodeURIComponent(openId)}`, { + headers: authHeaders("jwt", principals[0]!.jwt), + }); + return res.status === 200; + }, + 60_000, + ); + return ids; +} + +/** + * Steady-state op: mostly reads + in-place updates on the fixed corpus so ACL + * row count plateaus. Occasional bounded creates keep write ACL warm without + * unbounded cache growth (creates are capped by worker*loop modulo). + */ +async function steadyOp( + principal: Principal, + i: number, + counter: OpCounter, + latency: LatencyTracker, +): Promise { + const headers = { + ...authHeaders("jwt", principal.jwt), + "Content-Type": "application/json", + }; + const mode = i % 5; + const { ms } = await timed(async () => { + if (mode === 0) { + // Bounded create: overwrite a rotating slot id so ACL rows stay capped. + const slot = i % Math.max(CLIENTS * 8, 16); + const id = `mem-slot-${principal.name}-${suiteId}-${slot}`; + const existing = await fetch(`${PROXY}/${DB}/${encodeURIComponent(id)}`, { headers }); + let rev: string | undefined; + if (existing.status === 200) { + rev = ((await existing.json()) as { _rev?: string })._rev; + } + const res = await fetch(`${PROXY}/${DB}/${encodeURIComponent(id)}`, { + method: "PUT", + headers, + body: JSON.stringify({ + _id: id, + ...(rev ? { _rev: rev } : {}), + creator: principal.name, + kind: "slot-write", + body: `slot-${i}`, + n: i, + }), + }); + if (!res.ok) counter.add({ ops: 1, errors: 1 }); + else counter.add({ ops: 1, docsWritten: 1 }); + return; + } + + if (mode === 1) { + const chunk = 20; + const offset = (i * chunk) % seedIds.length; + const docs = seedIds.slice(offset, offset + chunk).map((id) => ({ id })); + if (docs.length === 0) { + counter.add({ ops: 1 }); + return; + } + const res = await fetch(`${PROXY}/${DB}/_bulk_get`, { + method: "POST", + headers, + body: JSON.stringify({ docs }), + }); + if (!res.ok) { + counter.add({ ops: 1, errors: 1 }); + return; + } + const body = (await res.json()) as { + results: Array<{ docs: Array<{ ok?: unknown }> }>; + }; + const readable = body.results.filter((r) => r.docs[0]?.ok).length; + counter.add({ ops: 1, docsRead: readable }); + return; + } + + const id = seedIds[i % seedIds.length]!; + const res = await fetch(`${PROXY}/${DB}/${encodeURIComponent(id)}`, { headers }); + if (res.status === 200) counter.add({ ops: 1, docsRead: 1 }); + else if (res.status === 404) counter.add({ ops: 1 }); + else counter.add({ ops: 1, errors: 1 }); + }); + latency.record(ms); +} + +async function runSoak(): Promise<{ + load: ReturnType; + samples: MemorySample[]; + gcAvailable: boolean; +}> { + const counter = new OpCounter(); + const latency = new LatencyTracker(); + const collected: MemorySample[] = []; + let sawGc = false; + const t0 = performance.now(); + const deadline = t0 + DURATION_SEC * 1000; + let stop = false; + + await resetServerProfile(); + + const sampler = (async () => { + while (!stop) { + if (FORCE_GC) { + const ran = await requestServerGc(); + if (ran) sawGc = true; + } + const sample = await scrapeMemorySample(performance.now()); + if (sample) collected.push(sample); + await sleep(SAMPLE_MS); + } + })(); + + const workers = Array.from({ length: CLIENTS }, async (_, clientIdx) => { + const principal = principals[clientIdx % principals.length]!; + let i = 0; + while (performance.now() < deadline) { + await steadyOp(principal, clientIdx * 1_000_000 + i, counter, latency); + i += 1; + } + }); + + await Promise.all(workers); + stop = true; + await sampler; + + // Settle sample after load stops (helps distinguish leak vs in-flight buffers). + await sleep(Math.min(SAMPLE_MS, 2000)); + if (FORCE_GC) { + const ran = await requestServerGc(); + if (ran) sawGc = true; + } + const settle = await scrapeMemorySample(performance.now()); + if (settle) collected.push(settle); + + const load = rateReport(counter, latency, performance.now() - t0); + console.log(`\n${formatRate(load, "memory soak load")}\n`); + return { load, samples: collected, gcAvailable: sawGc }; +} + +function renderMarkdownReport(opts: { + trend: MemoryTrendReport; + load: ReturnType; + gcAvailable: boolean; + sampleCount: number; +}): string { + const { trend: t, load, gcAvailable: gc, sampleCount } = opts; + const verdict = t.stable ? "STABLE — no evidence of a memory leak" : "UNSTABLE — investigate"; + const lines = [ + `# Memory stability report`, + ``, + `Generated: ${new Date().toISOString()}`, + ``, + `## Verdict`, + ``, + `**${verdict}**`, + ``, + t.reasons.length + ? t.reasons.map((r) => `- ${r}`).join("\n") + : `- Steady-state heap/rss slopes and median shifts stayed within budgets after warmup.`, + ``, + `## Configuration`, + ``, + `| Setting | Value |`, + `| --- | --- |`, + `| proxy | ${PROXY} |`, + `| db | ${DB} |`, + `| duration_sec | ${DURATION_SEC} |`, + `| sample_ms | ${SAMPLE_MS} |`, + `| clients | ${CLIENTS} |`, + `| seed_docs | ${SEED_DOCS} |`, + `| warmup_fraction | ${WARMUP_FRAC} |`, + `| force_gc | ${FORCE_GC} (available=${gc}) |`, + `| samples | ${sampleCount} |`, + ``, + `## Load summary`, + ``, + `| Metric | Value |`, + `| --- | --- |`, + `| duration_sec | ${load.durationSec.toFixed(2)} |`, + `| ops | ${load.ops} |`, + `| ops_per_sec | ${load.opsPerSec.toFixed(2)} |`, + `| docs_read | ${load.docsRead} |`, + `| docs_written | ${load.docsWritten} |`, + `| error_rate | ${(load.errorRate * 100).toFixed(2)}% |`, + `| latency p50/p95/p99 ms | ${load.latency.p50Ms.toFixed(1)} / ${load.latency.p95Ms.toFixed(1)} / ${load.latency.p99Ms.toFixed(1)} |`, + ``, + `## Memory trend (steady state)`, + ``, + `| Signal | Value |`, + `| --- | --- |`, + `| heap_used first → last | ${(t.heapUsed.first / (1024 * 1024)).toFixed(2)} → ${(t.heapUsed.last / (1024 * 1024)).toFixed(2)} MiB (Δ ${(t.heapUsed.netGrowth / (1024 * 1024)).toFixed(2)} MiB) |`, + `| heap_used slope | ${(t.heapUsed.fit.slopePerSec / 1024).toFixed(2)} KiB/s (r=${t.heapUsed.fit.r.toFixed(3)}) |`, + `| heap median 1st→3rd third | ${(t.heapUsed.medianFirstThird / (1024 * 1024)).toFixed(2)} → ${(t.heapUsed.medianLastThird / (1024 * 1024)).toFixed(2)} MiB |`, + `| rss first → last | ${(t.rss.first / (1024 * 1024)).toFixed(2)} → ${(t.rss.last / (1024 * 1024)).toFixed(2)} MiB |`, + `| rss slope | ${(t.rss.fit.slopePerSec / 1024).toFixed(2)} KiB/s (r=${t.rss.fit.r.toFixed(3)}) |`, + `| acl_rows first → last | ${t.aclRows.first} → ${t.aclRows.last} (Δ ${t.aclRows.netGrowth}) |`, + ``, + `## Method`, + ``, + `1. Seed a fixed mixed-ACL corpus through the proxy.`, + `2. Run concurrent HTTP readers/writers that reuse rotating document slots so the in-memory ACL map plateaus.`, + `3. While load runs, scrape \`GET /_couch-auth-proxy/profile\` (opt-in \`PROFILE=true\`) for \`process.memoryUsage()\` plus ACL/session resource sizes.`, + FORCE_GC + ? `4. Optionally \`POST /_couch-auth-proxy/profile/gc\` each sample when the proxy was started with \`--expose-gc\` (profile compose overlay).` + : `4. Forced GC disabled for this run.`, + `5. Discard the leading warmup fraction, fit heap/rss vs time, and compare median heap in the first vs last third of the steady window. Expected ACL-row growth is budgeted; unexplained growth fails the assessment.`, + ``, + `## Thresholds`, + ``, + `| Threshold | Value |`, + `| --- | --- |`, + `| max heap slope | ${(thresholds.maxHeapSlopeBytesPerSec / 1024).toFixed(1)} KiB/s |`, + `| max rss slope | ${(thresholds.maxRssSlopeBytesPerSec / 1024).toFixed(1)} KiB/s |`, + `| max unexplained heap median shift | ${(thresholds.maxHeapThirdGrowthBytes / (1024 * 1024)).toFixed(1)} MiB |`, + `| heap budget per new ACL row | ${thresholds.heapBytesPerAclRowBudget} B |`, + `| min steady samples | ${thresholds.minSteadySamples} |`, + ``, + ]; + return `${lines.join("\n")}\n`; +} + +describe("memory stability assessment", () => { + beforeAll(async () => { + await waitForReady(); + const ready = await memoryEndpointReady(); + if (!ready) { + throw new Error( + "PROFILE memory probe unavailable. Start with: pnpm docker:up:profile (or PROFILE=true), then pnpm test:perf:memory", + ); + } + const users = [ + { name: "alice", pass: "alice-pass", roles: ["readers"] }, + { name: "bob", pass: "bob-pass", roles: ["writers"] }, + { name: "carol", pass: "carol-pass", roles: ["readers"] }, + { name: "dave", pass: "dave-pass", roles: [] as string[] }, + ]; + for (const u of users) { + await createUserIfMissing(u.name, u.pass, u.roles); + principals.push({ ...u, jwt: await mintJwt(u.name, u.roles) }); + } + await ensureDbOpenForDemoUsers(DB); + const secRes = await fetch(`${PROXY}/${DB}/_security`, { headers: adminHeaders() }); + const sec = (await secRes.json()) as { + admins: unknown; + members: { names: string[]; roles: string[] }; + }; + if (!sec.members.names.includes("dave")) { + sec.members.names.push("dave"); + await fetch(`${PROXY}/${DB}/_security`, { + method: "PUT", + headers: { ...adminHeaders(), "Content-Type": "application/json" }, + body: JSON.stringify(sec), + }); + } + seedIds = await seedCorpus(); + console.log( + `\n(memory soak: duration=${DURATION_SEC}s sample=${SAMPLE_MS}ms clients=${CLIENTS} seed=${SEED_DOCS})\n`, + ); + await sleep(500); + }, 300_000); + + afterAll(async () => { + if (trend && loadReport) { + const summary = { + version: process.env.npm_package_version ?? "unknown", + at: new Date().toISOString(), + config: { + durationSec: DURATION_SEC, + sampleMs: SAMPLE_MS, + clients: CLIENTS, + seedDocs: SEED_DOCS, + warmupFraction: WARMUP_FRAC, + forceGc: FORCE_GC, + gcAvailable, + proxy: PROXY, + db: DB, + thresholds, + }, + load: loadReport, + trend, + samples, + }; + await mkdir(path.dirname(RESULTS_PATH), { recursive: true }); + await writeFile(RESULTS_PATH, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); + const md = renderMarkdownReport({ + trend, + load: loadReport, + gcAvailable, + sampleCount: samples.length, + }); + await writeFile(REPORT_PATH, md, "utf8"); + console.log(`\nWrote memory results → ${RESULTS_PATH}`); + console.log(`Wrote memory report → ${REPORT_PATH}\n`); + } + await fetch(`${PROXY}/${DB}`, { method: "DELETE", headers: adminHeaders() }).catch( + () => undefined, + ); + }); + + it( + "sustains ACL load without unbounded heap/rss growth", + async () => { + const result = await runSoak(); + samples.push(...result.samples); + loadReport = result.load; + gcAvailable = result.gcAvailable; + + trend = analyzeMemoryTrend(samples, thresholds); + console.log(`\n${formatTrendReport(trend, "memory stability")}\n`); + + expect(loadReport.errorRate).toBeLessThan(0.05); + expect(loadReport.ops).toBeGreaterThan(0); + expect(samples.length).toBeGreaterThanOrEqual(thresholds.minSteadySamples); + expect(trend.stable, trend.reasons.join("; ") || "unstable").toBe(true); + }, + Math.max(600_000, (DURATION_SEC + 120) * 1000), + ); +}); diff --git a/test/perf/memory.ts b/test/perf/memory.ts new file mode 100644 index 0000000..8b6c5b0 --- /dev/null +++ b/test/perf/memory.ts @@ -0,0 +1,358 @@ +/** + * Memory sampling + trend analysis for the long-running stability harness. + */ +import { + formatBytes, + formatMemorySnapshot, + type ProcessMemorySnapshot, + type ResourceStats, +} from "../../src/util/memory.js"; +import type { ProfileSnapshot } from "../../src/util/profile.js"; +import { PROXY } from "../integration/helpers.js"; +import { fetchServerProfile, profileEndpointAvailable } from "./profile.js"; + +export type MemorySample = { + tMs: number; + at: string; + memory: ProcessMemorySnapshot; + resources: ResourceStats; + requests: number; +}; + +export type LinearFit = { + /** Slope in units per millisecond. */ + slopePerMs: number; + /** Slope in units per second. */ + slopePerSec: number; + intercept: number; + /** Pearson r (−1..1); 0 when undefined. */ + r: number; + n: number; +}; + +export type MemoryTrendReport = { + samples: number; + steadySamples: number; + durationSec: number; + warmupSec: number; + heapUsed: { + first: number; + last: number; + min: number; + max: number; + mean: number; + medianFirstThird: number; + medianLastThird: number; + netGrowth: number; + thirdGrowth: number; + fit: LinearFit; + }; + rss: { + first: number; + last: number; + min: number; + max: number; + netGrowth: number; + fit: LinearFit; + }; + aclRows: { + first: number; + last: number; + netGrowth: number; + fit: LinearFit; + }; + /** Bytes of heap growth attributed per new ACL row (Infinity if rows flat). */ + heapBytesPerAclRow: number; + /** True when steady-state heap/rss trends look leak-free under configured thresholds. */ + stable: boolean; + reasons: string[]; +}; + +export type StabilityThresholds = { + /** Discard this leading fraction of the run as warmup (cache fill, JIT). */ + warmupFraction: number; + /** Max allowed heapUsed linear slope after warmup (bytes/sec). */ + maxHeapSlopeBytesPerSec: number; + /** Max allowed rss linear slope after warmup (bytes/sec). */ + maxRssSlopeBytesPerSec: number; + /** Max median(last third) − median(first third) heap growth (bytes). */ + maxHeapThirdGrowthBytes: number; + /** + * When ACL rows grow, allow this many extra heap bytes per new row before + * counting toward the absolute third-growth budget. + */ + heapBytesPerAclRowBudget: number; + /** Require at least this many steady-state samples. */ + minSteadySamples: number; +}; + +export const DEFAULT_STABILITY_THRESHOLDS: StabilityThresholds = { + warmupFraction: 0.25, + maxHeapSlopeBytesPerSec: 64 * 1024, // 64 KiB/s sustained + maxRssSlopeBytesPerSec: 128 * 1024, // 128 KiB/s sustained + maxHeapThirdGrowthBytes: 48 * 1024 * 1024, // 48 MiB median shift + heapBytesPerAclRowBudget: 2048, + minSteadySamples: 8, +}; + +export function linearFit(xs: number[], ys: number[]): LinearFit { + const n = Math.min(xs.length, ys.length); + if (n < 2) { + return { slopePerMs: 0, slopePerSec: 0, intercept: ys[0] ?? 0, r: 0, n }; + } + let sumX = 0; + let sumY = 0; + let sumXX = 0; + let sumYY = 0; + let sumXY = 0; + for (let i = 0; i < n; i++) { + const x = xs[i]!; + const y = ys[i]!; + sumX += x; + sumY += y; + sumXX += x * x; + sumYY += y * y; + sumXY += x * y; + } + const denom = n * sumXX - sumX * sumX; + const slopePerMs = denom === 0 ? 0 : (n * sumXY - sumX * sumY) / denom; + const intercept = (sumY - slopePerMs * sumX) / n; + const varX = n * sumXX - sumX * sumX; + const varY = n * sumYY - sumY * sumY; + const cov = n * sumXY - sumX * sumY; + const r = varX <= 0 || varY <= 0 ? 0 : cov / Math.sqrt(varX * varY); + return { + slopePerMs, + slopePerSec: slopePerMs * 1000, + intercept, + r, + n, + }; +} + +function median(values: number[]): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 0) return (sorted[mid - 1]! + sorted[mid]!) / 2; + return sorted[mid]!; +} + +function seriesStats(values: number[]): { + first: number; + last: number; + min: number; + max: number; + mean: number; +} { + if (values.length === 0) { + return { first: 0, last: 0, min: 0, max: 0, mean: 0 }; + } + let min = values[0]!; + let max = values[0]!; + let sum = 0; + for (const v of values) { + if (v < min) min = v; + if (v > max) max = v; + sum += v; + } + return { + first: values[0]!, + last: values[values.length - 1]!, + min, + max, + mean: sum / values.length, + }; +} + +/** Analyze sampled memory for steady-state leak signals. */ +export function analyzeMemoryTrend( + samples: MemorySample[], + thresholds: StabilityThresholds = DEFAULT_STABILITY_THRESHOLDS, +): MemoryTrendReport { + const reasons: string[] = []; + if (samples.length === 0) { + return { + samples: 0, + steadySamples: 0, + durationSec: 0, + warmupSec: 0, + heapUsed: { + first: 0, + last: 0, + min: 0, + max: 0, + mean: 0, + medianFirstThird: 0, + medianLastThird: 0, + netGrowth: 0, + thirdGrowth: 0, + fit: { slopePerMs: 0, slopePerSec: 0, intercept: 0, r: 0, n: 0 }, + }, + rss: { + first: 0, + last: 0, + min: 0, + max: 0, + netGrowth: 0, + fit: { slopePerMs: 0, slopePerSec: 0, intercept: 0, r: 0, n: 0 }, + }, + aclRows: { + first: 0, + last: 0, + netGrowth: 0, + fit: { slopePerMs: 0, slopePerSec: 0, intercept: 0, r: 0, n: 0 }, + }, + heapBytesPerAclRow: 0, + stable: false, + reasons: ["no samples collected"], + }; + } + + const t0 = samples[0]!.tMs; + const durationSec = (samples[samples.length - 1]!.tMs - t0) / 1000; + const warmupCount = Math.min( + samples.length - 1, + Math.max(0, Math.floor(samples.length * thresholds.warmupFraction)), + ); + const steady = samples.slice(warmupCount); + const warmupSec = steady.length ? (steady[0]!.tMs - t0) / 1000 : durationSec; + + const xs = steady.map((s) => s.tMs - t0); + const heapYs = steady.map((s) => s.memory.heapUsed); + const rssYs = steady.map((s) => s.memory.rss); + const aclYs = steady.map((s) => s.resources.aclRows); + + const heapFit = linearFit(xs, heapYs); + const rssFit = linearFit(xs, rssYs); + const aclFit = linearFit(xs, aclYs); + const heapBasic = seriesStats(heapYs); + const rssBasic = seriesStats(rssYs); + const aclBasic = seriesStats(aclYs); + + const third = Math.max(1, Math.floor(steady.length / 3)); + const medianFirstThird = median(heapYs.slice(0, third)); + const medianLastThird = median(heapYs.slice(Math.max(0, steady.length - third))); + const thirdGrowth = medianLastThird - medianFirstThird; + const aclGrowth = Math.max(0, aclBasic.last - aclBasic.first); + const heapBudgetFromAcl = aclGrowth * thresholds.heapBytesPerAclRowBudget; + const unexplainedThirdGrowth = thirdGrowth - heapBudgetFromAcl; + const heapNetGrowth = heapBasic.last - heapBasic.first; + const heapBytesPerAclRow = + aclGrowth === 0 + ? heapNetGrowth > 0 + ? Number.POSITIVE_INFINITY + : 0 + : heapNetGrowth / aclGrowth; + + if (steady.length < thresholds.minSteadySamples) { + reasons.push(`insufficient steady samples (${steady.length} < ${thresholds.minSteadySamples})`); + } + if (heapFit.slopePerSec > thresholds.maxHeapSlopeBytesPerSec) { + reasons.push( + `heapUsed slope ${formatBytes(heapFit.slopePerSec)}/s exceeds ${formatBytes(thresholds.maxHeapSlopeBytesPerSec)}/s`, + ); + } + if (rssFit.slopePerSec > thresholds.maxRssSlopeBytesPerSec) { + reasons.push( + `rss slope ${formatBytes(rssFit.slopePerSec)}/s exceeds ${formatBytes(thresholds.maxRssSlopeBytesPerSec)}/s`, + ); + } + if (unexplainedThirdGrowth > thresholds.maxHeapThirdGrowthBytes) { + reasons.push( + `unexplained heap median shift ${formatBytes(unexplainedThirdGrowth)} exceeds ${formatBytes(thresholds.maxHeapThirdGrowthBytes)} (aclRows +${aclGrowth}, budget ${formatBytes(heapBudgetFromAcl)})`, + ); + } + + return { + samples: samples.length, + steadySamples: steady.length, + durationSec, + warmupSec, + heapUsed: { + ...heapBasic, + medianFirstThird, + medianLastThird, + netGrowth: heapBasic.last - heapBasic.first, + thirdGrowth, + fit: heapFit, + }, + rss: { + ...rssBasic, + netGrowth: rssBasic.last - rssBasic.first, + fit: rssFit, + }, + aclRows: { + first: aclBasic.first, + last: aclBasic.last, + netGrowth: aclBasic.last - aclBasic.first, + fit: aclFit, + }, + heapBytesPerAclRow, + stable: reasons.length === 0, + reasons, + }; +} + +export function formatTrendReport(report: MemoryTrendReport, label = "memory trend"): string { + const lines = [ + `=== ${label} ===`, + `samples: ${report.samples} (steady ${report.steadySamples})`, + `duration_sec: ${report.durationSec.toFixed(1)} (warmup ${report.warmupSec.toFixed(1)})`, + `heap_used first/last: ${formatBytes(report.heapUsed.first)} → ${formatBytes(report.heapUsed.last)} (Δ ${formatBytes(report.heapUsed.netGrowth)})`, + `heap_used min/max: ${formatBytes(report.heapUsed.min)} / ${formatBytes(report.heapUsed.max)}`, + `heap_used slope: ${formatBytes(report.heapUsed.fit.slopePerSec)}/s (r=${report.heapUsed.fit.r.toFixed(3)})`, + `heap_used 1st→3rd med:${formatBytes(report.heapUsed.medianFirstThird)} → ${formatBytes(report.heapUsed.medianLastThird)} (Δ ${formatBytes(report.heapUsed.thirdGrowth)})`, + `rss first/last: ${formatBytes(report.rss.first)} → ${formatBytes(report.rss.last)} (Δ ${formatBytes(report.rss.netGrowth)})`, + `rss slope: ${formatBytes(report.rss.fit.slopePerSec)}/s (r=${report.rss.fit.r.toFixed(3)})`, + `acl_rows first/last: ${report.aclRows.first} → ${report.aclRows.last} (Δ ${report.aclRows.netGrowth})`, + `heap_per_acl_row: ${Number.isFinite(report.heapBytesPerAclRow) ? formatBytes(report.heapBytesPerAclRow) : "n/a"}`, + `stable: ${report.stable ? "yes" : "NO"}`, + ]; + if (report.reasons.length) { + lines.push(`reasons:`); + for (const r of report.reasons) lines.push(` - ${r}`); + } + return lines.join("\n"); +} + +export function sampleFromProfile(snap: ProfileSnapshot, tMs: number): MemorySample | null { + if (!snap.memory || !snap.resources) return null; + return { + tMs, + at: new Date().toISOString(), + memory: snap.memory, + resources: snap.resources, + requests: snap.requests, + }; +} + +/** Scrape one memory sample from the PROFILE probe (null when disabled). */ +export async function scrapeMemorySample( + tMs = performance.now(), + baseUrl = PROXY, +): Promise { + const snap = await fetchServerProfile(baseUrl); + if (!snap) return null; + return sampleFromProfile(snap, tMs); +} + +/** Request optional V8 GC via the PROFILE probe. */ +export async function requestServerGc(baseUrl = PROXY): Promise { + try { + const res = await fetch(`${baseUrl}/_couch-auth-proxy/profile/gc`, { method: "POST" }); + if (!res.ok) return false; + const body = (await res.json()) as { gc?: boolean }; + return body.gc === true; + } catch { + return false; + } +} + +export async function memoryEndpointReady(baseUrl = PROXY): Promise { + if (!(await profileEndpointAvailable(baseUrl))) return false; + const sample = await scrapeMemorySample(performance.now(), baseUrl); + return sample !== null; +} + +export { formatBytes, formatMemorySnapshot }; diff --git a/test/unit/memory-trend.test.ts b/test/unit/memory-trend.test.ts new file mode 100644 index 0000000..05a1b8f --- /dev/null +++ b/test/unit/memory-trend.test.ts @@ -0,0 +1,91 @@ +/** + * Unit tests for memory-stability trend analysis (no docker). + */ +import { describe, expect, it } from "vitest"; +import { formatBytes } from "../../src/util/memory.js"; +import { analyzeMemoryTrend, linearFit, type MemorySample } from "../perf/memory.js"; + +function sample(tMs: number, heapUsed: number, rss: number, aclRows: number): MemorySample { + return { + tMs, + at: new Date(tMs).toISOString(), + memory: { + rss, + heapTotal: heapUsed + 10, + heapUsed, + external: 1, + arrayBuffers: 0, + }, + resources: { + aclDbs: 1, + aclRows, + aclTombstones: 0, + aclInflightEnsures: 0, + aclInflightRefreshes: 0, + sessionCacheEntries: 2, + sessionInflight: 0, + }, + requests: 0, + }; +} + +describe("memory util", () => { + it("formats bytes", () => { + expect(formatBytes(512)).toBe("512B"); + expect(formatBytes(2048)).toBe("2.0KiB"); + expect(formatBytes(2 * 1024 * 1024)).toBe("2.00MiB"); + }); +}); + +describe("linearFit", () => { + it("recovers a known slope", () => { + const xs = [0, 1000, 2000, 3000]; + const ys = [100, 200, 300, 400]; // 0.1 per ms → 100/s + const fit = linearFit(xs, ys); + expect(fit.slopePerMs).toBeCloseTo(0.1, 6); + expect(fit.slopePerSec).toBeCloseTo(100, 3); + expect(fit.r).toBeCloseTo(1, 6); + }); +}); + +describe("analyzeMemoryTrend", () => { + it("marks a flat series stable", () => { + const samples = Array.from({ length: 20 }, (_, i) => + sample(i * 1000, 50 * 1024 * 1024, 80 * 1024 * 1024, 300), + ); + const report = analyzeMemoryTrend(samples, { + warmupFraction: 0.25, + maxHeapSlopeBytesPerSec: 64 * 1024, + maxRssSlopeBytesPerSec: 128 * 1024, + maxHeapThirdGrowthBytes: 48 * 1024 * 1024, + heapBytesPerAclRowBudget: 2048, + minSteadySamples: 8, + }); + expect(report.stable).toBe(true); + expect(report.heapUsed.fit.slopePerSec).toBeCloseTo(0, 3); + }); + + it("flags a steep heap climb as unstable", () => { + const samples = Array.from({ length: 20 }, (_, i) => + sample( + i * 1000, + 20 * 1024 * 1024 + i * 2 * 1024 * 1024, // +2MiB/s + 40 * 1024 * 1024 + i * 2 * 1024 * 1024, + 300, + ), + ); + const report = analyzeMemoryTrend(samples); + expect(report.stable).toBe(false); + expect(report.reasons.some((r) => r.includes("heapUsed slope"))).toBe(true); + }); + + it("budgets heap growth explained by ACL row growth", () => { + // Steady window grows ~1KiB heap per new ACL row — within 2KiB budget. + const samples = Array.from({ length: 20 }, (_, i) => + sample(i * 1000, 30 * 1024 * 1024 + i * 1000, 60 * 1024 * 1024, 100 + i), + ); + const report = analyzeMemoryTrend(samples); + expect(report.aclRows.netGrowth).toBeGreaterThan(0); + expect(report.stable).toBe(true); + }); +}); diff --git a/test/unit/profile.test.ts b/test/unit/profile.test.ts index bf3fd26..b38912f 100644 --- a/test/unit/profile.test.ts +++ b/test/unit/profile.test.ts @@ -127,6 +127,20 @@ describe("profile probes", () => { let snap = await (await app.request("http://localhost/_couch-auth-proxy/profile")).json(); expect(snap.enabled).toBe(true); expect(snap.requests).toBe(0); + expect(snap.memory).toMatchObject({ + rss: expect.any(Number), + heapUsed: expect.any(Number), + heapTotal: expect.any(Number), + external: expect.any(Number), + arrayBuffers: expect.any(Number), + }); + expect(snap.resources).toMatchObject({ + aclDbs: expect.any(Number), + aclRows: expect.any(Number), + aclTombstones: expect.any(Number), + sessionCacheEntries: expect.any(Number), + sessionInflight: expect.any(Number), + }); // Non-probe request should be recorded (404 catch-all still runs principal). const miss = await app.request("http://localhost/no-such-db-for-profile"); @@ -143,5 +157,28 @@ describe("profile probes", () => { expect(await reset.json()).toEqual({ ok: true }); snap = await (await app.request("http://localhost/_couch-auth-proxy/profile")).json(); expect(snap.requests).toBe(0); + + const gc = await app.request("http://localhost/_couch-auth-proxy/profile/gc", { + method: "POST", + }); + expect(gc.status).toBe(200); + const gcBody = await gc.json(); + expect(gcBody.ok).toBe(true); + expect(typeof gcBody.gc).toBe("boolean"); + expect(gcBody.memory.heapUsed).toBeGreaterThan(0); + }); + + it("returns 404 for gc probe when PROFILE is off", async () => { + const config = loadConfig({ + COUCH_URL: "http://127.0.0.1:5984", + RATE_LIMIT_ENABLED: "false", + }); + const services = createServices(config); + services.sessions.resolve = async () => anonymousPrincipal(); + const app = createApp(services); + const gc = await app.request("http://localhost/_couch-auth-proxy/profile/gc", { + method: "POST", + }); + expect(gc.status).toBe(404); }); }); diff --git a/vitest.perf.config.ts b/vitest.perf.config.ts index b3689d4..66a1eb5 100644 --- a/vitest.perf.config.ts +++ b/vitest.perf.config.ts @@ -7,7 +7,8 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { - include: ["test/perf/**/*.test.ts"], + // Throughput harness only — memory soak is opt-in via vitest.perf.memory.config.ts + include: ["test/perf/acl-sync-load.test.ts"], environment: "node", testTimeout: 600_000, hookTimeout: 300_000, diff --git a/vitest.perf.memory.config.ts b/vitest.perf.memory.config.ts new file mode 100644 index 0000000..d6fca4f --- /dev/null +++ b/vitest.perf.memory.config.ts @@ -0,0 +1,17 @@ +/** + * Vitest config for the long-running memory stability assessment. + * Requires PROFILE=true on the proxy (see `pnpm test:perf:memory`). + */ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["test/perf/memory-stability.test.ts"], + environment: "node", + // Soak length is env-tunable; allow long runs (default 5m + headroom). + testTimeout: 1_800_000, + hookTimeout: 300_000, + fileParallelism: false, + sequence: { concurrent: false }, + }, +});