From 6e8ae4a246901d9d88d1a93ce98b4ea0347f8e9c Mon Sep 17 00:00:00 2001 From: Anto Subash Date: Fri, 17 Apr 2026 17:08:23 +0200 Subject: [PATCH] feat(perf): add bench/memory/loadtest tooling + fix i18n hot-path allocation Introduces a first-class performance harness for the project: - pytest-benchmark (`make bench`) for micro-benchmarks, gated behind a `perf` marker so `make test-py` stays unchanged. - memray (`make memray-run`, `make memray-flamegraph`) for allocation profiling of arbitrary Python targets. - locust + memray load-profiling (`make loadtest`, `make loadtest-memray`) that starts uvicorn under memray, seeds a forged-session-cookie admin so traffic exercises real authenticated code paths, and renders a flamegraph. The load-profile also surfaced a real own-code allocation hotspot: `build_i18n_block` was calling `dict(registry.messages(locale))` on every Inertia page render to coerce a `MappingProxyType` into something the JSON encoder accepts. Across a 100-user / 30s run this was ~37K allocations and ~58 MB churn for data that never changes within a process. `I18nRegistry.messages_snapshot(locale)` builds the plain-dict copy once at `load()` time and hands out the same instance on every call. After the fix the function is no longer in the top-50 allocation sites; total allocations drop by ~35K and churn by ~59 MB while throughput and latency stay flat. --- .gitignore | 4 + Makefile | 31 ++++++- framework/core/simple_module_core/i18n.py | 31 ++++++- .../simple_module_hosting/_inertia_shared.py | 4 +- pyproject.toml | 13 ++- scripts/loadtest_memray.sh | 91 +++++++++++++++++++ scripts/loadtest_seed.py | 56 ++++++++++++ tests/benchmarks/__init__.py | 0 tests/benchmarks/test_session_cookie_bench.py | 25 +++++ tests/loadtest/locustfile.py | 65 +++++++++++++ 10 files changed, 311 insertions(+), 9 deletions(-) create mode 100755 scripts/loadtest_memray.sh create mode 100644 scripts/loadtest_seed.py create mode 100644 tests/benchmarks/__init__.py create mode 100644 tests/benchmarks/test_session_cookie_bench.py create mode 100644 tests/loadtest/locustfile.py diff --git a/.gitignore b/.gitignore index 2c87d614..203e4492 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,10 @@ host/client_app/modules.generated.css # Worktrees .worktrees/ +# Performance profiles +.memray/ +.benchmarks/ + # OS .DS_Store Thumbs.db diff --git a/Makefile b/Makefile index ee147cdf..e1b9db5a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: install install-py install-js dev dev-api dev-ui build test test-py test-js test-e2e lint doctor migrate migration downgrade migration-history docker-up docker-down kill new-module gen-pages sync-module-deps ci-python-lint ci-python-typecheck ci-js-lint ci-js-typecheck ci-check-file-size +.PHONY: install install-py install-js dev dev-api dev-ui build test test-py test-js test-e2e bench memray-run memray-flamegraph loadtest loadtest-memray lint doctor migrate migration downgrade migration-history docker-up docker-down kill new-module gen-pages sync-module-deps ci-python-lint ci-python-typecheck ci-js-lint ci-js-typecheck ci-check-file-size # Install install: @@ -48,6 +48,35 @@ test-js: test-e2e: ## Run end-to-end browser smoke tests (requires `make docker-up` + `make dev` and `uv run playwright install chromium`) uv run pytest -m e2e tests/e2e +# Performance +bench: ## Run pytest-benchmark suite (tests/benchmarks). Override args with BENCH_ARGS=... + uv run pytest -m perf --benchmark-enable --benchmark-columns=min,mean,median,max,stddev,ops,rounds $(BENCH_ARGS) tests/benchmarks + +# Memory profiling with memray. Point TARGET at any runnable script/module. +# Examples: +# make memray-run TARGET="-m pytest tests/benchmarks -m perf --benchmark-disable" +# make memray-run TARGET="scripts/new_module.py demo" +MEMRAY_OUT ?= .memray/profile.bin +TARGET ?= -m pytest tests/benchmarks -m perf --benchmark-disable +memray-run: ## Record an allocation profile into $(MEMRAY_OUT) + @mkdir -p $(dir $(MEMRAY_OUT)) + uv run memray run --force -o $(MEMRAY_OUT) $(TARGET) + @echo "Profile: $(MEMRAY_OUT) — render with 'make memray-flamegraph'" + +memray-flamegraph: ## Render $(MEMRAY_OUT) as an HTML flamegraph + uv run memray flamegraph --force $(MEMRAY_OUT) + +# Load testing. `make loadtest` assumes `make dev` is running separately. +# `make loadtest-memray` starts uvicorn under memray, runs locust headless, +# shuts down, and emits a flamegraph. Override locust args via LOCUST_ARGS=... +LOCUST_HOST ?= http://localhost:8000 +LOCUST_ARGS ?= -u 20 -r 5 -t 30s +loadtest: ## Run locust against a server already on $(LOCUST_HOST) + uv run locust -f tests/loadtest/locustfile.py --host $(LOCUST_HOST) --headless $(LOCUST_ARGS) + +loadtest-memray: ## Start uvicorn under memray, load-test, emit flamegraph + scripts/loadtest_memray.sh $(LOCUST_ARGS) + lint: ci-python-lint ci-python-typecheck ci-js-lint ci-js-typecheck ci-check-file-size # Kept granular so pr.yml can run them in parallel. diff --git a/framework/core/simple_module_core/i18n.py b/framework/core/simple_module_core/i18n.py index 9c8162bf..f2e97b49 100644 --- a/framework/core/simple_module_core/i18n.py +++ b/framework/core/simple_module_core/i18n.py @@ -83,9 +83,15 @@ def __init__(self, default_locale: str, supported_locales: list[str]) -> None: # Immutable views into ``_messages`` — handed out by ``messages()`` to # avoid a per-call dict copy. Rebuilt whenever ``load()`` runs. self._message_views: dict[str, MappingProxyType[str, str]] = {} + # Plain-dict snapshots for JSON-serializing callers (e.g. Inertia shared + # props). Built once per load; handing the same dict out on every request + # avoids per-request dict copies that used to dominate allocations on the + # Inertia render path. + self._message_snapshots: dict[str, dict[str, str]] = {} self._available_locales: tuple[str, ...] = () self._available_locales_list: list[str] = [] self._empty_view: MappingProxyType[str, str] = MappingProxyType({}) + self._empty_snapshot: dict[str, str] = {} self._loaded = False def add_source(self, namespace: str, locale_dir: Path) -> None: @@ -124,6 +130,9 @@ def load(self) -> None: self._message_views = { locale: MappingProxyType(msgs) for locale, msgs in self._messages.items() } + # Plain-dict snapshots for serialization callers. ``dict(msgs)`` runs + # once here rather than on every Inertia render. + self._message_snapshots = {locale: dict(msgs) for locale, msgs in self._messages.items()} self._available_locales = tuple(locale for locale, msgs in self._messages.items() if msgs) self._available_locales_list = list(self._available_locales) self._loaded = True @@ -144,8 +153,7 @@ def messages(self, locale: str) -> Mapping[str, str]: Returns an immutable view (``MappingProxyType``) into the cached message dict — zero-copy. Callers that JSON-serialize the result - (e.g. Inertia shared props) should wrap with ``dict(...)`` at the - boundary. + should use :meth:`messages_snapshot` instead. """ view = self._message_views.get(locale) if view is not None: @@ -157,6 +165,25 @@ def messages(self, locale: str) -> Mapping[str, str]: return self._empty_view return MappingProxyType(raw) + def messages_snapshot(self, locale: str) -> dict[str, str]: + """Plain-dict snapshot for callers that JSON-serialize the result. + + Built once at :meth:`load` time and handed out by reference on every + call. Callers must treat it as read-only — mutating the returned dict + corrupts subsequent responses. Used by the Inertia shared-props builder + where it sits on the request hot path; prior to this method, + ``dict(messages(locale))`` per request was the top own-code allocator. + """ + snapshot = self._message_snapshots.get(locale) + if snapshot is not None: + return snapshot + # Fallback for tests that skip ``load()``: synthesize the snapshot on + # demand from whatever ``_messages`` holds. + raw = self._messages.get(locale) + if raw is None: + return self._empty_snapshot + return dict(raw) + class _SafeFormatDict(dict): """Dict that returns ``{key}`` for missing keys so str.format_map doesn't raise.""" diff --git a/framework/hosting/simple_module_hosting/_inertia_shared.py b/framework/hosting/simple_module_hosting/_inertia_shared.py index fb90abb0..8ce90fb0 100644 --- a/framework/hosting/simple_module_hosting/_inertia_shared.py +++ b/framework/hosting/simple_module_hosting/_inertia_shared.py @@ -52,7 +52,5 @@ def build_i18n_block(scope: Scope, request: Request) -> dict: return { "locale": locale, "supportedLocales": registry.available_locales(), - # Inertia's JSON encoder doesn't accept the MappingProxyType view - # returned by messages(); copy to a plain dict at the boundary. - "messages": dict(registry.messages(locale)) if send_messages else None, + "messages": registry.messages_snapshot(locale) if send_messages else None, } diff --git a/pyproject.toml b/pyproject.toml index efd1a46a..478569dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,11 +16,14 @@ members = [ dev = [ "pytest>=8.0", "pytest-asyncio>=0.24", + "pytest-benchmark>=4.0", "httpx>=0.27", "ruff>=0.8", "playwright>=1.58.0", "pytest-playwright>=0.7.2", "ty>=0.0.29", + "memray>=1.14; sys_platform != 'win32'", + "locust>=2.31", ] [tool.ruff] @@ -92,8 +95,12 @@ invalid-argument-type = "ignore" [tool.pytest.ini_options] asyncio_mode = "auto" -testpaths = ["framework/core/tests", "framework/db/tests", "framework/hosting/tests", "framework/testing/tests", "host/tests", "modules/auth/tests", "modules/dashboard/tests", "modules/products/tests", "modules/users/tests", "tests/integration", "tests/e2e"] -markers = ["e2e: end-to-end tests requiring a live browser"] +testpaths = ["framework/core/tests", "framework/db/tests", "framework/hosting/tests", "framework/testing/tests", "host/tests", "modules/auth/tests", "modules/dashboard/tests", "modules/products/tests", "modules/users/tests", "tests/integration", "tests/e2e", "tests/benchmarks"] +markers = [ + "e2e: end-to-end tests requiring a live browser", + "perf: performance benchmarks (opt-in; run via `make bench`)", +] # --durations=20 prints the 20 slowest tests after every run so slow fixtures # (e.g. function-scoped ``app`` rebuilds) surface without needing an opt-in flag. -addopts = "-m 'not e2e' --durations=20" +# Benchmarks are opt-in — they run under `make bench`, not the default suite. +addopts = "-m 'not e2e and not perf' --durations=20 --benchmark-disable" diff --git a/scripts/loadtest_memray.sh b/scripts/loadtest_memray.sh new file mode 100755 index 00000000..355500f1 --- /dev/null +++ b/scripts/loadtest_memray.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Start uvicorn wrapped in memray, drive it with locust, then emit a flamegraph. +# +# memray instruments the parent python process, so we run uvicorn with a single +# worker and no --reload (reload spawns subprocesses memray cannot see). +# +# Usage: scripts/loadtest_memray.sh [locust-args...] +# Defaults: -u 20 -r 5 -t 30s +# Override: scripts/loadtest_memray.sh -u 50 -r 10 -t 60s +set -euo pipefail + +HOST=${HOST:-http://localhost:8000} +PORT=${PORT:-8000} +PROFILE=${PROFILE:-.memray/loadtest.bin} +LOCUSTFILE=${LOCUSTFILE:-tests/loadtest/locustfile.py} + +# Default locust scenario — override by passing any locust flags as arguments. +if [ "$#" -eq 0 ]; then + set -- -u 20 -r 5 -t 30s +fi + +mkdir -p "$(dirname "$PROFILE")" + +# Seed the load-test admin and export the signed session cookie + CSRF token +# so the locustfile's AuthedUser activates. Skip with SKIP_SEED=1. +if [ "${SKIP_SEED:-0}" != "1" ]; then + echo "[loadtest-memray] seeding load-test user" + eval "$(uv run python scripts/loadtest_seed.py)" + export SM_LOADTEST_COOKIE SM_LOADTEST_CSRF +fi + +echo "[loadtest-memray] starting uvicorn under memray → $PROFILE" +# --force overwrites any previous profile; --follow-fork picks up any children +# uvicorn might spawn (none in single-worker mode, but harmless). +uv run memray run --force --follow-fork -o "$PROFILE" \ + -m uvicorn host.main:app --host 127.0.0.1 --port "$PORT" --log-level warning & +SERVER_PID=$! + +# Signal the memray wrapper AND any descendants. uv + memray spawn uvicorn as +# a grandchild, so signaling just $SERVER_PID leaves uvicorn attached to the +# port. `pkill -P` walks one level of children; we also hit whatever is bound +# to the port as a belt-and-braces final check. +_signal_tree() { + local sig=$1 + kill "-$sig" "$SERVER_PID" 2>/dev/null || true + pkill "-$sig" -P "$SERVER_PID" 2>/dev/null || true +} + +cleanup() { + if kill -0 "$SERVER_PID" 2>/dev/null; then + echo "[loadtest-memray] stopping uvicorn (pid $SERVER_PID)" + _signal_tree INT + # Give memray a moment to flush the profile; escalate if it lingers. + for _ in $(seq 1 20); do + kill -0 "$SERVER_PID" 2>/dev/null || break + sleep 0.25 + done + if kill -0 "$SERVER_PID" 2>/dev/null; then + _signal_tree TERM + wait "$SERVER_PID" 2>/dev/null || true + fi + fi + # Last-ditch: anything still bound to the port is ours to reap. + if command -v lsof > /dev/null 2>&1; then + lsof -ti tcp:"$PORT" 2>/dev/null | xargs -r kill -9 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +# Wait for /health/live — fail loudly if the server doesn't come up. +echo "[loadtest-memray] waiting for $HOST/health/live" +for _ in $(seq 1 60); do + if curl -sf "$HOST/health/live" > /dev/null 2>&1; then + break + fi + sleep 0.5 +done +if ! curl -sf "$HOST/health/live" > /dev/null 2>&1; then + echo "[loadtest-memray] server did not become healthy in 30s" >&2 + exit 1 +fi + +echo "[loadtest-memray] running locust: $*" +uv run locust -f "$LOCUSTFILE" --headless --host "$HOST" "$@" + +cleanup +trap - EXIT INT TERM + +echo "[loadtest-memray] rendering flamegraph" +uv run memray flamegraph --force "$PROFILE" +echo "[loadtest-memray] done — open .memray/memray-flamegraph-loadtest.html" diff --git a/scripts/loadtest_seed.py b/scripts/loadtest_seed.py new file mode 100644 index 00000000..402e3abb --- /dev/null +++ b/scripts/loadtest_seed.py @@ -0,0 +1,56 @@ +"""Seed a load-test admin and mint a forged session cookie. + +Runs migrations-independent (assumes ``make migrate`` already ran), creates +an idempotent admin user, then prints shell-exportable env vars carrying a +signed Starlette session cookie and a matching CSRF token. Source the output +before launching locust: + + eval "$(uv run python scripts/loadtest_seed.py)" + +The cookie bypasses the real login flow so the load-test profile reflects +steady-state authenticated traffic — not login/CSRF-bootstrap overhead, and +not the login rate limiter (which would trip under 100 concurrent users). +""" + +from __future__ import annotations + +import asyncio + +from simple_module_db.session import init_db +from simple_module_hosting.csrf import SESSION_CSRF_TOKEN_KEY +from simple_module_hosting.settings import Settings +from simple_module_testing import forge_session_cookie +from users.bootstrap import create_admin + +LOAD_EMAIL = "loadtest@example.com" +LOAD_PASSWORD = "loadtest-password-x!1" +LOAD_CSRF_TOKEN = "loadtest-csrf-token" + + +async def _main() -> None: + settings = Settings() + db_state = init_db(settings.database_url) + try: + async with db_state.session_factory() as session: + result = await create_admin( + session, + email=LOAD_EMAIL, + password=LOAD_PASSWORD, + full_name="Load Test", + ) + user_id = str(result.user.id) + finally: + await db_state.engine.dispose() + + cookie = forge_session_cookie( + settings.secret_key, + {"user_id": user_id, SESSION_CSRF_TOKEN_KEY: LOAD_CSRF_TOKEN}, + ) + # Shell-sourceable output — quote for safety; itsdangerous cookies can + # contain dots and hyphens but no spaces. + print(f"export SM_LOADTEST_COOKIE='{cookie}'") + print(f"export SM_LOADTEST_CSRF='{LOAD_CSRF_TOKEN}'") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/benchmarks/test_session_cookie_bench.py b/tests/benchmarks/test_session_cookie_bench.py new file mode 100644 index 00000000..cd2e20d8 --- /dev/null +++ b/tests/benchmarks/test_session_cookie_bench.py @@ -0,0 +1,25 @@ +"""Sample pytest-benchmark suite. + +Run via ``make bench``. Gated by the ``benchmark`` marker so the default +``pytest`` run skips these. Use this file as a template — each benchmark is +a plain sync test that calls the ``benchmark`` fixture on the code under +test and lets pytest-benchmark handle round/warmup selection. +""" + +from __future__ import annotations + +import pytest +from simple_module_hosting.csrf import SESSION_CSRF_TOKEN_KEY +from simple_module_testing import forge_session_cookie + +pytestmark = pytest.mark.perf + + +def test_forge_session_cookie(benchmark): + """Signing a session cookie is on every authenticated request's hot path.""" + secret = "bench-secret-key" + payload = {"user_id": "00000000-0000-0000-0000-000000000001", SESSION_CSRF_TOKEN_KEY: "tok"} + + result = benchmark(forge_session_cookie, secret, payload) + + assert isinstance(result, str) and result diff --git a/tests/loadtest/locustfile.py b/tests/loadtest/locustfile.py new file mode 100644 index 00000000..b8a16c5c --- /dev/null +++ b/tests/loadtest/locustfile.py @@ -0,0 +1,65 @@ +"""Locust load-test scenarios. + +Run via ``make loadtest`` (requires a server already running at :8000, +e.g. ``make dev``) or ``make loadtest-memray`` which starts a profiled +uvicorn, runs this file headless, shuts down, and emits a flamegraph. + +Two user classes are defined. Locust runs **all** ``HttpUser`` subclasses +by default; filter with ``--class-picker`` or ``--tags`` if you only want +one: + +* :class:`AnonymousUser` — hits unauthenticated paths that redirect to + ``/users/login``. Useful for profiling middleware and the login page + render. Always enabled. +* :class:`AuthedUser` — seeded via ``scripts/loadtest_seed.py``. Only + activates when ``SM_LOADTEST_COOKIE`` and ``SM_LOADTEST_CSRF`` are + exported in the environment. Otherwise it disables itself so anonymous + runs still work. Drives real product-service code paths. +""" + +from __future__ import annotations + +import os + +from locust import HttpUser, between, tag, task + +_COOKIE = os.environ.get("SM_LOADTEST_COOKIE") +_CSRF = os.environ.get("SM_LOADTEST_CSRF") + + +@tag("anon") +class AnonymousUser(HttpUser): + wait_time = between(0.1, 0.5) + + @task(3) + def browse_products(self) -> None: + self.client.get("/products/", name="anon GET /products/") + + @task(1) + def readiness(self) -> None: + self.client.get("/health/ready", name="anon GET /health/ready") + + +@tag("authed") +class AuthedUser(HttpUser): + wait_time = between(0.1, 0.5) + + # ``abstract = True`` when credentials are missing so locust skips spawning + # this class rather than sending anonymous traffic that redirects. + abstract = not (_COOKIE and _CSRF) + + def on_start(self) -> None: + self.client.cookies.set("session", _COOKIE) + self.client.headers["X-CSRF-Token"] = _CSRF + + @task(5) + def browse_products_page(self) -> None: + self.client.get("/products/", name="auth GET /products/ (inertia)") + + @task(3) + def list_products_api(self) -> None: + self.client.get("/api/products/", name="auth GET /api/products/") + + @task(1) + def dashboard(self) -> None: + self.client.get("/", name="auth GET / (dashboard)")