From 7cbb48d84465c61cd91e92fdee09dfa0afe13d70 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Mon, 26 Oct 2026 15:51:22 +0100 Subject: [PATCH 1/2] test(backend): add CPU benchmarks for the upload and product tree paths - Image uploads: metadata stripping with rotation, and the thumbnail set, on a 12 MP phone JPEG so thumbnails take the JPEG draft path - Product trees: create-request validation, the component tree read at its 1,000-component cap, and the export's read models, JSON and CSV, all from rows built in memory - pytest-codspeed in the tests group and a `just bench` recipe that runs them once --- backend/README.md | 1 + backend/justfile | 5 + backend/pyproject.toml | 3 + backend/tests/README.md | 7 + backend/tests/benchmarks/__init__.py | 1 + backend/tests/benchmarks/conftest.py | 205 ++++++++++++++++++ backend/tests/benchmarks/test_bench_images.py | 42 ++++ .../tests/benchmarks/test_bench_products.py | 76 +++++++ backend/uv.lock | 27 +++ 9 files changed, 367 insertions(+) create mode 100644 backend/tests/benchmarks/__init__.py create mode 100644 backend/tests/benchmarks/conftest.py create mode 100644 backend/tests/benchmarks/test_bench_images.py create mode 100644 backend/tests/benchmarks/test_bench_products.py diff --git a/backend/README.md b/backend/README.md index 82b9c5913..2186fd1dd 100644 --- a/backend/README.md +++ b/backend/README.md @@ -34,6 +34,7 @@ just test # run all tests just test-unit # fast unit tests just test-cov # tests with coverage just refresh-disposable-email-domains # update the committed disposable-email fallback list +just bench # run the CPU benchmarks once (CodSpeed measures them in CI) just perf-baseline # run the k6 baseline suite and export a JSON summary (output under reports/performance/ is gitignored) just migrate # apply migrations just migrate-create "description" # create a migration from the current model changes diff --git a/backend/justfile b/backend/justfile index 6c96c18c4..122733b89 100644 --- a/backend/justfile +++ b/backend/justfile @@ -76,6 +76,11 @@ test-ci: mkdir -p reports/coverage {{ pytest }} tests/unit tests/integration -n auto --dist=loadgroup --durations=20 --junitxml=reports/junit.xml --cov --cov-report=xml --cov-report=term +# Run the CPU benchmarks once as plain tests. CodSpeed counts their instructions in CI; +# `just bench --codspeed` times them locally instead. +bench *args: + {{ pytest }} tests/benchmarks {{ args }} + # Mutation testing against one throwaway Postgres; the full app takes hours. Narrow # it with mutant globs: `just mutation 'app.core.images*'`. Survivors never fail the # run; it lists functions with more survivors than tests/mutation-baseline.txt. diff --git a/backend/pyproject.toml b/backend/pyproject.toml index ce28639f9..1d31efa00 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -123,6 +123,9 @@ # Mocking and assertions "dirty-equals>=0.8.0", # Flexible assertions for dynamic data "pytest-mock>=3.14.0", # Better mocking with cleaner pytest integration + + # CPU benchmarks (tests/benchmarks), measured by CodSpeed in CI + "pytest-codspeed>=5.0.3", ] mutation = [ "mutmut>=3.8.0", diff --git a/backend/tests/README.md b/backend/tests/README.md index f4840d2f0..8d28f660a 100644 --- a/backend/tests/README.md +++ b/backend/tests/README.md @@ -40,6 +40,13 @@ cache wiring, file cleanup, migration behavior. One subsystem per test. A few multi-step journeys that cross feature boundaries, such as authenticate -> mutate -> fetch or camera setup -> record -> persist. Keep them sparse; they are slower than the other tiers. +### `tests/benchmarks/` + +CPU benchmarks for backend hot paths (image uploads, product create validation, the component tree +read, and the export), written with `pytest-codspeed`. No database or network: they start from rows +built in memory. The CodSpeed workflow measures them on every pull request that touches the +backend. `just bench` runs them once as plain tests, and so does `just test`. + ## Assertions - Assert what tells the paths apart, not just the outcome. Several paths often converge on one diff --git a/backend/tests/benchmarks/__init__.py b/backend/tests/benchmarks/__init__.py new file mode 100644 index 000000000..f46051b6f --- /dev/null +++ b/backend/tests/benchmarks/__init__.py @@ -0,0 +1 @@ +"""CPU benchmarks for backend hot paths, measured by CodSpeed (pytest-codspeed).""" diff --git a/backend/tests/benchmarks/conftest.py b/backend/tests/benchmarks/conftest.py new file mode 100644 index 000000000..4bfc6ca2d --- /dev/null +++ b/backend/tests/benchmarks/conftest.py @@ -0,0 +1,205 @@ +"""Shared inputs for the benchmark suite: built once per session, with no database or network. + +Every input is deterministic, so CodSpeed's instruction counts change only when the code does. +""" + +import io +import random +import uuid +from collections import deque +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +import pytest +from PIL import Image as PILImage +from PIL.ExifTags import IFD + +from app.api.auth.models import User +from app.api.common.models.enums import Unit +from app.api.data_collection.crud.product_tree_queries import COMPONENT_TREE_MAX_COMPONENTS +from app.api.data_collection.models.product import MaterialProductLink, Product +from app.api.reference_data.models import Material, ProductType + +if TYPE_CHECKING: + from typing import Any + +_TIMESTAMP = datetime(2025, 1, 1, tzinfo=UTC) + +# A 12 MP phone photo, the most common upload. At this size thumbnail generation takes +# the JPEG draft path (decoding at half scale for the 1600 px width). +PHOTO_SIZE = (4032, 3024) + + +def make_phone_jpeg(width: int, height: int) -> bytes: + """Encode a JPEG shaped like a phone upload: camera EXIF, GPS, a serial number and a rotation. + + A smooth gradient compresses to a fraction of a real photo and decodes much faster, so + seeded noise is blended in to bring the file to a photo's size (about 4 MB at 12 MP). + """ + base = PILImage.effect_mandelbrot((width // 8, height // 8), (-2.0, -1.2, 1.0, 1.2), 100).resize( + (width, height), PILImage.Resampling.BICUBIC + ) + noise = PILImage.frombytes("L", (width, height), random.Random(0).randbytes(width * height)) # noqa: S311 -- seeded on purpose, not a secret + img = PILImage.merge( + "RGB", (base, PILImage.blend(base, noise, 0.2), PILImage.linear_gradient("L").resize((width, height))) + ) + exif = img.getexif() + exif[0x010F] = "Test Camera" # Make (kept) + exif[0x0110] = "Model X" # Model (kept) + exif[0x0131] = "Firmware 1.0" # Software (dropped) + exif[0x0112] = 6 # Orientation: rotate 90 degrees, so the rotation is baked in + sub_ifd = exif.get_ifd(IFD.Exif) + sub_ifd[0x920A] = 4.2 # FocalLength (kept) + sub_ifd[0x8827] = 400 # ISOSpeedRatings (kept) + sub_ifd[0xA431] = "SERIAL123" # BodySerialNumber (dropped) + gps = exif.get_ifd(IFD.GPSInfo) + gps[1] = "N" # GPSLatitudeRef (dropped with the whole GPS block) + gps[2] = (52.0, 9.0, 36.0) # GPSLatitude + buffer = io.BytesIO() + img.save(buffer, format="JPEG", exif=exif, quality=90) + return buffer.getvalue() + + +@pytest.fixture(scope="session") +def phone_jpeg() -> bytes: + """A 12 MP JPEG with identifying EXIF and a 90 degree rotation tag.""" + return make_phone_jpeg(*PHOTO_SIZE) + + +def _component_create(depth: int, breadth: int, prefix: str) -> dict[str, Any]: + node: dict[str, Any] = { + "name": f"Component {prefix}", + "description": "Sub-assembly removed with basic hand tools.", + "brand": "Brand", + "model": f"Model {prefix}", + "amount_in_parent": 2, + "weight_g": 120.5, + "height_cm": 10, + "width_cm": 5, + "depth_cm": 3, + "bill_of_materials": [ + {"quantity": 0.3, "unit": "g", "material_id": 1}, + {"quantity": 0.1, "unit": "g", "material_id": 2}, + ], + "components": [], + } + if depth > 0: + node["components"] = [_component_create(depth - 1, breadth, f"{prefix}.{i}") for i in range(breadth)] + return node + + +@pytest.fixture(scope="session") +def product_create_payload() -> dict[str, Any]: + """A product create request body with 84 components over 3 levels (4 x 4 x 4 + 4 x 4 + 4).""" + return { + "name": "Office Chair", + "description": "Complete chair assembly", + "brand": "Brand 1", + "model": "Model 1", + "product_type_id": 1, + "weight_g": 20000, + "height_cm": 150, + "width_cm": 70, + "depth_cm": 50, + "circularity_properties": { + "recyclability": "Metal frame and plastic shell can be separated with basic tools.", + "disassemblability": "Most fasteners are visible and non-destructive to remove.", + }, + "videos": [{"url": "https://www.youtube.com/watch?v=123456789", "description": "Disassembly video"}], + "bill_of_materials": [{"quantity": 0.3, "unit": "g", "material_id": 1}], + "components": [_component_create(2, 4, str(i)) for i in range(4)], + } + + +@dataclass(frozen=True, slots=True) +class ProductTree: + """A base product and its components as unsaved ORM rows, shaped like the tree loaders' output.""" + + root: Product + children_by_parent_id: dict[int, list[Product]] + + @property + def size(self) -> int: + """Number of components below the base product, every level counted.""" + return sum(len(children) for children in self.children_by_parent_id.values()) + + +def _owner(index: int, visibility: str) -> User: + return User( + id=uuid.UUID(int=index, version=4), + email=f"owner{index}@example.org", + username=f"owner{index}", + hashed_password="not-a-hash", + preferences={"profile_visibility": visibility}, + ) + + +@pytest.fixture(scope="session") +def product_tree() -> ProductTree: + """A base product with as many components as one tree read returns (1,000), eight levels deep. + + Four top-level components, each with two children, filled breadth first until the cap. + Rows alternate between a public and a private owner, so owner redaction runs on half of + them. They carry no images: image URLs are derived from a stat of the stored file, which + is I/O and outside what an instruction count can say anything about. + """ + owners = (_owner(1, "public"), _owner(2, "private")) + product_type = ProductType(id=1, name="Office chair", description="Swivel chair with armrests") + materials = ( + Material(id=1, name="Steel", density_kg_m3=7850.0, is_crm=False), + Material(id=2, name="Polypropylene", density_kg_m3=905.0, is_crm=False), + ) + next_id = iter(range(1, COMPONENT_TREE_MAX_COMPONENTS + 2)) + + def row(parent_id: int | None) -> Product: + product_id = next(next_id) + owner = owners[product_id % 2] + return Product( + id=product_id, + name=f"=Part {product_id}" if product_id % 7 == 0 else f"Part {product_id}", + description='Removed with a T10 screwdriver; held by "glue" at two points.', + brand="brand", + model=f"Model {product_id}", + product_type_id=product_type.id, + product_type=product_type, + weight_g=120.5, + height_cm=10.0, + width_cm=5.0, + depth_cm=3.0, + circularity_properties={"recyclability": "Separable", "remanufacturability": "Standard screws"}, + owner_id=owner.id, + owner=owner, + parent_id=parent_id, + amount_in_parent=None if parent_id is None else 2, + version=1, + created_at=_TIMESTAMP, + updated_at=_TIMESTAMP, + bill_of_materials=[ + MaterialProductLink( + material_id=material.id, + material=material, + product_id=product_id, + quantity=0.25, + unit=Unit.KILOGRAM, + created_at=_TIMESTAMP, + updated_at=_TIMESTAMP, + ) + for material in materials + ], + ) + + root = row(None) + children_by_parent_id: dict[int, list[Product]] = {} + queue = deque([root]) + made = 0 + while queue and made < COMPONENT_TREE_MAX_COMPONENTS: + parent = queue.popleft() + for _ in range(4 if parent is root else 2): + if made == COMPONENT_TREE_MAX_COMPONENTS: + break + child = row(parent.id) + children_by_parent_id.setdefault(parent.id, []).append(child) + queue.append(child) + made += 1 + return ProductTree(root=root, children_by_parent_id=children_by_parent_id) diff --git a/backend/tests/benchmarks/test_bench_images.py b/backend/tests/benchmarks/test_bench_images.py new file mode 100644 index 000000000..ba48fe7b3 --- /dev/null +++ b/backend/tests/benchmarks/test_bench_images.py @@ -0,0 +1,42 @@ +"""Benchmarks for the image upload pipeline, which runs on every image upload before the response.""" + +import io +from typing import TYPE_CHECKING + +import pytest +from PIL import Image as PILImage + +from app.core.images import THUMBNAIL_WIDTHS, generate_thumbnails, process_image_for_storage +from tests.benchmarks.conftest import PHOTO_SIZE + +if TYPE_CHECKING: + from pathlib import Path + + from pytest_codspeed import BenchmarkFixture + + +@pytest.mark.benchmark +def test_process_image_for_storage(benchmark: BenchmarkFixture, phone_jpeg: bytes) -> None: + """Strip identifying metadata from a phone photo and bake in its rotation.""" + + # The upload is rewritten in place, so each round gets a fresh copy, made outside the measurement. + def fresh_upload() -> tuple[tuple[io.BytesIO], dict[str, object]]: + return (io.BytesIO(phone_jpeg),), {} + + width, height = benchmark.pedantic(process_image_for_storage, setup=fresh_upload) + assert (height, width) == PHOTO_SIZE + + +@pytest.mark.benchmark +def test_generate_thumbnails(benchmark: BenchmarkFixture, phone_jpeg: bytes, tmp_path: Path) -> None: + """Write every thumbnail width for a stored phone photo.""" + image_path = tmp_path / "original.jpg" + image_path.write_bytes(phone_jpeg) + + paths = benchmark(generate_thumbnails, image_path) + + widths = [] + for path in paths: + with PILImage.open(path) as thumbnail: + widths.append(thumbnail.width) + assert sorted(widths) == sorted(THUMBNAIL_WIDTHS) diff --git a/backend/tests/benchmarks/test_bench_products.py b/backend/tests/benchmarks/test_bench_products.py new file mode 100644 index 000000000..3d63c359a --- /dev/null +++ b/backend/tests/benchmarks/test_bench_products.py @@ -0,0 +1,76 @@ +"""Benchmarks for product tree requests: create validation, the component tree read, and the export. + +Each runs the same functions as its route, from the point the rows are loaded, so the +database round trips stay out and the per-row Python work is what gets counted. +""" + +from typing import TYPE_CHECKING, Any + +import pytest +from pydantic import TypeAdapter + +from app.api.data_collection.presentation.product_export import build_product_exports, render_products_csv +from app.api.data_collection.presentation.product_reads import render_component_tree +from app.api.data_collection.schemas import ProductCreateWithComponents, ProductExportRead + +if TYPE_CHECKING: + from pytest_codspeed import BenchmarkFixture + + from tests.benchmarks.conftest import ProductTree + +# The deepest tree the route serves: recursion_depth=10 renders nine levels below the roots. +MAX_TREE_RENDER_DEPTH = 9 + + +@pytest.mark.benchmark +def test_validate_product_create_tree(benchmark: BenchmarkFixture, product_create_payload: dict[str, Any]) -> None: + """Validate a nested product create request (84 components over 3 levels).""" + product = benchmark(ProductCreateWithComponents.model_validate, product_create_payload) + assert len(product.components) == 4 + + +@pytest.mark.benchmark +def test_render_component_tree(benchmark: BenchmarkFixture, product_tree: ProductTree) -> None: + """Render the component tree read at its 1,000-component cap, as an anonymous viewer.""" + roots = product_tree.children_by_parent_id[product_tree.root.id] + + tree = benchmark( + render_component_tree, + roots, + children_by_parent_id=product_tree.children_by_parent_id, + max_depth=MAX_TREE_RENDER_DEPTH, + viewer=None, + visited=frozenset({product_tree.root.id}), + ) + + # Owner attribution alternates per row; the private owner's rows are redacted. + assert {tree[0].owner_username, tree[1].owner_username} == {"owner1", None} + + +@pytest.mark.benchmark +def test_build_product_exports(benchmark: BenchmarkFixture, product_tree: ProductTree) -> None: + """Build the export read models for one product and its 1,000 components.""" + exports = benchmark(build_product_exports, [product_tree.root], product_tree.children_by_parent_id, None) + assert len(exports[0].components) == 4 + + +@pytest.mark.benchmark +def test_dump_product_exports_json(benchmark: BenchmarkFixture, product_tree: ProductTree) -> None: + """Serialize the export to JSON the way the export route does.""" + exports = build_product_exports([product_tree.root], product_tree.children_by_parent_id, None) + + def dump() -> bytes: + return TypeAdapter(list[ProductExportRead]).dump_json(exports) + + assert benchmark(dump).count(b'"amount_in_parent":2') == product_tree.size + + +@pytest.mark.benchmark +def test_render_products_csv(benchmark: BenchmarkFixture, product_tree: ProductTree) -> None: + """Flatten the export into the CSV download, one row per product or component.""" + exports = build_product_exports([product_tree.root], product_tree.children_by_parent_id, None) + + csv_text = benchmark(render_products_csv, exports) + + # The header plus the base product plus every component. + assert csv_text.count("\r\n") == 2 + product_tree.size diff --git a/backend/uv.lock b/backend/uv.lock index bcc032bd5..8eb414b9b 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -2099,6 +2099,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, ] +[[package]] +name = "pytest-codspeed" +version = "5.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/b4/cf932fcd1960a2fd6d9b09eb403253a8709aeee975961afa6299239a830e/pytest_codspeed-5.0.3.tar.gz", hash = "sha256:91afef90e6a96b013495e4702ef5d6358614a449e71008cdc194ef668778b92f", size = 324571, upload-time = "2026-05-22T16:20:49.231Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/6a/fdcec19c7f267c195f147c51d3fd2245f6b8d09b80495ed0a90c008e0842/pytest_codspeed-5.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:25464363c7f9b9bd5022e969c0addba616fa40ac9b8f0fc9e030c4538863b32d", size = 366259, upload-time = "2026-05-22T16:21:06.039Z" }, + { url = "https://files.pythonhosted.org/packages/6a/96/c6b03b81dcd21ae3d6b32cca0b3c10149fa378eb21b338d4b63c9eb8050b/pytest_codspeed-5.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:efd43f82ea03ced8488a767ded9473f050791ab7783ea8654107e1e0ac66af40", size = 932395, upload-time = "2026-05-22T16:21:04.804Z" }, + { url = "https://files.pythonhosted.org/packages/96/08/56ad8f1cc7d6962f8a680141b361e93467a2abc53d976cd9d5e1edd740e3/pytest_codspeed-5.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:782f9985b6f6b45b8bc20152d206d3a52b56dd088ba81cb70a71f0b39841be9e", size = 934994, upload-time = "2026-05-22T16:20:28.809Z" }, + { url = "https://files.pythonhosted.org/packages/0b/54/9096c4545f09da94b1b00f3be2fe4952949e86c9bcafca9a29b26aed1a75/pytest_codspeed-5.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9aa0815b90196f3c20d736ea8691381e97f12bbe8c7d87af10a351e434b452cb", size = 366311, upload-time = "2026-05-22T16:20:41.791Z" }, + { url = "https://files.pythonhosted.org/packages/a7/3c/24c53f67a38ad48cb087105ac30a8aa0923223ee274ea9bf2dc705edaa59/pytest_codspeed-5.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:85505c96a3477c346ec2d2b7dced8478f4c651e2b1666ee102d53a832b511853", size = 933169, upload-time = "2026-05-22T16:20:43.178Z" }, + { url = "https://files.pythonhosted.org/packages/d1/de/2213f868fa7694f743f96cccbc07e757f45c920c523cccc2da97bc8652df/pytest_codspeed-5.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20eba63765be9d1b6cacbbfad84b87d49eb04b357a7045a0899880da181f81e3", size = 935522, upload-time = "2026-05-22T16:21:03.398Z" }, + { url = "https://files.pythonhosted.org/packages/df/85/5dfea1c031d6cccc11653464828edf205c30f798caf5b2a85375aacd914a/pytest_codspeed-5.0.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:ec9fa6f0af0a9feb0e0bd517fb59ef28f806fbd50c0c6900ac26cbb4d080eba5", size = 366275, upload-time = "2026-05-22T16:20:59.463Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2b/af4d1b612f03b98a6cf3c7d5f62678917a60110a8bf380d49ab408b31137/pytest_codspeed-5.0.3-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8df77b3409f54f4a268f77f3ff74992fe1d995cdbaf2cecf8ad74d32db217ce7", size = 932537, upload-time = "2026-05-22T16:20:54.945Z" }, + { url = "https://files.pythonhosted.org/packages/f5/a2/c7ec45e36a61b418efb2a3cccaa67a0c2fcf1f21d5880f64c33114f0c249/pytest_codspeed-5.0.3-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5d8695a227ea1c3a41d25db5b3fe720bf1b4808bd38862be811a4efd902c792", size = 934153, upload-time = "2026-05-22T16:21:07.494Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c7/d5bada9618a0af56a5c8065fc61280849cab8e7c1e24025807a51c3157ce/pytest_codspeed-5.0.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bf4cc4178cbace8f4d2bd240408276bc4da3850ac5fcb5fb5f8a74ab417615bb", size = 366339, upload-time = "2026-05-22T16:20:51.968Z" }, + { url = "https://files.pythonhosted.org/packages/a8/37/fb27aeb40a81320e7349553b877a21333c897b27c8dfe215630452908f36/pytest_codspeed-5.0.3-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:abe793da40f87295d33988673d34f06ea569848b44490b847552cd416816258a", size = 933055, upload-time = "2026-05-22T16:20:44.861Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d9/6f2d69e96deaf0475a695fc9195af59e7a3b5fab50782855e65c63a7bc28/pytest_codspeed-5.0.3-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3a9ed38dfa776443b86f4b49a982e8443d0953db4974bd2673d63cc904ae1ad", size = 934481, upload-time = "2026-05-22T16:20:58.264Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b2/1d2a993c532146dce9eca5b5942d51898021c3579ce18b2454f932a915f8/pytest_codspeed-5.0.3-py3-none-any.whl", hash = "sha256:fe2ea83c924c2250675b75686c3ee456b8cf0208d83d552e182a195fdf467378", size = 74033, upload-time = "2026-05-22T16:20:26.814Z" }, +] + [[package]] name = "pytest-cov" version = "7.1.0" @@ -2319,6 +2344,7 @@ tests = [ { name = "pytest" }, { name = "pytest-alembic" }, { name = "pytest-asyncio" }, + { name = "pytest-codspeed" }, { name = "pytest-cov" }, { name = "pytest-mock" }, { name = "pytest-timeout" }, @@ -2398,6 +2424,7 @@ tests = [ { name = "pytest", specifier = ">=8.4.1" }, { name = "pytest-alembic", specifier = ">=0.12.1" }, { name = "pytest-asyncio", specifier = ">=1.0.0" }, + { name = "pytest-codspeed", specifier = ">=5.0.3" }, { name = "pytest-cov", specifier = ">=6.2.1" }, { name = "pytest-mock", specifier = ">=3.14.0" }, { name = "pytest-timeout", specifier = ">=2.3.0" }, From f0d9564086a0546190db86172fa872d24ec75525 Mon Sep 17 00:00:00 2001 From: Simon van Lierde Date: Mon, 26 Oct 2026 15:51:24 +0100 Subject: [PATCH 2/2] ci(codspeed): measure the backend benchmarks on pull requests and main Runs in CodSpeed's CPU simulation mode on pushes to main and on pull requests that touch the backend, uploading with OIDC from a job that holds the only id-token permission in the workflow. --- .github/CONTRIBUTING.md | 3 +- .github/workflows/codspeed.yml | 65 ++++++++++++++++++++++++++++++++++ README.md | 1 + 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/codspeed.yml diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 7d321560a..76f80a287 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -144,7 +144,8 @@ ladder, cheapest first: Pull requests also run `cloudflare-check`, the www E2E suite, and `docker-smoke`, so a green `just ci` can still leave a red PR check. `just ci-full` covers them; it takes tens of minutes and needs Docker. Only CodeQL, Trivy, Scorecard, and dependency review stay GitHub-only: they scan built -images or the repository itself. +images or the repository itself. The CodSpeed benchmark comparison also runs only on GitHub, since +it needs CodSpeed's instrumented runner; `just backend/bench` runs the same benchmarks once as tests. On a pull request, the subrepo jobs, their Docker smoke legs, and the CodeQL languages run only for the subrepos the diff touches (`scripts/ci_changed_areas.py`). A change outside the four subrepos diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 000000000..a8e9bda01 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,65 @@ +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json + +# Backend CPU benchmarks (backend/tests/benchmarks), measured by CodSpeed in its CPU +# simulation mode: it counts instructions, so results hold steady across hosted runners. +# Lives outside ci.yml so the `id-token: write` the upload needs stays on this one job. +name: CodSpeed + +on: + # Every run on main is the baseline a pull request is compared against. A PR is + # compared with the newest main commit that has a run, so main can skip commits + # that do not touch the backend too. + push: + branches: [main] + paths: + - backend/** + - .github/actions/setup-runtime/** + - .github/workflows/codspeed.yml + - .tool-versions + pull_request: + branches: [main] + paths: + - backend/** + - .github/actions/setup-runtime/** + - .github/workflows/codspeed.yml + - .tool-versions + # Lets CodSpeed backfill results for older main commits. + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: {} + +jobs: + benchmarks: + name: Backend Benchmarks + runs-on: ubuntu-26.04 + timeout-minutes: 20 + permissions: + contents: read + # The upload authenticates with OIDC. Pull requests from forks get no token, and + # CodSpeed accepts their uploads without one because the repository is public. + id-token: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up runtime + uses: ./.github/actions/setup-runtime + with: + setup-python: "true" + uv-cache-dependency-glob: backend/uv.lock + + - name: Install dependencies + run: just backend/install + + - name: Run benchmarks + uses: CodSpeedHQ/action@78102a468f96d8e0ccb92d0ec49425753fc42062 # v5.6.0 + with: + mode: simulation + working-directory: backend + run: uv run pytest tests/benchmarks --codspeed diff --git a/README.md b/README.md index f35e63020..d7b35a595 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ [![OpenSSF Best Practices](https://www.bestpractices.dev/projects/14492/badge)](https://www.bestpractices.dev/projects/14492) [![FAIR checklist badge](https://fairsoftwarechecklist.net/badge.svg)](https://fairsoftwarechecklist.net/v0.2?f=31&a=32113&i=22322&r=123) [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](.github/CODE_OF_CONDUCT.md) +[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/CMLPlatform/r9lab?utm_source=badge) [![Deployed](https://img.shields.io/website?url=https%3A%2F%2Fr9lab.io&label=website)](https://r9lab.io) R9lab is an open-source research platform for collecting and publicly viewing data on the