diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index 7a222fd8..e121d7b6 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -1,14 +1,20 @@ name: Benchmarks on: + push: + branches: + - main pull_request: types: - opened - synchronize + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: jobs: benchmark: - name: Benchmark tests + name: Run benchmarks runs-on: ubuntu-latest permissions: contents: read @@ -17,66 +23,28 @@ jobs: matrix: python_version: [3.14] steps: - - name: Checkout branch + - name: Checkout uses: actions/checkout@v4 - with: - path: pr - - - name: Checkout main - uses: actions/checkout@v4 - with: - ref: main - path: main - name: Install python uses: actions/setup-python@v5 with: - python-version: ${{matrix.python_version}} + python-version: "3.12" - name: Install uv uses: astral-sh/setup-uv@v4 with: enable-cache: true - cache-dependency-glob: "main/uv.lock" - - - name: Setup benchmarks - run: | - echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV - echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV - echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV - - - name: Run benchmarks on PR - working-directory: ./pr - run: | - uv sync --group test - uv run pytest --benchmark-only --benchmark-save=pr - - - name: Run benchmarks on main - working-directory: ./main - continue-on-error: true - run: | - uv sync --group test - uv run pytest --benchmark-only --benchmark-save=base + cache-dependency-glob: "uv.lock" - - name: Compare results - continue-on-error: false - run: | - uvx pytest-benchmark compare **/.benchmarks/**/*.json | tee cmp_results + - name: Install project + run: uv sync --group test - echo 'Benchmark comparison for [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (base) vs [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR)' >> pr_comment - echo '```' >> pr_comment - cat cmp_results >> pr_comment - echo '```' >> pr_comment - cat pr_comment > ${{ env.PR_COMMENT }} - - - name: Comment on PR - uses: actions/github-script@v7 + - name: Run benchmarks + uses: CodSpeedHQ/action@v4 + env: + RAY_ENABLE_UV_RUN_RUNTIME_ENV: 0 + PLUGBOARD_IO_READ_TIMEOUT: 5.0 with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: require('fs').readFileSync('${{ env.PR_COMMENT }}').toString() - }); + mode: simulation + run: uv run pytest tests/benchmark/ --codspeed diff --git a/README.md b/README.md index 77e5c3c9..1f674767 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ CodeQL + + CodSpeed Badge
Docs diff --git a/pyproject.toml b/pyproject.toml index 28505a04..4314c861 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ test = [ "optuna>=3.0,<5", "pytest>=8.3,<10", "pytest-asyncio>=1.0,<2", - "pytest-benchmark>=5.1.0", + "pytest-codspeed>=4.4.0", "pytest-cases>=3.8,<4", "pytest-env>=1.1,<2", "pytest-rerunfailures>=15.0,<17", @@ -119,6 +119,10 @@ fallback_version = "0.0.0" [tool.uv] package = true default-groups = ["all"] +required-environments = [ + # AARCH64 linux support required for Codspeed benchmarks + "sys_platform == 'linux' and platform_machine == 'aarch64'" +] [tool.uv.workspace] members = ["plugboard-schemas"] diff --git a/tests/benchmark/conftest.py b/tests/benchmark/conftest.py new file mode 100644 index 00000000..b092679a --- /dev/null +++ b/tests/benchmark/conftest.py @@ -0,0 +1,17 @@ +"""Configuration for benchmark tests.""" + +import typing as _t + +import pytest +import ray + + +@pytest.fixture(scope="session") +def ray_ctx() -> _t.Iterator[None]: + """Initialises and shuts down Ray for benchmarks. + + Dashboard is disabled to avoid MetricsHead timeout in CI. + """ + ray.init(num_cpus=5, num_gpus=1, resources={"custom_hardware": 10}, include_dashboard=False) + yield + ray.shutdown() diff --git a/tests/benchmark/test_benchmarking.py b/tests/benchmark/test_benchmarking.py index 7554a7a0..4e57af27 100644 --- a/tests/benchmark/test_benchmarking.py +++ b/tests/benchmark/test_benchmarking.py @@ -1,35 +1,78 @@ -"""Simple benchmark tests for Plugboard models.""" +"""Benchmark tests for Plugboard processes.""" -import asyncio +import pytest +from pytest_codspeed import BenchmarkFixture +import uvloop -from pytest_benchmark.fixture import BenchmarkFixture - -from plugboard.connector import AsyncioConnector -from plugboard.process import LocalProcess, Process +from plugboard.connector import AsyncioConnector, Connector, RayConnector, ZMQConnector +from plugboard.process import LocalProcess, Process, RayProcess from plugboard.schemas import ConnectorSpec from tests.integration.test_process_with_components_run import A, B -def _setup_process() -> tuple[tuple[Process], dict]: - comp_a = A(name="comp_a", iters=1000) +ITERS = 1000 + +CONNECTOR_PROCESS_PARAMS = [ + (AsyncioConnector, LocalProcess), + (ZMQConnector, LocalProcess), + (RayConnector, RayProcess), +] +CONNECTOR_PROCESS_IDS = ["asyncio", "zmq", "ray"] + + +def _build_process(connector_cls: type[Connector], process_cls: type[Process]) -> Process: + """Build a process with the given connector and process class.""" + comp_a = A(name="comp_a", iters=ITERS) comp_b1 = B(name="comp_b1", factor=1) comp_b2 = B(name="comp_b2", factor=2) components = [comp_a, comp_b1, comp_b2] connectors = [ - AsyncioConnector(spec=ConnectorSpec(source="comp_a.out_1", target="comp_b1.in_1")), - AsyncioConnector(spec=ConnectorSpec(source="comp_b1.out_1", target="comp_b2.in_1")), + connector_cls(spec=ConnectorSpec(source="comp_a.out_1", target="comp_b1.in_1")), + connector_cls(spec=ConnectorSpec(source="comp_b1.out_1", target="comp_b2.in_1")), ] - process = LocalProcess(components=components, connectors=connectors) - # Initialise process so that this is excluded from the benchmark timing - asyncio.run(process.init()) - # Return args and kwargs tuple for benchmark.pedantic - return (process,), {} + return process_cls(components=components, connectors=connectors) + + +@pytest.mark.benchmark +@pytest.mark.parametrize( + "connector_cls, process_cls", + CONNECTOR_PROCESS_PARAMS, + ids=CONNECTOR_PROCESS_IDS, +) +@pytest.mark.asyncio +async def test_benchmark_process_lifecycle( + connector_cls: type[Connector], + process_cls: type[Process], + ray_ctx: None, +) -> None: + """Benchmark the full lifecycle (init, run, destroy) of a Plugboard Process.""" + process = _build_process(connector_cls, process_cls) + async with process: + await process.run() + + +@pytest.mark.parametrize( + "connector_cls, process_cls", + CONNECTOR_PROCESS_PARAMS, + ids=CONNECTOR_PROCESS_IDS, +) +def test_benchmark_process_run( + benchmark: BenchmarkFixture, + connector_cls: type[Connector], + process_cls: type[Process], + ray_ctx: None, +) -> None: + """Benchmark running of a Plugboard Process.""" + def _setup() -> tuple[tuple[Process], dict]: + async def _init() -> Process: + process = _build_process(connector_cls, process_cls) + await process.init() + return process -def _run_process(process: Process) -> None: - asyncio.run(process.run()) + return (uvloop.run(_init()),), {} + def _run(process: Process) -> None: + uvloop.run(process.run()) -def test_benchmark_process_run(benchmark: BenchmarkFixture) -> None: - """Benchmark the running of a Plugboard Process.""" - benchmark.pedantic(_run_process, setup=_setup_process, rounds=5) + benchmark.pedantic(_run, setup=_setup, rounds=5) diff --git a/uv.lock b/uv.lock index 5fe30e44..95314cfe 100644 --- a/uv.lock +++ b/uv.lock @@ -12,6 +12,9 @@ resolution-markers = [ "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version < '3.13' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] +required-markers = [ + "platform_machine == 'aarch64' and sys_platform == 'linux'", +] [manifest] members = [ @@ -3944,8 +3947,8 @@ all = [ { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, - { name = "pytest-benchmark" }, { name = "pytest-cases" }, + { name = "pytest-codspeed" }, { name = "pytest-env" }, { name = "pytest-rerunfailures" }, { name = "radon" }, @@ -3991,8 +3994,8 @@ test = [ { name = "optuna" }, { name = "pytest" }, { name = "pytest-asyncio" }, - { name = "pytest-benchmark" }, { name = "pytest-cases" }, + { name = "pytest-codspeed" }, { name = "pytest-env" }, { name = "pytest-rerunfailures" }, { name = "ray", extra = ["default", "tune"] }, @@ -4063,8 +4066,8 @@ all = [ { name = "pre-commit", specifier = ">=3.8,<5" }, { name = "pytest", specifier = ">=8.3,<10" }, { name = "pytest-asyncio", specifier = ">=1.0,<2" }, - { name = "pytest-benchmark", specifier = ">=5.1.0" }, { name = "pytest-cases", specifier = ">=3.8,<4" }, + { name = "pytest-codspeed", specifier = ">=4.4.0" }, { name = "pytest-env", specifier = ">=1.1,<2" }, { name = "pytest-rerunfailures", specifier = ">=15.0,<17" }, { name = "radon", specifier = ">=6.0.1,<7" }, @@ -4110,8 +4113,8 @@ test = [ { name = "optuna", specifier = ">=3.0,<5" }, { name = "pytest", specifier = ">=8.3,<10" }, { name = "pytest-asyncio", specifier = ">=1.0,<2" }, - { name = "pytest-benchmark", specifier = ">=5.1.0" }, { name = "pytest-cases", specifier = ">=3.8,<4" }, + { name = "pytest-codspeed", specifier = ">=4.4.0" }, { name = "pytest-env", specifier = ">=1.1,<2" }, { name = "pytest-rerunfailures", specifier = ">=15.0,<17" }, { name = "ray", extras = ["default", "tune"], specifier = ">=2.40.0,<3" }, @@ -4340,15 +4343,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - [[package]] name = "py-partiql-parser" version = "0.6.3" @@ -4633,19 +4627,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] -[[package]] -name = "pytest-benchmark" -version = "5.2.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "py-cpuinfo" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/24/34/9f732b76456d64faffbef6232f1f9dbec7a7c4999ff46282fa418bd1af66/pytest_benchmark-5.2.3.tar.gz", hash = "sha256:deb7317998a23c650fd4ff76e1230066a76cb45dcece0aca5607143c619e7779", size = 341340, upload-time = "2025-11-09T18:48:43.215Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/33/29/e756e715a48959f1c0045342088d7ca9762a2f509b945f362a316e9412b7/pytest_benchmark-5.2.3-py3-none-any.whl", hash = "sha256:bc839726ad20e99aaa0d11a127445457b4219bdb9e80a1afc4b51da7f96b0803", size = 45255, upload-time = "2025-11-09T18:48:39.765Z" }, -] - [[package]] name = "pytest-cases" version = "3.10.1" @@ -4661,6 +4642,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/f2/7a29fb0571562034b05c38dceabba48dcc622be5d6c5448db80779e55de7/pytest_cases-3.10.1-py2.py3-none-any.whl", hash = "sha256:0deb8a85b6132e44adbc1cfc57897c6a624ec23f48ab445a43c7d56a6b9315a4", size = 108870, upload-time = "2026-03-02T23:05:32.663Z" }, ] +[[package]] +name = "pytest-codspeed" +version = "4.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, + { name = "pytest" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/bc/9070fdbfb479a0e92a12652a68875de157dc9be7dc4865a06a519e3a1877/pytest_codspeed-4.4.0.tar.gz", hash = "sha256:edb7c101d9c50439a42cf02cfa9c0ac92da618841636bbebf87c3fa54669442a", size = 201093, upload-time = "2026-04-14T15:13:20.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/70/4a401b37f80aaebbcbfb2803b0fab75331af554cd75755bc2059f7809bb4/pytest_codspeed-4.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a5c1d51e7ca72ffe247c99b9a97a54191185e8f7a27528e2200d7416da2a68b", size = 820334, upload-time = "2026-04-14T15:13:03.605Z" }, + { url = "https://files.pythonhosted.org/packages/16/52/beb46293d414d65163f8f3218aaa2f05e53bdc5cf64f24cc3843c31d3ca4/pytest_codspeed-4.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:215170441e57bfcbefd179dfd86ccd54ed0ee235e0602a068ce4448b35f13cb2", size = 829269, upload-time = "2026-04-14T15:13:05.197Z" }, + { url = "https://files.pythonhosted.org/packages/78/53/031793dab3a0edbbcbbd8755648ace0853f4cfb92a0e09e620f301f9ef5d/pytest_codspeed-4.4.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee3e1964446011ca192eebf0350227df231a5b88af57e518f2a4328fc8ca5131", size = 820300, upload-time = "2026-04-14T15:13:06.791Z" }, + { url = "https://files.pythonhosted.org/packages/e7/66/0c3530c0dd9959b7f0930551b3de296db391040e5e8ad3e0cab917736980/pytest_codspeed-4.4.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:340dbb1cc5a21434e0e29bd68ab03c7dc7ad9bfde09d1980b7161352c4c2f048", size = 829201, upload-time = "2026-04-14T15:13:08Z" }, + { url = "https://files.pythonhosted.org/packages/f2/8a/24c7997d95f8bda081b8d4346750a5db0d9d8405183ee5cb9062f7381476/pytest_codspeed-4.4.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:413666266762f9cef1321ba971a9e127b97a1f1dad40ddfd2184c2bc5ac157f9", size = 820242, upload-time = "2026-04-14T15:13:09.191Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7f/3912bf6c2bcddb69189d23213f28e5bc058fd4c78fca15dd0010938154b0/pytest_codspeed-4.4.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e258e6c3d5a8a02ae02a64831be3acd44c19210ffbf13321bdbb8c111c5c6fe4", size = 829190, upload-time = "2026-04-14T15:13:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/d8/f4/2cc5e10847aee4233690aa511df6b6f1c2c09f9d8ae506628a138f4ba201/pytest_codspeed-4.4.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56d5dd94dcb69460f916acb9c69865d0171b98acec3ce256645d0c0275b553d7", size = 827557, upload-time = "2026-04-14T15:13:12.553Z" }, + { url = "https://files.pythonhosted.org/packages/7f/57/982ce8aa81089b285730dca8404c76af648af41e46d95012be54452913e6/pytest_codspeed-4.4.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:33c38e0e797c74506004f231fc53eab0e412987de281755f714018334381aa3a", size = 835388, upload-time = "2026-04-14T15:13:14.232Z" }, + { url = "https://files.pythonhosted.org/packages/99/36/9e84323c6be426728e897133f8e9f3e65a90c26c137e190ca9b27bf304c3/pytest_codspeed-4.4.0-py3-none-any.whl", hash = "sha256:a6aab2fa73523f538e7729c20ccf4a1e8e921324c9877a816b05334135950fd9", size = 203809, upload-time = "2026-04-14T15:13:18.72Z" }, +] + [[package]] name = "pytest-env" version = "1.6.0"