From 56481b1020428545b0041727094c9fd1736a77ab Mon Sep 17 00:00:00 2001 From: Jaye Norman Date: Fri, 4 Jul 2025 08:53:31 -0600 Subject: [PATCH 1/8] Adding github action to comparatively benchmark PR --- .github/workflows/benchmark-pr.yml | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/benchmark-pr.yml diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml new file mode 100644 index 00000000000..0f02a0b55ee --- /dev/null +++ b/.github/workflows/benchmark-pr.yml @@ -0,0 +1,32 @@ +name: Benchmark PR +on: + pull_request: + branches: [main] + workflow_dispatch: + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: false + +jobs: + benchmark: + runs-on: ubuntu-latest + timeout-minutes: 20 #times out after 20 minutes + defaults: + run: + working-directory: ./benchmarks #sets the default working directory to ./benchmarks + steps: + - uses: actions/setup-python@v5 #sets up python with version 3.12 + with: + python-version: "3.12" + - name: Checkout repo + uses: actions/checkout@v4 #checks out repository + with: + fetch-depth: 0 # Fetch full history + - run: pip install asv virtualenv #install asv + - run: python -m asv machine --yes #setup the asv machine settings with device-given defaults + - name: Run ASV + # runs asv on the PR's base sha (should always be main) and the PR branch's sha + run: python -m asv continuous ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} \ No newline at end of file From 05f0d931a293dc8e6b39334f2ccd8ee7ebf9377d Mon Sep 17 00:00:00 2001 From: Jaye Norman Date: Wed, 9 Jul 2025 11:00:32 -0600 Subject: [PATCH 2/8] Benchmark PR only if 'benchmark' label is present --- .github/workflows/benchmark-pr.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 0f02a0b55ee..d53e49149d0 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -4,15 +4,15 @@ on: branches: [main] workflow_dispatch: -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. + concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} - cancel-in-progress: false + cancel-in-progress: true jobs: benchmark: - runs-on: ubuntu-latest + runs-on: ubuntu-latest + if: contains (github.event.pull_request.labels.*.name, 'benchmark') timeout-minutes: 20 #times out after 20 minutes defaults: run: From 199ed0a1ec7f7bf389c08bcba42a73d5340fad9a Mon Sep 17 00:00:00 2001 From: blue-jaye-121 Date: Wed, 9 Jul 2025 13:34:11 -0600 Subject: [PATCH 3/8] Update benchmark-pr.yml - change to generating data array and only on 'benchmark' label --- .github/workflows/benchmark-pr.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index d53e49149d0..317018e66e6 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -2,6 +2,7 @@ name: Benchmark PR on: pull_request: branches: [main] + types: [opened, synchronize, reopened, labeled] #defaults and when labeled workflow_dispatch: @@ -12,7 +13,7 @@ concurrency: jobs: benchmark: runs-on: ubuntu-latest - if: contains (github.event.pull_request.labels.*.name, 'benchmark') + if: contains(github.event.pull_request.labels.*.name, 'benchmark') timeout-minutes: 20 #times out after 20 minutes defaults: run: @@ -25,8 +26,11 @@ jobs: uses: actions/checkout@v4 #checks out repository with: fetch-depth: 0 # Fetch full history + - run: python -m pip install numpy pandas xarray metpy netcdf4 + - name: Create data array + run: python data_array_generate.py - run: pip install asv virtualenv #install asv - run: python -m asv machine --yes #setup the asv machine settings with device-given defaults - name: Run ASV # runs asv on the PR's base sha (should always be main) and the PR branch's sha - run: python -m asv continuous ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} \ No newline at end of file + run: python -m asv continuous ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} From c655c9389fb49047a0c1232d224223e3eb024590 Mon Sep 17 00:00:00 2001 From: Jaye Norman Date: Mon, 28 Jul 2025 13:26:31 -0600 Subject: [PATCH 4/8] Commenting workflow for documentation --- .github/workflows/benchmark-pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 317018e66e6..14659e1a1cd 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -1,6 +1,7 @@ name: Benchmark PR on: pull_request: + #we don't need pull requests on the gh-pages branch to be benchmarked branches: [main] types: [opened, synchronize, reopened, labeled] #defaults and when labeled workflow_dispatch: @@ -13,6 +14,7 @@ concurrency: jobs: benchmark: runs-on: ubuntu-latest + #sets only to run when the GitHub PR is labeled with 'benchmark' if: contains(github.event.pull_request.labels.*.name, 'benchmark') timeout-minutes: 20 #times out after 20 minutes defaults: From 5cced9fdf59f513a107fe06e730b82e835155946 Mon Sep 17 00:00:00 2001 From: Jaye Norman Date: Mon, 28 Jul 2025 13:32:04 -0600 Subject: [PATCH 5/8] Adding workflow for testing --- .github/workflows/benchmark-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 14659e1a1cd..7c6e486f155 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -2,7 +2,7 @@ name: Benchmark PR on: pull_request: #we don't need pull requests on the gh-pages branch to be benchmarked - branches: [main] + branches: [main, benchmark_pr_action] types: [opened, synchronize, reopened, labeled] #defaults and when labeled workflow_dispatch: From 444baac340b748faab0ac88eda00c1b813794350 Mon Sep 17 00:00:00 2001 From: Jaye Norman Date: Mon, 28 Jul 2025 14:22:50 -0600 Subject: [PATCH 6/8] Documenting PR in benchmarking docs --- docs/devel/benchmarking.rst | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/docs/devel/benchmarking.rst b/docs/devel/benchmarking.rst index 7eb9cd751a9..21e5a1cceef 100644 --- a/docs/devel/benchmarking.rst +++ b/docs/devel/benchmarking.rst @@ -5,11 +5,11 @@ Performance Benchmarking This guide provides information on the implementation and management of benchmarking in MetPy. ----------------- -Airspeed Velocity +airspeed velocity ----------------- -MetPy's source code is benchmarked using `Airspeed Velocity `_. -ASV is an open source software which builds environments based on historical and current +MetPy's source code is benchmarked using `airspeed velocity `_. +asv is an open source software which builds environments based on historical and current iterations of software and runs benchmark functions before compiling the results into digestable html pages. MetPy's developers have used GitHub Actions and a Unidata Jenkins instance in order to automatically perform benchmarking as part of the continuous @@ -32,11 +32,32 @@ This performance history is run using the Unidata Jenkins instance. Upon run, th ``benchmarks/Jenkinsfile`` instructs the Jenkins instance to create a custom ``Docker container`` using the ``benchmarks/Dockerfile`` and runs the benchmark functions within it. Jenkins uses the same Unidata machine for each run in order to ensure -consistent benchmarking results. ASV is installed in this container and runs the benchmark +consistent benchmarking results. asv is installed in this container and runs the benchmark functions for the historical commits of interest. In the event that successful results already -exist for the requested commit hash, ASV will skip it and maintain the previous results. +exist for the requested commit hash, asv will skip it and maintain the previous results. Finally, Jenkins pushes the results to a separate `results repository `_ -where a GitHub Action uses an ASV command to generate and deploy the html. +where a GitHub Action uses an asv command to generate and deploy the html. + + +------------------------------------- +Pull Request Comparative Benchmarking +------------------------------------- + + +As part of the continuous integration workflow of MetPy, a GitHub Action has been implemented +which uses the ``asv continous`` command to automatically use asv to benchmark the PR's SHA +versus the current main branch. This comparative benchmark is only done when the pull request +is labeled ``benchmark`` by a MetPy maintainer. This is because the check takes about 10 +minutes and is not necessary for every pull request, only those that change the calc +module. + +Currently, the benchmark is set up to fail if any one benchmark takes 10% or more longer +on the PR branch. But failing this check doesn't mean you can't contribute! +Your PR might emphasize accuracy at the sacrifice of speed, and that might be ok in some cases. +The maintainers will work with each pull request on a case-by-case basis and can help you +if you're getting unexpected benchmarking failures. If you want to test out your performance +before opening a pull request, look into :ref:`local comparative benchmarking `. + ------------------- Benchmark Functions @@ -44,7 +65,7 @@ Benchmark Functions Located within the ``benchmarks/benchmarks`` directory are ``.py`` files each containing a class ``TimeSuite``, ``setup`` and ``setup_cache`` functions, and functions with the name -scheme ``time_example_metpy_function``. This is ASV's required `syntax `_ +scheme ``time_example_metpy_function``. This is asv's required `syntax `_ for writing benchmarks. The ``setup_cache`` function loads the artificial benchmarking dataset ``data_array_compressed.nc`` and prepares the dataset for use by the benchmarks. The ``setup`` function "slices" the 4D dataset into the appropriate dimensions to create variables that can @@ -68,6 +89,8 @@ follow these steps: a. To benchmark your code as is currently is, use ``python -m asv run`` + .. _local-benchmarking-reference-label: + b. To compare a working branch with *your version* of MetPy's main branch, use ``python -m asv continuous main `` where ```` is the name of your branch. You can also simply use two commit hashes in the place of the branch names. To view From 6899a749f007f6a72e46962562a63bf8c2b4dabe Mon Sep 17 00:00:00 2001 From: Jaye Norman Date: Mon, 28 Jul 2025 14:25:00 -0600 Subject: [PATCH 7/8] Only benchmarking main PR's --- .github/workflows/benchmark-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 7c6e486f155..14659e1a1cd 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -2,7 +2,7 @@ name: Benchmark PR on: pull_request: #we don't need pull requests on the gh-pages branch to be benchmarked - branches: [main, benchmark_pr_action] + branches: [main] types: [opened, synchronize, reopened, labeled] #defaults and when labeled workflow_dispatch: From cfb2ddd22e7981c38fe5c03849f778b01f3e6b79 Mon Sep 17 00:00:00 2001 From: Jaye Norman Date: Mon, 28 Jul 2025 14:36:55 -0600 Subject: [PATCH 8/8] fixing linting errors --- docs/devel/benchmarking.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/devel/benchmarking.rst b/docs/devel/benchmarking.rst index 21e5a1cceef..8484a6736a9 100644 --- a/docs/devel/benchmarking.rst +++ b/docs/devel/benchmarking.rst @@ -45,7 +45,7 @@ Pull Request Comparative Benchmarking As part of the continuous integration workflow of MetPy, a GitHub Action has been implemented -which uses the ``asv continous`` command to automatically use asv to benchmark the PR's SHA +which uses the ``asv continuous`` command to automatically use asv to benchmark the PR's SHA versus the current main branch. This comparative benchmark is only done when the pull request is labeled ``benchmark`` by a MetPy maintainer. This is because the check takes about 10 minutes and is not necessary for every pull request, only those that change the calc @@ -56,7 +56,8 @@ on the PR branch. But failing this check doesn't mean you can't contribute! Your PR might emphasize accuracy at the sacrifice of speed, and that might be ok in some cases. The maintainers will work with each pull request on a case-by-case basis and can help you if you're getting unexpected benchmarking failures. If you want to test out your performance -before opening a pull request, look into :ref:`local comparative benchmarking `. +before opening a pull request, look into +:ref:`local comparative benchmarking `. ------------------- @@ -90,7 +91,7 @@ follow these steps: use ``python -m asv run`` .. _local-benchmarking-reference-label: - + b. To compare a working branch with *your version* of MetPy's main branch, use ``python -m asv continuous main `` where ```` is the name of your branch. You can also simply use two commit hashes in the place of the branch names. To view