diff --git a/.github/python-packages.json b/.github/python-packages.json new file mode 100644 index 0000000..63f5f65 --- /dev/null +++ b/.github/python-packages.json @@ -0,0 +1,5 @@ +{ + "nisyscfg": { + "package-basepath": "." + } +} diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..16f684e --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,51 @@ +name: CI + +on: + push: + branches: + - main + workflow_call: + workflow_dispatch: + +permissions: {} + +jobs: + check_workflows: + name: Check workflows + uses: ./.github/workflows/check_workflows.yml + get_package_names: + name: Get package names + uses: ./.github/workflows/get_package_names.yml + check_packages: + name: Check ${{ matrix.package-name }} + needs: get_package_names + strategy: + matrix: + package-name: ${{ fromJson(needs.get_package_names.outputs.package-names) }} + uses: ./.github/workflows/check_package.yml + with: + package-name: ${{ matrix.package-name }} + test_packages: + name: Test ${{ matrix.package-name }} + needs: get_package_names + strategy: + matrix: + package-name: ${{ fromJson(needs.get_package_names.outputs.package-names) }} + uses: ./.github/workflows/test_package.yml + with: + package-name: ${{ matrix.package-name }} + + test_wheel_build: + name: Test wheel build + needs: check_packages + uses: ./.github/workflows/test_wheel_build.yml + + report_test_results: + name: Report test results + if: ${{ always() }} + needs: test_packages + uses: ./.github/workflows/report_test_results.yml + permissions: + checks: write # needed to push the aggregated report + pull-requests: write # needed to post the report to the PR + diff --git a/.github/workflows/PR.yml b/.github/workflows/PR.yml new file mode 100644 index 0000000..4841912 --- /dev/null +++ b/.github/workflows/PR.yml @@ -0,0 +1,24 @@ +name: PR + +on: + pull_request: + branches: + - main + workflow_call: + workflow_dispatch: + +permissions: {} +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + run_ci: + name: Run CI + uses: ./.github/workflows/CI.yml + permissions: + checks: write # needed to push the aggregated test results report + pull-requests: write # needed to post the test results report to the PR + + + diff --git a/.github/workflows/check_package.yml b/.github/workflows/check_package.yml new file mode 100644 index 0000000..49cf4f4 --- /dev/null +++ b/.github/workflows/check_package.yml @@ -0,0 +1,19 @@ +name: Check package + +on: + workflow_call: + inputs: + package-name: + description: 'The name of the package folder to check.' + default: '' + required: true + type: string + +permissions: {} + +jobs: + get_package_info: + name: Get package info for ${{ inputs.package-name }} + uses: ./.github/workflows/get_package_info.yml + with: + package-name: ${{ inputs.package-name }} diff --git a/.github/workflows/check_workflows.yml b/.github/workflows/check_workflows.yml new file mode 100644 index 0000000..c3ad500 --- /dev/null +++ b/.github/workflows/check_workflows.yml @@ -0,0 +1,26 @@ +name: Check workflows + +on: + workflow_call: + workflow_dispatch: + +permissions: {} + +jobs: + zizmor: + name: Run zizmor + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Run zizmor + uses: zizmorcore/zizmor-action@192e21d79ab29983730a13d1382995c2307fbcaa # v0.5.7 + with: + advanced-security: false # TODO: use the default when we publish the repo + inputs: | + .github/workflows + + + diff --git a/.github/workflows/get_package_info.yml b/.github/workflows/get_package_info.yml new file mode 100644 index 0000000..a911b8a --- /dev/null +++ b/.github/workflows/get_package_info.yml @@ -0,0 +1,64 @@ +name: Get package info + +on: + workflow_call: + inputs: + package-name: + description: 'The name of the package from .github/python-packages.json.' + default: '' + required: true + type: string + outputs: + package-basepath: + description: 'The package base path relative to the repo root.' + # NOTE: This is not the absolute path, because downstream actions are expected to be run on multiple platforms. + value: ${{ jobs.get_package_info.outputs.package-basepath }} + install-extras: + description: 'A space-separated list of package extras to install.' + value: ${{ jobs.get_package_info.outputs.install-extras }} +permissions: {} + +jobs: + get_package_info: + name: Get package info for ${{ inputs.package-name }} + runs-on: ubuntu-latest + outputs: + package-basepath: ${{ steps.parse_json.outputs.package-basepath }} + install-extras: ${{ steps.parse_json.outputs.install-extras }} + should-check-docs: ${{ steps.parse_json.outputs.should-check-docs }} + steps: + - name: Check out repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get package info from JSON + id: parse_json + env: + PACKAGE_NAME: ${{ inputs.package-name }} + # NOTE: jq only supports standard JSON, not JSONC or JSON5 + run: | + info=$(jq -c --arg name "$PACKAGE_NAME" '.[$name]' "./.github/python-packages.json") + if [ "$info" = "null" ]; then + echo "Error: Package '$PACKAGE_NAME' not found in .github/python-packages.json." >&2 + exit 1 + fi + + package_basepath=$(echo "$info" | jq -r '."package-basepath"') + install_extras=$(echo "$info" | jq -r '."install-extras"') + + should_check_docs=false + if [ -d "${package_basepath}/docs" ]; then + should_check_docs=true + fi + + if [ "${install_extras}" == "null" ]; then + install_extras="" + fi + echo "package-basepath=${package_basepath}" >> $GITHUB_OUTPUT + echo "install-extras=${install_extras}" >> $GITHUB_OUTPUT + echo "should-check-docs=${should_check_docs}" >> $GITHUB_OUTPUT + - name: Show package info + env: + STEP_OUTPUTS: ${{ toJSON(steps.parse_json.outputs) }} + run: | + echo "$STEP_OUTPUTS" diff --git a/.github/workflows/get_package_names.yml b/.github/workflows/get_package_names.yml new file mode 100644 index 0000000..714a062 --- /dev/null +++ b/.github/workflows/get_package_names.yml @@ -0,0 +1,33 @@ +name: Get package names + +on: + workflow_call: + outputs: + package-names: + description: "List of package names from .github/python-packages.json" + value: ${{ jobs.get_package_names.outputs.package-names }} + +permissions: {} + +jobs: + get_package_names: + name: Get package names + runs-on: ubuntu-latest + outputs: + package-names: ${{ steps.parse_json.outputs.package-names }} + steps: + - name: Check out repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Get package info from JSON + id: parse_json + # NOTE: jq only supports standard JSON, not JSONC or JSON5 + run: | + info=$(jq -c 'keys' ./.github/python-packages.json) + if [ -z "$info" ] || [ "$info" = "[]" ]; then + echo "Error: No package names found in .github/python-packages.json" >&2 + exit 1 + fi + echo "Package names: $info" + echo "package-names=$info" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml deleted file mode 100644 index eb2cbe3..0000000 --- a/.github/workflows/publish-to-pypi.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Publish Package to PyPI - -on: - release: - types: - - published - -jobs: - build-and-publish: - if: github.repository == 'ni/nisyscfg-python' - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Python 3.9 - uses: ni/python-actions/setup-python@dee640bba235ae28fdc6b7337c643bd06358ae90 # v0.9.0 - with: - python-version: 3.9 - - - name: Set up Poetry - uses: ni/python-actions/setup-poetry@aa64e60612cb078b0c2ada666becbd70d4817d55 # v0.7.1 - - - name: Build packages - run: poetry build - - - name: Publish distribution to Test PyPI - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 - with: - password: ${{ secrets.test_pypi_password }} - repository-url: https://test.pypi.org/legacy/ - continue-on-error: true - - - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 - with: - password: ${{ secrets.pypi_password }} \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..8438c1d --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,162 @@ +name: Publish to PyPI + +on: + release: + types: [published] + workflow_dispatch: + inputs: + environment: + description: The environment to publish to. + default: 'none' + required: true + type: choice + options: + - none + - pypi + - testpypi + package-name: + description: The name of the package to publish. + default: 'none' + required: true + type: choice + options: + - none + - nisyscfg + +permissions: {} + +env: + environment: ${{ case(github.event_name == 'release', 'pypi', inputs.environment) }} + environment-info: | + { + "pypi": { + "base-url": "https://pypi.org", + "upload-url": "https://upload.pypi.org/legacy/" + }, + "testpypi": { + "base-url": "https://test.pypi.org", + "upload-url": "https://test.pypi.org/legacy/" + } + } + +jobs: + get_publish_info: + name: Get publish info + runs-on: ubuntu-latest + permissions: {} + outputs: + package-name: ${{ steps.get_package_name_and_version.outputs.package-name }} + package-version: ${{ steps.get_package_name_and_version.outputs.package-version }} + steps: + - name: Get package name and package version + id: get_package_name_and_version + env: + EVENT_NAME: ${{ github.event_name }} + EVENT_TAG: ${{ github.event.release.tag_name }} + INPUT_PACKAGE_NAME: ${{ inputs.package-name }} + run: | + if [ "$EVENT_NAME" = "release" ]; then + if [[ ! "$EVENT_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Invalid release tag '$EVENT_TAG'. Expected format: v.." + exit 1 + fi + + echo "package-name=nisyscfg" >> "$GITHUB_OUTPUT" + echo "package-version=${EVENT_TAG#v}" >> "$GITHUB_OUTPUT" + else + echo "package-name=$INPUT_PACKAGE_NAME" >> "$GITHUB_OUTPUT" + echo "package-version=unused" >> "$GITHUB_OUTPUT" + fi + get_package_info: + name: Get package info for ${{ needs.get_publish_info.outputs.package-name }} + if: ${{ github.event_name == 'release' || inputs.package-name != 'none' }} + uses: ./.github/workflows/get_package_info.yml + needs: get_publish_info + permissions: + contents: read + with: + package-name: ${{ needs.get_publish_info.outputs.package-name }} + # TODO: Call analyze package post including analyze package file which uses ni/python-actions/analyze-project + # TODO: Call check docs post including check docs file which generates docs. + build_package: + name: Build ${{ needs.get_publish_info.outputs.package-name }} + if: ${{ github.event_name == 'release' || inputs.package-name != 'none' }} + runs-on: ubuntu-latest + needs: [get_publish_info, get_package_info] # TODO: Add analyze package and check docs to needs + steps: + - name: Check out repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Python + uses: ni/python-actions/setup-python@dee640bba235ae28fdc6b7337c643bd06358ae90 # v0.9.0 + - name: Set up Poetry + uses: ni/python-actions/setup-poetry@dee640bba235ae28fdc6b7337c643bd06358ae90 # v0.9.0 + with: + use-cache: false + - name: Check project version + if: github.event_name == 'release' + uses: ni/python-actions/check-project-version@dee640bba235ae28fdc6b7337c643bd06358ae90 # v0.9.0 + with: + project-directory: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }} + expected-version: ${{ needs.get_publish_info.outputs.package-version }} + - name: Build distribution packages + run: poetry build + working-directory: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }} + - name: Verify distribution packages + run: | + test -n "$(find dist -maxdepth 1 -type f -name '*.whl' -print -quit)" + test -n "$(find dist -maxdepth 1 -type f -name '*.tar.gz' -print -quit)" + ls -l dist + working-directory: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }} + - name: Upload build artifacts + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ needs.get_publish_info.outputs.package-name }}-distribution-packages + path: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }}/dist/* + publish_to_pypi: + name: Publish ${{ needs.get_publish_info.outputs.package-name }} + if: ${{ github.event_name == 'release' || inputs.environment != 'none' }} + runs-on: ubuntu-latest + needs: [get_publish_info, get_package_info, build_package] + environment: + name: ${{ case(github.event_name == 'release', 'pypi', inputs.environment) }} + url: ${{ fromJson(env.environment-info)[env.environment].base-url }}/p/${{ needs.get_publish_info.outputs.package-name }} + permissions: + id-token: write + steps: + - name: Download build artifacts + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ needs.get_publish_info.outputs.package-name }}-distribution-packages + path: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }}/dist/ + - name: Display distribution packages + run: ls -lR dist/ + working-directory: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }} + - name: Upload to ${{ env.environment }} + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + repository-url: ${{ fromJson(env.environment-info)[env.environment].upload-url }} + packages-dir: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }}/dist/ + update_version: + name: Update ${{ needs.get_publish_info.outputs.package-name }} version + if: ${{ github.event_name == 'release' || inputs.package-name != 'none' }} + runs-on: ubuntu-latest + needs: [get_publish_info, get_package_info, build_package] + permissions: + contents: write + pull-requests: write + steps: + - name: Check out repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Python + uses: ni/python-actions/setup-python@dee640bba235ae28fdc6b7337c643bd06358ae90 # v0.9.0 + - name: Set up Poetry + uses: ni/python-actions/setup-poetry@dee640bba235ae28fdc6b7337c643bd06358ae90 # v0.9.0 + - name: Update project version + uses: ni/python-actions/update-project-version@dee640bba235ae28fdc6b7337c643bd06358ae90 # v0.9.0 + with: + project-directory: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }} + branch-prefix: users/build/${{ needs.get_publish_info.outputs.package-name }}- \ No newline at end of file diff --git a/.github/workflows/report_test_results.yml b/.github/workflows/report_test_results.yml new file mode 100644 index 0000000..f144fd4 --- /dev/null +++ b/.github/workflows/report_test_results.yml @@ -0,0 +1,33 @@ +name: Report test results + +on: + workflow_call: + workflow_dispatch: + +permissions: {} + +jobs: + report_test_results: + name: Report test results + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + steps: + - name: Check out repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Download test results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: test_results + pattern: test_results_* + merge-multiple: true + - name: List downloaded files + run: ls -lR + - name: Publish test results + uses: EnricoMi/publish-unit-test-result-action@c950f6fb443cb5af20a377fd0dfaa78838901040 # v2.23.0 + with: + files: "test_results/**/*.xml" + if: always() diff --git a/.github/workflows/test-python-package.yml b/.github/workflows/test-python-package.yml deleted file mode 100644 index 072f303..0000000 --- a/.github/workflows/test-python-package.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Test Python Package - -on: [push, pull_request] - -jobs: - test: - - runs-on: ubuntu-latest - strategy: - max-parallel: 4 - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Set up Python ${{ matrix.python-version }} - uses: ni/python-actions/setup-python@dee640bba235ae28fdc6b7337c643bd06358ae90 # v0.9.0 - with: - python-version: ${{ matrix.python-version }} - - name: Set up Poetry - uses: ni/python-actions/setup-poetry@aa64e60612cb078b0c2ada666becbd70d4817d55 # v0.7.1 - - name: Install locked dependencies - run: poetry install -v - - name: Lint with flake8 - run: poetry run flake8 - - name: Test with pytest - run: poetry run pytest \ No newline at end of file diff --git a/.github/workflows/test_package.yml b/.github/workflows/test_package.yml new file mode 100644 index 0000000..32aa7a2 --- /dev/null +++ b/.github/workflows/test_package.yml @@ -0,0 +1,81 @@ +name: Test package +on: + workflow_call: + inputs: + package-name: + description: 'The name of the package folder to test.' + default: '' + required: true + type: string + workflow_dispatch: + inputs: + package-name: + description: 'The name of the package folder to test.' + default: '' + required: true + type: string + +permissions: {} + +jobs: + get_package_info: + name: Get package info - ${{ inputs.package-name }} + uses: ./.github/workflows/get_package_info.yml + with: + package-name: ${{ inputs.package-name }} + + run_unit_tests: + strategy: + # Fail-fast skews the pass/fail ratio and seems to make pytest produce + # incomplete JUnit XML results. + fail-fast: false + matrix: + os: [windows-latest, ubuntu-latest] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + name: Run unit tests - ${{ inputs.package-name }} (${{ matrix.python-version }} - ${{ matrix.os }}) + needs: get_package_info + runs-on: ${{ matrix.os }} + steps: + - name: Check out repo + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + submodules: true + - name: Get OS version + run: echo "osVersion=$ImageOS" >> "$GITHUB_ENV" + shell: bash + - name: Set up Python + uses: ni/python-actions/setup-python@aa64e60612cb078b0c2ada666becbd70d4817d55 # v0.7.1 + id: setup-python + with: + python-version: ${{ matrix.python-version }} + - name: Set up Poetry + uses: ni/python-actions/setup-poetry@aa64e60612cb078b0c2ada666becbd70d4817d55 # v0.7.1 + - name: Cache ${{ inputs.package-name }} virtualenv + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }}/.venv + key: ${{ inputs.package-name }}-${{ matrix.os }}-py${{ matrix.python-version }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install ${{ inputs.package-name }} + run: poetry install -v + working-directory: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }} + shell: bash + - name: Run ${{ inputs.package-name }} unit tests and code coverage + env: + PACKAGE_NAME: ${{ inputs.package-name }} + MATRIX_OS: ${{ matrix.os }} + PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }} + run: poetry run pytest -v --cov="$PACKAGE_NAME" --junitxml="test_results/$PACKAGE_NAME-$MATRIX_OS-py$PYTHON_VERSION.xml" + working-directory: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath }} + shell: bash + - name: Upload ${{ inputs.package-name }} test results + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: test_results_unit_${{ inputs.package-name }}_${{ matrix.os }}_py${{ steps.setup-python.outputs.python-version }} + path: ${{ github.workspace }}/${{ needs.get_package_info.outputs.package-basepath == '.' && 'test_results/*.xml' || format('{0}/test_results/*.xml', needs.get_package_info.outputs.package-basepath) }} + if: always() + + + + diff --git a/.github/workflows/test-wheel-build.yml b/.github/workflows/test_wheel_build.yml similarity index 86% rename from .github/workflows/test-wheel-build.yml rename to .github/workflows/test_wheel_build.yml index 9a59080..ad8e4bf 100644 --- a/.github/workflows/test-wheel-build.yml +++ b/.github/workflows/test_wheel_build.yml @@ -1,6 +1,10 @@ name: Test Building Python Package -on: [push, pull_request] +on: + workflow_call: + workflow_dispatch: + +permissions: {} jobs: build-and-upload-artifact: @@ -8,6 +12,8 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python 3.9 uses: ni/python-actions/setup-python@dee640bba235ae28fdc6b7337c643bd06358ae90 # v0.9.0