diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 179decec6..868e01cd6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -8,7 +8,6 @@ on: # yamllint disable-line rule:truthy push: branches: - main - paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: workflow_dispatch: @@ -41,7 +40,6 @@ concurrency: permissions: contents: read - id-token: write env: AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 @@ -76,6 +74,7 @@ jobs: uses: ./.github/workflows/_diskgraph-wheel.yml pytest: + name: pytest (${{ matrix.test_dir }}) needs: [diskgraph-wheel] if: ${{ !cancelled() }} runs-on: ubuntu-24.04 @@ -90,13 +89,9 @@ jobs: id-token: write env: KUMO_RUN_ADLS_TESTS: ${{ github.repository == 'kumo-ai/kumo' && contains(matrix.test_dir, 'test/io_utils/') && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/') || contains(matrix.test_dir, 'test/connector/') || contains(matrix.test_dir, 'test/rest/test_*.py')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} + KUMO_RUN_GCP_TESTS: '0' AZURE_STORAGE_ACCOUNT: ${{ vars.AZURE_STORAGE_ACCOUNT }} AZURE_CONTAINER_NAME: ${{ vars.AZURE_CONTAINER_NAME }} - GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} - GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT }} strategy: fail-fast: false @@ -144,7 +139,7 @@ jobs: "test/docker", "test/deploy/test_cloud_storage_preflight.py", "test/rest/integ/test_data_variety/test_table_lifecycle.py", - # GCS e2e tests moved to testing_gcs.yml (requires IS_GCP_DEPLOY=1) + # The profile plugin routes live GCP cases to testing_gcs.yml. "--splits 2 --group 1 test/rest/test_*.py", "--splits 2 --group 2 test/rest/test_*.py", "-k '[local]' test/rest/integ/test_diskgraph_e2e.py", @@ -159,7 +154,7 @@ jobs: outputs: # Push built coverage report to gh-pages only when pytest is triggered on main. - push-gh-pages: ${{ steps.non-pytest-changes.outputs.only_changed != 'true' }} + push-gh-pages: ${{ steps.test-impact.outputs.run == 'true' }} steps: - name: Checkout repository and submodules @@ -168,41 +163,35 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 + - name: Determine test impact + id: test-impact + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + HEAD_SHA: ${{ github.sha }} + run: | + # Missing history and manual runs execute conservatively. + if [ -z "$BASE_SHA" ] || ! git diff --name-only -z "$BASE_SHA" "$HEAD_SHA" -- > "$RUNNER_TEMP/test-profile-changes"; then + echo 'run=true' >> "$GITHUB_OUTPUT" + else + python3 -c 'import sys; from tools.ci_test_profiles import integration_required; paths = [p for p in sys.stdin.buffer.read().decode("utf-8", "surrogateescape").split("\0") if p]; print("run=" + str(integration_required(paths)).lower())' < "$RUNNER_TEMP/test-profile-changes" >> "$GITHUB_OUTPUT" + fi + - name: Set up Python + if: env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true' uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "3.14" activate-environment: true - # This enables us to skip the minimal python tests. - # Note, it's not possible use the standard `on.pull_request` method to - # do this, as we need to mark the overall workflow as success to satisfy - # the main branch protection rule, and this does not happen if we skip - # the entire workflow - # - # uv.lock is deliberately NOT in this list: `uv lock --check` (the - # uv_lock job) only proves the lock's metadata is fresh, not that the - # resolved versions actually install or import cleanly. A lock-only - # dependency bump must still run the real install + test suite. - - name: Check for only non pytest required file changes - id: non-pytest-changes - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ui/** - CHANGELOG.md - tools/** - benchmark/** - - name: Configure AWS credentials - if: env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true' + if: env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true' uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests aws-region: us-west-2 - name: Set up packages - if: env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true' + if: env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true' uses: ./.github/actions/setup with: github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} @@ -210,22 +199,15 @@ jobs: install-graphviz: ${{ matrix.test_dir == 'test/graph/' }} - name: Log in to Azure with GitHub OIDC - if: (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') && env.KUMO_RUN_ADLS_TESTS == '1' + if: (env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true') && env.KUMO_RUN_ADLS_TESTS == '1' uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 with: client-id: ${{ vars.AZURE_CLIENT_ID }} tenant-id: ${{ vars.AZURE_TENANT_ID }} subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} - - name: Authenticate to Google Cloud with GitHub OIDC - if: (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') && env.KUMO_RUN_GCP_TESTS == '1' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - project_id: ${{ vars.GCP_PROJECT_ID }} - workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - - name: Set up coverage artifact name + if: env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true' run: | # This name must avoid using special chars and must be unique per: # run id (to collect coverage results in the subsequent job) @@ -233,6 +215,7 @@ jobs: echo "ARTIFACT_NAME_SAFE=.coverage-${{ github.run_id }}-$(echo -n ${{ matrix.test_dir }} | sha256sum | awk '{print $1}')" >> $GITHUB_ENV - name: Set up flake diagnostics + if: env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true' run: | mkdir -p "$ARTIFACTS_DIR" # Ubuntu runners default to kernel.yama.ptrace_scope=1, which @@ -245,7 +228,7 @@ jobs: uv pip install py-spy || true - name: Run minimal test suite - if: env.RUN_FULL_TEST != 'true' && (env.FORCE_RUN_TESTS == 'true' || steps.non-pytest-changes.outputs.only_changed != 'true') + if: env.RUN_FULL_TEST != 'true' && (env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true') timeout-minutes: 60 run: | set -o pipefail @@ -258,7 +241,7 @@ jobs: # exceeds the per-group duration target (e.g. # test_db_file_storage::test_compression at ~511s in # test/rest/test_*.py). No-op when --splits is not in matrix.test_dir. - GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=1800 --timeout-method=signal --splitting-algorithm least_duration ${{ env.STORE_DURATIONS == 'true' && '--store-durations --durations-path=.test_durations' || '' }} ${{ matrix.test_dir }} + GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest -p tools.pytest_test_profiles --test-profile general "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=1800 --timeout-method=signal --splitting-algorithm least_duration ${{ env.STORE_DURATIONS == 'true' && '--store-durations --durations-path=.test_durations' || '' }} ${{ matrix.test_dir }} move_coverage_artifact # Mirror local-test-relevant additions in setup.cfg [tool:pytest] env. env: @@ -267,8 +250,7 @@ jobs: KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: "0" SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} + KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: 'false' AWS_DEFAULT_REGION: "us-west-2" DEPLOY_STAGE: "test" TEMPORAL_SERVER: "localhost" @@ -281,7 +263,7 @@ jobs: DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'test/rest/integ/test_diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Run full test suite - if: env.RUN_FULL_TEST == 'true' && steps.non-pytest-changes.outputs.only_changed != 'true' + if: env.RUN_FULL_TEST == 'true' && (env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true') timeout-minutes: 90 run: | set -o pipefail @@ -291,7 +273,7 @@ jobs: # --timeout=4500 fires below the 90-min step cap so pytest-timeout # gets a chance to run; --timeout-method=signal so the on-failure # flake-diagnostics hook runs before the process exits. - FULL_TEST=1 GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=4500 --timeout-method=signal --splitting-algorithm least_duration ${{ matrix.test_dir }} + FULL_TEST=1 GLOG_logtostdout=1 python -X faulthandler -m coverage run -m pytest -p tools.pytest_test_profiles --test-profile general "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=4500 --timeout-method=signal --splitting-algorithm least_duration ${{ matrix.test_dir }} move_coverage_artifact env: # Set to "1" in a PR branch to re-enable io_utils.fsutil DEBUG logs @@ -299,8 +281,7 @@ jobs: KUMO_PYTEST_ENABLE_FSUTIL_DEBUG_LOGS: "0" SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} + KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: 'false' AWS_DEFAULT_REGION: "us-west-2" DEPLOY_STAGE: "test" TEMPORAL_SERVER: "localhost" @@ -312,7 +293,7 @@ jobs: DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'test/rest/integ/test_diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Upload coverage artifact - if: always() + if: always() && env.ARTIFACT_NAME_SAFE != '' && hashFiles(env.ARTIFACT_NAME_SAFE) != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: ${{ env.ARTIFACT_NAME_SAFE }} @@ -783,6 +764,109 @@ jobs: path: .test_durations include-hidden-files: true + pytest-gcp: + environment: + name: gha-kumo-workflow-tests + # This job uses environment-scoped OIDC; it does not deploy an application. + deployment: false + name: pytest-gcp (${{ matrix.test_dir }}) + needs: [diskgraph-wheel] + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + env: + COVERAGE_PREFIX: .coverage-${{ github.run_id }}-gcp-${{ strategy.job-index }} + GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} + GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} + GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ matrix.restricted_identities && vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT || '' }} + GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ matrix.restricted_identities && vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT || '' }} + + strategy: + fail-fast: false + matrix: + include: + - test_dir: test/connector/test_bigquery_connector.py + profile: gcp + aws: false + restricted_identities: true + - test_dir: test/io_utils/fsutil/test_gcs.py test/io_utils/fsutil/test_all.py + profile: gcp + aws: false + restricted_identities: false + - test_dir: test/rest/test_tables.py::test_validate_size + profile: gcp + aws: false + restricted_identities: false + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + ref: ${{ github.sha }} + fetch-depth: 40 + + - name: Determine test impact + id: test-impact + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + HEAD_SHA: ${{ github.sha }} + run: | + # Missing history and manual runs execute conservatively. + if [ -z "$BASE_SHA" ] || ! git diff --name-only -z "$BASE_SHA" "$HEAD_SHA" -- > "$RUNNER_TEMP/test-profile-changes"; then + echo 'run=true' >> "$GITHUB_OUTPUT" + else + python3 -c 'import sys; from tools.ci_test_profiles import integration_required; paths = [p for p in sys.stdin.buffer.read().decode("utf-8", "surrogateescape").split("\0") if p]; print("run=" + str(integration_required(paths)).lower())' < "$RUNNER_TEMP/test-profile-changes" >> "$GITHUB_OUTPUT" + fi + + # Package/wheel bootstrap uses public indexes and GitHub cache/artifacts. + # Cloud credentials are acquired only after dependency installation. + - name: Set up packages + if: env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true' + uses: ./.github/actions/setup + with: + diskgraph-wheel-artifact: diskgraph-wheel + + - name: Authenticate to Google Cloud with GitHub OIDC + if: env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + project_id: ${{ vars.GCP_PROJECT_ID }} + workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} + + - name: Run GCP profile tests + if: env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true' + timeout-minutes: 60 + run: | + set -o pipefail + export SPARK_LOCAL_IP="127.0.0.1" + source ./tools/ci_pytest_log_controls.sh + configure_pytest_ci_output + python -X faulthandler -m tools.ci_test_profiles run "${{ matrix.profile }}" --coverage-file "$COVERAGE_PREFIX" "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=1800 --timeout-method=thread ${{ matrix.test_dir }} + env: + GCS_TEST_BUCKET: 'kumo-test-bucket' + GCP_STORAGE_TRANSFER_BUCKET: 'kumo-test-bucket' + FULL_TEST: ${{ env.RUN_FULL_TEST == 'true' && '1' || '0' }} + GLOG_logtostdout: '1' + DEPLOY_STAGE: "test" + TEMPORAL_SERVER: "localhost" + DATA_SPARK_MEM: "1g" + MAX_DATA_BACKEND_WORKER_PROCESSES: "1" + CUDA: "cpu" + KUMO_ENABLE_GLOVE: "False" + SPARK_VERSION: "3.5" + KUMO_TESTING: "True" + + - name: Upload GCP coverage artifact + if: always() && hashFiles(format('{0}*', env.COVERAGE_PREFIX)) != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ env.COVERAGE_PREFIX }} + path: ${{ env.COVERAGE_PREFIX }}* + include-hidden-files: true + coverage: runs-on: ubuntu-24.04 permissions: @@ -795,8 +879,8 @@ jobs: group: coverage-gh-pages cancel-in-progress: false continue-on-error: true - needs: [pytest] - if: ${{ always() && needs.pytest.result != 'skipped' && needs.pytest.result != 'cancelled' && needs.pytest.outputs.push-gh-pages == 'true'}} + needs: [pytest, pytest-gcp] + if: ${{ always() && needs.pytest-gcp.result == 'success' && needs.pytest.result != 'skipped' && needs.pytest.result != 'cancelled' && needs.pytest.outputs.push-gh-pages == 'true'}} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 diff --git a/.github/workflows/testing_gcs.yml b/.github/workflows/testing_gcs.yml index a49396bae..699721018 100644 --- a/.github/workflows/testing_gcs.yml +++ b/.github/workflows/testing_gcs.yml @@ -8,7 +8,6 @@ on: # yamllint disable-line rule:truthy push: branches: - main - paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: workflow_dispatch: @@ -26,16 +25,6 @@ concurrency: permissions: contents: read - id-token: write - -env: - AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - GITHUB_REF: ${{secrets.GITHUB_REF}} - GITHUB_API_URL: ${{secrets.GITHUB_API_URL}} - GITHUB_REPOSITORY: ${{secrets.GITHUB_REPOSITORY}} - GITHUB_REPOSITORY_OWNER: ${{secrets.GITHUB_REPOSITORY_OWNER}} - GITHUB_RUN_ID: ${{secrets.GITHUB_RUN_ID}} jobs: pytest-gcs: @@ -43,25 +32,38 @@ jobs: name: gha-kumo-workflow-tests # This job uses environment-scoped OIDC; it does not deploy an application. deployment: false - name: gcs-e2e-tests + # Keep the three existing e2e check labels when adding capability fields. + name: gcs-e2e-tests (${{ matrix.test_dir }}) runs-on: ubuntu-24.04 - continue-on-error: true permissions: contents: read id-token: write env: - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} + GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ matrix.restricted_identities && vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT || '' }} + GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ matrix.restricted_identities && vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT || '' }} strategy: fail-fast: false matrix: - test_dir: [ - test/rest/integ/test_gcs_e2e.py, - test/rest/integ/test_bigquery_diskgraph_e2e.py, - test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py, - ] + include: + - test_dir: test/rest/integ/test_gcs_e2e.py + profile: gcp + aws: false + restricted_identities: false + - test_dir: test/rest/integ/test_bigquery_diskgraph_e2e.py + profile: gcp-aws + aws: true + restricted_identities: false + - test_dir: test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py + profile: gcp + aws: false + restricted_identities: false + - test_dir: test/rest/integ/test_gcs_public_api_e2e.py + profile: gcp + aws: false + restricted_identities: false steps: - name: Checkout repository and submodules @@ -70,80 +72,56 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 - - name: Get GCS related changes - id: gcs-files-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47 - with: - files: | - .github/actions/setup/action.yml - .github/workflows/testing_gcs.yml - kumo/connector/** - kumo/testing/gcs.py - io_utils/** - rest/** - setup.py - test/rest/integ/test_gcs_e2e.py - test/rest/integ/test_bigquery_diskgraph_e2e.py - test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py - test/rest/integ/test_cases/test_bigquery_e2e_fixtures.json - rest/devserver/configs/diskgraph-test.json - kumo/testing/bigquery.py - kumo/testing/rest.py + - name: Determine test impact + id: test-impact + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + HEAD_SHA: ${{ github.sha }} + run: | + # Missing history and manual runs execute conservatively. + if [ -z "$BASE_SHA" ] || ! git diff --name-only -z "$BASE_SHA" "$HEAD_SHA" -- > "$RUNNER_TEMP/test-profile-changes"; then + echo 'run=true' >> "$GITHUB_OUTPUT" + else + python3 -c 'import sys; from tools.ci_test_profiles import integration_required; paths = [p for p in sys.stdin.buffer.read().decode("utf-8", "surrogateescape").split("\0") if p]; print("run=" + str(integration_required(paths)).lower())' < "$RUNNER_TEMP/test-profile-changes" >> "$GITHUB_OUTPUT" + fi - - name: Set up Python - if: steps.gcs-files-changed.outputs.any_changed == 'true' - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - with: - python-version: "3.14" - activate-environment: true + # Package/wheel bootstrap uses public indexes and GitHub cache/artifacts. + # Cloud credentials are acquired only after dependency installation. + - name: Set up packages + if: steps.test-impact.outputs.run == 'true' + uses: ./.github/actions/setup - - name: Configure AWS credentials - if: steps.gcs-files-changed.outputs.any_changed == 'true' + - name: Configure AWS credentials for the S3 dataplane + if: steps.test-impact.outputs.run == 'true' && matrix.aws uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests aws-region: us-west-2 - - name: Set up packages - if: steps.gcs-files-changed.outputs.any_changed == 'true' && !contains(matrix.test_dir, 'diskgraph') - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - - name: Set up packages (with diskgraph) - if: steps.gcs-files-changed.outputs.any_changed == 'true' && contains(matrix.test_dir, 'diskgraph') - uses: ./.github/actions/setup - with: - github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - name: Authenticate to Google Cloud with GitHub OIDC - if: steps.gcs-files-changed.outputs.any_changed == 'true' && env.KUMO_RUN_GCP_TESTS == '1' + if: steps.test-impact.outputs.run == 'true' uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 with: project_id: ${{ vars.GCP_PROJECT_ID }} workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - - name: Run E2E tests - if: steps.gcs-files-changed.outputs.any_changed == 'true' + - name: Run GCP profile tests + if: steps.test-impact.outputs.run == 'true' timeout-minutes: 60 run: | set -o pipefail export SPARK_LOCAL_IP="127.0.0.1" source ./tools/ci_pytest_log_controls.sh configure_pytest_ci_output - GLOG_logtostdout=1 python -X faulthandler -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=1800 --timeout-method=thread ${{ matrix.test_dir }} + python -X faulthandler -m tools.ci_test_profiles run "${{ matrix.profile }}" "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=50 --timeout=1800 --timeout-method=thread ${{ matrix.test_dir }} env: - # GCS-specific environment variables - IS_GCP_DEPLOY: "1" GCS_TEST_BUCKET: 'kumo-test-bucket' GCP_STORAGE_TRANSFER_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: "true" - DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - # Standard test environment variables - SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - AWS_DEFAULT_REGION: "us-west-2" + AWS_TEST_ROLE_ARN: ${{ matrix.aws && 'arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3' || '' }} + AWS_DEFAULT_REGION: ${{ matrix.aws && 'us-west-2' || '' }} + FULL_TEST: ${{ (github.event_name == 'push' || inputs.full_test == true) && '1' || '0' }} + GLOG_logtostdout: '1' DEPLOY_STAGE: "test" TEMPORAL_SERVER: "localhost" DATA_SPARK_MEM: "1g" diff --git a/.github/workflows/testing_snowflake.yml b/.github/workflows/testing_snowflake.yml index e665ef75e..6d9466bcf 100644 --- a/.github/workflows/testing_snowflake.yml +++ b/.github/workflows/testing_snowflake.yml @@ -8,7 +8,6 @@ on: # yamllint disable-line rule:truthy push: branches: - main - paths-ignore: ["ui/**", "tools/**"] pull_request: merge_group: workflow_dispatch: @@ -26,7 +25,6 @@ concurrency: permissions: contents: read - id-token: write env: AWS_TEST_ROLE_ARN: arn:aws:iam::926922431314:role/gha-kumo-preprod-test-s3 @@ -43,18 +41,14 @@ jobs: name: gha-kumo-workflow-tests # This job uses environment-scoped OIDC; it does not deploy an application. deployment: false - name: all-snowflake-tests + name: all-snowflake-tests (${{ matrix.test_name }}) runs-on: ubuntu-24.04 continue-on-error: true permissions: contents: read id-token: write env: - KUMO_RUN_GCP_TESTS: ${{ github.repository == 'kumo-ai/kumo' && (contains(matrix.test_dir, 'test/io_utils/fsutil') || contains(matrix.test_dir, 'test/connector') || contains(matrix.test_dir, 'test/rest')) && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && '1' || '0' }} - GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} - GCP_TEST_SERVICE_ACCOUNT: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_TABLE_SERVICE_ACCOUNT }} - GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT: ${{ vars.GCP_BIGQUERY_NO_JOB_SERVICE_ACCOUNT }} + KUMO_RUN_GCP_TESTS: '0' strategy: matrix: @@ -72,78 +66,20 @@ jobs: include: - test_name: 'connector (test/connector)' test_dir: 'test/connector' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/connector/** - setup.py - test/connector/** - test_name: 'rest (test/rest)' test_dir: 'test/rest --ignore=test/rest/integ/test_snowflake_diskgraph_e2e.py --ignore=test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - setup.py - test/rest/** - test_name: 'fsutil (test/io_utils/fsutil)' test_dir: 'test/io_utils/fsutil' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - io_utils/** - setup.py - test/io_utils/fsutil/** - test_name: 'backend - data (test/data/ test/graph/' test_dir: 'test/data/ test/graph/**snowflake**' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/data/** - kumo/stats/** - io_utils/** - setup.py - test/data/ - test/graph/**snowflake** - test_name: 'backend - pquery (test/pquery/bp_generator)' test_dir: 'test/pquery/test_pql_batch_pred_generator.py' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/data/** - kumo/pquery/** - io_utils/** - kumo/data/data_processing_backend/** - setup.py - test/pquery/test_pql_batch_pred_generator.py - test_name: 'backend - pquery (test/pquery/train_table_generator)' test_dir: 'test/pquery/train_table_generator' - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/data/** - kumo/pquery/** - io_utils/** - setup.py - test/pquery/train_table_generator/** - test_name: 'snowflake diskgraph e2e (test/rest/integ/test_snowflake_diskgraph_e2e.py)' test_dir: "test/rest/integ/test_snowflake_diskgraph_e2e.py" - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/connector/** - rest/backends/data_backend/activities/** - test/rest/integ/test_snowflake_diskgraph_e2e.py - test/rest/integ/test_cases/test_snowflake_e2e_fixtures.json - rest/devserver/configs/diskgraph-test.json - test_name: 'snowflake ingestion diskgraph e2e (test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py)' test_dir: "test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py" - files_changed: | - .github/actions/setup/action.yml - .github/workflows/testing_snowflake.yml - kumo/connector/** - rest/backends/data_backend/activities/** - test/rest/integ/test_snowflake_ingestion_diskgraph_e2e.py - rest/devserver/configs/diskgraph-test.json steps: @@ -153,72 +89,68 @@ jobs: ref: ${{ github.sha }} fetch-depth: 40 - - name: Get any data backend related changes - id: data-backend-files-changed - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47.0.0 - with: - files: | - ${{ matrix.files_changed }} + - name: Determine test impact + id: test-impact + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + HEAD_SHA: ${{ github.sha }} + run: | + # Missing history and manual runs execute conservatively. + if [ -z "$BASE_SHA" ] || ! git diff --name-only -z "$BASE_SHA" "$HEAD_SHA" -- > "$RUNNER_TEMP/test-profile-changes"; then + echo 'run=true' >> "$GITHUB_OUTPUT" + else + python3 -c 'import sys; from tools.ci_test_profiles import integration_required; paths = [p for p in sys.stdin.buffer.read().decode("utf-8", "surrogateescape").split("\0") if p]; print("run=" + str(integration_required(paths)).lower())' < "$RUNNER_TEMP/test-profile-changes" >> "$GITHUB_OUTPUT" + fi - name: Configure AWS credentials - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' + if: steps.test-impact.outputs.run == 'true' uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests aws-region: us-west-2 - name: Set up packages - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && !contains(matrix.test_dir, 'diskgraph') && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' + if: steps.test-impact.outputs.run == 'true' && !contains(matrix.test_dir, 'diskgraph') uses: ./.github/actions/setup with: github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - name: Set up packages (with diskgraph) - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && contains(matrix.test_dir, 'diskgraph') && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' + if: steps.test-impact.outputs.run == 'true' && contains(matrix.test_dir, 'diskgraph') uses: ./.github/actions/setup with: github_token: ${{ secrets.KUMO_GITHUB_BOT_TOKEN }} - - name: Authenticate to Google Cloud with GitHub OIDC - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' && env.KUMO_RUN_GCP_TESTS == '1' - uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 - with: - project_id: ${{ vars.GCP_PROJECT_ID }} - workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} - service_account: ${{ vars.GCP_TEST_SERVICE_ACCOUNT }} - # NOTE here we use coverage.py instead of pytest-cov to obtain JSON # outputs that are easier to parse and manage (pytest-cov/455). # coverage.py also has additional nice features that may be useful in the # future for better coverage reports. - name: Run minimal test suite - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && github.event_name != 'push' && inputs.full_test != true && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' + if: steps.test-impact.outputs.run == 'true' && github.event_name != 'push' && inputs.full_test != true timeout-minutes: 120 run: | set -o pipefail export SPARK_LOCAL_IP="127.0.0.1" source ./tools/ci_pytest_log_controls.sh configure_pytest_ci_output - GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=30 --runsnowflake ${{ matrix.test_dir }} + GLOG_logtostdout=1 python -m coverage run -m pytest -p tools.pytest_test_profiles --test-profile snowflake "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=30 --runsnowflake ${{ matrix.test_dir }} env: SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} + KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: 'false' AWS_DEFAULT_REGION: "us-west-2" DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} - name: Run full test suite - if: steps.data-backend-files-changed.outputs.any_changed == 'true' && (github.event_name == 'push' || inputs.full_test == true) && steps.only-ui-files-changed.outputs.only_changed != 'true' && steps.only-rest-files-changed.outputs.only_changed != 'true' + if: steps.test-impact.outputs.run == 'true' && (github.event_name == 'push' || inputs.full_test == true) timeout-minutes: 120 run: | set -o pipefail export SPARK_LOCAL_IP="127.0.0.1" source ./tools/ci_pytest_log_controls.sh configure_pytest_ci_output - FULL_TEST=1 GLOG_logtostdout=1 python -m coverage run -m pytest "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=30 --runsnowflake ${{ matrix.test_dir }} + FULL_TEST=1 GLOG_logtostdout=1 python -m coverage run -m pytest -p tools.pytest_test_profiles --test-profile snowflake "${PYTEST_CI_OUTPUT_ARGS[@]}" --durations=30 --runsnowflake ${{ matrix.test_dir }} env: SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} - GCS_TEST_BUCKET: 'kumo-test-bucket' - KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: ${{ env.KUMO_RUN_GCP_TESTS == '1' && 'true' || 'false' }} + KUMO_GCP_WORKLOAD_EXCHANGE_ENABLED: 'false' AWS_DEFAULT_REGION: "us-west-2" DEPLOY_ENV_CONFIG_FILE: ${{ contains(matrix.test_dir, 'diskgraph') && 'rest/devserver/configs/diskgraph-test.json' || '' }} diff --git a/test/scripts/test_ci_workflow_profiles.py b/test/scripts/test_ci_workflow_profiles.py new file mode 100644 index 000000000..786c8df96 --- /dev/null +++ b/test/scripts/test_ci_workflow_profiles.py @@ -0,0 +1,194 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import shlex +import subprocess + +import pytest +import yaml + +from io_utils import fsutil +from tools.ci_test_profiles import GCP_AWS_MODULE, permits_target, profile_for + +WORKFLOWS = ( + ('.github/workflows/testing.yml', 'pytest', 'general'), + ('.github/workflows/testing_snowflake.yml', 'pytest', 'snowflake'), + ('.github/workflows/testing_gcs.yml', 'pytest-gcs', 'gcp'), +) + + +@pytest.fixture +def workflows(): + return { + profile: yaml.safe_load(fsutil.read_text(path)) + for path, _, profile in WORKFLOWS + } + + +def test_existing_check_names_are_stable(workflows): + assert workflows['general']['name'] == 'Testing and Coverage' + assert workflows['snowflake']['name'] == 'Snowflake Testing and Coverage' + assert workflows['gcp']['name'] == 'GCS Testing' + assert workflows['general']['jobs']['pytest']['name'] == ( + 'pytest (${{ matrix.test_dir }})') + assert workflows['snowflake']['jobs']['pytest']['name'] == ( + 'all-snowflake-tests (${{ matrix.test_name }})') + gcp = workflows['gcp']['jobs']['pytest-gcs'] + assert gcp['name'] == 'gcs-e2e-tests (${{ matrix.test_dir }})' + labels = [row['test_dir'] for row in gcp['strategy']['matrix']['include']] + for existing in ( + 'test/rest/integ/test_gcs_e2e.py', + 'test/rest/integ/test_bigquery_diskgraph_e2e.py', + 'test/rest/integ/test_bigquery_ingestion_diskgraph_e2e.py'): + assert labels.count(existing) == 1 + + +def test_gcp_rows_cover_registry_once(workflows): + gcp = workflows['gcp']['jobs']['pytest-gcs'] + migrated = workflows['general']['jobs']['pytest-gcp'] + rows = (gcp['strategy']['matrix']['include'] + + migrated['strategy']['matrix']['include']) + assert len(rows) == 7 + selected = [] + for row in rows: + profile = profile_for(row['profile']) + assert row['aws'] == ('aws' in profile.capabilities) + for target in shlex.split(row['test_dir']): + assert target in profile.targets + assert permits_target(profile, target) + selected.append(target) + assert row['restricted_identities'] == ( + row['test_dir'] == 'test/connector/test_bigquery_connector.py') + assert len(selected) == len(set(selected)) + assert set(selected) == set(profile_for('gcp').targets) | {GCP_AWS_MODULE} + + +def test_google_auth_cannot_block_other_providers(workflows): + for name in ('general', 'snowflake'): + job = workflows[name]['jobs']['pytest'] + assert job['continue-on-error'] is True + assert job['env']['KUMO_RUN_GCP_TESTS'] == '0' + assert not any(step.get('uses', '').startswith('google-github-actions/') + for step in job['steps']) + commands = [step['run'] for step in job['steps'] + if '-m pytest' in step.get('run', '')] + assert len(commands) == 2 + for command in commands: + assert '-m coverage run -m pytest' in command + assert f'-p tools.pytest_test_profiles --test-profile {name}' in command + assert 'tools.ci_test_profiles run' not in command + assert 'gcp' not in str(job.get('needs', [])) + assert 'kumo-core-gpu-ci' not in str(workflows) + + +def test_selected_gcp_auth_failure_is_not_masked(workflows): + job = workflows['gcp']['jobs']['pytest-gcs'] + assert not job.get('continue-on-error', False) + assert job['strategy']['fail-fast'] is False + steps = job['steps'] + setup = next(i for i, step in enumerate(steps) + if step.get('uses') == './.github/actions/setup') + aws = next(i for i, step in enumerate(steps) + if step.get('uses', '').startswith('aws-actions/')) + google = next(i for i, step in enumerate(steps) + if step.get('uses', '').startswith('google-github-actions/')) + tests = next(i for i, step in enumerate(steps) + if 'tools.ci_test_profiles run' in step.get('run', '')) + assert setup < aws < google < tests + assert steps[aws]['if'] == "steps.test-impact.outputs.run == 'true' && matrix.aws" + for index in (setup, google, tests): + assert steps[index]['if'] == "steps.test-impact.outputs.run == 'true'" + assert not steps[index].get('continue-on-error', False) + assert 'SNOWFLAKE' not in str(job) + assert 'GOOGLE_CREDENTIALS_STR_JSON' not in str(job) + assert not any('upload-artifact' in step.get('uses', '') for step in steps) + + +def test_oidc_permission_is_scoped_to_cloud_jobs(workflows): + for _, job_id, profile in WORKFLOWS: + workflow = workflows[profile] + assert workflow['permissions'] == {'contents': 'read'} + assert workflow['jobs'][job_id]['permissions']['id-token'] == 'write' + events = workflow.get('on', workflow.get(True)) + assert 'paths-ignore' not in events['push'] + assert events['pull_request'] is None + + +@pytest.mark.parametrize('changed,missing,expected', [ + ('docs/development/testing.md', False, 'false'), + ('tools/ci_test_profiles.py', False, 'true'), + ('uv.lock', False, 'true'), + ('docs/executable.py', False, 'true'), + ('ui/application.ts', False, 'true'), + ('test/path with spaces.py', False, 'true'), + ('docs/development/testing.md', True, 'true'), +]) +def test_workflow_impact_gate(workflows, tmp_path, changed, missing, expected): + scripts = [ + next(step['run'] for step in workflows[profile]['jobs'][job]['steps'] + if step.get('id') == 'test-impact') + for _, job, profile in WORKFLOWS + ] + assert len(set(scripts)) == 1 + output = str(tmp_path / 'output') + fsutil.write_text(output, '') + environment = { + 'PATH': '/usr/bin:/bin', 'PYTHONPATH': os.getcwd(), + 'GITHUB_OUTPUT': output, 'RUNNER_TEMP': str(tmp_path), + 'BASE_SHA': 'base', 'HEAD_SHA': 'head', 'CHANGED_PATH': changed, + } + git = ('git() { return 1; }\n' if missing else + 'git() { printf "%s\\0" "$CHANGED_PATH"; }\n') + result = subprocess.run(['/bin/bash', '-e', '-c', git + scripts[0]], + env=environment, check=False, capture_output=True, + text=True) + assert result.returncode == 0, result.stderr + assert fsutil.read_text(output).strip() == f'run={expected}' + + +def test_missing_coverage_has_no_upload_error(workflows): + step = next(step for step in workflows['general']['jobs']['pytest']['steps'] + if step['name'] == 'Upload coverage artifact') + assert "env.ARTIFACT_NAME_SAFE != ''" in step['if'] + assert "hashFiles(env.ARTIFACT_NAME_SAFE) != ''" in step['if'] + + +def test_migrated_coverage_joins_existing_report(workflows): + jobs = workflows['general']['jobs'] + job = jobs['pytest-gcp'] + assert job['permissions'] == {'contents': 'read', 'id-token': 'write'} + assert not job.get('continue-on-error', False) + assert job['strategy']['fail-fast'] is False + assert job['needs'] == ['diskgraph-wheel'] + rows = job['strategy']['matrix']['include'] + assert len(rows) == 3 + assert all(row['profile'] == 'gcp' and not row['aws'] for row in rows) + prefix = job['env']['COVERAGE_PREFIX'] + assert prefix == '.coverage-${{ github.run_id }}-gcp-${{ strategy.job-index }}' + steps = job['steps'] + assert not any(step.get('uses', '').startswith('aws-actions/') for step in steps) + setup = next(i for i, step in enumerate(steps) + if step.get('uses') == './.github/actions/setup') + auth = next(i for i, step in enumerate(steps) + if step.get('uses', '').startswith('google-github-actions/')) + execute = next(i for i, step in enumerate(steps) + if 'tools.ci_test_profiles run' in step.get('run', '')) + assert setup < auth < execute + assert '--coverage-file "$COVERAGE_PREFIX"' in steps[execute]['run'] + for index in (setup, auth, execute): + assert steps[index]['if'] == ( + "env.FORCE_RUN_TESTS == 'true' || steps.test-impact.outputs.run == 'true'") + assert not steps[index].get('continue-on-error', False) + upload = next(step for step in steps + if step.get('name') == 'Upload GCP coverage artifact') + assert upload['with']['path'] == '${{ env.COVERAGE_PREFIX }}*' + assert upload['with']['include-hidden-files'] is True + assert 'hashFiles' in upload['if'] + coverage = jobs['coverage'] + assert coverage['needs'] == ['pytest', 'pytest-gcp'] + assert "needs.pytest-gcp.result == 'success'" in coverage['if'] + download = next(step for step in coverage['steps'] + if step.get('name') == 'Download coverage data') + assert download['with']['pattern'] == '.coverage-${{ github.run_id }}-*'