_example_tests_runner.yml

# Reusable workflow for running example tests
name: Example Tests Runner

on:
  workflow_call:
    inputs:
      docker_image:
        description: "Docker image to use for tests"
        required: true
        type: string
      container_options:
        description: "Docker options for the test container"
        required: false
        type: string
        default: "--shm-size=2gb"
      example:
        description: "Example name to test (e.g. 'hf_ptq')"
        required: true
        type: string
      timeout_minutes:
        description: "Timeout in minutes for the job"
        required: false
        type: number
        default: 60
      pip_install_extras:
        description: "Pip install extras (e.g. '[hf,dev-test]' or '[all,dev-test]')"
        required: false
        type: string
        default: "[all,dev-test]"
      runner:
        description: "GitHub runner to use"
        required: false
        type: string
        default: "linux-amd64-gpu-rtxpro6000-latest-1"
      allow_failure:
        description: "If true, test failures are reported as a warning and do not fail the job (used to keep a known-broken example non-blocking)"
        required: false
        type: boolean
        default: false

jobs:
  run-test:
    runs-on: ${{ inputs.runner }}
    timeout-minutes: ${{ inputs.timeout_minutes }}
    permissions:
      contents: read
    container:
      image: ${{ inputs.docker_image }}
      options: ${{ inputs.container_options }} # TRT-LLM tests on 2-GPU runner need more shared memory
      env:
        PIP_CONSTRAINT: "" # Disable pip constraint for upgrading packages
        HF_TOKEN: ${{ secrets.HF_TOKEN }}
        # Build CUDA kernels only for the runner's RTX PRO 6000 (sm_120), not the image's ~6 archs.
        TORCH_CUDA_ARCH_LIST: "12.0"
    steps:
      - name: Install git for vLLM image
        if: startsWith(inputs.docker_image, 'vllm/') || startsWith(inputs.docker_image, 'docker.io/vllm/')
        run: apt-get update && apt-get install -y git
      - uses: actions/checkout@v6
      - uses: nv-gha-runners/setup-proxy-cache@main
      - uses: ./.github/actions/cache-extensions
        id: ext-cache
        with:
          # Per example: jobs sharing an image build different extensions, and the first to save wins the key.
          cache-key: rtxpro6000-example-${{ inputs.example }}-${{ inputs.docker_image }}
      - name: Setup environment variables
        run: |
          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/include:/usr/lib/x86_64-linux-gnu:/usr/local/tensorrt/targets/x86_64-linux-gnu/lib" >> $GITHUB_ENV
          echo "PATH=${PATH}:/usr/local/tensorrt/targets/x86_64-linux-gnu/bin" >> $GITHUB_ENV
      - name: Install dependencies
        shell: bash
        run: |
          # Uninstall conflicting system-wide installed modelopt in nemo containers
          pip uninstall -y nvidia-modelopt || true

          # nvcr.io/nvidia/tensorrt:26.05-py3 ships cuDNN 9.22 with no preinstalled torch, and
          # torch 2.14 pins cuDNN 9.24: mixing them fails with CUDNN_SUBLIBRARY_LOADING_FAILED.
          if [[ "${{ inputs.docker_image }}" == *"/tensorrt:"* ]]; then
            echo "torch<2.14" > /tmp/pip-constraints.txt
            export PIP_CONSTRAINT=/tmp/pip-constraints.txt
          fi

          # Use `python3 -m pip` instead of `pip` to avoid conflicts with system pip for nemo containers
          # Editable install so example scripts launched as subprocesses resolve modelopt to the same source path as the test process
          python3 -m pip install -e ".${{ inputs.pip_install_extras }}"

          if [[ "${{ inputs.example }}" == *"diffusers"* ]]; then
            echo "Uninstalling apex for diffusers: T5 Int8 (PixArt) + Apex is not supported as per https://github.com/huggingface/transformers/issues/21391"
            python3 -m pip uninstall -y apex || true
          fi

          find examples/${{ inputs.example }} -name "requirements.txt" | while read req_file; do python3 -m pip install -r "$req_file" || exit 1; done
      - name: Run tests
        id: run_tests
        shell: bash
        continue-on-error: ${{ inputs.allow_failure }}
        env:
          # Absolute paths so subprocesses running from different working directories
          # all find the config and write .coverage.* files to the same location.
          COVERAGE_PROCESS_START: ${{ github.workspace }}/pyproject.toml
          COVERAGE_FILE: ${{ github.workspace }}/.coverage
          # sys.monitoring records the same lines as the default C tracer at a fraction of the
          # overhead; the tracer slowed each hf_ptq example run by ~60%. Needs Python 3.12+.
          COVERAGE_CORE: sysmon
        run: |
          echo "Running tests for: ${{ inputs.example }}"
          pytest_args=()
          if [[ "${{ inputs.example }}" == "diffusers" ]]; then
            # Keep legacy ONNX/TRT tests available manually during deprecation.
            pytest_args+=(--ignore=tests/examples/diffusers/test_diffusers_onnx_trt.py)
          fi
          if [[ "${{ inputs.example }}" == "hf_ptq" && "${{ github.ref }}" == refs/heads/pull-request/* ]]; then
            # One GPU on PR runners: small-model tests share it and the rest run alone on it
            # (tests/examples/hf_ptq/conftest.py). The 2-GPU nightly stays serial.
            pytest_args+=(-n 4)
          fi
          python3 -m pytest tests/examples/${{ inputs.example }} --cov "${pytest_args[@]}"
      - name: Save extensions
        # Not tied to the tests passing, see the action description
        if: always() && steps.ext-cache.outputs.cache-hit != 'true'
        uses: ./.github/actions/save-extensions
        with:
          cache-key: ${{ steps.ext-cache.outputs.cache-key }}
      - name: Flag allowed failure
        if: ${{ inputs.allow_failure && steps.run_tests.outcome == 'failure' }}
        run: |
          echo "::warning title=Allowed example failure::'${{ inputs.example }}' failed but is in the allow-failure list (vars.ALLOW_FAILURE_EXAMPLE_TESTS); not blocking. Remove it from the variable once fixed."
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v7
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: coverage.xml
          # One flag per example, not a shared `examples`: carryforward only applies to a flag
          # with no upload, so a shared flag would replace every lane's coverage with the subset
          # that ran once lanes are gated independently.
          flags: examples-${{ inputs.example }}
          fail_ci_if_error: false # test may be skipped if relevant file changes are not detected
          verbose: true