diff --git a/.github/workflows/example_tests.yml b/.github/workflows/example_tests.yml index 67b98cbd230..e646af86ac6 100644 --- a/.github/workflows/example_tests.yml +++ b/.github/workflows/example_tests.yml @@ -133,7 +133,7 @@ jobs: with: docker_image: "nvcr.io/nvidia/pytorch:26.09-py3" example: ${{ matrix.example }} - timeout_minutes: 30 + timeout_minutes: 45 pip_install_extras: "[hf,dev-test]" runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), format(',{0},', matrix.example)) }} @@ -191,8 +191,8 @@ jobs: with: docker_image: "nvcr.io/nvidia/nemo:26.08" example: megatron_bridge - # PR runs single-GPU (~11 min); the nightly runs every test multi-GPU (~58 min). - timeout_minutes: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 30 || 75 }} + # PR runs single-GPU (~20 min with the setup); the nightly runs every test multi-GPU (~45 min). + timeout_minutes: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 45 || 90 }} pip_install_extras: "[hf,puzzletron,dev-test]" runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} allow_failure: ${{ contains(format(',{0},', vars.ALLOW_FAILURE_EXAMPLE_TESTS), ',megatron_bridge,') }} diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index b73f85affeb..e4265ede836 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -44,13 +44,13 @@ jobs: matrix: include: - example: gpu - timeout: 60 + timeout: 75 container_image: nvcr.io/nvidia/pytorch:26.09-py3 - example: gpu_megatron - timeout: 75 + timeout: 90 container_image: nvcr.io/nvidia/nemo:26.08 - example: gpu_trtllm - timeout: 15 + timeout: 30 container_image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc20 - example: gpu_vllm timeout: 30 diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 9c1154a2b65..ea174f86557 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -41,7 +41,7 @@ jobs: needs: [pr-gate] if: needs.pr-gate.outputs.any_changed == 'true' runs-on: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }} - timeout-minutes: 15 + timeout-minutes: 30 container: image: nvcr.io/nvidia/pytorch:26.09-py3 env: diff --git a/tests/conftest.py b/tests/conftest.py index af77546a244..6046a3e8b2d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -52,11 +52,11 @@ def pytest_addoption(parser): # Every collectible test group must be listed here else collection errors occur # A test can override its cap by adding ``@pytest.mark.timeout(...)`` _DEFAULT_TIMEOUT = { - "examples": int(os.environ.get("MODELOPT_QA_TEST_TIMEOUT", 300)), - "gpu": 120, - "gpu_megatron": 120, + "examples": int(os.environ.get("MODELOPT_QA_TEST_TIMEOUT", 600)), + "gpu": 180, + "gpu_megatron": 180, "gpu_trtllm": 60, - "gpu_vllm": 60, + "gpu_vllm": 120, "regression": 180, "unit": 120 if platform.system() == "Windows" else 60, }