From 25c7693800e9ef0900decd70a24193a5b34cc1aa Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Fri, 9 Oct 2026 23:43:23 +0000 Subject: [PATCH 1/4] Run the tests marked release on the nightly and on-demand gpu runs Tests marked `release` collect only when pytest gets --run-release, and no workflow passes it, so a test marked `release` never runs in CI (the docs say the same of the example release suite). Set PYTEST_ADDOPTS=--run-release in the "Run gpu tests" step for every run that is not a pull-request run: the nightly schedule and manual dispatches. That lets slow or network-heavy checks leave the pull-request jobs without leaving CI. Pull-request runs are unchanged. Nothing under tests/gpu* is marked release yet; the next commit marks the first test. The release example suites (hf_ptq/test_deploy.py, gpt-oss QAT) belong to example_tests.yml, which does not get the flag, so the docs sentence that said no workflow passes it now says no example workflow does. Testing: actionlint is clean. A stub test marked `release`, collected under the repo's tests/conftest.py, is skipped with "Only run when --run-release is given" when PYTEST_ADDOPTS is empty and runs when it is --run-release. The expression is `!startsWith(github.ref, 'refs/heads/pull-request/') && '--run-release' || ''`, which gives '' for a pull-request ref and '--run-release' otherwise. Not run on GitHub. Signed-off-by: Shengliang Xu --- .github/workflows/gpu_tests.yml | 3 +++ docs/source/deployment/3_unified_hf.rst | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 5094075dafd..24b7fb90bde 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -99,6 +99,9 @@ jobs: # sys.monitoring records the same lines as the default C tracer at a fraction of the # overhead on import-heavy worker processes. Needs Python 3.12+. COVERAGE_CORE: sysmon + # Tests marked `release` (real-weight downloads and the like) run on nightly and + # on-demand runs, not on pull requests. + PYTEST_ADDOPTS: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') && '--run-release' || '' }} run: | # Use `python3` (the vllm image has no `python` on PATH) python3 -m pip install nox && nox -s ${{ matrix.example }} diff --git a/docs/source/deployment/3_unified_hf.rst b/docs/source/deployment/3_unified_hf.rst index d3dc840cc62..c231169cd23 100644 --- a/docs/source/deployment/3_unified_hf.rst +++ b/docs/source/deployment/3_unified_hf.rst @@ -144,8 +144,8 @@ prompts, asserting that each returns non-empty output. Two limits are worth stating plainly, because they bound what any ✅ below can mean: * **These are declared cases, not PR-gated coverage.** The suite is marked ``release`` and collects - only when pytest is given ``--run-release``, which no workflow in ``.github/workflows/`` currently - passes. A green check on a pull request does not mean these cases ran. + only when pytest is given ``--run-release``, which no example workflow in ``.github/workflows/`` + currently passes. A green check on a pull request does not mean these cases ran. * **Each case is a load-and-generate smoke check on the text path.** It does not verify accuracy, image or audio inputs, diffusion output, or that speculative decoding actually engages. From 98cd4e942a2260c7599683680098ee950a7c52cb Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Fri, 9 Oct 2026 23:43:45 +0000 Subject: [PATCH 2/4] Run the real-weights Megatron MMLU test only with --run-release test_megatron_generate_and_mmlu[tp] builds a 28-layer Qwen3-0.6B Megatron model, downloads 1.2-1.5 GB of real weights into a fresh container cache and scores about 1,400 MMLU prompts: a 32 s call in every pull-request gpu_megatron job. It is the only test in the lane that downloads real weights. Mark it `release` so pytest skips it unless --run-release is given. The importer shape and name tests and the generate control-flow tests in the same module keep running on pull requests. This test is also the only one that checks real-weight accuracy (0.36 < MMLU < 0.39), so a numeric regression in the import or generation path is no longer caught on a pull request. The previous commit passes --run-release on the nightly and on-demand runs, so the real-weight accuracy check still runs there, a day after a change instead of on the pull request. Testing: ruff check and ruff format with the pinned ruff 0.15.20, and py_compile. Ran a stub module with the same decorators under the repo's tests/conftest.py on CPU: without the flag the four parametrized ids are skipped with "Only run when --run-release is given", with it they all run. Not run: the test itself, because this machine has no GPU and its Megatron is too old to import the module, and the flag on GitHub (the previous commit checked it with a stub test only). Signed-off-by: Shengliang Xu --- tests/gpu_megatron/torch/utils/plugins/test_utils_megatron.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/gpu_megatron/torch/utils/plugins/test_utils_megatron.py b/tests/gpu_megatron/torch/utils/plugins/test_utils_megatron.py index 8d3af2d7f3c..286eea97a5b 100644 --- a/tests/gpu_megatron/torch/utils/plugins/test_utils_megatron.py +++ b/tests/gpu_megatron/torch/utils/plugins/test_utils_megatron.py @@ -64,6 +64,9 @@ def _test_megatron_generate_and_mmlu(rank, size, parallelism): assert 0.36 < megatron_mmlu(model, tokenizer, fraction=0.1, batch_size=16) < 0.39 +@pytest.mark.release( + reason="Real Qwen3-0.6B weights and a 1.2-1.5 GB download; run with --run-release." +) @pytest.mark.parametrize("parallelism", ["tp", "pp", "cp", "dp"]) def test_megatron_generate_and_mmlu(dist_workers, parallelism, num_gpus): if num_gpus == 1 and parallelism != "tp": From f17420ce3ec7b3859748fabbe442a98cd80853a4 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Sat, 10 Oct 2026 00:07:06 +0000 Subject: [PATCH 3/4] Run the gated Nemotron v3 dataset smoke test only on the nightly test_get_dataset_dataloader_nemotron_v3_chat_template streams the seven gated Nemotron post-training v3 datasets from the Hub and applies a chat template: a 24 s call in the PR run (run 37703177445) that needs an HF token and the network, which also makes it a source of Hub flakes. It checks the live gated data, not modelopt logic: the combo expansion and sample distribution of get_dataset_dataloader are covered by the CPU unit tests in tests/unit/torch/utils/test_dataset_utils.py, which record the per-dataset calls instead of fetching. Mark it `release`. The nightly and on-demand gpu runs pass --run-release, so it still runs there every day, and it is skipped on pull requests. Testing: ruff check and format with the pinned ruff 0.15.20. Without the flag the test is skipped with "Only run when --run-release is given". Not run: the test itself or the nightly. Expected saving is about 24 s per pull-request gpu job. Signed-off-by: Shengliang Xu --- tests/gpu/torch/utils/test_dataset_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/gpu/torch/utils/test_dataset_utils.py b/tests/gpu/torch/utils/test_dataset_utils.py index bc9451ca03a..f83266762b6 100644 --- a/tests/gpu/torch/utils/test_dataset_utils.py +++ b/tests/gpu/torch/utils/test_dataset_utils.py @@ -42,6 +42,9 @@ def qwen3_tokenizer(): return tokenizer +@pytest.mark.release( + reason="Streams the 7 gated Nemotron v3 datasets; runs on the nightly schedule." +) def test_get_dataset_dataloader_nemotron_v3_chat_template(qwen3_tokenizer): if not get_token(): pytest.skip( From df4638728c9687c7404ac532c843d2131eb552c2 Mon Sep 17 00:00:00 2001 From: Shengliang Xu Date: Tue, 20 Oct 2026 18:57:02 +0000 Subject: [PATCH 4/4] Use the manual marker for the nightly-only GPU tests Review feedback on this PR: `release` is the QA team's marker (real large models in their QA pipeline), while `manual` is what the unit tests already use for tests that run nightly or on demand. Use `manual` here too: - the real-weights Megatron MMLU test and the gated Nemotron v3 dataset test carry `manual` instead of `release`; - the gpu workflow passes `--run-manual` instead of `--run-release` on nightly and on-demand runs. It keys on the event (copy-pr-bot runs pull requests as `push` events), so a manual dispatch from a pull-request branch also runs them; - the docs sentence about `--run-release` is back to main's wording, since no workflow passes that flag. `--run-manual` also starts the existing manual-marked tests of tests/gpu on those runs: 12 NAS vision-model cases, 6 puzzletron cases (about 40 s each by their marker), and the TRT client and distributed NAS cases that do not collect on a CPU-only host. The unit lane already runs its manual tests this way. Testing: pre-commit and actionlint pass. On a CPU host the Nemotron v3 test is skipped with "Only run when --run-manual is given" and runs with the flag; `-m manual` selects it. The Megatron MMLU test needs megatron.core.models.hybrid, which that host lacks, so its marker change was not exercised. Signed-off-by: Shengliang Xu --- .github/workflows/gpu_tests.yml | 6 +++--- docs/source/deployment/3_unified_hf.rst | 4 ++-- tests/gpu/torch/utils/test_dataset_utils.py | 2 +- .../gpu_megatron/torch/utils/plugins/test_utils_megatron.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/gpu_tests.yml b/.github/workflows/gpu_tests.yml index 24b7fb90bde..ca18afd0e65 100644 --- a/.github/workflows/gpu_tests.yml +++ b/.github/workflows/gpu_tests.yml @@ -99,9 +99,9 @@ jobs: # sys.monitoring records the same lines as the default C tracer at a fraction of the # overhead on import-heavy worker processes. Needs Python 3.12+. COVERAGE_CORE: sysmon - # Tests marked `release` (real-weight downloads and the like) run on nightly and - # on-demand runs, not on pull requests. - PYTEST_ADDOPTS: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') && '--run-release' || '' }} + # Tests marked `manual` (real-weight downloads and the like) run on nightly and on-demand + # runs, as in the unit tests, not on pull requests (copy-pr-bot runs those as `push`). + PYTEST_ADDOPTS: ${{ github.event_name != 'push' && '--run-manual' || '' }} run: | # Use `python3` (the vllm image has no `python` on PATH) python3 -m pip install nox && nox -s ${{ matrix.example }} diff --git a/docs/source/deployment/3_unified_hf.rst b/docs/source/deployment/3_unified_hf.rst index c231169cd23..d3dc840cc62 100644 --- a/docs/source/deployment/3_unified_hf.rst +++ b/docs/source/deployment/3_unified_hf.rst @@ -144,8 +144,8 @@ prompts, asserting that each returns non-empty output. Two limits are worth stating plainly, because they bound what any ✅ below can mean: * **These are declared cases, not PR-gated coverage.** The suite is marked ``release`` and collects - only when pytest is given ``--run-release``, which no example workflow in ``.github/workflows/`` - currently passes. A green check on a pull request does not mean these cases ran. + only when pytest is given ``--run-release``, which no workflow in ``.github/workflows/`` currently + passes. A green check on a pull request does not mean these cases ran. * **Each case is a load-and-generate smoke check on the text path.** It does not verify accuracy, image or audio inputs, diffusion output, or that speculative decoding actually engages. diff --git a/tests/gpu/torch/utils/test_dataset_utils.py b/tests/gpu/torch/utils/test_dataset_utils.py index f83266762b6..35aa3663def 100644 --- a/tests/gpu/torch/utils/test_dataset_utils.py +++ b/tests/gpu/torch/utils/test_dataset_utils.py @@ -42,7 +42,7 @@ def qwen3_tokenizer(): return tokenizer -@pytest.mark.release( +@pytest.mark.manual( reason="Streams the 7 gated Nemotron v3 datasets; runs on the nightly schedule." ) def test_get_dataset_dataloader_nemotron_v3_chat_template(qwen3_tokenizer): diff --git a/tests/gpu_megatron/torch/utils/plugins/test_utils_megatron.py b/tests/gpu_megatron/torch/utils/plugins/test_utils_megatron.py index 286eea97a5b..acec7515087 100644 --- a/tests/gpu_megatron/torch/utils/plugins/test_utils_megatron.py +++ b/tests/gpu_megatron/torch/utils/plugins/test_utils_megatron.py @@ -64,8 +64,8 @@ def _test_megatron_generate_and_mmlu(rank, size, parallelism): assert 0.36 < megatron_mmlu(model, tokenizer, fraction=0.1, batch_size=16) < 0.39 -@pytest.mark.release( - reason="Real Qwen3-0.6B weights and a 1.2-1.5 GB download; run with --run-release." +@pytest.mark.manual( + reason="Real Qwen3-0.6B weights and a 1.2-1.5 GB download; run with --run-manual." ) @pytest.mark.parametrize("parallelism", ["tp", "pp", "cp", "dp"]) def test_megatron_generate_and_mmlu(dist_workers, parallelism, num_gpus):