test(e2e): add opt-in Qwen3.5-122B CUDA coverage

.agents/skills/run-e2e/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: run-e2e
-description: Use when the user asks to run, validate, or diagnose the AFD plugin's DeepSeek-V2-Lite GPU/NPU, Qwen3 MoE GPU, or Qwen3.6 MoE CUDA end-to-end tests through the Qwen3.5/3.6 adapter family, including PR-gate E2E, GSM8K-7 accuracy, graph, eager, DBO, or 2A2F scenarios.
+description: Use when the user asks to run, validate, or diagnose the AFD plugin's DeepSeek-V2-Lite GPU/NPU, Qwen3 MoE GPU, Qwen3.6 MoE CUDA, or opt-in Qwen3.5-122B CUDA end-to-end tests, including PR-gate E2E, GSM8K accuracy, graph, eager, DBO, or AFD topology scenarios.
 ---
 
 # Run AFD E2E Tests
@@ -13,6 +13,8 @@ Run one of the model suites:
 - `tests/e2e/models/qwen3_moe/test_qwen3_moe.py` on GPU
 - `tests/e2e/models/qwen3_6/test_qwen3_6.py` on CUDA (text-only Qwen3.6
   evidence for the Qwen3.5/3.6 adapter family)
+- `tests/e2e/models/qwen3_5/test_qwen3_5_122b.py` on eight CUDA devices
+  (manual, eager-only large-model profile)
 
 Each suite contains four gate scenarios:
 
@@ -51,8 +53,9 @@ or uv.lock.
 
 Fail before pytest when a prerequisite is missing; never turn it into a skip.
 
-Set HF_HOME before every run. The pytest entrypoint downloads/caches GSM8K and
-the model when the backend model env var is unset.
+Set HF_HOME before every run. Default pytest entrypoints download/cache GSM8K
+and the model when the backend model env var is unset. The Qwen3.5-122B
+profile only caches GSM8K and never downloads its checkpoint.
 
 ### 3. Configure the run
 
@@ -120,6 +123,26 @@ Multimodal, NPU, `compute_gate_on_attention=true`, asynchronous,
 pipeline-parallel, and multi-node execution are not covered; quantization is
 unverified.
 
+For the Qwen3.5-122B large-model profile, run:
+
+~~~bash
+export AFD_E2E_BACKEND=gpu
+export AFD_E2E_LARGE_MODEL=1
+export AFD_GPU_E2E_MODEL=/path/to/Qwen3.5-122B-A10B
+export AFD_E2E_DEVICES=0,2,4,6,1,3,5,7
+python -m pytest -q -s \
+  tests/e2e/models/qwen3_5/test_qwen3_5_122b.py
+~~~
+
+Verify the model path and eight unique devices before pytest. Device order is
+part of the profile contract: the first four run native DP4 or AFD Attention
+DP4, and the last four run AFD FFN DP4/EP4. Both cases are BF16, text-only,
+vLLM V1, eager, and natural-routing GSM8K-7 checks. Do not enable graph, DBO,
+or benchmark-only forced routing. The profile disables the FlashInfer sampler
+because vLLM 0.26.0 rejects Blackwell SM12 during its capability check. Report
+`baseline-eager` and `afd-eager-4a4f` separately, including cleanup and
+released GPU memory.
+
 Do not add backend markers or run scenarios in parallel; they share devices.
 
 For the local DeepSeek-V2-Lite 2A1F cases, run the same pytest entrypoint with
@@ -129,17 +152,18 @@ On cancellation, forward SIGTERM and allow over 90 seconds for cleanup.
 
 ### 5. Report
 
-Success means the selected suite reports 4 passed and 0 skipped. Report the
-failed scenario, first actionable error, and cleanup status. Any skip is a
-gate failure.
+Success means a default suite reports 4 passed and 0 skipped; the Qwen3.5-122B
+profile reports 2 passed and 0 skipped. Report the failed scenario, first
+actionable error, and cleanup status. Any skip is a gate failure.
 
 ## Environment reference
 
 | Variable | Backend | Required |
 |---|---|---|
 | AFD_E2E_BACKEND | both | yes: gpu or npu |
 | AFD_E2E_DEVICES | both | yes: four unique IDs for the default suite |
-| AFD_GPU_E2E_MODEL | GPU | no; downloads the selected suite's model when unset |
+| AFD_E2E_LARGE_MODEL | GPU | Qwen3.5-122B only: must equal 1 |
+| AFD_GPU_E2E_MODEL | GPU | Qwen3.5-122B: yes; default suites download when unset |
 | AFD_GPU_E2E_VLLM_BIN | GPU | no; defaults to vllm |
 | AFD_NPU_E2E_MODEL | NPU | no; downloads the selected suite's model when unset |
 | AFD_NPU_E2E_VLLM_BIN | NPU | no; defaults to vllm |

docs/design/module/e2e_testing.md

@@ -27,6 +27,7 @@ validation_paths:
   - "tests/e2e/models/deepseek_v2_lite/test_async_cam_npu.py"
   - "tests/e2e/models/qwen3_moe/test_qwen3_moe.py"
   - "tests/e2e/models/qwen3_6/test_qwen3_6.py"
+  - "tests/e2e/models/qwen3_5/test_qwen3_5_122b.py"
 upstream_refs:
   - "vLLM 0.26.0 serving and shutdown interfaces"
   - "lm-evaluation-harness GSM8K task and local-completions API"
@@ -36,6 +37,7 @@ verified_platform_refs:
   - "Ascend NPU DeepSeek-V2-Lite"
   - "CUDA Qwen3 MoE"
   - "CUDA Qwen3.6 MoE"
+  - "CUDA Qwen3.5-122B-A10B"
 related_issues: []
 last_reviewed: 2026-08-19
 ---
@@ -68,9 +70,11 @@ cleanup. Production code does not depend on the E2E harness.
 
 - `E2E-INV-001` — A case **MUST** have a stable lower-kebab-case ID and cover
   behavior not already covered by an existing case.
-- `E2E-INV-002` — A PR case **MUST NOT** use more than four unique devices.
-  Gate AFD cases **MUST** use 2 Attention ranks and 2 FFN ranks; 2A1F cases
-  are local-only.
+- `E2E-INV-002` — A default PR-gate case **MUST NOT** use more than four
+  unique devices. Gate AFD cases **MUST** use 2 Attention ranks and 2 FFN
+  ranks; 2A1F cases are local-only. Larger hardware profiles **MUST** be
+  explicit opt-ins selected by exact pytest node ID, require an explicit model
+  path, and **MUST NOT** enter a gate without matching hardware.
 - `E2E-INV-003` — Cases sharing devices or ports **MUST** run sequentially,
   remain order-independent, and release owned process groups before the next
   case.
@@ -120,6 +124,14 @@ Multimodal, NPU, `compute_gate_on_attention=true`, pipeline-parallel,
 asynchronous, and multi-node execution are outside this case; quantization is
 unverified.
 
+The opt-in Qwen3.5-122B-A10B profile covers a checkpoint size and 256-expert
+configuration that the default family case does not. It adds two eager-only
+cases: native DP4/TP1/EP4 and synchronous AFD 4A4F, with Attention DP4/TP1 and
+FFN DP4/TP1/EP4. The profile requires eight explicit devices,
+`AFD_E2E_LARGE_MODEL=1`, and an existing `AFD_GPU_E2E_MODEL`; it never
+downloads the checkpoint. It is manual hardware coverage, not a default PR or
+merge gate. Graph and DBO are outside this profile.
+
 ## Accuracy gate
 
 | Setting | PR | Weekly |

tests/e2e/README.md

@@ -30,9 +30,11 @@ Run from the repository root. The environment needs `vllm`, `pytest`,
 `afd_plugin`, `lm_eval`, `datasets`, and `huggingface_hub`. NPU also needs
 `torch_npu`.
 
-The selected test downloads/caches `openai/gsm8k` and its Hugging Face model
-when the backend model env var is unset. Point `HF_HOME` at a persistent cache
-if you want to reuse downloads across runs.
+The default model suites download/cache `openai/gsm8k` and their Hugging Face
+model when the backend model env var is unset. Point `HF_HOME` at a persistent
+cache if you want to reuse downloads across runs. Large-model profiles are an
+exception: they require an explicit local model path and never download a
+checkpoint.
 
 GPU:
 
@@ -89,6 +91,34 @@ by AFD scenarios. The suite uses the same GSM8K-7, eight-shot, 4096-token,
 `compute_gate_on_attention=true`, pipeline-parallel, asynchronous, and
 multi-node execution are not covered; quantization is unverified.
 
+### Qwen3.5-122B large-model CUDA profile
+
+`Qwen/Qwen3.5-122B-A10B` has a separate, manual hardware profile. It is not a
+default PR or merge gate and never downloads the roughly 234 GiB checkpoint.
+Running it requires an explicit opt-in, an existing model path, and exactly
+eight unique CUDA device IDs:
+
+```bash
+export AFD_E2E_BACKEND=gpu
+export AFD_E2E_LARGE_MODEL=1
+export AFD_GPU_E2E_MODEL=/path/to/Qwen3.5-122B-A10B
+# First four: Native or AFD Attention. Last four: AFD FFN.
+export AFD_E2E_DEVICES=0,2,4,6,1,3,5,7
+python -m pytest -q -s \
+  tests/e2e/models/qwen3_5/test_qwen3_5_122b.py
+```
+
+The two sequential cases are `baseline-eager` (native DP4/TP1/EP4 on the
+first four devices) and `afd-eager-4a4f` (Attention DP4/TP1 on the first four,
+FFN DP4/TP1/EP4 on the last four). Both use BF16, text-only vLLM V1, natural
+routing, GSM8K-7 with eight-shot prompting, and a 4096-token model length.
+The profile removes benchmark-only forced-routing variables from child server
+environments. It also disables the optional FlashInfer sampler because vLLM
+0.26.0 rejects Blackwell SM12 during that sampler's capability check; greedy
+GSM8K does not require it. Graph, DBO, asynchronous, multi-node, quantized,
+and multimodal coverage are out of scope; graph coverage remains excluded
+while #261 is unresolved.
+
 ### Local 2A1F cases
 
 The 2 Attention + 1 FFN scenarios are local-only cases; CI gates do not run
@@ -155,10 +185,11 @@ Use run-e2e to run the Qwen3 MoE GPU E2E tests with HF_HOME
 /data/huggingface.
 ```
 
-Provide `HF_HOME` and `AFD_E2E_BACKEND`. `AFD_E2E_DEVICES` is optional; when
-unset, the test module picks the defaults above. The model path is optional
-when Hugging Face download is available. The skill checks prerequisites, runs
-the same four tests, and reports failures and process cleanup.
+Provide `HF_HOME` and `AFD_E2E_BACKEND`. For default suites,
+`AFD_E2E_DEVICES` is optional and the model path is optional when Hugging Face
+download is available. The Qwen3.5-122B profile instead requires the explicit
+large-model variables documented above. The skill checks prerequisites, runs
+the selected cases, and reports failures and process cleanup.
 
 ## NPU async CAM smoke test
 

tests/e2e/models/qwen3_5/test_qwen3_5_122b.py

@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the AFD plugin project
+"""Opt-in CUDA Qwen3.5-122B-A10B E2E coverage."""
+
+from __future__ import annotations
+
+import os
+import sys
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+
+from tests.conftest import download_dataset, run_runner
+from tests.e2e.runner import AFD_EAGER_4A4F_SCENARIO, BASELINE_EAGER_SCENARIO
+
+GSM8K_DATASET_ID = "openai/gsm8k"
+GSM8K_DATASET_CONFIG = "main"
+LARGE_MODEL_OPT_IN_ENV = "AFD_E2E_LARGE_MODEL"
+DEVICE_COUNT = 8
+ROLE_DEVICE_COUNT = 4
+SCENARIOS = (BASELINE_EAGER_SCENARIO, AFD_EAGER_4A4F_SCENARIO)
+COMMON_VLLM_ARGS = (
+    "--dtype=bfloat16",
+    "--language-model-only",
+    "--max-model-len=4096",
+    "--max-num-seqs=1",
+    "--max-num-batched-tokens=4096",
+    "--mamba-cache-mode=align",
+    "--all2all-backend=allgather_reducescatter",
+    "--seed=0",
+)
+CONTROLLED_ROUTING_ENV_VARS = (
+    "VLLM_MOE_ROUTING_SIMULATION_STRATEGY",
+    "AFD_BENCHMARK_FORCE_LB_TOPN_PER_RANK",
+)
+FLASHINFER_SAMPLER_ENV = "VLLM_USE_FLASHINFER_SAMPLER"
+
+
+def _required_env(name: str) -> str:
+    value = os.environ.get(name)
+    if not value:
+        raise RuntimeError(f"{name} must be set")
+    return value
+
+
+def _devices() -> list[str]:
+    raw_devices = _required_env("AFD_E2E_DEVICES")
+    devices = [item.strip() for item in raw_devices.split(",") if item.strip()]
+    if len(devices) != DEVICE_COUNT:
+        raise RuntimeError(
+            f"AFD_E2E_DEVICES must contain exactly {DEVICE_COUNT} devices",
+        )
+    if len(devices) != len(set(devices)):
+        raise RuntimeError("AFD_E2E_DEVICES must contain unique devices")
+    return devices
+
+
+def prepare_e2e_assets() -> None:
+    """Validate the explicit large-model contract, then cache GSM8K."""
+    if _required_env("AFD_E2E_BACKEND") != "gpu":
+        raise RuntimeError("Qwen3.5-122B E2E supports only the 'gpu' backend")
+    if _required_env(LARGE_MODEL_OPT_IN_ENV) != "1":
+        raise RuntimeError(f"{LARGE_MODEL_OPT_IN_ENV} must be set to 1")
+    model_path = Path(_required_env("AFD_GPU_E2E_MODEL")).expanduser()
+    if not model_path.is_dir():
+        raise RuntimeError(
+            f"AFD_GPU_E2E_MODEL must be an existing directory: {model_path}"
+        )
+    _devices()
+    download_dataset(GSM8K_DATASET_ID, GSM8K_DATASET_CONFIG)
+
+
+def build_runner_command(scenario: str, gsm8k_output_path: Path) -> list[str]:
+    if scenario not in SCENARIOS:
+        raise ValueError(f"unsupported Qwen3.5-122B scenario: {scenario}")
+    if _required_env("AFD_E2E_BACKEND") != "gpu":
+        raise RuntimeError("Qwen3.5-122B E2E supports only the 'gpu' backend")
+
+    devices = _devices()
+    command = [
+        sys.executable,
+        "-m",
+        "tests.e2e.runner",
+        "--model",
+        _required_env("AFD_GPU_E2E_MODEL"),
+        "--vllm-bin",
+        os.environ.get("AFD_GPU_E2E_VLLM_BIN", "vllm"),
+        "--device-backend",
+        "gpu",
+        "--attention-devices",
+        ",".join(devices[:ROLE_DEVICE_COUNT]),
+    ]
+    command.extend(f"--common-vllm-arg={arg}" for arg in COMMON_VLLM_ARGS)
+    if scenario == AFD_EAGER_4A4F_SCENARIO:
+        command.extend(
+            ["--ffn-devices", ",".join(devices[ROLE_DEVICE_COUNT:])],
+        )
+    command.extend(
+        [
+            "--scenario",
+            scenario,
+            "--gsm8k-output-path",
+            str(gsm8k_output_path),
+            "--served-model-name-prefix",
+            "qwen3-5-122b-afd",
+        ],
+    )
+    return command
+
+
+def natural_routing_env() -> dict[str, str]:
+    """Return the natural-routing environment validated by this profile."""
+    env = os.environ.copy()
+    for name in CONTROLLED_ROUTING_ENV_VARS:
+        env.pop(name, None)
+    # vLLM 0.26.0's FlashInfer sampler rejects Blackwell SM12 during device
+    # capability detection. Greedy GSM8K does not require that sampler.
+    env[FLASHINFER_SAMPLER_ENV] = "0"
+    return env
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _prepare_e2e_assets() -> Iterator[None]:
+    prepare_e2e_assets()
+    yield
+
+
+@pytest.mark.e2e
+@pytest.mark.parametrize("scenario", SCENARIOS, ids=SCENARIOS)
+def test_qwen3_5_122b(scenario: str, tmp_path: Path) -> None:
+    command = build_runner_command(scenario, tmp_path / scenario)
+    run_runner(command, env=natural_routing_env())

tests/e2e/runner.py

@@ -38,6 +38,8 @@
 ASYNC_UBATCH_NUM_STAGES = 2
 ASYNC_UBATCH_BATCH_SIZE = 2
 V2_SYNC_CONNECTOR = "P2pNcclAFDConnector"
+BASELINE_EAGER_SCENARIO = "baseline-eager"
+AFD_EAGER_4A4F_SCENARIO = "afd-eager-4a4f"
 V2_SCENARIOS = (
     "afd-v2-eager-1a1f",
     "afd-v2-eager-dp2",
@@ -104,6 +106,7 @@ def main() -> int:
     previous_handlers = {signum: signal.getsignal(signum) for signum in handled_signals}
     received_signal: int | None = None
     cleanup_in_progress = False
+    launch_order: tuple[tuple[str, str], ...]
 
     def exit_after_cleanup(signum: int, _frame: Any) -> None:
         nonlocal received_signal
@@ -210,6 +213,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--scenario",
         choices=[
+            BASELINE_EAGER_SCENARIO,
             "baseline-graph",
             "afd-eager-2a1f",
             "afd-graph-2a1f",
@@ -219,6 +223,7 @@ def parse_args() -> argparse.Namespace:
             "afd-graph-dbo-2a2f",
             ASYNC_CAM_SCENARIO,
             ASYNC_UBATCH_SCENARIO,
+            AFD_EAGER_4A4F_SCENARIO,
             *V2_SCENARIOS,
         ],
         required=True,
@@ -323,6 +328,7 @@ def configure_scenario(args: argparse.Namespace) -> None:
     is_async_cam = args.scenario == ASYNC_CAM_SCENARIO
     is_async_ubatch = args.scenario == ASYNC_UBATCH_SCENARIO
     scenario_settings = {
+        BASELINE_EAGER_SCENARIO: (True, False, False, 4, 0),
         "baseline-graph": (True, True, False, 4, 0),
         "afd-eager-2a1f": (False, False, False, 2, 1),
         "afd-graph-2a1f": (False, True, False, 2, 1),
@@ -344,6 +350,7 @@ def configure_scenario(args: argparse.Namespace) -> None:
             ASYNC_UBATCH_ATTENTION_RANKS,
             ASYNC_UBATCH_FFN_RANKS,
         ),
+        AFD_EAGER_4A4F_SCENARIO: (False, False, False, 4, 4),
         "afd-v2-eager-1a1f": (False, False, False, 1, 1),
         "afd-v2-eager-dp2": (False, False, False, 2, 2),
         "afd-v2-eager-tp2": (False, False, False, 2, 2),