From 8ea9b350be5d7ed30f2c09535668b04b0397b40a Mon Sep 17 00:00:00 2001 From: Yue Date: Thu, 15 Oct 2026 03:50:26 +0000 Subject: [PATCH 1/4] Skip chat rows that cannot be converted instead of failing the source A row that decodes but cannot be converted (e.g. a string messages field) is skipped with a warning and counted as rejected, as the tokenizing reader skips malformed records; the give-up rule still bounds an unusable source. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Yue --- .../utils/plugins/prepare_megatron_data_blend.py | 15 +++++++++++---- .../plugins/test_prepare_megatron_data_blend.py | 8 +++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/modelopt/torch/utils/plugins/prepare_megatron_data_blend.py b/modelopt/torch/utils/plugins/prepare_megatron_data_blend.py index 2bc7aacb8c9..74cff163fa1 100644 --- a/modelopt/torch/utils/plugins/prepare_megatron_data_blend.py +++ b/modelopt/torch/utils/plugins/prepare_megatron_data_blend.py @@ -277,9 +277,16 @@ def _take_chat_rows( max_rejected = 10 * want + 100 try: for row in _iter_rows(source, output_dir, shuffle, seed): - chat = _to_chat(row, source["content_field"], reasoning_content) - # Judge the source row's top-level media keys too, which the conversion does not keep. - if chat is None or not is_text_only_chat_example({**row, **chat}): + try: + chat = _to_chat(row, source["content_field"], reasoning_content) + # Judge the source row's top-level media keys too, which the conversion drops. + if chat is not None and not is_text_only_chat_example({**row, **chat}): + chat = None + except (AttributeError, KeyError, TypeError, ValueError) as e: + # As in the tokenizing reader: one malformed record must not lose the source. + print(f"[WARN] {name}: skipping a row that cannot be converted: {e!r}", flush=True) + chat = None + if chat is None: rejected += 1 if rejected > max_rejected: break @@ -288,7 +295,7 @@ def _take_chat_rows( if len(rows) >= want: break except Exception as e: # e.g. a gated split, or a schema the streaming reader cannot cast - raise RuntimeError(f"{name}: failed to read or convert") from e + raise RuntimeError(f"{name}: failed to read") from e if len(rows) < want: gave_up = rejected > max_rejected reason = f"gave up after {rejected} rejected rows" if gave_up else "source exhausted" diff --git a/tests/gpu_megatron/torch/utils/plugins/test_prepare_megatron_data_blend.py b/tests/gpu_megatron/torch/utils/plugins/test_prepare_megatron_data_blend.py index 6306ecef292..f1ab4dba29c 100644 --- a/tests/gpu_megatron/torch/utils/plugins/test_prepare_megatron_data_blend.py +++ b/tests/gpu_megatron/torch/utils/plugins/test_prepare_megatron_data_blend.py @@ -342,7 +342,7 @@ def iter_rows(source, output_dir, shuffle, seed): raise TypeError("cannot cast") monkeypatch.setattr(blend_module, "_iter_rows", iter_rows) - with pytest.raises(RuntimeError, match="org/a train: failed to read or convert"): + with pytest.raises(RuntimeError, match="org/a train: failed to read"): blend_module.prepare_chat_data_blend(write_config()) assert not (tmp_path / "blend" / "train.jsonl").exists() @@ -357,8 +357,10 @@ def test_write_jsonl_is_atomic(tmp_path: Path): def test_chat_blend_reads_files_sources(tmp_path: Path, monkeypatch): plain, packed = tmp_path / "a.jsonl", tmp_path / "b.jsonl.gz" lines = [json.dumps(_chat(i)) for i in range(20)] - # A malformed record is skipped with a warning, as in the tokenizing reader. - plain.write_text("\n".join([*lines[:10], '{"messages": [', *lines[10:]]) + "\n", "utf-8") + # Malformed records, unparseable or unconvertible, are skipped with a warning as in the + # tokenizing reader. + bad = ['{"messages": [', '{"messages": "hello"}'] + plain.write_text("\n".join([*lines[:10], *bad, *lines[10:]]) + "\n", "utf-8") with gzip.open(packed, "wt", encoding="utf-8") as f: f.writelines(json.dumps(_chat(i)) + "\n" for i in range(20, 40)) paths = {"data/a.jsonl": str(plain), "data/b.jsonl.gz": str(packed)} From 54a6ce0325307e3490d8b0f4f74ebb371de55ce1 Mon Sep 17 00:00:00 2001 From: Yue Date: Sat, 17 Oct 2026 15:53:05 +0000 Subject: [PATCH 2/4] Address post-merge review of the chat JSONL blend and the Super VL QAD tutorial - Drop chat rows whose text field is not a string. - Rename the NemotronH omni QAD tutorial to the released checkpoint name, NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16. - Give the exact vLLM flags behind the results: --mamba-cache-dtype, --enable-mamba-cache-stochastic-rounding and --mamba-cache-philox-rounds 5, and the 32k max-model-len used for MMLU-ProX. - Add a scope note (text-only QAD at 32k; AA-LCR at 64k; no vision benchmarks) and a CHANGELOG entry. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Yue --- CHANGELOG.rst | 1 + examples/megatron_bridge/README.md | 2 +- .../README.md | 27 ++++++++++--------- .../data_blend.yaml | 0 examples/megatron_bridge/tutorials/README.md | 2 +- .../plugins/prepare_megatron_data_blend.py | 5 ++-- .../test_prepare_megatron_data_blend.py | 1 + 7 files changed, 22 insertions(+), 16 deletions(-) rename examples/megatron_bridge/tutorials/{NemotronH-Omni-W4A4-QAD => NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16}/README.md (86%) rename examples/megatron_bridge/tutorials/{NemotronH-Omni-W4A4-QAD => NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16}/data_blend.yaml (100%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 448f085adb7..edfec36beb8 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -89,6 +89,7 @@ Changelog - Add mergeable LoRA QAD for fake-quantized dense Megatron/Transformer Engine linear layers. Enable it with ``--lora_rank`` in Megatron-Bridge PTQ, train through the normal QAD flow, and export a standalone quantized checkpoint without deployment adapters. - Add optional Top-P (nucleus) truncation to the Megatron top-k logits KD loss (now ``TopLogitsKLLoss``) via ``logit_kl_top_p`` and ``logit_kl_top_p_min_k`` in ``DistillationConfig``: after the global Top-K selection, only the smallest prefix whose cumulative teacher probability reaches ``top_p`` (with a floor of ``min_k`` entries) contributes to the KL, mirroring ``--logits-save-top-p`` / ``--logits-save-top-p-min-k`` in Megatron-LM's logits saver. - Add an end-to-end W4A4 NVFP4 PTQ and QAD tutorial for Qwen3.6-35B-A3B also covering evaluation and vLLM throughput benchmarking. See `examples/megatron_bridge/tutorials/Qwen3.6-35B-A3B/README.md `_ for details. +- Add an end-to-end W4A4 NVFP4 PTQ and QAD tutorial for NVIDIA Nemotron 3.5 Super VL, covering chat-data blend preparation, assistant-only distillation, Hugging Face export and vLLM serving. See `examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md `_ for details. - Add ``--mlflow `` to the ``examples/megatron_bridge`` scripts that write a checkpoint -- ``prune_minitron.py``, ``quantize.py``, ``distill.py``, ``export_quantized_megatron_to_hf.py`` and ``export_distilled_megatron_to_hf.py`` (MLflow's own ``MLFLOW_TRACKING_URI`` is honoured too). Each run records the invocation, its arguments as searchable params and its log -- a distillation records its training metrics instead of the rank-0 log -- and writes ``.experiment.json`` into the checkpoint it produced, so a pruning, a quantization, the distillation that refines its checkpoint and the export that deploys it can be traced to one another; uploading the checkpoints themselves stays off unless ``--mlflow_log_checkpoints`` is passed. - Add ``--ep_size`` to ``examples/megatron_bridge/export_quantized_megatron_to_hf.py`` so large MoE models with grouped-GEMM experts can be exported with their experts sharded across GPUs. Checkpoints built with ``--no_moe_grouped_gemm`` must still be exported at ``--ep_size 1``. - Add unified HF export of GLM-5 / GLM-5.2 (``glm_moe_dsa``) checkpoints quantized with ``examples/megatron_bridge``. The MTP layer, which Megatron-Bridge does not build for these models, is copied through unquantized from the source checkpoint. DSA sparse-attention models also gain FP8 KV-cache quantization on Megatron. diff --git a/examples/megatron_bridge/README.md b/examples/megatron_bridge/README.md index a6428d79022..19deab559ba 100644 --- a/examples/megatron_bridge/README.md +++ b/examples/megatron_bridge/README.md @@ -20,7 +20,7 @@ This directory contains examples of using Model Optimizer with the [NeMo Megatro > > Or the [Qwen3.6-35B-A3B W4A4 NVFP4 + QAD tutorial](tutorials/Qwen3.6-35B-A3B/README.md) for an end-to-end quantization-aware distillation workflow. > -> Or the [NemotronH omni VL W4A4 + QAD tutorial](tutorials/NemotronH-Omni-W4A4-QAD/README.md) for a Mamba-Transformer MoE vision-language model. +> Or the [NVIDIA Nemotron 3.5 Super VL W4A4 + QAD tutorial](tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md) for a Mamba-Transformer MoE vision-language model. ## Pre-Requisites diff --git a/examples/megatron_bridge/tutorials/NemotronH-Omni-W4A4-QAD/README.md b/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md similarity index 86% rename from examples/megatron_bridge/tutorials/NemotronH-Omni-W4A4-QAD/README.md rename to examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md index 461f2e201c9..929bdd03701 100644 --- a/examples/megatron_bridge/tutorials/NemotronH-Omni-W4A4-QAD/README.md +++ b/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md @@ -1,6 +1,6 @@ -# NemotronH Omni VL: W4A4 NVFP4 PTQ + QAD + Deployment +# NVIDIA Nemotron 3.5 Super VL: W4A4 NVFP4 PTQ + QAD + Deployment -End-to-end W4A4 quantization of a NemotronH omni vision-language checkpoint (architecture `NemotronH_Omni_Reasoning_V3`), a Mamba-Transformer MoE language model with a vision tower: mixed NVFP4/FP8 post-training quantization → Quantization-Aware Distillation (QAD) → Hugging Face export → vLLM serving. This document covers: +End-to-end W4A4 quantization of [NVIDIA Nemotron 3.5 Super VL](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16) (architecture `NemotronH_Omni_Reasoning_V3`), a Mamba-Transformer MoE language model with a vision tower: mixed NVFP4/FP8 post-training quantization → Quantization-Aware Distillation (QAD) → Hugging Face export → vLLM serving. This document covers: 1. **[Data Preparation](#1-data-preparation)**: the chat-format QAD blend, from `data_blend.yaml` 2. **[Quantization](#2-quantization)**: W4A4 PTQ with `examples/megatron_bridge/quantize.py` @@ -28,6 +28,8 @@ All checkpoints were served with the same vLLM settings (see [Section 5](#5-eval MMLU-ProX scores a response only if it ends with the answer phrase in the question's language. Most of the QAD gain comes from fewer responses missing that phrase (11.3% for PTQ, 5.4% after QAD); on the responses with a readable answer, accuracy rises from 83.9 to 85.8. +**Scope:** QAD trains on text only, at a 32k sequence length, and only the language model is quantized. AA-LCR is the one long-context benchmark here (prompts average about 108k tokens); a 64k-sequence QAD run scored 67.31 on it, no better than 32k. Vision benchmarks were not run. + The exported W4A4 checkpoint is **70 GB**. --- @@ -72,14 +74,14 @@ PTQ runs on **one 4-GPU GB300 node** with 512 calibration samples. ```bash torchrun --nproc_per_node 4 /opt/Model-Optimizer/examples/megatron_bridge/quantize.py \ - --hf_model_name_or_path \ + --hf_model_name_or_path nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16 \ --trust_remote_code \ --recipe models/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/ptq/nvfp4-mse \ --tp_size 1 --ep_size 4 --pp_size 1 \ --calib_dataset_name cnn_nemotron_v2_mix \ --calib_num_samples 512 \ --calib_batch_size 1 \ - --export_megatron_path /path/to/nemotron_h_omni_w4a4_megatron \ + --export_megatron_path /path/to/nemotron_3_5_super_vl_w4a4_megatron \ --skip_generate ``` @@ -108,9 +110,9 @@ We used **8 nodes × 4 GB300 (32 GPUs)** with context parallelism 4 and expert p export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1) MASTER_PORT=29500 srun bash -c 'export RANK=$SLURM_PROCID WORLD_SIZE=$SLURM_NTASKS LOCAL_RANK=$SLURM_LOCALID python -u /opt/Model-Optimizer/examples/megatron_bridge/distill.py \ - --teacher_hf_path \ - --student_hf_path \ - --student_megatron_path /path/to/nemotron_h_omni_w4a4_megatron \ + --teacher_hf_path nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16 \ + --student_hf_path nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16 \ + --student_megatron_path /path/to/nemotron_3_5_super_vl_w4a4_megatron \ --trust_remote_code \ --tp_size 1 --pp_size 1 --cp_size 4 --ep_size 32 \ --sft --sft_hf_dataset /path/to/qad_blend/train.jsonl \ @@ -145,10 +147,10 @@ Export the QAD checkpoint to a Hugging Face checkpoint on one 4-GPU node (pipeli ```bash torchrun --nproc_per_node 4 /opt/Model-Optimizer/examples/megatron_bridge/export_quantized_megatron_to_hf.py \ - --hf_model_name_or_path \ + --hf_model_name_or_path nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16 \ --trust_remote_code \ --megatron_path /path/to/qad_output/checkpoints \ - --export_unified_hf_path /path/to/nemotron_h_omni_w4a4_hf \ + --export_unified_hf_path /path/to/nemotron_3_5_super_vl_w4a4_hf \ --pp_size 4 ``` @@ -167,9 +169,10 @@ Serve with vLLM on one 4-GPU GB300 node, TP=4 with expert parallelism. Quantized | Setting | Value | | --- | --- | | `--kv-cache-dtype` | `fp8` | -| `--max-model-len` | 262144 | +| `--max-model-len` | 262144 (32768 for MMLU-ProX) | | `--mamba-ssm-cache-dtype` | `float16` | -| `--mamba-conv-cache-dtype` | `float32` | -| Mamba cache rounding | stochastic rounding, 5 Philox rounds | +| `--mamba-cache-dtype` | `float32` | +| `--enable-mamba-cache-stochastic-rounding` | on | +| `--mamba-cache-philox-rounds` | 5 | | Reasoning parser | `nemotron_v3` | | Chat template | the checkpoint's `chat_template.jinja` | diff --git a/examples/megatron_bridge/tutorials/NemotronH-Omni-W4A4-QAD/data_blend.yaml b/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/data_blend.yaml similarity index 100% rename from examples/megatron_bridge/tutorials/NemotronH-Omni-W4A4-QAD/data_blend.yaml rename to examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/data_blend.yaml diff --git a/examples/megatron_bridge/tutorials/README.md b/examples/megatron_bridge/tutorials/README.md index 454403083e8..ca03a85c993 100644 --- a/examples/megatron_bridge/tutorials/README.md +++ b/examples/megatron_bridge/tutorials/README.md @@ -8,5 +8,5 @@ Each one walks through a complete workflow using the scripts in [examples/megatr | Tutorial | What it covers | | --- | --- | | [NVIDIA-Nemotron-3-Nano-30B-A3B-BF16](NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/README.md) | End-to-end optimization of the Nemotron-3-Nano-30B-A3B-BF16 (MoE + Mamba-Transformer hybrid) model: Minitron structured pruning (31.6B/A3.6B → 22B/A3.0B) → two-phase knowledge distillation (100B tokens, 8K then 32K seq length) → quantization → vLLM deployment. Includes data-blend preparation, evaluation setup, and detailed pruning / data-blend / long-context ablations. | -| [NemotronH-Omni-W4A4-QAD](NemotronH-Omni-W4A4-QAD/README.md) | **W4A4** quantization of the language model of a NemotronH omni vision-language checkpoint (`NemotronH_Omni_Reasoning_V3`, Mamba-Transformer MoE): mixed NVFP4/FP8 PTQ → KD-only QAD on chat data → Hugging Face export → vLLM serving. | +| [NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16](NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md) | **W4A4** quantization of the language model of NVIDIA Nemotron 3.5 Super VL (`NemotronH_Omni_Reasoning_V3`, Mamba-Transformer MoE): mixed NVFP4/FP8 PTQ → KD-only QAD on chat data → Hugging Face export → vLLM serving. | | [Qwen3.6-35B-A3B](Qwen3.6-35B-A3B/README.md) | End-to-end **W4A4 NVFP4** optimization of the Qwen3.6-35B-A3B (MoE + hybrid linear/full attention VLM) model: NVFP4 W4A4 post-training quantization → quantization-aware distillation (QAD) to recover the accuracy W4A4 costs → evaluation → vLLM deployment. | diff --git a/modelopt/torch/utils/plugins/prepare_megatron_data_blend.py b/modelopt/torch/utils/plugins/prepare_megatron_data_blend.py index 74cff163fa1..86f45089a43 100644 --- a/modelopt/torch/utils/plugins/prepare_megatron_data_blend.py +++ b/modelopt/torch/utils/plugins/prepare_megatron_data_blend.py @@ -199,8 +199,9 @@ def _to_chat( if row.get(key): chat[key] = row[key] else: - text = row.get("text") or "" - if not text: + text = row.get("text") + # A non-string value (e.g. a list) would otherwise pass as list content. + if not isinstance(text, str) or not text: return None if "\nraw body\n"}, "text") is None # A role without a mapping would cut its turn out of the middle: the whole row is dropped. From 0f2d986f41a1e318eb6d0efa3636a64740affa4c Mon Sep 17 00:00:00 2001 From: Yue Date: Sat, 17 Oct 2026 16:23:56 +0000 Subject: [PATCH 3/4] Drop the scope note from the Super VL QAD tutorial Co-Authored-By: Claude Opus 5.5 Signed-off-by: Yue --- .../NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md b/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md index 929bdd03701..d974379604a 100644 --- a/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md +++ b/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md @@ -28,8 +28,6 @@ All checkpoints were served with the same vLLM settings (see [Section 5](#5-eval MMLU-ProX scores a response only if it ends with the answer phrase in the question's language. Most of the QAD gain comes from fewer responses missing that phrase (11.3% for PTQ, 5.4% after QAD); on the responses with a readable answer, accuracy rises from 83.9 to 85.8. -**Scope:** QAD trains on text only, at a 32k sequence length, and only the language model is quantized. AA-LCR is the one long-context benchmark here (prompts average about 108k tokens); a 64k-sequence QAD run scored 67.31 on it, no better than 32k. Vision benchmarks were not run. - The exported W4A4 checkpoint is **70 GB**. --- From c28c50c6c67c6a101283135a9b55fb1d75eb1b53 Mon Sep 17 00:00:00 2001 From: Yue Date: Sat, 17 Oct 2026 16:26:00 +0000 Subject: [PATCH 4/4] State the tutorial's recipe, data and evaluation relative to the released checkpoint Co-Authored-By: Claude Opus 5.5 Signed-off-by: Yue --- .../NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md b/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md index d974379604a..8df736d75a4 100644 --- a/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md +++ b/examples/megatron_bridge/tutorials/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16/README.md @@ -10,6 +10,8 @@ End-to-end W4A4 quantization of [NVIDIA Nemotron 3.5 Super VL](https://huggingfa Only the language model (including its MTP head) is quantized; the vision tower and projector stay BF16 and are copied from the source checkpoint. +The PTQ recipe is the same as the one used for the released [`nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4`](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4) checkpoint. The calibration data and the QAD training data are all public Hugging Face datasets. The evaluation results below differ from the official model card scores. + ## Results All checkpoints were served with the same vLLM settings (see [Section 5](#5-evaluation-and-serving)); the quantized ones with an FP8 KV cache. BF16 is the mean of two runs.