From 2eae29b6d3507ee983559ed753ff2b1ec82d4e81 Mon Sep 17 00:00:00 2001 From: Zhiyu Cheng Date: Tue, 29 Sep 2026 17:43:00 -0700 Subject: [PATCH 1/4] [OMNIML-5935] Fix hf_ptq overwriting valid pad_token with eos_token get_tokenizer() had an inverted condition: it replaced pad_token whenever it was NOT "" or was None, which overwrote any valid existing token. For Qwen3-0.6B the pad_token <|endoftext|> (151643) was silently replaced with eos_token <|im_end|> (151645) in the exported tokenizer while generation_config.json retained the original ID, creating a mismatch. Fix the condition to only replace when pad_token is None (missing) or is "" (unusable as a pad token), preserving all other existing tokens. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Zhiyu Cheng --- CHANGELOG.rst | 1 + examples/hf_ptq/example_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index be9eeeeb48f..2f4633f73e9 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -110,6 +110,7 @@ Changelog - Fix Megatron unified HF export of MoE models with grouped-GEMM experts when only the experts are quantized (e.g. ``nvfp4_experts_only-*`` recipes): ``hf_quant_config.json`` and the ``quantization_config`` in ``config.json`` were not written, so the quantized experts were served as unquantized weights. Re-export such checkpoints. - Fix DFlash conversion on NoPE targets whose config leaves ``rope_theta`` unset. - Fix offline DFlash training failing to reconstruct the target logits when the captured hidden states are stored in a different dtype than the target's weights. +- Fix ``hf_ptq`` overwriting a model's existing ``pad_token`` with ``eos_token`` when the model already has a valid padding token. The exported tokenizer now preserves the source model's padding configuration. - Fix Megatron-Core checkpoint saving for quantized grouped MoE experts when tensor and expert parallelism are both enabled. - Fix unified HuggingFace export of RADIO-based VLMs retaining post-conversion vision and projector names instead of restoring the hub layout; deployment loaders could skip those weights. diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index ca6fa88f26f..62452e1d9f9 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -376,8 +376,8 @@ def get_tokenizer(ckpt_path, trust_remote_code=False, **kwargs) -> PreTrainedTok ckpt_path, trust_remote_code=trust_remote_code, **kwargs ) - # can't set attribute 'pad_token' for "" - if tokenizer.pad_token != "" or tokenizer.pad_token is None: + # can't set attribute 'pad_token' for ""; leave other valid pad tokens unchanged + if tokenizer.pad_token is None or tokenizer.pad_token == "": tokenizer.pad_token = tokenizer.eos_token assert tokenizer.pad_token is not None, f"Pad token for {ckpt_path} cannot be set!" From 56467ed98d71db3845e895a867c175be4af13628 Mon Sep 17 00:00:00 2001 From: Zhiyu Cheng Date: Fri, 2 Oct 2026 16:29:28 -0700 Subject: [PATCH 2/4] [OMNIML-5935] Simplify pad_token guard: only replace when None Per review feedback: replace pad_token with eos_token only when the tokenizer has no pad_token at all. This matches get_processor() and avoids silently replacing any existing pad token, including . Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Zhiyu Cheng --- examples/hf_ptq/example_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index 62452e1d9f9..2ffba9a4f69 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -376,8 +376,7 @@ def get_tokenizer(ckpt_path, trust_remote_code=False, **kwargs) -> PreTrainedTok ckpt_path, trust_remote_code=trust_remote_code, **kwargs ) - # can't set attribute 'pad_token' for ""; leave other valid pad tokens unchanged - if tokenizer.pad_token is None or tokenizer.pad_token == "": + if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token assert tokenizer.pad_token is not None, f"Pad token for {ckpt_path} cannot be set!" From cbf620f6f049914209472c2a10efe34d50d5f5c9 Mon Sep 17 00:00:00 2001 From: Zhiyu Cheng Date: Fri, 2 Oct 2026 17:27:07 -0700 Subject: [PATCH 3/4] [OMNIML-5935] Add get_tokenizer unit tests for pad_token handling Tests three cases: valid pad token preserved, preserved, and None falls back to eos_token. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Zhiyu Cheng --- tests/examples/hf_ptq/test_example_utils.py | 31 +++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/examples/hf_ptq/test_example_utils.py b/tests/examples/hf_ptq/test_example_utils.py index 8651183cb83..ec5145f7cc7 100644 --- a/tests/examples/hf_ptq/test_example_utils.py +++ b/tests/examples/hf_ptq/test_example_utils.py @@ -436,6 +436,37 @@ def from_pretrained(*args, output_loading_info=False, **kwargs): assert used["path"] == ("bundled" if expect_bundled_code else "builtin") +# ---------- get_tokenizer ---------------------------------------------------- + + +class _StubTokenizer: + """Minimal stand-in for a PreTrainedTokenizer.""" + + def __init__(self, pad_token, eos_token="<|eos|>"): + self.pad_token = pad_token + self.eos_token = eos_token + + +@pytest.mark.parametrize( + ("pad_token", "eos_token", "expected_pad"), + [ + # Valid existing pad token must be preserved unchanged. + ("<|endoftext|>", "<|im_end|>", "<|endoftext|>"), + # as pad must also be preserved (only None triggers the fallback). + ("", "<|im_end|>", ""), + # Missing pad token falls back to eos_token. + (None, "<|im_end|>", "<|im_end|>"), + ], +) +def test_get_tokenizer_pad_token(monkeypatch, pad_token, eos_token, expected_pad): + stub = _StubTokenizer(pad_token=pad_token, eos_token=eos_token) + monkeypatch.setattr( + example_utils.AutoTokenizer, "from_pretrained", lambda *a, **k: stub + ) + tok = example_utils.get_tokenizer("/fake/path") + assert tok.pad_token == expected_pad + + def _layerwise(**kwargs): return {"enable": True, **kwargs} From 54fc258340e505caebcb93599bd629f3a414fa20 Mon Sep 17 00:00:00 2001 From: Zhiyu Cheng Date: Tue, 6 Oct 2026 10:32:15 -0700 Subject: [PATCH 4/4] [OMNIML-5935] Fix ruff format in get_tokenizer test Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: Zhiyu Cheng --- tests/examples/hf_ptq/test_example_utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/examples/hf_ptq/test_example_utils.py b/tests/examples/hf_ptq/test_example_utils.py index ec5145f7cc7..cedf8ac29c5 100644 --- a/tests/examples/hf_ptq/test_example_utils.py +++ b/tests/examples/hf_ptq/test_example_utils.py @@ -460,9 +460,7 @@ def __init__(self, pad_token, eos_token="<|eos|>"): ) def test_get_tokenizer_pad_token(monkeypatch, pad_token, eos_token, expected_pad): stub = _StubTokenizer(pad_token=pad_token, eos_token=eos_token) - monkeypatch.setattr( - example_utils.AutoTokenizer, "from_pretrained", lambda *a, **k: stub - ) + monkeypatch.setattr(example_utils.AutoTokenizer, "from_pretrained", lambda *a, **k: stub) tok = example_utils.get_tokenizer("/fake/path") assert tok.pad_token == expected_pad