diff --git a/CHANGELOG.rst b/CHANGELOG.rst index be9eeeeb48f..2f4633f73e9 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -110,6 +110,7 @@ Changelog - Fix Megatron unified HF export of MoE models with grouped-GEMM experts when only the experts are quantized (e.g. ``nvfp4_experts_only-*`` recipes): ``hf_quant_config.json`` and the ``quantization_config`` in ``config.json`` were not written, so the quantized experts were served as unquantized weights. Re-export such checkpoints. - Fix DFlash conversion on NoPE targets whose config leaves ``rope_theta`` unset. - Fix offline DFlash training failing to reconstruct the target logits when the captured hidden states are stored in a different dtype than the target's weights. +- Fix ``hf_ptq`` overwriting a model's existing ``pad_token`` with ``eos_token`` when the model already has a valid padding token. The exported tokenizer now preserves the source model's padding configuration. - Fix Megatron-Core checkpoint saving for quantized grouped MoE experts when tensor and expert parallelism are both enabled. - Fix unified HuggingFace export of RADIO-based VLMs retaining post-conversion vision and projector names instead of restoring the hub layout; deployment loaders could skip those weights. diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index ca6fa88f26f..2ffba9a4f69 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -376,8 +376,7 @@ def get_tokenizer(ckpt_path, trust_remote_code=False, **kwargs) -> PreTrainedTok ckpt_path, trust_remote_code=trust_remote_code, **kwargs ) - # can't set attribute 'pad_token' for "" - if tokenizer.pad_token != "" or tokenizer.pad_token is None: + if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token assert tokenizer.pad_token is not None, f"Pad token for {ckpt_path} cannot be set!" diff --git a/tests/examples/hf_ptq/test_example_utils.py b/tests/examples/hf_ptq/test_example_utils.py index 8651183cb83..cedf8ac29c5 100644 --- a/tests/examples/hf_ptq/test_example_utils.py +++ b/tests/examples/hf_ptq/test_example_utils.py @@ -436,6 +436,35 @@ def from_pretrained(*args, output_loading_info=False, **kwargs): assert used["path"] == ("bundled" if expect_bundled_code else "builtin") +# ---------- get_tokenizer ---------------------------------------------------- + + +class _StubTokenizer: + """Minimal stand-in for a PreTrainedTokenizer.""" + + def __init__(self, pad_token, eos_token="<|eos|>"): + self.pad_token = pad_token + self.eos_token = eos_token + + +@pytest.mark.parametrize( + ("pad_token", "eos_token", "expected_pad"), + [ + # Valid existing pad token must be preserved unchanged. + ("<|endoftext|>", "<|im_end|>", "<|endoftext|>"), + # as pad must also be preserved (only None triggers the fallback). + ("", "<|im_end|>", ""), + # Missing pad token falls back to eos_token. + (None, "<|im_end|>", "<|im_end|>"), + ], +) +def test_get_tokenizer_pad_token(monkeypatch, pad_token, eos_token, expected_pad): + stub = _StubTokenizer(pad_token=pad_token, eos_token=eos_token) + monkeypatch.setattr(example_utils.AutoTokenizer, "from_pretrained", lambda *a, **k: stub) + tok = example_utils.get_tokenizer("/fake/path") + assert tok.pad_token == expected_pad + + def _layerwise(**kwargs): return {"enable": True, **kwargs}