[OMNIML-5935] Fix hf_ptq overwriting valid pad_token with eos_token

CHANGELOG.rst

@@ -110,6 +110,7 @@ Changelog
 - Fix Megatron unified HF export of MoE models with grouped-GEMM experts when only the experts are quantized (e.g. ``nvfp4_experts_only-*`` recipes): ``hf_quant_config.json`` and the ``quantization_config`` in ``config.json`` were not written, so the quantized experts were served as unquantized weights. Re-export such checkpoints.
 - Fix DFlash conversion on NoPE targets whose config leaves ``rope_theta`` unset.
 - Fix offline DFlash training failing to reconstruct the target logits when the captured hidden states are stored in a different dtype than the target's weights.
+- Fix ``hf_ptq`` overwriting a model's existing ``pad_token`` with ``eos_token`` when the model already has a valid padding token. The exported tokenizer now preserves the source model's padding configuration.
 - Fix Megatron-Core checkpoint saving for quantized grouped MoE experts when tensor and expert parallelism are both enabled.
 - Fix unified HuggingFace export of RADIO-based VLMs retaining post-conversion vision and
   projector names instead of restoring the hub layout; deployment loaders could skip those weights.

examples/hf_ptq/example_utils.py

@@ -376,8 +376,7 @@ def get_tokenizer(ckpt_path, trust_remote_code=False, **kwargs) -> PreTrainedTok
         ckpt_path, trust_remote_code=trust_remote_code, **kwargs
     )
 
-    # can't set attribute 'pad_token' for "<unk>"
-    if tokenizer.pad_token != "<unk>" or tokenizer.pad_token is None:
+    if tokenizer.pad_token is None:
         tokenizer.pad_token = tokenizer.eos_token
 
     assert tokenizer.pad_token is not None, f"Pad token for {ckpt_path} cannot be set!"

tests/examples/hf_ptq/test_example_utils.py

@@ -436,6 +436,35 @@ def from_pretrained(*args, output_loading_info=False, **kwargs):
     assert used["path"] == ("bundled" if expect_bundled_code else "builtin")
 
 
+# ---------- get_tokenizer ----------------------------------------------------
+
+
+class _StubTokenizer:
+    """Minimal stand-in for a PreTrainedTokenizer."""
+
+    def __init__(self, pad_token, eos_token="<|eos|>"):
+        self.pad_token = pad_token
+        self.eos_token = eos_token
+
+
+@pytest.mark.parametrize(
+    ("pad_token", "eos_token", "expected_pad"),
+    [
+        # Valid existing pad token must be preserved unchanged.
+        ("<|endoftext|>", "<|im_end|>", "<|endoftext|>"),
+        # <unk> as pad must also be preserved (only None triggers the fallback).
+        ("<unk>", "<|im_end|>", "<unk>"),
+        # Missing pad token falls back to eos_token.
+        (None, "<|im_end|>", "<|im_end|>"),
+    ],
+)
+def test_get_tokenizer_pad_token(monkeypatch, pad_token, eos_token, expected_pad):
+    stub = _StubTokenizer(pad_token=pad_token, eos_token=eos_token)
+    monkeypatch.setattr(example_utils.AutoTokenizer, "from_pretrained", lambda *a, **k: stub)
+    tok = example_utils.get_tokenizer("/fake/path")
+    assert tok.pad_token == expected_pad
+
+
 def _layerwise(**kwargs):
     return {"enable": True, **kwargs}