From 4b6f5d229618bb5a1455f54169879e4dd19a1b9f Mon Sep 17 00:00:00 2001 From: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:07:36 -0400 Subject: [PATCH 1/6] fix(autocast): clamp FP64 values that FP32 cannot represent GraphSanitizer narrowed FP64 initializers and Constant values with a bare astype(np.float32), so anything past the FP32 range became +/-inf. numpy raises "overflow encountered in cast" for it, but nothing acted on that, and an infinity then poisons whatever arithmetic the tensor feeds. DBL_MAX is an ordinary way to spell "no bound" on a Clip, so this is reachable from a valid model rather than only a contrived one. Clamp to the FP32 range instead and name the tensor in a warning, matching what PrecisionConverter._convert_initializer already does when it narrows FP32 to FP16. Values too small for FP32 still flush to zero, which is ordinary narrowing and does not propagate the way an infinity does. Signed-off-by: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> --- CHANGELOG.rst | 1 + modelopt/onnx/autocast/graphsanitizer.py | 19 +++++- .../unit/onnx/autocast/test_graphsanitizer.py | 66 +++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4f621df5dea1..787abc831d4b 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -94,6 +94,7 @@ Changelog - Fix ``megatron_generate`` dropping the VLM vision inputs (``pixel_values`` / ``image_grid_thw`` / ``image_sizes``) after the first generated token when KV-cache decoding is off, including the automatic fallback under sequence parallelism, which made generation silently ignore the image. No other ModelOpt feature is affected. - Fix two issues in the vLLM offline hidden-state dump (``examples/speculative_decoding/collect_hidden_states/compute_hidden_states_vllm.py``) that only surface on large runs. **Resume:** the filter that skips conversations whose ``.pt`` already exists now runs with ``load_from_cache_file=False``. It depends on on-disk state, which is not part of the fingerprint ``datasets`` computes from the function and the dataset, so with a persistent HF cache reused across a resumed or requeued run the cached "keep everything" result from an earlier run was replayed and the dump re-generated and overwrote conversations it had already finished (observed: tens of thousands of ``.pt`` rewritten while the output count stayed flat). **Staging:** generation is now chunked (``--save-chunk-size``, default 256), so each chunk is saved and its staged hidden states freed before the next chunk is generated. Previously the whole dataset was generated before anything was saved, which kept every conversation staged in the connector's ``shared_storage_path`` (``/dev/shm``, i.e. RAM, by default) at once and exhausted it partway through large dumps. Chunking also makes the dump incrementally durable, so an interrupted run keeps its finished conversations and resumes from them. The save path now also frees each conversation's staged hidden states in a ``finally``, so a conversation skipped mid-loop (e.g. a short ``loss_mask``) can no longer leak its staging file, and conversation ids are validated as plain filenames before being used to build output paths. - Hybrid (e.g. Nemotron-H) checkpoints saved by the ``examples/megatron_bridge`` scripts now record their layer spec in ``run_config.yaml`` in a form that reloads, so they can be converted to HuggingFace; a checkpoint saved by an earlier release still needs its ``model.hybrid_stack_spec`` block replaced by hand. +- Fix AutoCast turning FP64 initializers and ``Constant`` values that exceed the FP32 range into infinities. They are now clamped to the largest FP32 value and the affected tensor is named in a warning. 0.47.0 (2026-09-23) ^^^^^^^^^^^^^^^^^^^ diff --git a/modelopt/onnx/autocast/graphsanitizer.py b/modelopt/onnx/autocast/graphsanitizer.py index 58bd678159c5..80ef79019fe7 100644 --- a/modelopt/onnx/autocast/graphsanitizer.py +++ b/modelopt/onnx/autocast/graphsanitizer.py @@ -517,6 +517,21 @@ def _get_initializer_value(self, name: str, return_array: bool = False) -> np.nd return value if return_array else value.item() return None + def _narrow_to_fp32(self, fp64_data: np.ndarray, name: str) -> np.ndarray: + """Converts FP64 data to FP32, clamping values that FP32 cannot represent. + + A bare cast turns anything beyond the FP32 range into +/-inf, which then poisons + the arithmetic it feeds. + """ + fp32_max = np.finfo(np.float32).max + if np.any(np.isfinite(fp64_data) & (np.abs(fp64_data) > fp32_max)): + logger.warning( + f"Initializer {name} contains values outside the FP32 range, " + f"they will be clamped to {fp32_max}." + ) + fp64_data = np.clip(fp64_data, -fp32_max, fp32_max) + return fp64_data.astype(np.float32) + def _convert_fp64_initializers(self) -> bool: """Convert FP64 initializers to FP32. @@ -529,7 +544,7 @@ def _convert_fp64_initializers(self) -> bool: if initializer.data_type == onnx.TensorProto.DOUBLE: # Convert the data to FP32 fp64_data = numpy_helper.to_array(initializer, base_dir=self.external_data_dir) - fp32_data = fp64_data.astype(np.float32) + fp32_data = self._narrow_to_fp32(fp64_data, initializer.name) # Create new initializer with FP32 data new_initializer = numpy_helper.from_array(fp32_data, name=initializer.name) @@ -588,7 +603,7 @@ def _convert_fp64_nodes(self) -> bool: if attr.name == "value" and attr.t.data_type == onnx.TensorProto.DOUBLE: # Convert the tensor value to FP32 fp64_data = numpy_helper.to_array(attr.t, base_dir=self.external_data_dir) - fp32_data = fp64_data.astype(np.float32) + fp32_data = self._narrow_to_fp32(fp64_data, node.name) new_tensor = numpy_helper.from_array(fp32_data) attr.t.CopyFrom(new_tensor) modified = True diff --git a/tests/unit/onnx/autocast/test_graphsanitizer.py b/tests/unit/onnx/autocast/test_graphsanitizer.py index f42324a6338b..c5d1f0ea39b3 100644 --- a/tests/unit/onnx/autocast/test_graphsanitizer.py +++ b/tests/unit/onnx/autocast/test_graphsanitizer.py @@ -235,6 +235,72 @@ def test_convert_fp64_initializers(): assert init.data_type == TensorProto.FLOAT +def test_convert_fp64_initializers_clamps_out_of_range_values(): + """Values FP32 cannot represent must be clamped, not cast to infinity.""" + x = helper.make_tensor_value_info("X", TensorProto.FLOAT, [2]) + y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [2]) + fp64_max = np.finfo(np.float64).max + + graph = helper.make_graph( + nodes=[ + helper.make_node("Clip", ["X", "lower", "upper"], ["Y"], name="clip"), + ], + name="fp64_out_of_range", + inputs=[x], + outputs=[y], + initializer=[ + numpy_helper.from_array(np.array([-fp64_max], np.float64), name="lower"), + numpy_helper.from_array(np.array([fp64_max], np.float64), name="upper"), + ], + ) + sanitizer = GraphSanitizer(helper.make_model(graph)) + + assert sanitizer._convert_fp64_initializers() is True + + fp32_max = np.finfo(np.float32).max + converted = { + init.name: numpy_helper.to_array(init) for init in sanitizer.model.graph.initializer + } + for name in ("lower", "upper"): + assert np.all(np.isfinite(converted[name])), f"{name} was cast to infinity" + assert converted["lower"] == -fp32_max + assert converted["upper"] == fp32_max + + +def test_convert_fp64_nodes_clamps_out_of_range_values(): + """A Constant node's FP64 value must be clamped the same way an initializer is.""" + x = helper.make_tensor_value_info("X", TensorProto.FLOAT, [2]) + y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [2]) + fp64_max = np.finfo(np.float64).max + + constant = helper.make_node( + "Constant", + inputs=[], + outputs=["big"], + name="big_constant", + value=numpy_helper.from_array(np.array([fp64_max], np.float64), name="big_value"), + ) + graph = helper.make_graph( + nodes=[constant, helper.make_node("Mul", ["X", "big"], ["Y"], name="mul")], + name="fp64_constant_out_of_range", + inputs=[x], + outputs=[y], + ) + sanitizer = GraphSanitizer(helper.make_model(graph)) + + assert sanitizer._convert_fp64_nodes() is True + + value = next( + numpy_helper.to_array(attr.t) + for node in sanitizer.model.graph.node + if node.name == "big_constant" + for attr in node.attribute + if attr.name == "value" + ) + assert np.all(np.isfinite(value)), "Constant value was cast to infinity" + assert value == np.finfo(np.float32).max + + def test_convert_fp64_io_types(): """Test conversion of FP64 input/output types to FP32.""" # Create inputs and outputs with FP64 types From 2182769b3f5f7ea266f0973d1c4c1b25ac324b6a Mon Sep 17 00:00:00 2001 From: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> Date: Mon, 28 Sep 2026 19:40:23 -0400 Subject: [PATCH 2/6] fix(autocast): keep the sign of an initializer that underflows When an initializer value is too small for the target precision, _convert_initializer replaces it with that type's smallest subnormal. The replacement was the positive subnormal regardless of the original sign, so a tiny negative value came back positive -- -1e-10 became +5.96e-08 for FP16. The magnitude is negligible either way; the sign is not, for anything that compares against zero. Signed-off-by: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> --- CHANGELOG.rst | 1 + modelopt/onnx/autocast/precisionconverter.py | 2 +- .../onnx/autocast/test_precisionconverter.py | 32 +++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 787abc831d4b..b5970a21e039 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -95,6 +95,7 @@ Changelog - Fix two issues in the vLLM offline hidden-state dump (``examples/speculative_decoding/collect_hidden_states/compute_hidden_states_vllm.py``) that only surface on large runs. **Resume:** the filter that skips conversations whose ``.pt`` already exists now runs with ``load_from_cache_file=False``. It depends on on-disk state, which is not part of the fingerprint ``datasets`` computes from the function and the dataset, so with a persistent HF cache reused across a resumed or requeued run the cached "keep everything" result from an earlier run was replayed and the dump re-generated and overwrote conversations it had already finished (observed: tens of thousands of ``.pt`` rewritten while the output count stayed flat). **Staging:** generation is now chunked (``--save-chunk-size``, default 256), so each chunk is saved and its staged hidden states freed before the next chunk is generated. Previously the whole dataset was generated before anything was saved, which kept every conversation staged in the connector's ``shared_storage_path`` (``/dev/shm``, i.e. RAM, by default) at once and exhausted it partway through large dumps. Chunking also makes the dump incrementally durable, so an interrupted run keeps its finished conversations and resumes from them. The save path now also frees each conversation's staged hidden states in a ``finally``, so a conversation skipped mid-loop (e.g. a short ``loss_mask``) can no longer leak its staging file, and conversation ids are validated as plain filenames before being used to build output paths. - Hybrid (e.g. Nemotron-H) checkpoints saved by the ``examples/megatron_bridge`` scripts now record their layer spec in ``run_config.yaml`` in a form that reloads, so they can be converted to HuggingFace; a checkpoint saved by an earlier release still needs its ``model.hybrid_stack_spec`` block replaced by hand. - Fix AutoCast turning FP64 initializers and ``Constant`` values that exceed the FP32 range into infinities. They are now clamped to the largest FP32 value and the affected tensor is named in a warning. +- Fix AutoCast flipping the sign of an initializer value too small for the target precision; such values now keep their sign. 0.47.0 (2026-09-23) ^^^^^^^^^^^^^^^^^^^ diff --git a/modelopt/onnx/autocast/precisionconverter.py b/modelopt/onnx/autocast/precisionconverter.py index a78249fdca4c..32e5c758eaeb 100644 --- a/modelopt/onnx/autocast/precisionconverter.py +++ b/modelopt/onnx/autocast/precisionconverter.py @@ -1240,7 +1240,7 @@ def _convert_initializer_data( self._warned_values_clamp_min = True np_array = np.where( (np_array != 0.0) & (np.abs(np_array) < data_lowest), - data_lowest, + np.copysign(data_lowest, np_array), np_array, ) new_array = np_array.astype(to_type.numpy_type) diff --git a/tests/unit/onnx/autocast/test_precisionconverter.py b/tests/unit/onnx/autocast/test_precisionconverter.py index d3e0a786a65a..d3c672a18b9d 100644 --- a/tests/unit/onnx/autocast/test_precisionconverter.py +++ b/tests/unit/onnx/autocast/test_precisionconverter.py @@ -624,6 +624,38 @@ def test_clamping_fp16_initializers_out_of_range( assert np.all(add_init_fp32_array == add_init_out_of_range) +@pytest.mark.parametrize("use_standalone_type_inference", [True, False]) +def test_underflowing_fp16_initializers_keep_their_sign( + model_with_multiple_consumers, use_standalone_type_inference +): + """A value too small for FP16 is replaced by the smallest subnormal of the same sign.""" + model, value_info_map, initializer_map, node_to_init_map = model_with_multiple_consumers + + smallest = np.finfo(np.float16).smallest_subnormal + add_init_underflow = np.array([[-1e-10, 1e-10]], dtype=np.float32) + model.graph.initializer[1].CopyFrom( + numpy_helper.from_array(add_init_underflow, name="add_init") + ) + + converter = PrecisionConverter( + model, + value_info_map, + initializer_map, + node_to_init_map, + use_standalone_type_inference=use_standalone_type_inference, + ) + converter._convert_initializers(low_precision_nodes=["add1", "add2"], high_precision_nodes=[]) + + converted = next( + numpy_helper.to_array(init) + for init in converter.model.graph.initializer + if init.name == "add_init" + ) + assert converted.dtype == np.float16 + assert converted[0, 0] == -smallest, "a negative value must not come back positive" + assert converted[0, 1] == smallest + + @pytest.mark.parametrize("use_standalone_type_inference", [True, False]) def test_bf16_no_clamping_initializers_out_of_range( model_with_multiple_consumers, use_standalone_type_inference From 65d1989a8f80be6b6d9a02b789728be028ea56f0 Mon Sep 17 00:00:00 2001 From: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:02:43 -0400 Subject: [PATCH 3/6] Leave an existing infinity alone when narrowing, and name the tensor The guard tested only finite out-of-range values but np.clip then flattened everything, so an infinity the model already held survived on its own and was turned into the FP32 maximum as soon as the same tensor also carried an oversized finite value. Clamp only the finite out-of-range elements. Narrowing an infinity the author put there is not this function's decision; the cast maps it to an FP32 infinity, which is what the model asked for. The warning also said "Initializer" for a Constant node's value. It now names the tensor, which is the initializer name or the node's output. Signed-off-by: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> --- modelopt/onnx/autocast/graphsanitizer.py | 18 ++++++++----- .../unit/onnx/autocast/test_graphsanitizer.py | 26 +++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/modelopt/onnx/autocast/graphsanitizer.py b/modelopt/onnx/autocast/graphsanitizer.py index 80ef79019fe7..ddec94092c4c 100644 --- a/modelopt/onnx/autocast/graphsanitizer.py +++ b/modelopt/onnx/autocast/graphsanitizer.py @@ -518,18 +518,20 @@ def _get_initializer_value(self, name: str, return_array: bool = False) -> np.nd return None def _narrow_to_fp32(self, fp64_data: np.ndarray, name: str) -> np.ndarray: - """Converts FP64 data to FP32, clamping values that FP32 cannot represent. + """Converts FP64 data to FP32, clamping finite values that FP32 cannot represent. - A bare cast turns anything beyond the FP32 range into +/-inf, which then poisons - the arithmetic it feeds. + A bare cast turns a finite value beyond the FP32 range into +/-inf, which then + poisons the arithmetic it feeds. An infinity the model already held is left alone, + since narrowing it is not this function's decision to make. """ fp32_max = np.finfo(np.float32).max - if np.any(np.isfinite(fp64_data) & (np.abs(fp64_data) > fp32_max)): + out_of_range = np.isfinite(fp64_data) & (np.abs(fp64_data) > fp32_max) + if np.any(out_of_range): logger.warning( - f"Initializer {name} contains values outside the FP32 range, " + f"Tensor {name} contains values outside the FP32 range, " f"they will be clamped to {fp32_max}." ) - fp64_data = np.clip(fp64_data, -fp32_max, fp32_max) + fp64_data = np.where(out_of_range, np.copysign(fp32_max, fp64_data), fp64_data) return fp64_data.astype(np.float32) def _convert_fp64_initializers(self) -> bool: @@ -603,7 +605,9 @@ def _convert_fp64_nodes(self) -> bool: if attr.name == "value" and attr.t.data_type == onnx.TensorProto.DOUBLE: # Convert the tensor value to FP32 fp64_data = numpy_helper.to_array(attr.t, base_dir=self.external_data_dir) - fp32_data = self._narrow_to_fp32(fp64_data, node.name) + fp32_data = self._narrow_to_fp32( + fp64_data, node.output[0] if node.output else node.name + ) new_tensor = numpy_helper.from_array(fp32_data) attr.t.CopyFrom(new_tensor) modified = True diff --git a/tests/unit/onnx/autocast/test_graphsanitizer.py b/tests/unit/onnx/autocast/test_graphsanitizer.py index c5d1f0ea39b3..d403d04b3e4c 100644 --- a/tests/unit/onnx/autocast/test_graphsanitizer.py +++ b/tests/unit/onnx/autocast/test_graphsanitizer.py @@ -267,6 +267,32 @@ def test_convert_fp64_initializers_clamps_out_of_range_values(): assert converted["upper"] == fp32_max +def test_convert_fp64_initializers_preserves_infinity(): + """An infinity the model already held is not a value FP32 cannot represent.""" + x = helper.make_tensor_value_info("X", TensorProto.FLOAT, [3]) + y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [3]) + # An oversized finite value alongside the infinities must not drag them down with it. + values = np.array([np.inf, -np.inf, np.finfo(np.float64).max], dtype=np.float64) + + graph = helper.make_graph( + nodes=[helper.make_node("Mul", ["X", "mixed"], ["Y"], name="mul")], + name="fp64_with_infinity", + inputs=[x], + outputs=[y], + initializer=[numpy_helper.from_array(values, name="mixed")], + ) + sanitizer = GraphSanitizer(helper.make_model(graph)) + + assert sanitizer._convert_fp64_initializers() is True + + converted = numpy_helper.to_array( + next(init for init in sanitizer.model.graph.initializer if init.name == "mixed") + ) + assert converted[0] == np.inf + assert converted[1] == -np.inf + assert converted[2] == np.finfo(np.float32).max + + def test_convert_fp64_nodes_clamps_out_of_range_values(): """A Constant node's FP64 value must be clamped the same way an initializer is.""" x = helper.make_tensor_value_info("X", TensorProto.FLOAT, [2]) From 02c1fe4174479855e296ea14397944f492a5a9d8 Mon Sep 17 00:00:00 2001 From: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:13:57 -0400 Subject: [PATCH 4/6] Cover ConstantOfShape in the node overflow test _convert_fp64_nodes sends Constant and ConstantOfShape through the same helper, but only Constant was asserted, so a dispatch regression could send a ConstantOfShape value to infinity without failing anything. Parametrize over both rather than add a near-duplicate test. Signed-off-by: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> --- modelopt/onnx/autocast/graphsanitizer.py | 23 +---------- .../unit/onnx/autocast/test_graphsanitizer.py | 38 +++++++++++-------- 2 files changed, 24 insertions(+), 37 deletions(-) diff --git a/modelopt/onnx/autocast/graphsanitizer.py b/modelopt/onnx/autocast/graphsanitizer.py index ddec94092c4c..58bd678159c5 100644 --- a/modelopt/onnx/autocast/graphsanitizer.py +++ b/modelopt/onnx/autocast/graphsanitizer.py @@ -517,23 +517,6 @@ def _get_initializer_value(self, name: str, return_array: bool = False) -> np.nd return value if return_array else value.item() return None - def _narrow_to_fp32(self, fp64_data: np.ndarray, name: str) -> np.ndarray: - """Converts FP64 data to FP32, clamping finite values that FP32 cannot represent. - - A bare cast turns a finite value beyond the FP32 range into +/-inf, which then - poisons the arithmetic it feeds. An infinity the model already held is left alone, - since narrowing it is not this function's decision to make. - """ - fp32_max = np.finfo(np.float32).max - out_of_range = np.isfinite(fp64_data) & (np.abs(fp64_data) > fp32_max) - if np.any(out_of_range): - logger.warning( - f"Tensor {name} contains values outside the FP32 range, " - f"they will be clamped to {fp32_max}." - ) - fp64_data = np.where(out_of_range, np.copysign(fp32_max, fp64_data), fp64_data) - return fp64_data.astype(np.float32) - def _convert_fp64_initializers(self) -> bool: """Convert FP64 initializers to FP32. @@ -546,7 +529,7 @@ def _convert_fp64_initializers(self) -> bool: if initializer.data_type == onnx.TensorProto.DOUBLE: # Convert the data to FP32 fp64_data = numpy_helper.to_array(initializer, base_dir=self.external_data_dir) - fp32_data = self._narrow_to_fp32(fp64_data, initializer.name) + fp32_data = fp64_data.astype(np.float32) # Create new initializer with FP32 data new_initializer = numpy_helper.from_array(fp32_data, name=initializer.name) @@ -605,9 +588,7 @@ def _convert_fp64_nodes(self) -> bool: if attr.name == "value" and attr.t.data_type == onnx.TensorProto.DOUBLE: # Convert the tensor value to FP32 fp64_data = numpy_helper.to_array(attr.t, base_dir=self.external_data_dir) - fp32_data = self._narrow_to_fp32( - fp64_data, node.output[0] if node.output else node.name - ) + fp32_data = fp64_data.astype(np.float32) new_tensor = numpy_helper.from_array(fp32_data) attr.t.CopyFrom(new_tensor) modified = True diff --git a/tests/unit/onnx/autocast/test_graphsanitizer.py b/tests/unit/onnx/autocast/test_graphsanitizer.py index d403d04b3e4c..2670bb1cab42 100644 --- a/tests/unit/onnx/autocast/test_graphsanitizer.py +++ b/tests/unit/onnx/autocast/test_graphsanitizer.py @@ -293,38 +293,44 @@ def test_convert_fp64_initializers_preserves_infinity(): assert converted[2] == np.finfo(np.float32).max -def test_convert_fp64_nodes_clamps_out_of_range_values(): - """A Constant node's FP64 value must be clamped the same way an initializer is.""" +@pytest.mark.parametrize("op_type", ["Constant", "ConstantOfShape"]) +def test_convert_fp64_nodes_clamps_out_of_range_values(op_type): + """A node's FP64 value attribute must be clamped the same way an initializer is.""" x = helper.make_tensor_value_info("X", TensorProto.FLOAT, [2]) y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [2]) - fp64_max = np.finfo(np.float64).max - - constant = helper.make_node( - "Constant", - inputs=[], - outputs=["big"], - name="big_constant", - value=numpy_helper.from_array(np.array([fp64_max], np.float64), name="big_value"), + value = numpy_helper.from_array( + np.array([np.finfo(np.float64).max], np.float64), name="big_value" ) + + initializer = [] + if op_type == "ConstantOfShape": + producer = helper.make_node( + "ConstantOfShape", ["shape"], ["big"], name="big_node", value=value + ) + initializer.append(numpy_helper.from_array(np.array([2], dtype=np.int64), name="shape")) + else: + producer = helper.make_node("Constant", [], ["big"], name="big_node", value=value) + graph = helper.make_graph( - nodes=[constant, helper.make_node("Mul", ["X", "big"], ["Y"], name="mul")], - name="fp64_constant_out_of_range", + nodes=[producer, helper.make_node("Mul", ["X", "big"], ["Y"], name="mul")], + name="fp64_node_out_of_range", inputs=[x], outputs=[y], + initializer=initializer, ) sanitizer = GraphSanitizer(helper.make_model(graph)) assert sanitizer._convert_fp64_nodes() is True - value = next( + converted = next( numpy_helper.to_array(attr.t) for node in sanitizer.model.graph.node - if node.name == "big_constant" + if node.name == "big_node" for attr in node.attribute if attr.name == "value" ) - assert np.all(np.isfinite(value)), "Constant value was cast to infinity" - assert value == np.finfo(np.float32).max + assert np.all(np.isfinite(converted)), f"{op_type} value was cast to infinity" + assert converted == np.finfo(np.float32).max def test_convert_fp64_io_types(): From acbd79f8f4eb56dacacf707883ae6dd7429f7e26 Mon Sep 17 00:00:00 2001 From: SID-6921 <99672439+SID-6921@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:06:20 -0400 Subject: [PATCH 5/6] Clamp FP64 values exceeding FP32 range instead of casting to infinity _convert_fp64_initializers() and _convert_fp64_nodes() cast FP64 data straight to FP32, silently turning any finite value larger than FP32's max into infinity. Add a shared _clamp_to_fp32_range() helper that clamps only such out-of-range finite values (preserving any infinity already present) and warns with the affected tensor's name, matching the behavior the existing test suite expects. Signed-off-by: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> --- modelopt/onnx/autocast/graphsanitizer.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/modelopt/onnx/autocast/graphsanitizer.py b/modelopt/onnx/autocast/graphsanitizer.py index 58bd678159c5..708a80d77233 100644 --- a/modelopt/onnx/autocast/graphsanitizer.py +++ b/modelopt/onnx/autocast/graphsanitizer.py @@ -517,6 +517,23 @@ def _get_initializer_value(self, name: str, return_array: bool = False) -> np.nd return value if return_array else value.item() return None + @staticmethod + def _clamp_to_fp32_range(data: np.ndarray, name: str) -> np.ndarray: + """Clamp finite values outside the FP32 range, leaving existing +/-inf untouched. + + Casting a finite FP64 value larger than FP32's max straight to FP32 silently turns it + into infinity. Genuine infinities already in the data are not "too large for FP32" and + must be preserved as-is. + """ + fp32_max = np.finfo(np.float32).max + out_of_range = np.isfinite(data) & (np.abs(data) > fp32_max) + if np.any(out_of_range): + logger.warning( + f"{name} contains FP64 values outside the FP32 range; clamping to {fp32_max}." + ) + data = np.where(out_of_range, np.copysign(fp32_max, data), data) + return data + def _convert_fp64_initializers(self) -> bool: """Convert FP64 initializers to FP32. @@ -529,6 +546,7 @@ def _convert_fp64_initializers(self) -> bool: if initializer.data_type == onnx.TensorProto.DOUBLE: # Convert the data to FP32 fp64_data = numpy_helper.to_array(initializer, base_dir=self.external_data_dir) + fp64_data = self._clamp_to_fp32_range(fp64_data, initializer.name) fp32_data = fp64_data.astype(np.float32) # Create new initializer with FP32 data @@ -588,6 +606,9 @@ def _convert_fp64_nodes(self) -> bool: if attr.name == "value" and attr.t.data_type == onnx.TensorProto.DOUBLE: # Convert the tensor value to FP32 fp64_data = numpy_helper.to_array(attr.t, base_dir=self.external_data_dir) + fp64_data = self._clamp_to_fp32_range( + fp64_data, f"{node.op_type} node {node.name}" + ) fp32_data = fp64_data.astype(np.float32) new_tensor = numpy_helper.from_array(fp32_data) attr.t.CopyFrom(new_tensor) From aef1e48c6a41cc94a26b5d1ba0ebd2e4c68a14fd Mon Sep 17 00:00:00 2001 From: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> Date: Mon, 26 Oct 2026 15:46:15 -0400 Subject: [PATCH 6/6] Move CHANGELOG entries to the 0.49.0 section Signed-off-by: Siddhardha Nanda <99672439+SID-6921@users.noreply.github.com> --- CHANGELOG.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f32f477d49b5..f632f4591c13 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -60,6 +60,8 @@ Changelog - Fix Megatron Top-K/Top-P distillation gradients being multiplied by the tensor-parallel size when tensor parallelism is enabled; reported loss values are unchanged. - Fix the Puzzletron memory objective pricing the KV cache with ``hidden_size // num_attention_heads`` instead of the model's ``head_dim``, which misestimated it for models such as Mistral-Small, GPT-OSS and Gemma-3 and for every candidate with a pruned hidden size. Recompute subblock statistics produced with earlier versions. - Fix ONNX INT4 quantization ending the Python process when the INT4 packing extension fails to build, for example without a C++ compiler. It now falls back to the Python implementation. +- Fix AutoCast turning FP64 initializers and ``Constant`` values that exceed the FP32 range into infinities. They are now clamped to the largest FP32 value and the affected tensor is named in a warning. +- Fix AutoCast flipping the sign of an initializer value too small for the target precision; such values now keep their sign. 0.48.0 (2026-10-xx) ^^^^^^^^^^^^^^^^^^^ @@ -254,8 +256,6 @@ Changelog - Fix YAML config I/O decoding with the locale codepage instead of UTF-8, which made a config containing any non-ASCII byte fail to load on a machine whose locale is not UTF-8 (notably Windows, where the default is cp1252). ``modelopt/recipe/loader.py``, the two ONNX autotune state files, the two transformers config readers, the distill config and the puzzletron profile now pass ``encoding="utf-8"`` explicitly. Only the YAML config paths are covered: these are the files most likely to carry non-ASCII text in comments, model names or paths, and the only ones read inside a user's process. - Hybrid (e.g. Nemotron-H) checkpoints saved by the ``examples/megatron_bridge`` scripts now record their layer spec in ``run_config.yaml`` in a form that reloads, so they can be converted to HuggingFace; a checkpoint saved by an earlier release still needs its ``model.hybrid_stack_spec`` block replaced by hand. - Fix ONNX export of quantized models failing with ``Unsupported op_type for real weight quantization: Transpose`` when a weight reaches its consumer transposed, as in MaxViT's attention blocks. The same export now also quantizes each weight along the axis its Q/DQ pair declares, correcting silently wrong ``dq_only`` output when that axis differed from the consumer's default. -- Fix AutoCast turning FP64 initializers and ``Constant`` values that exceed the FP32 range into infinities. They are now clamped to the largest FP32 value and the affected tensor is named in a warning. -- Fix AutoCast flipping the sign of an initializer value too small for the target precision; such values now keep their sign. - Fix the ``evaluation`` agent skill's LiveCodeBench recipe scoring correct solutions that use binary standard I/O (``sys.stdin.buffer`` / ``sys.stdout.buffer``) as failures: the recipe now pins a fixed LiveCodeBench judge in the NeMo-Skills container. LiveCodeBench scores change as a result, so re-run earlier baselines before comparing. - Fix GPTQ with Megatron data parallelism: each data-parallel replica of a module now builds its Hessian from the whole calibration set instead of only its own shard, so all replicas reach the same quantized weights. - Fix ``examples/llm_sparsity/weight_sparsity/finetune.py`` saving a sharded state dict under FSDP2, the default FSDP version in Transformers 5, which made ``export_hf_ckpt.py`` fail to restore it. Re-run finetuning to produce a loadable ``finetuned_modelopt_state.pth``.