From 6f1ed5985a5cbc1cb609823de1c7ddd5752cdb05 Mon Sep 17 00:00:00 2001 From: roman-janik-nxp Date: Wed, 29 Jul 2026 12:09:27 +0200 Subject: [PATCH] NXP backend: Enable Hardswish with new Neutron flow --- .../nxp/backend/edge_program_converter.py | 1 + .../ops_converters/__init__.py | 4 + .../ops_converters/hardswish_converter.py | 50 ++++++ backends/nxp/neutron_partitioner.py | 1 + backends/nxp/quantizer/neutron_quantizer.py | 4 + backends/nxp/quantizer/patterns.py | 18 +++ backends/nxp/tests/executorch_pipeline.py | 2 +- .../test_hardswish_converter.py | 152 ++++++++++++++++++ backends/nxp/tests/models.py | 36 +++++ backends/nxp/tests/ops_aliases.py | 1 + docs/source/backends/nxp/op-support.csv | 1 + 11 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 backends/nxp/backend/ir/converter/node_converters/ops_converters/hardswish_converter.py create mode 100644 backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py diff --git a/backends/nxp/backend/edge_program_converter.py b/backends/nxp/backend/edge_program_converter.py index a12915e18af..ad9dd2a49da 100644 --- a/backends/nxp/backend/edge_program_converter.py +++ b/backends/nxp/backend/edge_program_converter.py @@ -43,6 +43,7 @@ exir_ops.edge.aten.convolution.default: ConvolutionConverter, # noqa F405 exir_ops.edge.aten.convolution.out: ConvolutionConverter, # noqa F405 exir_ops.edge.aten.exp.default: ExpConverter, # noqa F405 + exir_ops.edge.aten.hardswish.default: HardswishConverter, # noqa F405 exir_ops.edge.aten.hardtanh.default: HardTanhConverter, # noqa F405 exir_ops.edge.aten.leaky_relu.default: LeakyReluConverter, # noqa F405 exir_ops.edge.aten.log.default: LogConverter, # noqa F405 diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/__init__.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/__init__.py index b651b304f69..e21647f6e90 100755 --- a/backends/nxp/backend/ir/converter/node_converters/ops_converters/__init__.py +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/__init__.py @@ -43,6 +43,9 @@ from executorch.backends.nxp.backend.ir.converter.node_converters.ops_converters.getitem_converter import ( GetItemConverter, ) +from executorch.backends.nxp.backend.ir.converter.node_converters.ops_converters.hardswish_converter import ( + HardswishConverter, +) from executorch.backends.nxp.backend.ir.converter.node_converters.ops_converters.hardtanh_converter import ( HardTanhConverter, ) @@ -137,6 +140,7 @@ "ConvolutionConverter", "ExpConverter", "GetItemConverter", + "HardswishConverter", "HardTanhConverter", "LeakyReluConverter", "LogConverter", diff --git a/backends/nxp/backend/ir/converter/node_converters/ops_converters/hardswish_converter.py b/backends/nxp/backend/ir/converter/node_converters/ops_converters/hardswish_converter.py new file mode 100644 index 00000000000..454e941242b --- /dev/null +++ b/backends/nxp/backend/ir/converter/node_converters/ops_converters/hardswish_converter.py @@ -0,0 +1,50 @@ +# Copyright 2026 NXP +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import torch + +from executorch.backends.nxp.backend.ir.converter.node_converter import ( + CustomDelegationOptions, + NodeConverter, +) +from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options import ( + hard_swish_options, +) +from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec +from torch.fx import Node +from torch.nn import Parameter + + +class HardswishConverter(NodeConverter): + @staticmethod + def _is_supported_on_target( + node: Node, + neutron_target_spec: NeutronTargetSpec, + parameters_mapping: dict[str, Parameter], + custom_delegation_options: CustomDelegationOptions, + ) -> bool: + supported_types = [torch.int8, torch.uint8] + if not NodeConverter.uses_quantization_type_for_io( + node, supported_types, [0], [0] + ): + return False + + return True + + @staticmethod + def _is_supported_in_IR( + node: Node, + parameters_mapping: dict[str, Parameter], + custom_delegation_options: CustomDelegationOptions, + ) -> bool: + return True + + def convert(self, node: Node): + self.assert_convertible(node) + + t_op = self._create_tflite_op_with_io_tensors(node) + t_op.builtin_options = hard_swish_options.HardSwish() + + self.builder.append_operators([t_op]) diff --git a/backends/nxp/neutron_partitioner.py b/backends/nxp/neutron_partitioner.py index eb23888aad3..e244cb22b57 100644 --- a/backends/nxp/neutron_partitioner.py +++ b/backends/nxp/neutron_partitioner.py @@ -216,6 +216,7 @@ def tag_qdq_clusters(self, nodes: list[torch.fx.Node]): exir_ops.edge.aten.convolution.default: ConvolutionConverter, # noqa F405 exir_ops.edge.aten.convolution.out: ConvolutionConverter, # noqa F405 exir_ops.edge.aten.exp.default: ExpConverter, # noqa F405 + exir_ops.edge.aten.hardswish.default: HardswishConverter, # noqa F405 exir_ops.edge.aten.hardtanh.default: HardTanhConverter, # noqa F405 exir_ops.edge.aten.leaky_relu.default: LeakyReluConverter, # noqa F405 exir_ops.edge.aten.log.default: LogConverter, # noqa F405 diff --git a/backends/nxp/quantizer/neutron_quantizer.py b/backends/nxp/quantizer/neutron_quantizer.py index 5218d4d9c86..1f1d26ce229 100644 --- a/backends/nxp/quantizer/neutron_quantizer.py +++ b/backends/nxp/quantizer/neutron_quantizer.py @@ -30,6 +30,8 @@ DropoutPattern, ExpPattern, FlattenPattern, + HardSwishInPlacePattern, + HardSwishPattern, HardTanhInPlacePattern, HardTanhPattern, LeakyReluInPlacePattern, @@ -284,6 +286,8 @@ def __init__(self, neutron_target_spec: NeutronTargetSpec, is_qat: bool = False) OpQuantizer(DropoutPattern(is_qat=is_qat), static_qconfig), OpQuantizer(ExpPattern(is_qat=is_qat), static_qconfig), OpQuantizer(FlattenPattern(is_qat=is_qat), static_qconfig), + OpQuantizer(HardSwishPattern(is_qat=is_qat), static_qconfig), + OpQuantizer(HardSwishInPlacePattern(is_qat=is_qat), static_qconfig), OpQuantizer(HardTanhPattern(is_qat=is_qat), static_qconfig), OpQuantizer(HardTanhInPlacePattern(is_qat=is_qat), static_qconfig), OpQuantizer(LeakyReluPattern(is_qat=is_qat), static_fc_qconfig), diff --git a/backends/nxp/quantizer/patterns.py b/backends/nxp/quantizer/patterns.py index 937fe7c3fe4..3bbd2bed54e 100644 --- a/backends/nxp/quantizer/patterns.py +++ b/backends/nxp/quantizer/patterns.py @@ -763,6 +763,24 @@ def partition_types(self): return [torch.ops.aten.flatten.using_ints] +class HardSwishPattern(SingleInputBasicPattern): + """ + Quantizer for HardSwish operator. + """ + + def partition_types(self): + return [torch.ops.aten.hardswish.default] + + +class HardSwishInPlacePattern(SingleInputBasicPattern): + """ + Quantizer for HardSwish operator with param inplace=True. + """ + + def partition_types(self): + return [torch.ops.aten.hardswish_.default] + + class HardTanhPattern(SingleInputBasicPattern): """ Quantizer for HardTanh operator. diff --git a/backends/nxp/tests/executorch_pipeline.py b/backends/nxp/tests/executorch_pipeline.py index 1309e019428..a186535e53d 100644 --- a/backends/nxp/tests/executorch_pipeline.py +++ b/backends/nxp/tests/executorch_pipeline.py @@ -216,7 +216,7 @@ def to_quantized_edge_program( ) # List of operators to not decompose during the lowering. - preserve_ops = [torch.ops.aten.prelu.default] + preserve_ops = [torch.ops.aten.prelu.default, torch.ops.aten.hardswish.default] compile_spec = generate_neutron_compile_spec( target, intermediates_dir=intermediates_dir, diff --git a/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py b/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py new file mode 100644 index 00000000000..deff4e12f0c --- /dev/null +++ b/backends/nxp/tests/ir/converter/node_converter/test_hardswish_converter.py @@ -0,0 +1,152 @@ +# Copyright 2026 NXP +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import numpy as np + +# noinspection PyUnusedImports +import pytest +import torch + +from executorch.backends.nxp.tests.dataset_creator import RandomDatasetCreator +from executorch.backends.nxp.tests.graph_verifier import DetailedGraphVerifier +from executorch.backends.nxp.tests.model_output_comparator import ( + AllCloseOutputComparator, +) +from executorch.backends.nxp.tests.models import ( + ConvHardswishModule, + HardswishModule, + LinearHardswishModule, +) +from executorch.backends.nxp.tests.nsys_testing import lower_run_compare +from executorch.backends.nxp.tests.ops_aliases import ( + AddMM, + Convolution, + Hardswish, + PermuteCopy, + ViewCopy, +) +from executorch.backends.nxp.tests.use_qat import * # noqa F403 + + +@pytest.fixture(autouse=True) +def reseed_model_per_test_run(): + torch.manual_seed(23) + np.random.seed(23) + + +class TestHardswishConverter: + # noinspection PyMethodMayBeStatic + def assert_delegated( + self, + model, + input_shape, + mocker, + request, + expected_delegated_ops=None, + use_qat=False, + ): + rank = len(input_shape) + graph_verifier = DetailedGraphVerifier( + mocker, + expected_delegated_ops=expected_delegated_ops + or { + Hardswish: 1, + AddMM: 1, + PermuteCopy: 1, + ViewCopy: 0 if rank == 2 else 2, + }, + expected_non_delegated_ops={}, + ) + + # Cover also negative values to thoroughly test the operator. + dataset_creator = RandomDatasetCreator(low=-4, high=4) + comparator = AllCloseOutputComparator(atol=1) + + lower_run_compare( + model, + input_shape, + graph_verifier, + request, + dataset_creator, + output_comparator=comparator, + remove_quant_io_ops=True, + use_qat=use_qat, + ) + + @pytest.mark.parametrize( + "input_shape", + [ + pytest.param((1,), id="1D."), + pytest.param((7, 83), id="2D."), + pytest.param((7, 8, 12), id="3D."), + pytest.param((1, 4, 7, 8), id="4D."), + pytest.param((5, 4, 7, 8), id="4D batchsize != 1."), + pytest.param((1, 4, 3, 4, 14), id="5D."), + ], + ) + def test__basic_nsys_inference(self, mocker, request, input_shape): + channels = input_shape[-1] + model = LinearHardswishModule(in_features=channels, out_features=channels) + + self.assert_delegated(model, input_shape, mocker, request) + + def test__basic_nsys_inference_qat(self, mocker, request): + input_shape = (2, 4, 6, 7) + channels = input_shape[-1] + model = LinearHardswishModule(in_features=channels, out_features=channels) + + self.assert_delegated(model, input_shape, mocker, request, use_qat=True) + + def test__basic_nsys_inference_inplace(self, mocker, request, use_qat): + input_shape = (2, 4, 6, 7) + channels = input_shape[-1] + model = LinearHardswishModule( + in_features=channels, out_features=channels, inplace=True + ) + + self.assert_delegated(model, input_shape, mocker, request, use_qat=use_qat) + + @pytest.mark.parametrize( + "input_shape", + [ + pytest.param((3,), id="1D."), + pytest.param((1, 4), id="2D."), + pytest.param((4, 7, 4), id="3D."), + pytest.param((1, 6, 4, 4), id="4D."), + pytest.param((5, 4, 7, 8), id="4D batchsize != 1."), + pytest.param((2, 3, 8, 3, 11), id="5D."), + ], + ) + def test__single_hardswish(self, mocker, request, input_shape): + model = HardswishModule() + expected_delegated_ops = { + Hardswish: 1, + } + + self.assert_delegated( + model, + input_shape, + mocker, + request, + expected_delegated_ops=expected_delegated_ops, + ) + + @pytest.mark.parametrize( + "input_shape", + [ + pytest.param((1, 8, 4, 4), id="4D."), + ], + ) + def test__channels_first(self, mocker, request, input_shape): + channels = input_shape[1] + model = ConvHardswishModule(in_channels=channels) + expected_delegated_ops = { + Hardswish: 1, + Convolution: 1, + } + + self.assert_delegated( + model, input_shape, mocker, request, expected_delegated_ops + ) diff --git a/backends/nxp/tests/models.py b/backends/nxp/tests/models.py index 80e5b4e89a2..bcf485acfe6 100644 --- a/backends/nxp/tests/models.py +++ b/backends/nxp/tests/models.py @@ -1081,3 +1081,39 @@ def __init__(self): def forward(self, x, y): x = self.max_pool2d(x) return torch.minimum(x, y) + + +class LinearHardswishModule(torch.nn.Module): + def __init__(self, in_features, out_features, inplace=False): + super().__init__() + + self.linear = nn.Linear(in_features=in_features, out_features=out_features) + self.hardswish = torch.nn.Hardswish(inplace=inplace) + + def forward(self, x): + x = self.linear(x) + return self.hardswish(x) + + +class HardswishModule(torch.nn.Module): + def __init__(self, inplace=False): + super().__init__() + + self.hardswish = torch.nn.Hardswish(inplace=inplace) + + def forward(self, x): + return self.hardswish(x) + + +class ConvHardswishModule(torch.nn.Module): + def __init__(self, in_channels, inplace=False): + super().__init__() + + self.conv = Conv2dModule( + in_channels=in_channels, out_channels=in_channels, stride=1, padding=1 + ) + self.hardswish = torch.nn.Hardswish(inplace=inplace) + + def forward(self, x): + x = self.conv(x) + return self.hardswish(x) diff --git a/backends/nxp/tests/ops_aliases.py b/backends/nxp/tests/ops_aliases.py index 4bada8baed5..d71c5c9ab44 100644 --- a/backends/nxp/tests/ops_aliases.py +++ b/backends/nxp/tests/ops_aliases.py @@ -31,6 +31,7 @@ Exp = exir_ops.edge.aten.exp.default GetItem = operator.getitem GtScalar = exir_ops.edge.aten.gt.Scalar +Hardswish = exir_ops.edge.aten.hardswish.default HardTanh = exir_ops.edge.aten.hardtanh.default HardTanh_ = exir_ops.edge.aten.hardtanh_.default LeakyRelu = exir_ops.edge.aten.leaky_relu.default diff --git a/docs/source/backends/nxp/op-support.csv b/docs/source/backends/nxp/op-support.csv index e66df92dc3b..2aea6dba65a 100644 --- a/docs/source/backends/nxp/op-support.csv +++ b/docs/source/backends/nxp/op-support.csv @@ -16,6 +16,7 @@ aten.convolution.default,int8,static int8,"1D or 2D convolution, constant weight aten.dim_order_ops._clone_dim_order.default,,, "See aten.clone.default" aten.div.Tensor,int8,static int8,"divisor - static tensor or scalar value, one dimension must satisfy %8 = 0 or scalar division (all dims = 1)" aten.exp.default,int8,static int8, +aten.hardswish.default,int8,static int8, aten.hardtanh.default,int8,static int8,"Bounds = (-1, 1) or (0, 1) or (0, 6) or (0, None)" aten.leaky_relu.default,int8,static int8, aten.log.default,int8,static int8,