From e62e93e65cfbf92c4b5ff8add6420196e83c88f0 Mon Sep 17 00:00:00 2001 From: Prashant Rawat Date: Wed, 28 Oct 2026 23:25:07 -0700 Subject: [PATCH 1/2] Keep a folded int8 dtype through every retrace (#24239) Summary: Once Q/DQ is folded, the LUT activations (`tanh`, `sigmoid`, `silu`, `gelu`, `log*`, `sqrt`, `rsqrt`) and `div` take int8, but their ATen fake kernels promote it to float. Every pass retraces the graph, so each retrace labels these ops, and every view, permute, clamp and layout copy behind them. The dtype-based checks downstream then decline tensors that are int8 on device: e.g., `ValidateCortexMExplicitLayoutPass` rejects a convolution behind `tanh`. Fix the dtype during tracing, as the Arm backend does for ops whose fake kernel cannot handle quantized inputs. For these ops, when a node carries output qparams, the retrace takes its output dtype from them instead of from the fake kernel. Every pass that rebuilds the graph recomputes these types, and most passes in the Cortex-M lists are shared passes that don't know about this fix. So CortexMPassManager attaches it to every pass it runs. The one exception is AtenToCortexMPass, which replaces these ops with Cortex-M kernels that already report int8. The ops stay ATen until `AtenToCortexMPass`, and permute and layout-copy substitution still reads only the fake-tensor dtype, so an int8 graph that was never quantized lowers too. Authored with an AI coding assistant (Claude). Differential Revision: D128001102 --- .../cortex_m/passes/cortex_m_pass_manager.py | 37 +++- .../test/test_explicit_layout_pipeline.py | 181 ++++++++++++++++++ 2 files changed, 217 insertions(+), 1 deletion(-) diff --git a/backends/cortex_m/passes/cortex_m_pass_manager.py b/backends/cortex_m/passes/cortex_m_pass_manager.py index 1bd12cbaab0..90f6e941bd8 100644 --- a/backends/cortex_m/passes/cortex_m_pass_manager.py +++ b/backends/cortex_m/passes/cortex_m_pass_manager.py @@ -8,12 +8,15 @@ import inspect from typing import Any, Optional, Type +import torch from executorch.backends.arm._passes import ( DeduplicateGetAttrPass, FoldAndAnnotateQParamsPass, ScalarsToAttributePass, ) from executorch.backends.cortex_m.target_config import CortexM, CortexMTargetConfig +from executorch.backends.cortex_m.utils import _ACTIVATION_FNS +from executorch.backends.transforms.aten_to_dialect_pass import AtenToDialectPass from executorch.backends.transforms.convert_conv1d_to_conv2d_pass import ( ConvertConv1dToConv2dPass, ) @@ -30,10 +33,12 @@ from executorch.backends.transforms.replace_squeeze_unsqueeze_with_view import ( ReplaceSqueezeAndUnsqueezeWithViewPass, ) +from executorch.exir.dialects._ops import ops as exir_ops from executorch.exir.pass_base import ( ExportedProgramPassBase, ExportedProgramPassResult, ExportPass, + ProxyValue, ) from executorch.exir.pass_manager import ExportedProgramPassManager, PassType from executorch.exir.program._program import _transform, lift_constant_tensor_pass @@ -82,6 +87,36 @@ def call(self, exported_program: ExportedProgram) -> ExportedProgramPassResult: ) +# Once Q/DQ is folded these ops take int8, but their ATen fake kernels promote +# it to float, so each retrace would label them and everything behind them float. +_FLOAT_PROMOTING_OPS = frozenset({*_ACTIVATION_FNS, exir_ops.edge.aten.div.Tensor}) + + +class _PreserveQuantizedDtype(ExportPass): + def call_operator(self, op, args, kwargs, meta) -> ProxyValue: + result = super().call_operator(op, args, kwargs, meta) + input_qparams = meta.data.get("input_qparams") + output_qparams = meta.data.get("output_qparams") + # Match AtenToCortexMPass, which only lowers these ops with both qparams. + if op not in _FLOAT_PROMOTING_OPS or not input_qparams or not output_qparams: + return result + dtype = next(iter(output_qparams.values())).dtype + data = torch.empty_like(result.data, dtype=dtype) + self.tracer.set_metadata(result.node, data) + return ProxyValue(data, result.proxy) + + +def _preserving_quantized_dtype(pass_cls: PassClass) -> PassClass: + # Shared passes retrace too, so every pass needs this, except dialect + # lowering passes: they lower these ops themselves, and subclassing one + # would give it an empty substitution registry. + if not issubclass(pass_cls, ExportPass) or issubclass( + pass_cls, AtenToDialectPass + ): + return pass_cls + return type(pass_cls.__name__, (_PreserveQuantizedDtype, pass_cls), {}) + + class _CortexMLoweringPass(ExportedProgramPassBase): def __init__( self, pass_classes: list[PassClass], target_config: CortexMTargetConfig @@ -99,7 +134,7 @@ def call(self, exported_program: ExportedProgram) -> ExportedProgramPassResult: if "target_config" in signature.parameters: kwargs["target_config"] = self.target_config - transform_pass = pass_cls(**kwargs) + transform_pass = _preserving_quantized_dtype(pass_cls)(**kwargs) transformed = _transform(exported_program, transform_pass) modified |= transformed is not exported_program exported_program = transformed diff --git a/backends/cortex_m/test/test_explicit_layout_pipeline.py b/backends/cortex_m/test/test_explicit_layout_pipeline.py index 30528fff9ed..7a30a2e5ac0 100644 --- a/backends/cortex_m/test/test_explicit_layout_pipeline.py +++ b/backends/cortex_m/test/test_explicit_layout_pipeline.py @@ -26,6 +26,7 @@ ) from executorch.exir import to_edge from executorch.exir.dialects._ops import ops as exir_ops +from torch.export import ExportedProgram from torch.fx import Node # Temporary opt-in coverage. Move these invariants into the standard Cortex-M @@ -67,6 +68,35 @@ def forward(self, x): ) +class ThenConv1d(torch.nn.Module): + def __init__(self, fn) -> None: + super().__init__() + self.fn = fn + self.conv = torch.nn.Conv1d(8, 8, 1) + + def forward(self, x): + return self.conv(self.fn(x)) + + +class TanhConvReshapeConv(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.conv1 = torch.nn.Conv1d(8, 8, 1) + self.conv2 = torch.nn.Conv1d(4, 4, 1) + + def forward(self, x): + return self.conv2(self.conv1(torch.tanh(x)).view(1, 4, 32)) + + +class Apply(torch.nn.Module): + def __init__(self, fn) -> None: + super().__init__() + self.fn = fn + + def forward(self, *inputs): + return self.fn(*inputs) + + class NHWCPaddedConv(torch.nn.Module): def __init__(self, channels, shared_pad): super().__init__() @@ -382,3 +412,154 @@ def test_explicit_layout_rejects_unsupported_spatial_operator(): while error.__cause__ is not None: error = error.__cause__ assert "NHWC-eligible" in str(error) + + +def _run_passes(tester: CortexMTester, use_explicit_layout: bool) -> ExportedProgram: + tester.run_passes( + CortexMRunPasses( + target_config=CortexMTargetConfig(cpu=CortexM.M55), + use_explicit_layout=use_explicit_layout, + ) + ) + return tester.get_artifact(StageType.RUN_PASSES).exported_program() + + +@pytest.mark.parametrize( + "fn,shape", + [ + (torch.tanh, (1, 8, 16)), + (lambda x: torch.tanh(x).view(1, 8, 16), (1, 128)), + (lambda x: torch.relu(torch.sigmoid(x)), (1, 8, 16)), + (lambda x: torch.clamp(torch.tanh(x), -0.5, 0.5), (1, 8, 16)), + ], + ids=["tanh", "tanh_view", "sigmoid_relu", "tanh_clamp"], +) +def test_conv_behind_lut_activation_lowers_to_cmsis_nn(fn, shape): + torch.manual_seed(0) + inputs = (torch.randn(shape),) + tester = _run_explicit_layout_passes(CortexMTester(ThenConv1d(fn).eval(), inputs)) + program = tester.get_artifact(StageType.RUN_PASSES).exported_program() + + assert _count(program, exir_ops.edge.cortex_m.quantized_activation.default) == 1 + assert _count(program, exir_ops.edge.cortex_m.quantized_conv2d_nhwc.default) == 1 + assert _count(program, exir_ops.edge.cortex_m.transpose.default) == 2 + assert _count(program, exir_ops.edge.aten.permute_copy.default) == 0 + for node in program.graph.nodes: + if node.target == exir_ops.edge.aten.clamp.default: + assert node.meta["val"].dtype == torch.int8 + tester.run_method_and_compare_outputs(inputs=inputs, qtol=1) + + +@pytest.mark.parametrize( + "fn", + [ + torch.nn.functional.silu, + torch.nn.functional.gelu, + torch.log, + torch.log2, + torch.log10, + torch.log1p, + torch.sqrt, + torch.rsqrt, + ], + ids=["silu", "gelu", "log", "log2", "log10", "log1p", "sqrt", "rsqrt"], +) +def test_conv_behind_each_lut_activation_lowers_to_cmsis_nn(fn): + torch.manual_seed(0) + inputs = (torch.rand(1, 8, 16) + 0.5,) + tester = _run_explicit_layout_passes(CortexMTester(ThenConv1d(fn).eval(), inputs)) + program = tester.get_artifact(StageType.RUN_PASSES).exported_program() + + assert _count(program, exir_ops.edge.cortex_m.quantized_activation.default) == 1 + assert _count(program, exir_ops.edge.cortex_m.quantized_conv2d_nhwc.default) == 1 + tester.run_method_and_compare_outputs(inputs=inputs, qtol=1) + + +@pytest.mark.parametrize( + "fn,cortex_m_op,aten_op", + [ + ( + lambda x, y: torch.sigmoid(x).permute(0, 2, 1), + exir_ops.edge.cortex_m.transpose, + exir_ops.edge.aten.permute_copy, + ), + ( + lambda x, y: (x / y).permute(0, 2, 1), + exir_ops.edge.cortex_m.transpose, + exir_ops.edge.aten.permute_copy, + ), + ( + lambda x, y: (x / 2.0).permute(0, 2, 1), + exir_ops.edge.cortex_m.transpose, + exir_ops.edge.aten.permute_copy, + ), + ( + lambda x, y: torch.minimum(torch.tanh(x), y), + exir_ops.edge.cortex_m.minimum, + exir_ops.edge.aten.minimum, + ), + ( + lambda x, y: torch.maximum(torch.tanh(x), y), + exir_ops.edge.cortex_m.maximum, + exir_ops.edge.aten.maximum, + ), + ], + ids=[ + "sigmoid_permute", + "div_permute", + "div_scalar_permute", + "tanh_minimum", + "tanh_maximum", + ], +) +@pytest.mark.parametrize("use_explicit_layout", [False, True]) +def test_op_behind_float_promoting_op_lowers_to_cortex_m( + use_explicit_layout, fn, cortex_m_op, aten_op +): + torch.manual_seed(0) + inputs = (torch.randn(1, 8, 16), torch.rand(1, 8, 16) + 0.5) + tester = CortexMTester(Apply(fn).eval(), inputs) + tester.quantize(Quantize(CortexMQuantizer(use_explicit_layout=use_explicit_layout))) + tester.export().to_edge() + program = _run_passes(tester, use_explicit_layout) + + assert _count(program, cortex_m_op.default) == 1 + assert _count(program, aten_op.default) == 0 + tester.run_method_and_compare_outputs(inputs=inputs, qtol=1) + + +@pytest.mark.parametrize("use_explicit_layout", [False, True]) +def test_unquantized_int8_permute_lowers_to_transpose(use_explicit_layout): + inputs = (torch.randint(-128, 127, (1, 8, 16), dtype=torch.int8),) + tester = CortexMTester(Apply(lambda x: x.permute(0, 2, 1)).eval(), inputs) + tester.export().to_edge() + program = _run_passes(tester, use_explicit_layout) + + assert _count(program, exir_ops.edge.cortex_m.transpose.default) == 1 + assert _count(program, exir_ops.edge.aten.permute_copy.default) == 0 + + +def test_convs_around_a_reshape_behind_lut_activation_lower_to_cmsis_nn(): + torch.manual_seed(0) + inputs = (torch.randn(1, 8, 16),) + tester = _run_explicit_layout_passes( + CortexMTester(TanhConvReshapeConv().eval(), inputs) + ) + program = tester.get_artifact(StageType.RUN_PASSES).exported_program() + + assert _count(program, exir_ops.edge.cortex_m.quantized_conv2d_nhwc.default) == 2 + assert _count(program, exir_ops.edge.aten.convolution.default) == 0 + tester.run_method_and_compare_outputs(inputs=inputs, qtol=1) + + +def test_explicit_layout_rejects_float_input_to_int8_conv(): + model = ThenConv1d(lambda x: torch.full_like(x, 0.5, dtype=torch.float32)) + tester = CortexMTester(model.eval(), (torch.randn(1, 8, 16),)) + + with pytest.raises(Exception) as caught: + _run_explicit_layout_passes(tester) + + error = caught.value + while error.__cause__ is not None: + error = error.__cause__ + assert "must move an int8 tensor" in str(error) From 74f93e38bba3f7f307358ce30c4cbd115d461e75 Mon Sep 17 00:00:00 2001 From: Prashant Rawat Date: Wed, 28 Oct 2026 23:25:07 -0700 Subject: [PATCH 2/2] Precompute quantized constant ops after Q/DQ folding (#24258) Summary: Q/DQ folding (`FoldAndAnnotateQParamsPass`) always folds the quantize/dequantize pair around a constant-producing op (`full`, `full_like`, `arange`, `eye`, `linspace`, `scalar_tensor`) into the op: it removes the Q node and records the output qparams, but the op itself keeps producing float. The Arm backend follows folding with `ComputeConstantOpsAOTPass`, which evaluates such ops in float and quantizes the result. Cortex-M did not, so in `x + torch.full(shape, 0.5)` the int8 `quantized_add` kernel read a float32 constant, and the output was silently wrong. Add `ComputeQuantizedConstantOpsPass` and run it right after folding on both Cortex-M pass lists. - a. It only touches nodes with output qparams, so unquantized constant ops are unchanged. A quantized `full` stays a runtime fill, with its fill value quantized and its dtype set to the qparams' dtype, so a large fill does not turn into a large constant; - b.`full_like` is first converted to `full` with the Arm backend's `ConvertFullLikeToFullPass`. - c. Since `ComputeConstantOpsAOTPass` only logs a failed precompute, the new pass raises if a quantized constant op is still in the graph afterwards, as it would feed float data to an int8 kernel. Differential Revision: D128002647 --- backends/cortex_m/passes/BUCK | 1 + .../compute_quantized_constant_ops_pass.py | 63 +++++++++++++ .../cortex_m/passes/cortex_m_pass_manager.py | 3 + .../test/test_explicit_layout_pipeline.py | 93 ++++++++++++++++++- 4 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 backends/cortex_m/passes/compute_quantized_constant_ops_pass.py diff --git a/backends/cortex_m/passes/BUCK b/backends/cortex_m/passes/BUCK index 2bf0892a7e9..e5103635f89 100644 --- a/backends/cortex_m/passes/BUCK +++ b/backends/cortex_m/passes/BUCK @@ -31,6 +31,7 @@ fbcode_target(_kind = runtime.python_library, "activation_fusion_pass.py", "aten_to_cortex_m_pass.py", "clamp_hardswish_pass.py", + "compute_quantized_constant_ops_pass.py", "cortex_m_pass.py", "cortex_m_pass_manager.py", "decompose_hardswish_pass.py", diff --git a/backends/cortex_m/passes/compute_quantized_constant_ops_pass.py b/backends/cortex_m/passes/compute_quantized_constant_ops_pass.py new file mode 100644 index 00000000000..26518d3f687 --- /dev/null +++ b/backends/cortex_m/passes/compute_quantized_constant_ops_pass.py @@ -0,0 +1,63 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import torch +from executorch.backends.arm._passes import ( + ComputeConstantOpsAOTPass, + ConvertFullLikeToFullPass, +) +from executorch.exir.dialects._ops import ops as exir_ops +from executorch.exir.pass_base import PassResult +from torch.fx import GraphModule + + +def _is_quantized_fill(node: torch.fx.Node) -> bool: + return ( + node.target == exir_ops.edge.aten.full.default + and node.kwargs.get("dtype") == node.meta["output_qparams"][0].dtype + ) + + +class ComputeQuantizedConstantOpsPass(ComputeConstantOpsAOTPass): + """Give the quantized constant ops their int8 values. + + Folding removes the Q node after these ops but leaves them producing float, + so the int8 kernels they feed would read float data. A ``full`` stays a + runtime fill with its fill value quantized, so it adds no constant data; the + other ops are computed in float and stored as quantized constants. + ``full_like`` is converted to ``full`` first and handled the same way. + """ + + def compute_node_aot(self, node: torch.fx.Node) -> bool: + output_qparams = node.meta.get("output_qparams") + if not output_qparams: + return False + if node.target == exir_ops.edge.aten.full.default: + qparams = output_qparams[0] + fill = qparams.quantize_value(torch.tensor(float(node.args[1]))) + node.args = (node.args[0], int(fill.item())) + node.kwargs = {"dtype": qparams.dtype} + return True + return super().compute_node_aot(node) + + def call(self, graph_module: GraphModule) -> PassResult: + converted = ConvertFullLikeToFullPass()(graph_module) + result = super().call(converted.graph_module) + # The base pass only logs a failure; a quantized constant left behind + # would feed float data to an int8 kernel without any error. + remaining = [ + node.name + for node in result.graph_module.graph.nodes + if node.op == "call_function" + and node.target in self.targeted_ops + and node.meta.get("output_qparams") + and not _is_quantized_fill(node) + ] + if remaining: + raise RuntimeError( + f"Could not precompute quantized constant ops: {remaining}" + ) + return PassResult(result.graph_module, converted.modified or result.modified) diff --git a/backends/cortex_m/passes/cortex_m_pass_manager.py b/backends/cortex_m/passes/cortex_m_pass_manager.py index 90f6e941bd8..d5313b5fbaf 100644 --- a/backends/cortex_m/passes/cortex_m_pass_manager.py +++ b/backends/cortex_m/passes/cortex_m_pass_manager.py @@ -48,6 +48,7 @@ from .activation_fusion_pass import ActivationFusionPass from .aten_to_cortex_m_pass import AtenToCortexMPass from .clamp_hardswish_pass import ClampHardswishPass +from .compute_quantized_constant_ops_pass import ComputeQuantizedConstantOpsPass from .decompose_hardswish_pass import DecomposeHardswishPass from .decompose_mean_pass import DecomposeMeanPass from .decompose_sdpa_pass import DecomposeSDPAPass @@ -149,6 +150,7 @@ class CortexMPassManager(ExportedProgramPassManager): # Retrace scalar ops while the graph still has floating-point Q/DQ inputs. ReplaceScalarWithTensorArgPass, FoldAndAnnotateQParamsPass, + ComputeQuantizedConstantOpsPass, ActivationFusionPass, QuantizedClampActivationPass, DecomposeHardswishPass, @@ -164,6 +166,7 @@ class CortexMPassManager(ExportedProgramPassManager): explicit_layout_pass_list: list[PassClass] = [ RemoveGetItemPass, FoldAndAnnotateQParamsPass, + ComputeQuantizedConstantOpsPass, ReplaceScalarWithTensorArgPass, ActivationFusionPass, QuantizedClampActivationPass, diff --git a/backends/cortex_m/test/test_explicit_layout_pipeline.py b/backends/cortex_m/test/test_explicit_layout_pipeline.py index 7a30a2e5ac0..79becb4149f 100644 --- a/backends/cortex_m/test/test_explicit_layout_pipeline.py +++ b/backends/cortex_m/test/test_explicit_layout_pipeline.py @@ -9,6 +9,7 @@ import pytest import torch +from executorch.backends.arm._passes import ComputeConstantOpsAOTPass from executorch.backends.cortex_m.passes.cortex_m_pass_manager import ( CortexMPassManager, LiftConstantTensorsPass, @@ -552,9 +553,82 @@ def test_convs_around_a_reshape_behind_lut_activation_lower_to_cmsis_nn(): tester.run_method_and_compare_outputs(inputs=inputs, qtol=1) -def test_explicit_layout_rejects_float_input_to_int8_conv(): +def test_explicit_layout_lowers_conv_on_a_quantized_constant(): + inputs = (torch.randn(1, 8, 16),) model = ThenConv1d(lambda x: torch.full_like(x, 0.5, dtype=torch.float32)) - tester = CortexMTester(model.eval(), (torch.randn(1, 8, 16),)) + tester = _run_explicit_layout_passes(CortexMTester(model.eval(), inputs)) + program = tester.get_artifact(StageType.RUN_PASSES).exported_program() + + assert _count(program, exir_ops.edge.aten.full_like.default) == 0 + assert _count(program, exir_ops.edge.aten.full.default) == 1 + assert _count(program, exir_ops.edge.cortex_m.quantized_conv2d_nhwc.default) == 1 + tester.run_method_and_compare_outputs(inputs=inputs, qtol=1) + + +@pytest.mark.parametrize("use_explicit_layout", [False, True]) +@pytest.mark.parametrize( + "constant,constant_op,shape", + [ + ( + lambda: torch.arange(-1.0, 1.0, 0.125), + exir_ops.edge.aten.arange.start_step, + (16,), + ), + ( + lambda: torch.scalar_tensor(0.5), + exir_ops.edge.aten.scalar_tensor.default, + (), + ), + ], + ids=["arange", "scalar_tensor"], +) +def test_quantized_constant_is_computed_before_quantizing( + use_explicit_layout, constant, constant_op, shape +): + torch.manual_seed(0) + inputs = (torch.randn(shape),) + tester = CortexMTester(Apply(lambda x: x + constant()).eval(), inputs) + tester.quantize(Quantize(CortexMQuantizer(use_explicit_layout=use_explicit_layout))) + tester.export().to_edge() + program = _run_passes(tester, use_explicit_layout) + + assert _count(program, constant_op) == 0 + assert _count(program, exir_ops.edge.cortex_m.quantized_add.default) == 1 + tester.run_method_and_compare_outputs(inputs=inputs, qtol=1) + + +@pytest.mark.parametrize("use_explicit_layout", [False, True]) +@pytest.mark.parametrize( + "fill", + [lambda x: torch.full((1, 8, 16), 0.5), lambda x: torch.full_like(x, 0.5)], + ids=["full", "full_like"], +) +def test_quantized_fill_stays_a_runtime_int8_full(use_explicit_layout, fill): + torch.manual_seed(0) + inputs = (torch.randn(1, 8, 16),) + tester = CortexMTester(Apply(lambda x: x + fill(x)).eval(), inputs) + tester.quantize(Quantize(CortexMQuantizer(use_explicit_layout=use_explicit_layout))) + tester.export().to_edge() + program = _run_passes(tester, use_explicit_layout) + + (full,) = program.graph.find_nodes( + op="call_function", target=exir_ops.edge.aten.full.default + ) + assert full.meta["val"].dtype == torch.int8 + assert _count(program, exir_ops.edge.aten.full_like.default) == 0 + assert _count(program, exir_ops.edge.cortex_m.quantized_add.default) == 1 + tester.run_method_and_compare_outputs(inputs=inputs, qtol=1) + + +def test_failed_constant_precompute_is_an_error(monkeypatch): + def fail(self, node): + raise ValueError("cannot evaluate") + + monkeypatch.setattr(ComputeConstantOpsAOTPass, "compute_node_aot", fail) + tester = CortexMTester( + Apply(lambda x: x + torch.arange(-1.0, 1.0, 0.125)).eval(), + (torch.randn(16),), + ) with pytest.raises(Exception) as caught: _run_explicit_layout_passes(tester) @@ -562,4 +636,17 @@ def test_explicit_layout_rejects_float_input_to_int8_conv(): error = caught.value while error.__cause__ is not None: error = error.__cause__ - assert "must move an int8 tensor" in str(error) + assert "Could not precompute quantized constant ops" in str(error) + + +@pytest.mark.parametrize("use_explicit_layout", [False, True]) +def test_unquantized_constant_is_not_precomputed(use_explicit_layout): + inputs = (torch.randn(1, 8, 16),) + tester = CortexMTester( + Apply(lambda x: x + torch.full((1, 8, 16), 0.5)).eval(), inputs + ) + tester.export().to_edge() + program = _run_passes(tester, use_explicit_layout) + + assert _count(program, exir_ops.edge.aten.full.default) == 1 + tester.run_method_and_compare_outputs(inputs=inputs)