Qualcomm AI Engine Direct - Default use_16a16w to per-tensor conv weights

backends/qualcomm/export_utils.py

@@ -731,7 +731,7 @@ def build_executorch_binary(
 def make_quantizer(
     quant_dtype: Optional[QuantDtype] = QuantDtype.use_8a8w,
     custom_annotations=(),
-    per_channel_conv=True,
+    per_channel_conv: Optional[bool] = None,
     per_channel_linear=False,
     per_channel_embedding=False,
     act_observer=MovingAverageMinMaxObserver,
@@ -745,6 +745,11 @@ def make_quantizer(
     def to_chipset(model: Union[str, QcomChipset]) -> QcomChipset:
         return model if isinstance(model, QcomChipset) else getattr(QcomChipset, model)
 
+    if per_channel_conv is None:
+        # Per-channel int16 conv weights make the derived int32 bias inaccurate
+        # (pytorch/executorch#23108), so 16a16w defaults to per-tensor.
+        per_channel_conv = quant_dtype != QuantDtype.use_16a16w
+
     soc_models = (
         [to_chipset(model) for model in soc_model]
         if not isinstance(soc_model, (str, QcomChipset))

backends/qualcomm/quantizer/quantizer.py

@@ -813,8 +813,15 @@ def set_default_quant_config(
             is_linear_per_channel (bool, optional): Enables per-channel quantization for linear (fully connected) operations.
             is_embedding_per_channel (bool, optional): Enables per-channel quantization for embedding operations.
             act_observer (Optional[UniformQuantizationObserverBase], optional): Custom observer for activation quantization. If not specified, the default observer is determined by `QUANT_CONFIG_DICT`.
-
         """
+        if quant_dtype == QuantDtype.use_16a16w and is_conv_per_channel:
+            # Per-channel int16 conv weight scales are small enough that the
+            # derived int32 bias loses accuracy (pytorch/executorch#23108).
+            logging.warning(
+                "use_16a16w with per-channel conv weights can be much less "
+                "accurate than per-tensor; consider is_conv_per_channel=False "
+                "(per_channel_conv=False in make_quantizer)."
+            )
         self.default_quant_config = ModuleQConfig(
             quant_dtype,
             is_qat=is_qat,

backends/qualcomm/tests/rework/htp/op/v73/test.py

@@ -426,6 +426,10 @@ def test_conv2d_transpose(request, kwargs):
     Conv.test_2d_transpose(request, kwargs)  # noqa: F405
 
 
+def test_conv2d_16a16w_default(subtests, caplog):
+    Conv.test_2d_16a16w_default(subtests, caplog)  # noqa: F405
+
+
 # LPBQ (QNN_QUANTIZATION_ENCODING_BLOCKWISE_EXPANSION) requires V69+; enabled here
 @pytest.mark.parametrize(
     "kwargs",

backends/qualcomm/tests/rework/htp/op/v75/test.py

@@ -426,6 +426,10 @@ def test_conv2d_transpose(request, kwargs):
     Conv.test_2d_transpose(request, kwargs)  # noqa: F405
 
 
+def test_conv2d_16a16w_default(subtests, caplog):
+    Conv.test_2d_16a16w_default(subtests, caplog)  # noqa: F405
+
+
 # LPBQ (QNN_QUANTIZATION_ENCODING_BLOCKWISE_EXPANSION) requires V69+; enabled here
 @pytest.mark.parametrize(
     "kwargs",

backends/qualcomm/tests/rework/htp/op/v79/test.py

@@ -426,6 +426,10 @@ def test_conv2d_transpose(request, kwargs):
     Conv.test_2d_transpose(request, kwargs)  # noqa: F405
 
 
+def test_conv2d_16a16w_default(subtests, caplog):
+    Conv.test_2d_16a16w_default(subtests, caplog)  # noqa: F405
+
+
 # LPBQ (QNN_QUANTIZATION_ENCODING_BLOCKWISE_EXPANSION) requires V69+; enabled here
 @pytest.mark.parametrize(
     "kwargs",

backends/qualcomm/tests/rework/htp/op/v81/test.py

@@ -426,6 +426,10 @@ def test_conv2d_transpose(request, kwargs):
     Conv.test_2d_transpose(request, kwargs)  # noqa: F405
 
 
+def test_conv2d_16a16w_default(subtests, caplog):
+    Conv.test_2d_16a16w_default(subtests, caplog)  # noqa: F405
+
+
 # LPBQ (QNN_QUANTIZATION_ENCODING_BLOCKWISE_EXPANSION) requires V69+; enabled here
 @pytest.mark.parametrize(
     "kwargs",

backends/qualcomm/tests/rework/src/op.py

@@ -6,6 +6,7 @@
 
 import inspect
 import itertools
+import logging
 import math
 import random
 from functools import partial, reduce
@@ -15,6 +16,11 @@
 
 # Also registers torch.ops.qnn_custom.hadamard_transform (asserted in Hadamard.test).
 from executorch.backends.qualcomm.builders.custom_ops import _hadamard_matrix
+from executorch.backends.qualcomm.export_utils import (
+    make_quantizer,
+    prepare_pt2e,
+    QuantDtype,
+)
 
 from executorch.backends.qualcomm.tests.rework.conftest import (
     export_and_verify,
@@ -1370,6 +1376,50 @@ def test_2d_transpose(subtests, qnn_config, quantizer, compile_spec, expected):
             transpose=True,
         )
 
+    @staticmethod
+    def test_2d_16a16w_default(subtests, caplog):
+        # use_16a16w defaults to per-tensor conv weights, since per-channel int16
+        # weights make the derived int32 bias inaccurate (pytorch/executorch#23108).
+        # An explicit per_channel_conv=True is respected, with a warning.
+        def weight_qscheme(**make_quantizer_kwargs):
+            module = __class__(
+                in_channels=4,
+                out_channels=8,
+                kernel_size=3,
+                stride=1,
+                padding=1,
+                dilation=1,
+                groups=1,
+                bias=True,
+                variant=2,
+                transpose=False,
+            ).eval()
+            inputs = (torch.randn(1, 4, 8, 8),)
+            quantizer = make_quantizer(
+                quant_dtype=QuantDtype.use_16a16w,
+                **make_quantizer_kwargs,
+            )
+            exported = torch.export.export(module, inputs, strict=True).module()
+            prepared = prepare_pt2e(exported, quantizer)
+            [conv] = [
+                n
+                for n in prepared.graph.nodes
+                if n.target == torch.ops.aten.conv2d.default
+            ]
+            # prepare_pt2e inserts the weight observer between weight and conv.
+            return prepared.get_submodule(conv.args[1].target).qscheme
+
+        with subtests.test(msg="default_is_per_tensor"):
+            qscheme = weight_qscheme()
+            assert qscheme == torch.per_tensor_symmetric, qscheme
+
+        with subtests.test(msg="explicit_per_channel_is_respected_and_warns"):
+            with caplog.at_level(logging.WARNING):
+                caplog.clear()
+                qscheme = weight_qscheme(per_channel_conv=True)
+            assert qscheme == torch.per_channel_symmetric, qscheme
+            assert "use_16a16w with per-channel conv weights" in caplog.text
+
     @staticmethod
     @unpack_fixtures
     def test_2d_linear_like(subtests, qnn_config, quantizer, compile_spec, expected):

backends/test/suite/flows/qualcomm.py

@@ -67,6 +67,7 @@ def create_quantize_stage() -> Quantize:
     quantize=True,
     quant_dtype=QuantDtype.use_16a16w,
     use_fp16=False,
+    per_channel_conv=False,
     xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS,
 )
 QNN_16A8W_TEST_FLOW = _create_qnn_flow(

docs/source/backends-qualcomm.md

@@ -909,6 +909,12 @@ Supports:
 
 For details, see: backends/qualcomm/quantizer/quantizer.py
 
+Known limitation: with 16a16w, per-channel int16 conv weights make the derived
+int32 bias scale too small, which can severely reduce accuracy
+([#23108](https://github.com/pytorch/executorch/issues/23108)). For this reason
+`make_quantizer` defaults `per_channel_conv` to `False` for `QuantDtype.use_16a16w`.
+Passing `per_channel_conv=True` explicitly is still allowed, but the quantizer warns.
+
 ### Operator Support
 [The full operator support matrix](https://github.com/pytorch/executorch/tree/main/backends/qualcomm/builders#operator-support-status) is tracked and frequently updated in the ExecuTorch repository.