Mark the 3-D convolution cases that fail inside a backend as expected failures

backends/arm/test/setup_testing.sh

@@ -44,6 +44,7 @@ ${build_executor_runner} --pte=semihosting --target=ethos-u85-128 --system_confi
 # Ethos-U55/U65
 # aten::avg_pool2d.out                         - test_adaptive_avgpool2d_batch_sizes[arm_ethos_u55]
 # aten::gt.Tensor_out                          - test_divide_f32_trunc[arm_ethos_u55]
+# quantized_decomposed::dequantize_per_channel.out - test_conv3d_padding_reflect[arm_ethos_u55]
 #
 # Ethos-U85:
 # aten::_native_batch_norm_legit_no_training.out - test_swin_v2_t[arm_ethos_u85-static_shapes-float32]
@@ -75,6 +76,7 @@ ops_list_u55=(
     aten::bmm.out
     dim_order_ops::_clone_dim_order.out
     dim_order_ops::_to_dim_order_copy.out
+    quantized_decomposed::dequantize_per_channel.out
     "${ops_list_quantized_decomposed[@]}"
 )
 ops_list_u65=("${ops_list_u55[@]}")

backends/test/suite/flows/arm.py

@@ -25,6 +25,7 @@ def _create_arm_flow(
     per_channel_quantization: bool = True,
     use_portable_ops: bool = True,
     timeout: int = 1200,
+    xfail_patterns: list[str] | None = None,
 ) -> TestFlow:
 
     def _create_arm_tester(*args, **kwargs) -> ArmTester:
@@ -59,6 +60,7 @@ def create_quantize_stage() -> Quantize:
                 "TOSA/Ethos-U/VGF flows."
             }
         },
+        xfail_patterns=xfail_patterns or [],
     )
 
 
@@ -83,6 +85,8 @@ def create_quantize_stage() -> Quantize:
     "arm_ethos_u85",
     lambda: common.get_u85_compile_spec(),
     quantize=True,
+    # Ethos-U85 takes these 3-D convolutions and its output does not match.
+    xfail_patterns=["test_conv3d_groups", "test_conv3d_stride"],
 )
 ARM_VGF_FP_FLOW = _create_arm_flow(
     "arm_vgf_fp",

backends/test/suite/flows/coreml.py

@@ -11,6 +11,7 @@ def _create_coreml_flow(
     name: str,
     quantize: bool = False,
     minimum_deployment_target: Any = coremltools.target.iOS15,
+    xfail_patterns: list[str] | None = None,
 ) -> TestFlow:
     return TestFlow(
         name,
@@ -20,10 +21,20 @@ def _create_coreml_flow(
         ),
         quantize=quantize,
         skip_patterns=["test_argmin", "test_argmax"],
+        xfail_patterns=xfail_patterns or [],
     )
 
 
-COREML_TEST_FLOW = _create_coreml_flow("coreml")
+COREML_TEST_FLOW = _create_coreml_flow(
+    "coreml",
+    # Core ML fails to compile the model for the reflect padding case, and
+    # Core ML lowering of a transposed 3-D convolution with output_padding
+    # raises "'NoneType' object is not iterable".
+    xfail_patterns=[
+        "test_conv3d_padding_reflect",
+        "test_convtranspose3d_output_padding",
+    ],
+)
 COREML_STATIC_INT8_TEST_FLOW = _create_coreml_flow(
     "coreml_static_int8",
     quantize=True,

backends/test/suite/flows/qualcomm.py

@@ -12,6 +12,7 @@ def _create_qnn_flow(
     per_channel_linear=False,
     is_qat=False,
     use_fp16=True,
+    xfail_patterns: list[str] | None = None,
 ) -> TestFlow:
     if quantize and quant_dtype is None:
         raise RuntimeError("Quant dtype must be provided when quantize is true.")
@@ -37,25 +38,66 @@ def create_quantize_stage() -> Quantize:
         tester_factory=create_tester,
         quantize=quantize,
         quantize_stage_factory=create_quantize_stage if quantize else None,
+        xfail_patterns=xfail_patterns or [],
     )
 
 
+# QNN fails to lower these 3-D convolutions instead of leaving them to the
+# portable kernel. Partitioning asserts "DepthWise only supports Conv2d" on the
+# depthwise one and raises "'NoneType' object has no attribute 'op'" on the
+# replicate-padded one. For the circular-padded one, the QNN graph's inputs and
+# outputs do not match the delegated program.
+_QNN_CONV3D_LOWERING_XFAILS = [
+    "test_conv3d_depthwise",
+    "test_conv3d_padding_circular",
+    "test_conv3d_padding_replicate",
+]
+# With 8-bit weights, QNN takes these transposed 3-D convolutions and its output
+# does not match.
+_QNN_8BIT_WEIGHT_CONV3D_XFAILS = [
+    "test_convtranspose3d_depthwise",
+    "test_convtranspose3d_groups",
+    "test_convtranspose3d_padding",
+]
+
+
 QNN_TEST_FLOW = _create_qnn_flow("qnn")
 QNN_16A16W_TEST_FLOW = _create_qnn_flow(
-    "qnn_16a16w", quantize=True, quant_dtype=QuantDtype.use_16a16w, use_fp16=False
+    "qnn_16a16w",
+    quantize=True,
+    quant_dtype=QuantDtype.use_16a16w,
+    use_fp16=False,
+    xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS,
 )
 QNN_16A8W_TEST_FLOW = _create_qnn_flow(
-    "qnn_16a8w", quantize=True, quant_dtype=QuantDtype.use_16a8w, use_fp16=False
+    "qnn_16a8w",
+    quantize=True,
+    quant_dtype=QuantDtype.use_16a8w,
+    use_fp16=False,
+    xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS + _QNN_8BIT_WEIGHT_CONV3D_XFAILS,
 )
 QNN_16A4W_TEST_FLOW = _create_qnn_flow(
-    "qnn_16a4w", quantize=True, quant_dtype=QuantDtype.use_16a4w, use_fp16=False
+    "qnn_16a4w",
+    quantize=True,
+    quant_dtype=QuantDtype.use_16a4w,
+    use_fp16=False,
+    # Every 3-D convolution fails in this flow, most of them when QNN generates
+    # the context binary.
+    xfail_patterns=["test_conv3d", "test_convtranspose3d"],
 )
 QNN_16A4W_BLOCK_TEST_FLOW = _create_qnn_flow(
     "qnn_16a4w_block",
     quantize=True,
     quant_dtype=QuantDtype.use_8a8w,
     use_fp16=False,
+    # This flow still quantizes as 8a8w, not 16a4w block, so it fails the same
+    # cases as qnn_8a8w. Recheck the list when the dtype is corrected.
+    xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS + _QNN_8BIT_WEIGHT_CONV3D_XFAILS,
 )
 QNN_8A8W_TEST_FLOW = _create_qnn_flow(
-    "qnn_8a8w", quantize=True, quant_dtype=QuantDtype.use_8a8w, use_fp16=False
+    "qnn_8a8w",
+    quantize=True,
+    quant_dtype=QuantDtype.use_8a8w,
+    use_fp16=False,
+    xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS + _QNN_8BIT_WEIGHT_CONV3D_XFAILS,
 )

backends/test/suite/flows/webgpu.py

@@ -20,6 +20,9 @@ def _create_webgpu_flow() -> TestFlow:
             "lstm_batch_sizes",
             "upsample_nearest2d",
         ],
+        # WebGPU fails to build the graph for replicate padding ("clamp: int
+        # dtype unsupported").
+        xfail_patterns=["test_conv3d_padding_replicate"],
     )
 
 

backends/test/suite/generate_markdown_summary_json.py

@@ -13,18 +13,25 @@ class ResultCounts:
     total: int = 0
     passes: int = 0
     fails: int = 0
+    expected_fails: int = 0
     skips: int = 0
     by_detail: dict[str, int] = field(default_factory=lambda: {})
 
-    def add_row(self, result_value: str, result_detail: str) -> None:
+    def add_row(
+        self, result_value: str, result_detail: str, expected_fail: bool = False
+    ) -> None:
         """
-        Update the result counts for the specified row.
+        Update the result counts for the specified row. A failure in a test that
+        a flow lists in xfail_patterns counts as an expected failure.
         """
 
         self.total += 1
 
         if result_value == "Pass":
             self.passes += 1
+        elif result_value == "Fail" and expected_fail:
+            self.expected_fails += 1
+            return
         elif result_value == "Fail":
             self.fails += 1
         elif result_value == "Skip":
@@ -48,6 +55,8 @@ class AggregatedSummary:
     counts: ResultCounts
     counts_by_params: dict[str, ResultCounts]
     failed_tests: list[list[str]]
+    expected_failures: list[list[str]]
+    unexpected_passes: list[str]
 
 
 #
@@ -68,14 +77,27 @@ def aggregate_results(json_path: str) -> AggregatedSummary:
     counts = ResultCounts()
     failed_tests = []
     counts_by_param = {}
+    expected_failures = []
+    unexpected_passes = []
 
     for test_data in data["tests"]:
+        # The subtest rows hold the runner's own result. Only pytest's outcome
+        # knows whether the test is in a flow's xfail_patterns.
+        outcome = test_data.get("outcome")
+        expected_fail = outcome == "xfailed"
+        # Under pytest-xdist the first line names the worker, so check every line.
+        longrepr = str((test_data.get("call") or {}).get("longrepr") or "")
+        if outcome == "failed" and any(
+            line.startswith("[XPASS(strict)]") for line in longrepr.splitlines()
+        ):
+            unexpected_passes.append(test_data["nodeid"].split("::")[-1])
+
         result_meta = test_data["metadata"]
         for subtest_meta in result_meta["subtests"]:
             result = subtest_meta["Result"]
             result_detail = subtest_meta.get("Result Detail") or ""
 
-            counts.add_row(result, result_detail)
+            counts.add_row(result, result_detail, expected_fail)
 
             test_id = subtest_meta["Test ID"]
             base_test = subtest_meta["Test Case"]
@@ -84,15 +106,20 @@ def aggregate_results(json_path: str) -> AggregatedSummary:
             if params:
                 if params not in counts_by_param:
                     counts_by_param[params] = ResultCounts()
-                counts_by_param[params].add_row(result, result_detail)
+                counts_by_param[params].add_row(result, result_detail, expected_fail)
 
             if result.lower() == "fail":
-                failed_tests.append(subtest_meta)
+                if expected_fail:
+                    expected_failures.append(subtest_meta)
+                else:
+                    failed_tests.append(subtest_meta)
 
     return AggregatedSummary(
         counts=counts,
         failed_tests=failed_tests,
         counts_by_params=counts_by_param,
+        expected_failures=expected_failures,
+        unexpected_passes=unexpected_passes,
     )
 
 
@@ -120,7 +147,9 @@ def generate_markdown(json_path: str, exit_code: int = 0):  # noqa (C901)
 
     # Generate Summary section
     print("# Summary\n")
-    total_excluding_skips = results.counts.passes + results.counts.fails
+    total_excluding_skips = (
+        results.counts.passes + results.counts.fails + results.counts.expected_fails
+    )
     pass_fraction = results.counts.passes / total_excluding_skips
     fail_fraction = results.counts.fails / total_excluding_skips
     print(
@@ -129,14 +158,18 @@ def generate_markdown(json_path: str, exit_code: int = 0):  # noqa (C901)
     print(
         f"- **Fail**: {results.counts.fails}/{total_excluding_skips} ({fail_fraction*100:.2f}%)"
     )
+    expected_fail_fraction = results.counts.expected_fails / total_excluding_skips
+    print(
+        f"- **Expected Fail**: {results.counts.expected_fails}/{total_excluding_skips} ({expected_fail_fraction*100:.2f}%)"
+    )
     print(f"- **Skip**: {results.counts.skips}")
 
     if results.counts_by_params:
         print("\n## Results by Parameters\n")
 
         if len(results.counts_by_params) > 0:
             # Create table header
-            header_cols = ["Params", "Pass", "Fail", "Skip", "Pass %"]
+            header_cols = ["Params", "Pass", "Fail", "Expected Fail", "Skip", "Pass %"]
             print("| " + " | ".join(header_cols) + " |")
             print("|" + "|".join(["---"] * len(header_cols)) + "|")
 
@@ -145,13 +178,16 @@ def generate_markdown(json_path: str, exit_code: int = 0):  # noqa (C901)
                 row_values = [params_str]
 
                 # Add parameter values
-                pass_fraction = counts.passes / (counts.passes + counts.fails)
+                pass_fraction = counts.passes / (
+                    counts.passes + counts.fails + counts.expected_fails
+                )
 
                 # Add count values
                 row_values.extend(
                     [
                         str(counts.passes),
                         str(counts.fails),
+                        str(counts.expected_fails),
                         str(counts.skips),
                         f"{pass_fraction*100:.2f}%",
                     ]
@@ -166,6 +202,15 @@ def generate_markdown(json_path: str, exit_code: int = 0):  # noqa (C901)
     for detail, count in sorted(results.counts.by_detail.items()):
         print(f"- **{detail}**: {count}/{total_rows_with_result_detail}")
 
+    if results.unexpected_passes:
+        print("# Unexpected Passes\n")
+        print(
+            "These tests are in a flow's xfail_patterns but passed. Remove them from the list."
+        )
+        for test_id in results.unexpected_passes:
+            print(f"- {test_id}")
+        print()
+
     # Generate Failed Tests section
     print("# Failed Tests\n")
     print(
@@ -175,17 +220,27 @@ def generate_markdown(json_path: str, exit_code: int = 0):  # noqa (C901)
     print('pytest -c /dev/nul backends/test/suite/ -k "<test_id>"')
     print("```")
     if results.failed_tests:
-        header = build_header(results.failed_tests)
-
-        escaped_header = [escape_for_markdown(col) for col in header.keys()]
-        print("| " + " | ".join(escaped_header) + " |")
-        print("|" + "|".join(["---"] * len(escaped_header)) + "|")
-        for rec in results.failed_tests:
-            row = build_row(rec, header)
-            print("| " + " | ".join(row) + " |")
+        print_table(results.failed_tests)
     else:
         print("No failed tests.\n")
 
+    if results.expected_failures:
+        # Shown with their details, so an expected failure that starts failing
+        # in a different way is still visible.
+        print("# Expected Failures\n")
+        print_table(results.expected_failures)
+
+
+def print_table(records) -> None:
+    header = build_header(records)
+
+    escaped_header = [escape_for_markdown(col) for col in header.keys()]
+    print("| " + " | ".join(escaped_header) + " |")
+    print("|" + "|".join(["---"] * len(escaped_header)) + "|")
+    for rec in records:
+        row = build_row(rec, header)
+        print("| " + " | ".join(row) + " |")
+
 
 def build_header(data) -> dict[str, int]:
     """

backends/test/suite/operators/test_conv3d.py

@@ -134,13 +134,26 @@ def test_conv3d_no_bias(self, flow: TestFlow) -> None:
             flow,
         )
 
-    def test_conv3d_padding_modes(self, flow: TestFlow) -> None:
-        for mode in ["zeros", "reflect", "replicate", "circular"]:
-            self._test_op(
-                Model(padding=1, padding_mode=mode),
-                (torch.randn(4, 3, 8, 8, 8),),
-                flow,
-            )
+    def test_conv3d_padding_reflect(self, flow: TestFlow) -> None:
+        self._test_op(
+            Model(padding=1, padding_mode="reflect"),
+            (torch.randn(4, 3, 8, 8, 8),),
+            flow,
+        )
+
+    def test_conv3d_padding_replicate(self, flow: TestFlow) -> None:
+        self._test_op(
+            Model(padding=1, padding_mode="replicate"),
+            (torch.randn(4, 3, 8, 8, 8),),
+            flow,
+        )
+
+    def test_conv3d_padding_circular(self, flow: TestFlow) -> None:
+        self._test_op(
+            Model(padding=1, padding_mode="circular"),
+            (torch.randn(4, 3, 8, 8, 8),),
+            flow,
+        )
 
     def test_conv3d_channels(self, flow: TestFlow) -> None:
         self._test_op(