From af5a91ac2a0c05908468ae41c3a5286ec7b7d9e4 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 2 Nov 2026 11:10:34 -0800 Subject: [PATCH 1/7] Mark the 3-D convolution cases that fail inside a backend as expected failures Running 5-D convolutions in the backend test suite shows some cases that fail inside the backend that takes them, not in the portable kernel: - QNN quantized flows: depthwise 3-D convolutions and the padded input of padding_modes fail to lower, and the delegated transposed groups, depthwise and padding cases do not match. With 16-bit activations and 4-bit weights, QNN fails to build a context binary for every 3-D case. - Ethos-U85: the delegated stride and groups cases do not match. - Ethos-U55: the runner is built without the quantized_decomposed::dequantize_per_channel.out kernel that padding_modes leaves outside the delegate. - Core ML: the reflect padding case fails to compile, and lowering a transposed convolution with output_padding raises a TypeError. - WebGPU: replicate padding fails to build the graph. Each flow lists those tests as strict expected failures, so the report keeps recording how they fail, and a fix in the backend shows up as an unexpected pass. --- backends/test/suite/flows/arm.py | 8 ++++++ backends/test/suite/flows/coreml.py | 13 ++++++++- backends/test/suite/flows/qualcomm.py | 41 ++++++++++++++++++++++++--- backends/test/suite/flows/webgpu.py | 3 ++ 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/backends/test/suite/flows/arm.py b/backends/test/suite/flows/arm.py index 7e357f0fde9..f4a736735e2 100644 --- a/backends/test/suite/flows/arm.py +++ b/backends/test/suite/flows/arm.py @@ -25,6 +25,7 @@ def _create_arm_flow( per_channel_quantization: bool = True, use_portable_ops: bool = True, timeout: int = 1200, + xfail_patterns: list[str] | None = None, ) -> TestFlow: def _create_arm_tester(*args, **kwargs) -> ArmTester: @@ -59,6 +60,7 @@ def create_quantize_stage() -> Quantize: "TOSA/Ethos-U/VGF flows." } }, + xfail_patterns=xfail_patterns or [], ) @@ -78,11 +80,17 @@ def create_quantize_stage() -> Quantize: "arm_ethos_u55", lambda: common.get_u55_compile_spec(), quantize=True, + # The Corstone runner is built without the + # quantized_decomposed::dequantize_per_channel.out kernel that this case + # leaves outside the delegate. + xfail_patterns=["test_conv3d_padding_modes"], ) ARM_ETHOS_U85_FLOW = _create_arm_flow( "arm_ethos_u85", lambda: common.get_u85_compile_spec(), quantize=True, + # Ethos-U85 takes these 3-D convolutions and its output does not match. + xfail_patterns=["test_conv3d_groups", "test_conv3d_stride"], ) ARM_VGF_FP_FLOW = _create_arm_flow( "arm_vgf_fp", diff --git a/backends/test/suite/flows/coreml.py b/backends/test/suite/flows/coreml.py index 8a532ff0003..e9e2dc8e2f0 100644 --- a/backends/test/suite/flows/coreml.py +++ b/backends/test/suite/flows/coreml.py @@ -11,6 +11,7 @@ def _create_coreml_flow( name: str, quantize: bool = False, minimum_deployment_target: Any = coremltools.target.iOS15, + xfail_patterns: list[str] | None = None, ) -> TestFlow: return TestFlow( name, @@ -20,10 +21,20 @@ def _create_coreml_flow( ), quantize=quantize, skip_patterns=["test_argmin", "test_argmax"], + xfail_patterns=xfail_patterns or [], ) -COREML_TEST_FLOW = _create_coreml_flow("coreml") +COREML_TEST_FLOW = _create_coreml_flow( + "coreml", + # Core ML fails to compile the model for the reflect padding case, and + # Core ML lowering of a transposed 3-D convolution with output_padding + # raises "'NoneType' object is not iterable". + xfail_patterns=[ + "test_conv3d_padding_modes", + "test_convtranspose3d_output_padding", + ], +) COREML_STATIC_INT8_TEST_FLOW = _create_coreml_flow( "coreml_static_int8", quantize=True, diff --git a/backends/test/suite/flows/qualcomm.py b/backends/test/suite/flows/qualcomm.py index 99deb3d4877..87b4d1d785a 100644 --- a/backends/test/suite/flows/qualcomm.py +++ b/backends/test/suite/flows/qualcomm.py @@ -12,6 +12,7 @@ def _create_qnn_flow( per_channel_linear=False, is_qat=False, use_fp16=True, + xfail_patterns: list[str] | None = None, ) -> TestFlow: if quantize and quant_dtype is None: raise RuntimeError("Quant dtype must be provided when quantize is true.") @@ -37,25 +38,57 @@ def create_quantize_stage() -> Quantize: tester_factory=create_tester, quantize=quantize, quantize_stage_factory=create_quantize_stage if quantize else None, + xfail_patterns=xfail_patterns or [], ) +# 3-D convolutions that QNN takes and fails. The portable kernel runs none of +# them: QNN refuses to lower depthwise 3-D convolutions ("DepthWise only +# supports Conv2d") and the padded input of padding_modes ("'NoneType' object +# has no attribute 'op'"), and the delegated transposed cases do not match. +_QNN_QUANTIZED_CONV3D_XFAILS = [ + "test_conv3d_depthwise", + "test_conv3d_padding_modes", + "test_convtranspose3d_depthwise", + "test_convtranspose3d_groups", + "test_convtranspose3d_padding", +] + + QNN_TEST_FLOW = _create_qnn_flow("qnn") QNN_16A16W_TEST_FLOW = _create_qnn_flow( - "qnn_16a16w", quantize=True, quant_dtype=QuantDtype.use_16a16w, use_fp16=False + "qnn_16a16w", + quantize=True, + quant_dtype=QuantDtype.use_16a16w, + use_fp16=False, + xfail_patterns=["test_conv3d_depthwise", "test_conv3d_padding_modes"], ) QNN_16A8W_TEST_FLOW = _create_qnn_flow( - "qnn_16a8w", quantize=True, quant_dtype=QuantDtype.use_16a8w, use_fp16=False + "qnn_16a8w", + quantize=True, + quant_dtype=QuantDtype.use_16a8w, + use_fp16=False, + xfail_patterns=_QNN_QUANTIZED_CONV3D_XFAILS, ) QNN_16A4W_TEST_FLOW = _create_qnn_flow( - "qnn_16a4w", quantize=True, quant_dtype=QuantDtype.use_16a4w, use_fp16=False + "qnn_16a4w", + quantize=True, + quant_dtype=QuantDtype.use_16a4w, + use_fp16=False, + # QNN fails to generate a context binary for every 3-D convolution here. + xfail_patterns=["test_conv3d", "test_convtranspose3d"], ) QNN_16A4W_BLOCK_TEST_FLOW = _create_qnn_flow( "qnn_16a4w_block", quantize=True, quant_dtype=QuantDtype.use_8a8w, use_fp16=False, + xfail_patterns=_QNN_QUANTIZED_CONV3D_XFAILS, ) QNN_8A8W_TEST_FLOW = _create_qnn_flow( - "qnn_8a8w", quantize=True, quant_dtype=QuantDtype.use_8a8w, use_fp16=False + "qnn_8a8w", + quantize=True, + quant_dtype=QuantDtype.use_8a8w, + use_fp16=False, + xfail_patterns=_QNN_QUANTIZED_CONV3D_XFAILS, ) diff --git a/backends/test/suite/flows/webgpu.py b/backends/test/suite/flows/webgpu.py index ee0c24f221a..34c5bbb4940 100644 --- a/backends/test/suite/flows/webgpu.py +++ b/backends/test/suite/flows/webgpu.py @@ -20,6 +20,9 @@ def _create_webgpu_flow() -> TestFlow: "lstm_batch_sizes", "upsample_nearest2d", ], + # WebGPU fails to build the graph for replicate padding ("clamp: int + # dtype unsupported"). + xfail_patterns=["test_conv3d_padding_modes"], ) From dbc96f6180611d78c27f32f5f12041952b24ebe1 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 2 Nov 2026 13:26:39 -0800 Subject: [PATCH 2/7] Split the 3-D padding modes test and build the Ethos-U55 runner with per-channel dequantize test_conv3d_padding_modes ran four padding modes in one test, and the first failing mode stopped the rest. Each mode is now its own test, so a flow lists only the mode that fails there, and the other modes keep running. The Ethos-U55 reflect padding case failed only because the Corstone test runner was built without quantized_decomposed::dequantize_per_channel.out. The runner now includes that kernel, the same way the Arm test setup already adds other kernels that flow-suite cases leave outside the delegate, so the case runs instead of being listed as an expected failure. The QNN lists are split into the cases that crash QNN partitioning, which fail in every quantized flow, and the transposed cases that do not match with 8-bit weights. --- backends/arm/test/setup_testing.sh | 2 ++ backends/test/suite/flows/arm.py | 4 --- backends/test/suite/flows/coreml.py | 2 +- backends/test/suite/flows/qualcomm.py | 25 ++++++++------ backends/test/suite/flows/webgpu.py | 2 +- backends/test/suite/operators/test_conv3d.py | 34 ++++++++++++++++---- 6 files changed, 46 insertions(+), 23 deletions(-) diff --git a/backends/arm/test/setup_testing.sh b/backends/arm/test/setup_testing.sh index 731164c6c87..eac1a6272bf 100755 --- a/backends/arm/test/setup_testing.sh +++ b/backends/arm/test/setup_testing.sh @@ -44,6 +44,7 @@ ${build_executor_runner} --pte=semihosting --target=ethos-u85-128 --system_confi # Ethos-U55/U65 # aten::avg_pool2d.out - test_adaptive_avgpool2d_batch_sizes[arm_ethos_u55] # aten::gt.Tensor_out - test_divide_f32_trunc[arm_ethos_u55] +# quantized_decomposed::dequantize_per_channel.out - test_conv3d_padding_reflect[arm_ethos_u55] # # Ethos-U85: # aten::_native_batch_norm_legit_no_training.out - test_swin_v2_t[arm_ethos_u85-static_shapes-float32] @@ -75,6 +76,7 @@ ops_list_u55=( aten::bmm.out dim_order_ops::_clone_dim_order.out dim_order_ops::_to_dim_order_copy.out + quantized_decomposed::dequantize_per_channel.out "${ops_list_quantized_decomposed[@]}" ) ops_list_u65=("${ops_list_u55[@]}") diff --git a/backends/test/suite/flows/arm.py b/backends/test/suite/flows/arm.py index f4a736735e2..a356bc5778b 100644 --- a/backends/test/suite/flows/arm.py +++ b/backends/test/suite/flows/arm.py @@ -80,10 +80,6 @@ def create_quantize_stage() -> Quantize: "arm_ethos_u55", lambda: common.get_u55_compile_spec(), quantize=True, - # The Corstone runner is built without the - # quantized_decomposed::dequantize_per_channel.out kernel that this case - # leaves outside the delegate. - xfail_patterns=["test_conv3d_padding_modes"], ) ARM_ETHOS_U85_FLOW = _create_arm_flow( "arm_ethos_u85", diff --git a/backends/test/suite/flows/coreml.py b/backends/test/suite/flows/coreml.py index e9e2dc8e2f0..b2cb9367380 100644 --- a/backends/test/suite/flows/coreml.py +++ b/backends/test/suite/flows/coreml.py @@ -31,7 +31,7 @@ def _create_coreml_flow( # Core ML lowering of a transposed 3-D convolution with output_padding # raises "'NoneType' object is not iterable". xfail_patterns=[ - "test_conv3d_padding_modes", + "test_conv3d_padding_reflect", "test_convtranspose3d_output_padding", ], ) diff --git a/backends/test/suite/flows/qualcomm.py b/backends/test/suite/flows/qualcomm.py index 87b4d1d785a..33af830d79e 100644 --- a/backends/test/suite/flows/qualcomm.py +++ b/backends/test/suite/flows/qualcomm.py @@ -42,13 +42,17 @@ def create_quantize_stage() -> Quantize: ) -# 3-D convolutions that QNN takes and fails. The portable kernel runs none of -# them: QNN refuses to lower depthwise 3-D convolutions ("DepthWise only -# supports Conv2d") and the padded input of padding_modes ("'NoneType' object -# has no attribute 'op'"), and the delegated transposed cases do not match. -_QNN_QUANTIZED_CONV3D_XFAILS = [ +# QNN partitioning crashes on these 3-D convolutions instead of leaving them to +# the portable kernel: the depthwise one asserts "DepthWise only supports +# Conv2d", and the replicate-padded one raises "'NoneType' object has no +# attribute 'op'". +_QNN_CONV3D_LOWERING_XFAILS = [ "test_conv3d_depthwise", - "test_conv3d_padding_modes", + "test_conv3d_padding_replicate", +] +# With 8-bit weights, QNN takes these transposed 3-D convolutions and its output +# does not match. +_QNN_8BIT_WEIGHT_CONV3D_XFAILS = [ "test_convtranspose3d_depthwise", "test_convtranspose3d_groups", "test_convtranspose3d_padding", @@ -61,14 +65,14 @@ def create_quantize_stage() -> Quantize: quantize=True, quant_dtype=QuantDtype.use_16a16w, use_fp16=False, - xfail_patterns=["test_conv3d_depthwise", "test_conv3d_padding_modes"], + xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS, ) QNN_16A8W_TEST_FLOW = _create_qnn_flow( "qnn_16a8w", quantize=True, quant_dtype=QuantDtype.use_16a8w, use_fp16=False, - xfail_patterns=_QNN_QUANTIZED_CONV3D_XFAILS, + xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS + _QNN_8BIT_WEIGHT_CONV3D_XFAILS, ) QNN_16A4W_TEST_FLOW = _create_qnn_flow( "qnn_16a4w", @@ -83,12 +87,13 @@ def create_quantize_stage() -> Quantize: quantize=True, quant_dtype=QuantDtype.use_8a8w, use_fp16=False, - xfail_patterns=_QNN_QUANTIZED_CONV3D_XFAILS, + # This flow quantizes as 8a8w, so it fails the same cases as qnn_8a8w. + xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS + _QNN_8BIT_WEIGHT_CONV3D_XFAILS, ) QNN_8A8W_TEST_FLOW = _create_qnn_flow( "qnn_8a8w", quantize=True, quant_dtype=QuantDtype.use_8a8w, use_fp16=False, - xfail_patterns=_QNN_QUANTIZED_CONV3D_XFAILS, + xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS + _QNN_8BIT_WEIGHT_CONV3D_XFAILS, ) diff --git a/backends/test/suite/flows/webgpu.py b/backends/test/suite/flows/webgpu.py index 34c5bbb4940..3bde683f4c5 100644 --- a/backends/test/suite/flows/webgpu.py +++ b/backends/test/suite/flows/webgpu.py @@ -22,7 +22,7 @@ def _create_webgpu_flow() -> TestFlow: ], # WebGPU fails to build the graph for replicate padding ("clamp: int # dtype unsupported"). - xfail_patterns=["test_conv3d_padding_modes"], + xfail_patterns=["test_conv3d_padding_replicate"], ) diff --git a/backends/test/suite/operators/test_conv3d.py b/backends/test/suite/operators/test_conv3d.py index 01ffa4942df..a6cea5ca757 100644 --- a/backends/test/suite/operators/test_conv3d.py +++ b/backends/test/suite/operators/test_conv3d.py @@ -134,13 +134,33 @@ def test_conv3d_no_bias(self, flow: TestFlow) -> None: flow, ) - def test_conv3d_padding_modes(self, flow: TestFlow) -> None: - for mode in ["zeros", "reflect", "replicate", "circular"]: - self._test_op( - Model(padding=1, padding_mode=mode), - (torch.randn(4, 3, 8, 8, 8),), - flow, - ) + def test_conv3d_padding_zeros(self, flow: TestFlow) -> None: + self._test_op( + Model(padding=1, padding_mode="zeros"), + (torch.randn(4, 3, 8, 8, 8),), + flow, + ) + + def test_conv3d_padding_reflect(self, flow: TestFlow) -> None: + self._test_op( + Model(padding=1, padding_mode="reflect"), + (torch.randn(4, 3, 8, 8, 8),), + flow, + ) + + def test_conv3d_padding_replicate(self, flow: TestFlow) -> None: + self._test_op( + Model(padding=1, padding_mode="replicate"), + (torch.randn(4, 3, 8, 8, 8),), + flow, + ) + + def test_conv3d_padding_circular(self, flow: TestFlow) -> None: + self._test_op( + Model(padding=1, padding_mode="circular"), + (torch.randn(4, 3, 8, 8, 8),), + flow, + ) def test_conv3d_channels(self, flow: TestFlow) -> None: self._test_op( From c4a699e6090d82c120e038ccee25e4a38060ecc3 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 2 Nov 2026 13:56:15 -0800 Subject: [PATCH 3/7] Show expected failures and unexpected passes in the backend test summary The step summary counted each subtest by the runner's own result and never read pytest's outcome. So a test in a flow's xfail_patterns still counted as a failure and was listed under Failed Tests, and one that started passing showed up as an ordinary pass, although pytest reports it as a strict unexpected pass. The summary now counts failures of listed tests as expected failures, keeps them out of Failed Tests, and lists strict unexpected passes in their own section, so a stale entry is easy to spot. --- .../suite/generate_markdown_summary_json.py | 55 ++++++++++++++++--- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/backends/test/suite/generate_markdown_summary_json.py b/backends/test/suite/generate_markdown_summary_json.py index 4b6edc2a635..5ea7b31bbd6 100644 --- a/backends/test/suite/generate_markdown_summary_json.py +++ b/backends/test/suite/generate_markdown_summary_json.py @@ -13,18 +13,25 @@ class ResultCounts: total: int = 0 passes: int = 0 fails: int = 0 + expected_fails: int = 0 skips: int = 0 by_detail: dict[str, int] = field(default_factory=lambda: {}) - def add_row(self, result_value: str, result_detail: str) -> None: + def add_row( + self, result_value: str, result_detail: str, expected_fail: bool = False + ) -> None: """ - Update the result counts for the specified row. + Update the result counts for the specified row. A failure in a test that + a flow lists in xfail_patterns counts as an expected failure. """ self.total += 1 if result_value == "Pass": self.passes += 1 + elif result_value == "Fail" and expected_fail: + self.expected_fails += 1 + return elif result_value == "Fail": self.fails += 1 elif result_value == "Skip": @@ -48,6 +55,7 @@ class AggregatedSummary: counts: ResultCounts counts_by_params: dict[str, ResultCounts] failed_tests: list[list[str]] + unexpected_passes: list[str] # @@ -68,14 +76,26 @@ def aggregate_results(json_path: str) -> AggregatedSummary: counts = ResultCounts() failed_tests = [] counts_by_param = {} + unexpected_passes = [] for test_data in data["tests"]: + # The subtest rows hold the runner's own result. Only pytest's outcome + # knows whether the test is in a flow's xfail_patterns. + outcome = test_data.get("outcome") + expected_fail = outcome == "xfailed" + # Under pytest-xdist the first line names the worker, so check every line. + longrepr = str((test_data.get("call") or {}).get("longrepr") or "") + if outcome == "failed" and any( + line.startswith("[XPASS(strict)]") for line in longrepr.splitlines() + ): + unexpected_passes.append(test_data["nodeid"].split("::")[-1]) + result_meta = test_data["metadata"] for subtest_meta in result_meta["subtests"]: result = subtest_meta["Result"] result_detail = subtest_meta.get("Result Detail") or "" - counts.add_row(result, result_detail) + counts.add_row(result, result_detail, expected_fail) test_id = subtest_meta["Test ID"] base_test = subtest_meta["Test Case"] @@ -84,15 +104,16 @@ def aggregate_results(json_path: str) -> AggregatedSummary: if params: if params not in counts_by_param: counts_by_param[params] = ResultCounts() - counts_by_param[params].add_row(result, result_detail) + counts_by_param[params].add_row(result, result_detail, expected_fail) - if result.lower() == "fail": + if result.lower() == "fail" and not expected_fail: failed_tests.append(subtest_meta) return AggregatedSummary( counts=counts, failed_tests=failed_tests, counts_by_params=counts_by_param, + unexpected_passes=unexpected_passes, ) @@ -120,7 +141,9 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) # Generate Summary section print("# Summary\n") - total_excluding_skips = results.counts.passes + results.counts.fails + total_excluding_skips = ( + results.counts.passes + results.counts.fails + results.counts.expected_fails + ) pass_fraction = results.counts.passes / total_excluding_skips fail_fraction = results.counts.fails / total_excluding_skips print( @@ -129,6 +152,10 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) print( f"- **Fail**: {results.counts.fails}/{total_excluding_skips} ({fail_fraction*100:.2f}%)" ) + expected_fail_fraction = results.counts.expected_fails / total_excluding_skips + print( + f"- **Expected Fail**: {results.counts.expected_fails}/{total_excluding_skips} ({expected_fail_fraction*100:.2f}%)" + ) print(f"- **Skip**: {results.counts.skips}") if results.counts_by_params: @@ -136,7 +163,7 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) if len(results.counts_by_params) > 0: # Create table header - header_cols = ["Params", "Pass", "Fail", "Skip", "Pass %"] + header_cols = ["Params", "Pass", "Fail", "Expected Fail", "Skip", "Pass %"] print("| " + " | ".join(header_cols) + " |") print("|" + "|".join(["---"] * len(header_cols)) + "|") @@ -145,13 +172,16 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) row_values = [params_str] # Add parameter values - pass_fraction = counts.passes / (counts.passes + counts.fails) + pass_fraction = counts.passes / ( + counts.passes + counts.fails + counts.expected_fails + ) # Add count values row_values.extend( [ str(counts.passes), str(counts.fails), + str(counts.expected_fails), str(counts.skips), f"{pass_fraction*100:.2f}%", ] @@ -166,6 +196,15 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) for detail, count in sorted(results.counts.by_detail.items()): print(f"- **{detail}**: {count}/{total_rows_with_result_detail}") + if results.unexpected_passes: + print("# Unexpected Passes\n") + print( + "These tests are in a flow's xfail_patterns but passed. Remove them from the list." + ) + for test_id in results.unexpected_passes: + print(f"- {test_id}") + print() + # Generate Failed Tests section print("# Failed Tests\n") print( From e1273a90b003a978a8d9edf5ec8bd95d115e0c28 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 2 Nov 2026 13:56:24 -0800 Subject: [PATCH 4/7] Drop the zeros padding test and note the 16a4w block dtype gap test_conv3d_padding already runs a zeros-padded 3-D convolution on the same input, so a separate zeros test adds nothing. The qnn_16a4w_block flow still quantizes as 8a8w; the comment now says so as a gap, so whoever fixes the dtype knows to recheck its list. --- backends/test/suite/flows/qualcomm.py | 3 ++- backends/test/suite/operators/test_conv3d.py | 7 ------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/backends/test/suite/flows/qualcomm.py b/backends/test/suite/flows/qualcomm.py index 33af830d79e..fadfa740d75 100644 --- a/backends/test/suite/flows/qualcomm.py +++ b/backends/test/suite/flows/qualcomm.py @@ -87,7 +87,8 @@ def create_quantize_stage() -> Quantize: quantize=True, quant_dtype=QuantDtype.use_8a8w, use_fp16=False, - # This flow quantizes as 8a8w, so it fails the same cases as qnn_8a8w. + # This flow still quantizes as 8a8w, not 16a4w block, so it fails the same + # cases as qnn_8a8w. Recheck the list when the dtype is corrected. xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS + _QNN_8BIT_WEIGHT_CONV3D_XFAILS, ) QNN_8A8W_TEST_FLOW = _create_qnn_flow( diff --git a/backends/test/suite/operators/test_conv3d.py b/backends/test/suite/operators/test_conv3d.py index a6cea5ca757..e393934cd44 100644 --- a/backends/test/suite/operators/test_conv3d.py +++ b/backends/test/suite/operators/test_conv3d.py @@ -134,13 +134,6 @@ def test_conv3d_no_bias(self, flow: TestFlow) -> None: flow, ) - def test_conv3d_padding_zeros(self, flow: TestFlow) -> None: - self._test_op( - Model(padding=1, padding_mode="zeros"), - (torch.randn(4, 3, 8, 8, 8),), - flow, - ) - def test_conv3d_padding_reflect(self, flow: TestFlow) -> None: self._test_op( Model(padding=1, padding_mode="reflect"), From 947dd0092a9864831be0eda01a1736839d2d3392 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 2 Nov 2026 14:28:03 -0800 Subject: [PATCH 5/7] List the circular padding case for the QNN quantized flows The nightly run of the split padding tests shows that every quantized QNN flow fails to lower the circular-padded 3-D convolution: the QNN graph's inputs and outputs do not match the delegated program. Ethos-U55, Ethos-U85 and WebGPU pass it, and Ethos-U55 now also passes reflect padding with the per-channel dequantize kernel in its runner. --- backends/test/suite/flows/qualcomm.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backends/test/suite/flows/qualcomm.py b/backends/test/suite/flows/qualcomm.py index fadfa740d75..1dae3a25893 100644 --- a/backends/test/suite/flows/qualcomm.py +++ b/backends/test/suite/flows/qualcomm.py @@ -42,12 +42,14 @@ def create_quantize_stage() -> Quantize: ) -# QNN partitioning crashes on these 3-D convolutions instead of leaving them to -# the portable kernel: the depthwise one asserts "DepthWise only supports -# Conv2d", and the replicate-padded one raises "'NoneType' object has no -# attribute 'op'". +# QNN fails to lower these 3-D convolutions instead of leaving them to the +# portable kernel. Partitioning asserts "DepthWise only supports Conv2d" on the +# depthwise one and raises "'NoneType' object has no attribute 'op'" on the +# replicate-padded one. For the circular-padded one, the QNN graph's inputs and +# outputs do not match the delegated program. _QNN_CONV3D_LOWERING_XFAILS = [ "test_conv3d_depthwise", + "test_conv3d_padding_circular", "test_conv3d_padding_replicate", ] # With 8-bit weights, QNN takes these transposed 3-D convolutions and its output From addd40ea5cffc581ebafb7ceefeafef33c0b2ffb Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 2 Nov 2026 15:33:35 -0800 Subject: [PATCH 6/7] List expected failures in the backend test summary and test the summary The summary counted expected failures but no longer showed their details, so a test in xfail_patterns that started failing in a different way, for example a lowering failure instead of an output mismatch, was visible only in the JSON report. Expected failures now get their own table, like Failed Tests. New unit tests check the counts, which records go to the expected failure and failed test lists, and the strict unexpected pass detection, both for a serial run and for a pytest-xdist run, where the report text starts with the worker's name. --- .../suite/generate_markdown_summary_json.py | 36 ++++++--- .../suite/tests/test_markdown_summary_json.py | 81 +++++++++++++++++++ 2 files changed, 107 insertions(+), 10 deletions(-) create mode 100644 backends/test/suite/tests/test_markdown_summary_json.py diff --git a/backends/test/suite/generate_markdown_summary_json.py b/backends/test/suite/generate_markdown_summary_json.py index 5ea7b31bbd6..fa1fd482992 100644 --- a/backends/test/suite/generate_markdown_summary_json.py +++ b/backends/test/suite/generate_markdown_summary_json.py @@ -55,6 +55,7 @@ class AggregatedSummary: counts: ResultCounts counts_by_params: dict[str, ResultCounts] failed_tests: list[list[str]] + expected_failures: list[list[str]] unexpected_passes: list[str] @@ -76,6 +77,7 @@ def aggregate_results(json_path: str) -> AggregatedSummary: counts = ResultCounts() failed_tests = [] counts_by_param = {} + expected_failures = [] unexpected_passes = [] for test_data in data["tests"]: @@ -106,13 +108,17 @@ def aggregate_results(json_path: str) -> AggregatedSummary: counts_by_param[params] = ResultCounts() counts_by_param[params].add_row(result, result_detail, expected_fail) - if result.lower() == "fail" and not expected_fail: - failed_tests.append(subtest_meta) + if result.lower() == "fail": + if expected_fail: + expected_failures.append(subtest_meta) + else: + failed_tests.append(subtest_meta) return AggregatedSummary( counts=counts, failed_tests=failed_tests, counts_by_params=counts_by_param, + expected_failures=expected_failures, unexpected_passes=unexpected_passes, ) @@ -214,17 +220,27 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) print('pytest -c /dev/nul backends/test/suite/ -k ""') print("```") if results.failed_tests: - header = build_header(results.failed_tests) - - escaped_header = [escape_for_markdown(col) for col in header.keys()] - print("| " + " | ".join(escaped_header) + " |") - print("|" + "|".join(["---"] * len(escaped_header)) + "|") - for rec in results.failed_tests: - row = build_row(rec, header) - print("| " + " | ".join(row) + " |") + print_table(results.failed_tests) else: print("No failed tests.\n") + if results.expected_failures: + # Shown with their details, so an expected failure that starts failing + # in a different way is still visible. + print("# Expected Failures\n") + print_table(results.expected_failures) + + +def print_table(records) -> None: + header = build_header(records) + + escaped_header = [escape_for_markdown(col) for col in header.keys()] + print("| " + " | ".join(escaped_header) + " |") + print("|" + "|".join(["---"] * len(escaped_header)) + "|") + for rec in records: + row = build_row(rec, header) + print("| " + " | ".join(row) + " |") + def build_header(data) -> dict[str, int]: """ diff --git a/backends/test/suite/tests/test_markdown_summary_json.py b/backends/test/suite/tests/test_markdown_summary_json.py new file mode 100644 index 00000000000..4bf509afeab --- /dev/null +++ b/backends/test/suite/tests/test_markdown_summary_json.py @@ -0,0 +1,81 @@ +import json +import os +import tempfile +import unittest + +from ..generate_markdown_summary_json import aggregate_results + + +def _test(name: str, outcome: str, results: list[str], longrepr: str = "") -> dict: + subtests = [ + { + "Test ID": f"{name}[flow1]", + "Test Case": name, + "Subtest": i, + "Flow": "flow1", + "Result": result, + "Result Detail": "Output Mismatch" if result == "Fail" else "", + } + for i, result in enumerate(results) + ] + test = { + "nodeid": f"test_file.py::{name}[flow1]", + "outcome": outcome, + "metadata": {"subtests": subtests}, + } + if longrepr: + test["call"] = {"longrepr": longrepr} + return test + + +class AggregateResultsTest(unittest.TestCase): + def _aggregate(self, tests: list[dict]): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "report.json") + with open(path, "w") as f: + json.dump({"tests": tests}, f) + return aggregate_results(path) + + def test_expected_failures_are_not_failures(self): + summary = self._aggregate( + [ + _test("test_listed", "xfailed", ["Pass", "Fail"]), + _test("test_broken", "failed", ["Fail"]), + ] + ) + self.assertEqual(summary.counts.passes, 1) + self.assertEqual(summary.counts.fails, 1) + self.assertEqual(summary.counts.expected_fails, 1) + self.assertEqual( + [r["Test ID"] for r in summary.failed_tests], ["test_broken[flow1]"] + ) + self.assertEqual( + [r["Test ID"] for r in summary.expected_failures], ["test_listed[flow1]"] + ) + self.assertEqual(summary.counts.by_detail, {"Output Mismatch": 1}) + + def test_strict_unexpected_passes_are_listed(self): + summary = self._aggregate( + [ + _test( + "test_serial", + "failed", + ["Pass"], + "[XPASS(strict)] Expected to fail by flow1 xfail_patterns", + ), + # Under pytest-xdist the first line names the worker. + _test( + "test_xdist", + "failed", + ["Pass"], + "[gw0] linux -- Python 3.11 /usr/bin/python\n" + "[XPASS(strict)] Expected to fail by flow1 xfail_patterns", + ), + _test("test_plain_pass", "passed", ["Pass"]), + ] + ) + self.assertEqual( + summary.unexpected_passes, ["test_serial[flow1]", "test_xdist[flow1]"] + ) + self.assertEqual(summary.counts.passes, 3) + self.assertEqual(summary.failed_tests, []) From a2aac04a82811da755b5431c39a9ef43dbad4a71 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 2 Nov 2026 15:33:36 -0800 Subject: [PATCH 7/7] Correct the cause in the qnn_16a4w expected failure comment The depthwise 3-D convolution case fails in QNN partitioning in this flow too, before any context binary is built, so the comment no longer says that every case fails while generating the context binary. --- backends/test/suite/flows/qualcomm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backends/test/suite/flows/qualcomm.py b/backends/test/suite/flows/qualcomm.py index 1dae3a25893..1c671705c7d 100644 --- a/backends/test/suite/flows/qualcomm.py +++ b/backends/test/suite/flows/qualcomm.py @@ -81,7 +81,8 @@ def create_quantize_stage() -> Quantize: quantize=True, quant_dtype=QuantDtype.use_16a4w, use_fp16=False, - # QNN fails to generate a context binary for every 3-D convolution here. + # Every 3-D convolution fails in this flow, most of them when QNN generates + # the context binary. xfail_patterns=["test_conv3d", "test_convtranspose3d"], ) QNN_16A4W_BLOCK_TEST_FLOW = _create_qnn_flow(