diff --git a/backends/arm/test/setup_testing.sh b/backends/arm/test/setup_testing.sh index 731164c6c87..eac1a6272bf 100755 --- a/backends/arm/test/setup_testing.sh +++ b/backends/arm/test/setup_testing.sh @@ -44,6 +44,7 @@ ${build_executor_runner} --pte=semihosting --target=ethos-u85-128 --system_confi # Ethos-U55/U65 # aten::avg_pool2d.out - test_adaptive_avgpool2d_batch_sizes[arm_ethos_u55] # aten::gt.Tensor_out - test_divide_f32_trunc[arm_ethos_u55] +# quantized_decomposed::dequantize_per_channel.out - test_conv3d_padding_reflect[arm_ethos_u55] # # Ethos-U85: # aten::_native_batch_norm_legit_no_training.out - test_swin_v2_t[arm_ethos_u85-static_shapes-float32] @@ -75,6 +76,7 @@ ops_list_u55=( aten::bmm.out dim_order_ops::_clone_dim_order.out dim_order_ops::_to_dim_order_copy.out + quantized_decomposed::dequantize_per_channel.out "${ops_list_quantized_decomposed[@]}" ) ops_list_u65=("${ops_list_u55[@]}") diff --git a/backends/test/suite/flows/arm.py b/backends/test/suite/flows/arm.py index 7e357f0fde9..a356bc5778b 100644 --- a/backends/test/suite/flows/arm.py +++ b/backends/test/suite/flows/arm.py @@ -25,6 +25,7 @@ def _create_arm_flow( per_channel_quantization: bool = True, use_portable_ops: bool = True, timeout: int = 1200, + xfail_patterns: list[str] | None = None, ) -> TestFlow: def _create_arm_tester(*args, **kwargs) -> ArmTester: @@ -59,6 +60,7 @@ def create_quantize_stage() -> Quantize: "TOSA/Ethos-U/VGF flows." } }, + xfail_patterns=xfail_patterns or [], ) @@ -83,6 +85,8 @@ def create_quantize_stage() -> Quantize: "arm_ethos_u85", lambda: common.get_u85_compile_spec(), quantize=True, + # Ethos-U85 takes these 3-D convolutions and its output does not match. + xfail_patterns=["test_conv3d_groups", "test_conv3d_stride"], ) ARM_VGF_FP_FLOW = _create_arm_flow( "arm_vgf_fp", diff --git a/backends/test/suite/flows/coreml.py b/backends/test/suite/flows/coreml.py index 8a532ff0003..b2cb9367380 100644 --- a/backends/test/suite/flows/coreml.py +++ b/backends/test/suite/flows/coreml.py @@ -11,6 +11,7 @@ def _create_coreml_flow( name: str, quantize: bool = False, minimum_deployment_target: Any = coremltools.target.iOS15, + xfail_patterns: list[str] | None = None, ) -> TestFlow: return TestFlow( name, @@ -20,10 +21,20 @@ def _create_coreml_flow( ), quantize=quantize, skip_patterns=["test_argmin", "test_argmax"], + xfail_patterns=xfail_patterns or [], ) -COREML_TEST_FLOW = _create_coreml_flow("coreml") +COREML_TEST_FLOW = _create_coreml_flow( + "coreml", + # Core ML fails to compile the model for the reflect padding case, and + # Core ML lowering of a transposed 3-D convolution with output_padding + # raises "'NoneType' object is not iterable". + xfail_patterns=[ + "test_conv3d_padding_reflect", + "test_convtranspose3d_output_padding", + ], +) COREML_STATIC_INT8_TEST_FLOW = _create_coreml_flow( "coreml_static_int8", quantize=True, diff --git a/backends/test/suite/flows/qualcomm.py b/backends/test/suite/flows/qualcomm.py index 99deb3d4877..1c671705c7d 100644 --- a/backends/test/suite/flows/qualcomm.py +++ b/backends/test/suite/flows/qualcomm.py @@ -12,6 +12,7 @@ def _create_qnn_flow( per_channel_linear=False, is_qat=False, use_fp16=True, + xfail_patterns: list[str] | None = None, ) -> TestFlow: if quantize and quant_dtype is None: raise RuntimeError("Quant dtype must be provided when quantize is true.") @@ -37,25 +38,66 @@ def create_quantize_stage() -> Quantize: tester_factory=create_tester, quantize=quantize, quantize_stage_factory=create_quantize_stage if quantize else None, + xfail_patterns=xfail_patterns or [], ) +# QNN fails to lower these 3-D convolutions instead of leaving them to the +# portable kernel. Partitioning asserts "DepthWise only supports Conv2d" on the +# depthwise one and raises "'NoneType' object has no attribute 'op'" on the +# replicate-padded one. For the circular-padded one, the QNN graph's inputs and +# outputs do not match the delegated program. +_QNN_CONV3D_LOWERING_XFAILS = [ + "test_conv3d_depthwise", + "test_conv3d_padding_circular", + "test_conv3d_padding_replicate", +] +# With 8-bit weights, QNN takes these transposed 3-D convolutions and its output +# does not match. +_QNN_8BIT_WEIGHT_CONV3D_XFAILS = [ + "test_convtranspose3d_depthwise", + "test_convtranspose3d_groups", + "test_convtranspose3d_padding", +] + + QNN_TEST_FLOW = _create_qnn_flow("qnn") QNN_16A16W_TEST_FLOW = _create_qnn_flow( - "qnn_16a16w", quantize=True, quant_dtype=QuantDtype.use_16a16w, use_fp16=False + "qnn_16a16w", + quantize=True, + quant_dtype=QuantDtype.use_16a16w, + use_fp16=False, + xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS, ) QNN_16A8W_TEST_FLOW = _create_qnn_flow( - "qnn_16a8w", quantize=True, quant_dtype=QuantDtype.use_16a8w, use_fp16=False + "qnn_16a8w", + quantize=True, + quant_dtype=QuantDtype.use_16a8w, + use_fp16=False, + xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS + _QNN_8BIT_WEIGHT_CONV3D_XFAILS, ) QNN_16A4W_TEST_FLOW = _create_qnn_flow( - "qnn_16a4w", quantize=True, quant_dtype=QuantDtype.use_16a4w, use_fp16=False + "qnn_16a4w", + quantize=True, + quant_dtype=QuantDtype.use_16a4w, + use_fp16=False, + # Every 3-D convolution fails in this flow, most of them when QNN generates + # the context binary. + xfail_patterns=["test_conv3d", "test_convtranspose3d"], ) QNN_16A4W_BLOCK_TEST_FLOW = _create_qnn_flow( "qnn_16a4w_block", quantize=True, quant_dtype=QuantDtype.use_8a8w, use_fp16=False, + # This flow still quantizes as 8a8w, not 16a4w block, so it fails the same + # cases as qnn_8a8w. Recheck the list when the dtype is corrected. + xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS + _QNN_8BIT_WEIGHT_CONV3D_XFAILS, ) QNN_8A8W_TEST_FLOW = _create_qnn_flow( - "qnn_8a8w", quantize=True, quant_dtype=QuantDtype.use_8a8w, use_fp16=False + "qnn_8a8w", + quantize=True, + quant_dtype=QuantDtype.use_8a8w, + use_fp16=False, + xfail_patterns=_QNN_CONV3D_LOWERING_XFAILS + _QNN_8BIT_WEIGHT_CONV3D_XFAILS, ) diff --git a/backends/test/suite/flows/webgpu.py b/backends/test/suite/flows/webgpu.py index ee0c24f221a..3bde683f4c5 100644 --- a/backends/test/suite/flows/webgpu.py +++ b/backends/test/suite/flows/webgpu.py @@ -20,6 +20,9 @@ def _create_webgpu_flow() -> TestFlow: "lstm_batch_sizes", "upsample_nearest2d", ], + # WebGPU fails to build the graph for replicate padding ("clamp: int + # dtype unsupported"). + xfail_patterns=["test_conv3d_padding_replicate"], ) diff --git a/backends/test/suite/generate_markdown_summary_json.py b/backends/test/suite/generate_markdown_summary_json.py index 4b6edc2a635..fa1fd482992 100644 --- a/backends/test/suite/generate_markdown_summary_json.py +++ b/backends/test/suite/generate_markdown_summary_json.py @@ -13,18 +13,25 @@ class ResultCounts: total: int = 0 passes: int = 0 fails: int = 0 + expected_fails: int = 0 skips: int = 0 by_detail: dict[str, int] = field(default_factory=lambda: {}) - def add_row(self, result_value: str, result_detail: str) -> None: + def add_row( + self, result_value: str, result_detail: str, expected_fail: bool = False + ) -> None: """ - Update the result counts for the specified row. + Update the result counts for the specified row. A failure in a test that + a flow lists in xfail_patterns counts as an expected failure. """ self.total += 1 if result_value == "Pass": self.passes += 1 + elif result_value == "Fail" and expected_fail: + self.expected_fails += 1 + return elif result_value == "Fail": self.fails += 1 elif result_value == "Skip": @@ -48,6 +55,8 @@ class AggregatedSummary: counts: ResultCounts counts_by_params: dict[str, ResultCounts] failed_tests: list[list[str]] + expected_failures: list[list[str]] + unexpected_passes: list[str] # @@ -68,14 +77,27 @@ def aggregate_results(json_path: str) -> AggregatedSummary: counts = ResultCounts() failed_tests = [] counts_by_param = {} + expected_failures = [] + unexpected_passes = [] for test_data in data["tests"]: + # The subtest rows hold the runner's own result. Only pytest's outcome + # knows whether the test is in a flow's xfail_patterns. + outcome = test_data.get("outcome") + expected_fail = outcome == "xfailed" + # Under pytest-xdist the first line names the worker, so check every line. + longrepr = str((test_data.get("call") or {}).get("longrepr") or "") + if outcome == "failed" and any( + line.startswith("[XPASS(strict)]") for line in longrepr.splitlines() + ): + unexpected_passes.append(test_data["nodeid"].split("::")[-1]) + result_meta = test_data["metadata"] for subtest_meta in result_meta["subtests"]: result = subtest_meta["Result"] result_detail = subtest_meta.get("Result Detail") or "" - counts.add_row(result, result_detail) + counts.add_row(result, result_detail, expected_fail) test_id = subtest_meta["Test ID"] base_test = subtest_meta["Test Case"] @@ -84,15 +106,20 @@ def aggregate_results(json_path: str) -> AggregatedSummary: if params: if params not in counts_by_param: counts_by_param[params] = ResultCounts() - counts_by_param[params].add_row(result, result_detail) + counts_by_param[params].add_row(result, result_detail, expected_fail) if result.lower() == "fail": - failed_tests.append(subtest_meta) + if expected_fail: + expected_failures.append(subtest_meta) + else: + failed_tests.append(subtest_meta) return AggregatedSummary( counts=counts, failed_tests=failed_tests, counts_by_params=counts_by_param, + expected_failures=expected_failures, + unexpected_passes=unexpected_passes, ) @@ -120,7 +147,9 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) # Generate Summary section print("# Summary\n") - total_excluding_skips = results.counts.passes + results.counts.fails + total_excluding_skips = ( + results.counts.passes + results.counts.fails + results.counts.expected_fails + ) pass_fraction = results.counts.passes / total_excluding_skips fail_fraction = results.counts.fails / total_excluding_skips print( @@ -129,6 +158,10 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) print( f"- **Fail**: {results.counts.fails}/{total_excluding_skips} ({fail_fraction*100:.2f}%)" ) + expected_fail_fraction = results.counts.expected_fails / total_excluding_skips + print( + f"- **Expected Fail**: {results.counts.expected_fails}/{total_excluding_skips} ({expected_fail_fraction*100:.2f}%)" + ) print(f"- **Skip**: {results.counts.skips}") if results.counts_by_params: @@ -136,7 +169,7 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) if len(results.counts_by_params) > 0: # Create table header - header_cols = ["Params", "Pass", "Fail", "Skip", "Pass %"] + header_cols = ["Params", "Pass", "Fail", "Expected Fail", "Skip", "Pass %"] print("| " + " | ".join(header_cols) + " |") print("|" + "|".join(["---"] * len(header_cols)) + "|") @@ -145,13 +178,16 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) row_values = [params_str] # Add parameter values - pass_fraction = counts.passes / (counts.passes + counts.fails) + pass_fraction = counts.passes / ( + counts.passes + counts.fails + counts.expected_fails + ) # Add count values row_values.extend( [ str(counts.passes), str(counts.fails), + str(counts.expected_fails), str(counts.skips), f"{pass_fraction*100:.2f}%", ] @@ -166,6 +202,15 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) for detail, count in sorted(results.counts.by_detail.items()): print(f"- **{detail}**: {count}/{total_rows_with_result_detail}") + if results.unexpected_passes: + print("# Unexpected Passes\n") + print( + "These tests are in a flow's xfail_patterns but passed. Remove them from the list." + ) + for test_id in results.unexpected_passes: + print(f"- {test_id}") + print() + # Generate Failed Tests section print("# Failed Tests\n") print( @@ -175,17 +220,27 @@ def generate_markdown(json_path: str, exit_code: int = 0): # noqa (C901) print('pytest -c /dev/nul backends/test/suite/ -k ""') print("```") if results.failed_tests: - header = build_header(results.failed_tests) - - escaped_header = [escape_for_markdown(col) for col in header.keys()] - print("| " + " | ".join(escaped_header) + " |") - print("|" + "|".join(["---"] * len(escaped_header)) + "|") - for rec in results.failed_tests: - row = build_row(rec, header) - print("| " + " | ".join(row) + " |") + print_table(results.failed_tests) else: print("No failed tests.\n") + if results.expected_failures: + # Shown with their details, so an expected failure that starts failing + # in a different way is still visible. + print("# Expected Failures\n") + print_table(results.expected_failures) + + +def print_table(records) -> None: + header = build_header(records) + + escaped_header = [escape_for_markdown(col) for col in header.keys()] + print("| " + " | ".join(escaped_header) + " |") + print("|" + "|".join(["---"] * len(escaped_header)) + "|") + for rec in records: + row = build_row(rec, header) + print("| " + " | ".join(row) + " |") + def build_header(data) -> dict[str, int]: """ diff --git a/backends/test/suite/operators/test_conv3d.py b/backends/test/suite/operators/test_conv3d.py index 01ffa4942df..e393934cd44 100644 --- a/backends/test/suite/operators/test_conv3d.py +++ b/backends/test/suite/operators/test_conv3d.py @@ -134,13 +134,26 @@ def test_conv3d_no_bias(self, flow: TestFlow) -> None: flow, ) - def test_conv3d_padding_modes(self, flow: TestFlow) -> None: - for mode in ["zeros", "reflect", "replicate", "circular"]: - self._test_op( - Model(padding=1, padding_mode=mode), - (torch.randn(4, 3, 8, 8, 8),), - flow, - ) + def test_conv3d_padding_reflect(self, flow: TestFlow) -> None: + self._test_op( + Model(padding=1, padding_mode="reflect"), + (torch.randn(4, 3, 8, 8, 8),), + flow, + ) + + def test_conv3d_padding_replicate(self, flow: TestFlow) -> None: + self._test_op( + Model(padding=1, padding_mode="replicate"), + (torch.randn(4, 3, 8, 8, 8),), + flow, + ) + + def test_conv3d_padding_circular(self, flow: TestFlow) -> None: + self._test_op( + Model(padding=1, padding_mode="circular"), + (torch.randn(4, 3, 8, 8, 8),), + flow, + ) def test_conv3d_channels(self, flow: TestFlow) -> None: self._test_op( diff --git a/backends/test/suite/tests/test_markdown_summary_json.py b/backends/test/suite/tests/test_markdown_summary_json.py new file mode 100644 index 00000000000..4bf509afeab --- /dev/null +++ b/backends/test/suite/tests/test_markdown_summary_json.py @@ -0,0 +1,81 @@ +import json +import os +import tempfile +import unittest + +from ..generate_markdown_summary_json import aggregate_results + + +def _test(name: str, outcome: str, results: list[str], longrepr: str = "") -> dict: + subtests = [ + { + "Test ID": f"{name}[flow1]", + "Test Case": name, + "Subtest": i, + "Flow": "flow1", + "Result": result, + "Result Detail": "Output Mismatch" if result == "Fail" else "", + } + for i, result in enumerate(results) + ] + test = { + "nodeid": f"test_file.py::{name}[flow1]", + "outcome": outcome, + "metadata": {"subtests": subtests}, + } + if longrepr: + test["call"] = {"longrepr": longrepr} + return test + + +class AggregateResultsTest(unittest.TestCase): + def _aggregate(self, tests: list[dict]): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "report.json") + with open(path, "w") as f: + json.dump({"tests": tests}, f) + return aggregate_results(path) + + def test_expected_failures_are_not_failures(self): + summary = self._aggregate( + [ + _test("test_listed", "xfailed", ["Pass", "Fail"]), + _test("test_broken", "failed", ["Fail"]), + ] + ) + self.assertEqual(summary.counts.passes, 1) + self.assertEqual(summary.counts.fails, 1) + self.assertEqual(summary.counts.expected_fails, 1) + self.assertEqual( + [r["Test ID"] for r in summary.failed_tests], ["test_broken[flow1]"] + ) + self.assertEqual( + [r["Test ID"] for r in summary.expected_failures], ["test_listed[flow1]"] + ) + self.assertEqual(summary.counts.by_detail, {"Output Mismatch": 1}) + + def test_strict_unexpected_passes_are_listed(self): + summary = self._aggregate( + [ + _test( + "test_serial", + "failed", + ["Pass"], + "[XPASS(strict)] Expected to fail by flow1 xfail_patterns", + ), + # Under pytest-xdist the first line names the worker. + _test( + "test_xdist", + "failed", + ["Pass"], + "[gw0] linux -- Python 3.11 /usr/bin/python\n" + "[XPASS(strict)] Expected to fail by flow1 xfail_patterns", + ), + _test("test_plain_pass", "passed", ["Pass"]), + ] + ) + self.assertEqual( + summary.unexpected_passes, ["test_serial[flow1]", "test_xdist[flow1]"] + ) + self.assertEqual(summary.counts.passes, 3) + self.assertEqual(summary.failed_tests, [])