From a032856c7c6c1ab5533c18a0d2d19ee6f13fda08 Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Thu, 3 Sep 2026 09:24:17 +0200 Subject: [PATCH] Add explicit backend coverage --- examples/mobilenetv2_from_hub.py | 31 +++++- src/python/context.rs | 10 +- src/python/context_state.rs | 3 + tests/test_backend_coverage.py | 183 +++++++++++++++++++++++++++++++ tests/test_coreml_basic.py | 2 +- tests/test_performance.py | 12 +- 6 files changed, 230 insertions(+), 11 deletions(-) create mode 100644 tests/test_backend_coverage.py diff --git a/examples/mobilenetv2_from_hub.py b/examples/mobilenetv2_from_hub.py index c62ced0..211dd91 100755 --- a/examples/mobilenetv2_from_hub.py +++ b/examples/mobilenetv2_from_hub.py @@ -96,11 +96,29 @@ def main(): # Backend settings if args.backend == "cpu": - accelerated, power, backend_name = False, "default", "ONNX CPU" + accelerated, power, device_type, backend, backend_name = ( + False, + "default", + "cpu", + "onnx", + "ONNX CPU", + ) elif args.backend == "gpu": - accelerated, power, backend_name = True, "high-performance", "ONNX GPU" + accelerated, power, device_type, backend, backend_name = ( + True, + "high-performance", + "gpu", + "onnx", + "ONNX GPU", + ) else: - accelerated, power, backend_name = True, "high-performance", "CoreML (Neural Engine)" + accelerated, power, device_type, backend, backend_name = ( + True, + "high-performance", + "npu", + "coreml", + "CoreML (Neural Engine)", + ) print("=" * 70) print("MobileNetV2 Image Classification (Hugging Face Hub)") @@ -149,7 +167,12 @@ def main(): # Create context print("Creating WebNN context...") ml = webnn.ML() - context = ml.create_context(power_preference=power, accelerated=accelerated) + context = ml.create_context( + power_preference=power, + accelerated=accelerated, + device_type=device_type, + backend=backend, + ) print(f" [OK] Context created (accelerated={context.accelerated})") print() diff --git a/src/python/context.rs b/src/python/context.rs index bf8c9df..28511d0 100644 --- a/src/python/context.rs +++ b/src/python/context.rs @@ -1,4 +1,4 @@ -//! ML context and backend selection for WebNN API +//! ML context and backend selection for WebNN API //! //! PyO3 macros generate unsafe code that triggers unsafe_op_in_unsafe_fn warnings. //! This is expected behavior from the macro-generated code. @@ -728,11 +728,17 @@ impl PyMLContext { &backend, )?; let state = ContextState::new(options)?; + let resolved_backend = + if backend == "auto" && (device_type == "cpu" || !accelerated_requested) { + "onnx".to_string() + } else { + backend + }; Ok(Self { power_preference, accelerated_requested, device_type, - backend, + backend: resolved_backend, state: Mutex::new(state), }) } diff --git a/src/python/context_state.rs b/src/python/context_state.rs index 66e40e5..fbfbaf7 100644 --- a/src/python/context_state.rs +++ b/src/python/context_state.rs @@ -82,6 +82,9 @@ pub(crate) fn build_context_options( }; let options = MLContextOptions::new(power_preference, accelerated); match backend { + "auto" if device_type == "cpu" || !accelerated => { + Ok(options.with_rustnn_backend_hint(Backend::Onnx)) + } "auto" => Ok(options), "onnx" => Ok(options.with_rustnn_backend_hint(Backend::Onnx)), "trtx" => Ok(options.with_rustnn_backend_hint(Backend::Trtx)), diff --git a/tests/test_backend_coverage.py b/tests/test_backend_coverage.py new file mode 100644 index 0000000..c4d99e5 --- /dev/null +++ b/tests/test_backend_coverage.py @@ -0,0 +1,183 @@ +"""Execution coverage for explicit CPU/ONNX and CoreML backends.""" + +import numpy as np +import pytest + +from runtime_support import COREML_BACKEND_AVAILABLE, EXECUTION_BACKEND_AVAILABLE + +try: + import webnn +except ImportError: + pytestmark = pytest.mark.skip(reason="webnn not built yet") + + +def _context_for_backend(backend_name): + ml = webnn.ML() + if backend_name == "cpu": + return ml.create_context( + power_preference="default", + accelerated=False, + device_type="cpu", + backend="onnx", + ) + if backend_name == "coreml": + return ml.create_context( + power_preference="default", + accelerated=True, + device_type="npu", + backend="coreml", + ) + raise ValueError(f"unsupported backend: {backend_name}") + + +def _skip_if_unavailable(backend_name): + if backend_name == "cpu" and not EXECUTION_BACKEND_AVAILABLE: + pytest.skip("No working CPU execution backend is available") + if backend_name == "coreml" and not COREML_BACKEND_AVAILABLE: + pytest.skip("No working CoreML execution backend is available") + + +@pytest.mark.parametrize("backend_name", ["cpu", "coreml"]) +def test_add_relu_execute_on_backend(backend_name): + """Basic numeric graph executes on both backend families.""" + _skip_if_unavailable(backend_name) + context = _context_for_backend(backend_name) + builder = context.create_graph_builder() + x = builder.input("x", [2, 2], "float32") + y = builder.input("y", [2, 2], "float32") + output = builder.relu(builder.add(x, y)) + graph = builder.build({"output": output}) + + x_data = np.array([[1.0, -4.0], [2.0, -3.0]], dtype=np.float32) + y_data = np.array([[2.0, 1.0], [-5.0, 5.0]], dtype=np.float32) + result = context.compute(graph, {"x": x_data, "y": y_data}) + + np.testing.assert_allclose(result["output"], np.maximum(x_data + y_data, 0.0)) + + +@pytest.mark.parametrize("backend_name", ["cpu", "coreml"]) +def test_comparison_execute_on_backend(backend_name): + """Comparison outputs execute and round-trip as WebNN uint8 booleans.""" + _skip_if_unavailable(backend_name) + context = _context_for_backend(backend_name) + builder = context.create_graph_builder() + x = builder.input("x", [2, 2], "float32") + y = builder.input("y", [2, 2], "float32") + output = builder.greater(x, y) + graph = builder.build({"output": output}) + + x_data = np.array([[1.0, 4.0], [2.0, 3.0]], dtype=np.float32) + y_data = np.array([[2.0, 1.0], [2.0, 5.0]], dtype=np.float32) + result = context.compute(graph, {"x": x_data, "y": y_data}) + + np.testing.assert_array_equal(result["output"], (x_data > y_data).astype(np.uint8)) + + +@pytest.mark.parametrize("backend_name", ["cpu", "coreml"]) +def test_quantize_linear_with_constant_zero_point_execute_on_backend(backend_name): + """Quantization executes on both backends when CoreML-required constants are used.""" + _skip_if_unavailable(backend_name) + context = _context_for_backend(backend_name) + builder = context.create_graph_builder() + x = builder.input("x", [2, 2], "float32") + scale = builder.constant(np.array([[0.5]], dtype=np.float32)) + zero_point = builder.constant(np.array([[0]], dtype=np.int8)) + output = builder.quantize_linear(x, scale, zero_point) + graph = builder.build({"output": output}) + + x_data = np.array([[0.0, 1.0], [2.0, -1.0]], dtype=np.float32) + result = context.compute(graph, {"x": x_data}) + + np.testing.assert_array_equal(result["output"], np.array([[0, 2], [4, -2]], dtype=np.int8)) + + +@pytest.mark.xfail( + strict=True, + reason="CoreML logical ops currently require bool inputs; WebNN numeric truthiness is not lowered yet.", +) +def test_coreml_logical_numeric_inputs(): + _skip_if_unavailable("coreml") + context = _context_for_backend("coreml") + builder = context.create_graph_builder() + x = builder.input("x", [2, 2], "float32") + y = builder.input("y", [2, 2], "float32") + graph = builder.build({"output": builder.logical_and(x, y)}) + + context.compute( + graph, + { + "x": np.array([[0.0, 1.0], [2.0, 0.0]], dtype=np.float32), + "y": np.array([[1.0, 1.0], [0.0, 0.0]], dtype=np.float32), + }, + ) + + +@pytest.mark.xfail( + strict=True, + reason="CoreML comparison-to-logical chains currently emit a duplicate bool temporary name.", +) +def test_coreml_logical_comparison_inputs(): + _skip_if_unavailable("coreml") + context = _context_for_backend("coreml") + builder = context.create_graph_builder() + x = builder.input("x", [2, 2], "float32") + y = builder.input("y", [2, 2], "float32") + z = builder.input("z", [2, 2], "float32") + lhs = builder.greater(x, y) + rhs = builder.lesser(x, z) + graph = builder.build({"output": builder.logical_and(lhs, rhs)}) + + context.compute( + graph, + { + "x": np.array([[1.0, 4.0], [2.0, 3.0]], dtype=np.float32), + "y": np.ones((2, 2), dtype=np.float32), + "z": np.full((2, 2), 3.0, dtype=np.float32), + }, + ) + + +@pytest.mark.xfail( + strict=True, + reason="CoreML where/select currently requires bool conditions; WebNN integer truthiness is not lowered yet.", +) +def test_coreml_where_integer_condition(): + _skip_if_unavailable("coreml") + context = _context_for_backend("coreml") + builder = context.create_graph_builder() + condition = builder.input("condition", [2, 2], "int32") + true_value = builder.input("true_value", [2, 2], "float32") + false_value = builder.input("false_value", [2, 2], "float32") + graph = builder.build({"output": builder.where_(condition, true_value, false_value)}) + + context.compute( + graph, + { + "condition": np.array([[0, 1], [1, 0]], dtype=np.int32), + "true_value": np.ones((2, 2), dtype=np.float32), + "false_value": np.zeros((2, 2), dtype=np.float32), + }, + ) + + +@pytest.mark.xfail( + strict=True, + reason="CoreML quantize/dequantize currently require zero_point to be constant.", +) +def test_coreml_quantize_linear_dynamic_zero_point(): + _skip_if_unavailable("coreml") + context = _context_for_backend("coreml") + builder = context.create_graph_builder() + x = builder.input("x", [2, 2], "float32") + scale = builder.input("scale", [1, 1], "float32") + zero_point = builder.input("zero_point", [1, 1], "int8") + graph = builder.build({"output": builder.quantize_linear(x, scale, zero_point)}) + + context.compute( + graph, + { + "x": np.array([[0.0, 1.0], [2.0, -1.0]], dtype=np.float32), + "scale": np.array([[0.5]], dtype=np.float32), + "zero_point": np.array([[0]], dtype=np.int8), + }, + ) diff --git a/tests/test_coreml_basic.py b/tests/test_coreml_basic.py index a40c0da..01509f5 100755 --- a/tests/test_coreml_basic.py +++ b/tests/test_coreml_basic.py @@ -174,7 +174,7 @@ def main() -> int: # Initialize print_step(0, "Initializing ML context...") ml = webnn.ML() - context = ml.create_context(device_type="cpu") + context = ml.create_context(device_type="npu", accelerated=True, backend="coreml") print_success("Context created") # Build graph diff --git a/tests/test_performance.py b/tests/test_performance.py index 752e847..168b42d 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -97,7 +97,7 @@ def create_complex_model(builder): # @pytest.mark.benchmark # Reserved for a future pytest-benchmark integration. def test_performance_simple_onnx_cpu(ml): """Test ONNX CPU performance with simple model (baseline)""" - context = ml.create_context(power_preference="default", accelerated=False) + context = ml.create_context(power_preference="default", accelerated=False, backend="onnx") builder = context.create_graph_builder() graph = create_simple_model(builder) @@ -125,7 +125,9 @@ def test_performance_simple_onnx_cpu(ml): ) def test_performance_simple_coreml(ml): """Test CoreML performance with simple model""" - context = ml.create_context(power_preference="default", accelerated=True) + context = ml.create_context( + power_preference="default", accelerated=True, device_type="npu", backend="coreml" + ) builder = context.create_graph_builder() graph = create_simple_model(builder) @@ -150,7 +152,7 @@ def test_performance_simple_coreml(ml): @pytest.mark.slow def test_performance_complex_onnx_cpu(ml): """Test ONNX CPU performance with complex model""" - context = ml.create_context(power_preference="default", accelerated=False) + context = ml.create_context(power_preference="default", accelerated=False, backend="onnx") builder = context.create_graph_builder() graph = create_complex_model(builder) @@ -179,7 +181,9 @@ def test_performance_complex_onnx_cpu(ml): ) def test_performance_complex_coreml(ml): """Test CoreML performance with complex model - validates warm-up speedup""" - context = ml.create_context(power_preference="default", accelerated=True) + context = ml.create_context( + power_preference="default", accelerated=True, device_type="npu", backend="coreml" + ) builder = context.create_graph_builder() graph = create_complex_model(builder)