diff --git a/backends/transforms/test/test_create_delete_constant_placeholder.py b/backends/transforms/test/test_create_delete_constant_placeholder.py index b85a14b036f..b215d2578f0 100644 --- a/backends/transforms/test/test_create_delete_constant_placeholder.py +++ b/backends/transforms/test/test_create_delete_constant_placeholder.py @@ -13,6 +13,8 @@ duplicate_constant_node, ) from executorch.exir.dialects._ops import ops as exir_ops +from executorch.exir.pass_base import PassResult +from executorch.exir.pass_manager import ExportedProgramPassManager from torch.export import export from torch.export.graph_signature import InputKind @@ -247,3 +249,59 @@ def test_create_delete_non_persistent_buffer(): def test_create_delete_constant_tensor(): _test_create_delete(InputKind.CONSTANT_TENSOR) + + +def _test_delete_inside_a_pass(kind: InputKind, persistent_buffer: bool = None): + """Backend passes keep a reference to the program they were built from, + while ExportedProgramPassManager runs them on a shallow copy of it. The + deletion has to edit the shared signature in place, or the copy that the + manager verifies and returns still lists the deleted placeholder.""" + module = EmptyNetwork() + exported_program = export(module, args=module.test_data, strict=True) + exported_program = to_edge(exported_program).exported_program() + graph = exported_program.graph_module.graph + + const_name = "test_node" + input_node = list(graph.nodes)[0] + with graph.inserting_before(input_node): + const_node = create_constant_placeholder( + exp_program=exported_program, + graph=graph, + kind=kind, + name=const_name, + data=torch.ones(1), + persistent_buffer=persistent_buffer, + ) + assert len(exported_program.graph_signature.input_specs) == 2 + + def delete_unused_constant(graph_module: torch.fx.GraphModule) -> PassResult: + delete_constant_placeholder(exported_program, const_node) + return PassResult(graph_module, True) + + result = ExportedProgramPassManager([delete_unused_constant])(exported_program) + transformed = result.exported_program + signature = transformed.graph_signature + + assert len(signature.input_specs) == 1 + assert const_name not in signature.inputs_to_parameters + assert const_name not in signature.inputs_to_buffers + assert const_name not in signature.inputs_to_lifted_tensor_constants + assert len(transformed.state_dict) == 0 + assert len(transformed.constants) == 0 + assert transformed.module()(torch.zeros(1)) == 0 + + +def test_delete_inside_a_pass_parameter(): + _test_delete_inside_a_pass(InputKind.PARAMETER) + + +def test_delete_inside_a_pass_persistent_buffer(): + _test_delete_inside_a_pass(InputKind.BUFFER, True) + + +def test_delete_inside_a_pass_non_persistent_buffer(): + _test_delete_inside_a_pass(InputKind.BUFFER, False) + + +def test_delete_inside_a_pass_constant_tensor(): + _test_delete_inside_a_pass(InputKind.CONSTANT_TENSOR) diff --git a/backends/transforms/utils.py b/backends/transforms/utils.py index 92794364cb0..fcb5b1584ab 100644 --- a/backends/transforms/utils.py +++ b/backends/transforms/utils.py @@ -266,16 +266,10 @@ def delete_constant_placeholder(exp_program: ExportedProgram, node: torch.fx.Nod f"Cannot delete input node {node.name} since it is not a parameter, a buffer, nor a lifted tensor constant." ) - # Remove input from graph signature - input_specs = [ - spec - for spec in exp_program.graph_signature.input_specs - if spec.arg.name != node.name - ] - new_graph_signature = ExportGraphSignature( - input_specs, exp_program.graph_signature.output_specs - ) - exp_program._graph_signature = new_graph_signature + # Edit the spec list in place: ExportedProgramPassManager runs passes on a + # shallow copy of the program, which would not see a replaced signature. + input_specs = exp_program.graph_signature.input_specs + input_specs[:] = [spec for spec in input_specs if spec.arg.name != node.name] # Copies made by duplicate_constant_node keep the original target, so keep # the tensor while another placeholder still refers to it. diff --git a/backends/xnnpack/BUCK b/backends/xnnpack/BUCK index 1e12548bc4c..f26b7fb6b0c 100644 --- a/backends/xnnpack/BUCK +++ b/backends/xnnpack/BUCK @@ -45,3 +45,14 @@ fbcode_target(_kind = runtime.python_library, "//executorch/backends/xnnpack/utils:xnnpack_utils", ], ) + +fbcode_target(_kind = runtime.python_library, + name = "custom_ops_lib", + srcs = [ + "custom_ops_lib.py", + ], + visibility = ["PUBLIC"], + deps = [ + "//caffe2:torch", + ], +) diff --git a/backends/xnnpack/_passes/BUCK b/backends/xnnpack/_passes/BUCK index 2edf294fa4e..081b0ff31b2 100644 --- a/backends/xnnpack/_passes/BUCK +++ b/backends/xnnpack/_passes/BUCK @@ -11,6 +11,7 @@ fbcode_target(_kind = runtime.python_library, "//executorch/backends/transforms:addmm_mm_to_linear", "//executorch/backends/transforms:remove_clone_ops", "//executorch/backends/transforms:lib", + "//executorch/backends/xnnpack:custom_ops_lib", "//executorch/backends/xnnpack/partition:partitioner_graphs", "//executorch/backends/xnnpack/serialization:xnnpack_schema", "//executorch/backends/xnnpack/utils:xnnpack_utils", diff --git a/backends/xnnpack/_passes/__init__.py b/backends/xnnpack/_passes/__init__.py index 0051f8ceb88..6866f67d857 100644 --- a/backends/xnnpack/_passes/__init__.py +++ b/backends/xnnpack/_passes/__init__.py @@ -26,6 +26,9 @@ ) from executorch.backends.xnnpack._passes.decompose_batch_norm import DecomposeBatchNorm from executorch.backends.xnnpack._passes.decompose_cat import DecomposeConcatenate +from executorch.backends.xnnpack._passes.decompose_depthwise_conv_transpose import ( + DecomposeDepthwiseConvTransposePass, +) from executorch.backends.xnnpack._passes.fuse_activation_pass import FuseActivationPass from executorch.backends.xnnpack._passes.fuse_batch_norm import FuseBatchNormPass from executorch.backends.xnnpack._passes.fuse_pad_into_conv_pass import ( @@ -85,6 +88,7 @@ def __init__( ConstPropPass, FuseBatchNormPass, DecomposeBatchNorm, + DecomposeDepthwiseConvTransposePass, FuseActivationPass, DecomposeConcatenate, RemoveGetItemPass, diff --git a/backends/xnnpack/_passes/channels_last_tagged_reshape_pass.py b/backends/xnnpack/_passes/channels_last_tagged_reshape_pass.py index d5d1dc3ef3e..717b7b4649e 100644 --- a/backends/xnnpack/_passes/channels_last_tagged_reshape_pass.py +++ b/backends/xnnpack/_passes/channels_last_tagged_reshape_pass.py @@ -9,6 +9,7 @@ from enum import Enum from typing import List, Optional, Tuple +import executorch.backends.xnnpack.custom_ops_lib # noqa: F401 registers et_xnnpack ops import torch from executorch.backends.xnnpack._passes.xnnpack_pass import XNNPACKPass from executorch.backends.xnnpack.utils.quant_utils import ( @@ -62,6 +63,7 @@ class ChannelsLastTaggedReshapePass(XNNPACKPass): exir_ops.edge.aten.convolution.default, exir_ops.edge.aten.upsample_bilinear2d.vec, exir_ops.edge.aten.upsample_nearest2d.vec, + exir_ops.edge.et_xnnpack.depth_to_space_2d.default, exir_ops.edge.aten.mean.dim, exir_ops.edge.aten.max_pool2d.default, exir_ops.edge.aten.amax.default, diff --git a/backends/xnnpack/_passes/decompose_depthwise_conv_transpose.py b/backends/xnnpack/_passes/decompose_depthwise_conv_transpose.py new file mode 100644 index 00000000000..c41e0be5eeb --- /dev/null +++ b/backends/xnnpack/_passes/decompose_depthwise_conv_transpose.py @@ -0,0 +1,230 @@ +# Copyright (c) 2026 Software Mansion +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import math +from typing import cast, List, Optional, Tuple + +import executorch.backends.xnnpack.custom_ops_lib # noqa: F401 registers et_xnnpack ops +import torch +from executorch.backends.transforms.utils import ( + create_constant_placeholder, + delete_constant_placeholder, + get_param_tensor, + is_param_node, +) +from executorch.backends.xnnpack._passes.xnnpack_pass import XNNPACKPass +from executorch.exir.dialects._ops import ops as exir_ops +from executorch.exir.pass_base import PassResult + + +class DecomposeDepthwiseConvTransposePass(XNNPACKPass): + """ + Lower a depthwise transposed conv (groups == in_channels == out_channels) + with stride s > 1 into s*s depthwise convs on the low-resolution input, a + channel concatenation and a block-major depth_to_space_2d. + + XNNPACK runs transposed convs through its subconvolution IGEMM path, one + group at a time; with one channel per group that is far slower than its + depthwise conv microkernels. Polyphase decomposition: output phase a of a + stride-s transposed conv is an ordinary stride-1 correlation of the input + with every s-th tap of the kernel, + + y[s*m + a] = sum_t x[m - t] * w[s*t + a + p], 0 <= s*t + a + p < k, + + per axis. The s*s phase outputs concatenated along channels in (ay, ax) + order are exactly the block-major input depth_to_space_2d expects. Each + phase reads the input through a (possibly asymmetric) zero pad. + + Only applied when the transposed conv is a clean x s upscale, i.e. + (H - 1)*s - 2p + k + output_padding == s*H on both axes with the same s, + dilation is 1 and every phase's taps stay inside the input; any other + transposed conv is left to XNNPACK's own node. + """ + + def call(self, graph_module: torch.fx.GraphModule): + graph = graph_module.graph + # Plan every conv before rewriting any: the nodes created by + # _decompose have no meta["val"] until the retrace below. + plans = [] + for node in graph.nodes: + if ( + node.op == "call_function" + and node.target == exir_ops.edge.aten.convolution.default + ): + plan = self._plan(node) + if plan is not None: + plans.append((node, plan)) + for node, plan in plans: + self._decompose(graph, node, *plan) + + modified = bool(plans) + if modified: + graph_module.recompile() + graph_module = super().call(graph_module).graph_module + return PassResult(graph_module, modified) + + # (t_min, t_max) per phase for one axis, None when the axis + # does not decompose. + @staticmethod + def _axis_phases( + size: int, kernel: int, stride: int, padding: int, output_padding: int + ) -> Optional[List[Tuple[int, int]]]: + if stride < 2: + return None + if (size - 1) * stride - 2 * padding + kernel + output_padding != size * stride: + return None + phases = [] + for a in range(stride): + # integer t with 0 <= stride*t + a + padding <= kernel - 1 + t_min = math.ceil(-(a + padding) / stride) + t_max = math.floor((kernel - 1 - a - padding) / stride) + if t_min > 0 or t_max < 0: + return None + phases.append((t_min, t_max)) + return phases + + def _plan(self, node: torch.fx.Node): + ( + x, + weight, + _bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + ) = node.args + if not transposed: + return None + if not is_param_node(self.exported_program, weight): + return None + x_shape = list(x.meta["val"].shape) + w_shape = list(weight.meta["val"].shape) + if len(x_shape) != 4 or len(w_shape) != 4: + return None + channels = x_shape[1] + # transposed conv weight is (in, out / groups, kh, kw) + if groups != channels or w_shape[0] != channels or w_shape[1] != 1: + return None + if any(d != 1 for d in dilation): + return None + # ATen broadcasts a single spatial value over both spatial dims. + stride, padding, output_padding = ( + p * 2 if len(p) == 1 else p + for p in cast(List[List[int]], [stride, padding, output_padding]) + ) + if stride[0] != stride[1]: + return None + phases_y = self._axis_phases( + x_shape[2], w_shape[2], stride[0], padding[0], output_padding[0] + ) + phases_x = self._axis_phases( + x_shape[3], w_shape[3], stride[1], padding[1], output_padding[1] + ) + if phases_y is None or phases_x is None: + return None + return stride[0], padding, phases_y, phases_x + + def _decompose( + self, + graph: torch.fx.Graph, + node: torch.fx.Node, + s: int, + padding: List[int], + phases_y: List[Tuple[int, int]], + phases_x: List[Tuple[int, int]], + ) -> None: + x, weight_node, bias_node = node.args[0], node.args[1], node.args[2] + groups = node.args[8] + weight = get_param_tensor(self.exported_program, weight_node) + assert weight is not None + weight_spec = next( + spec + for spec in self.exported_program.graph_signature.input_specs + if spec.arg.name == weight_node.name + ) + + phase_convs = [] + for ay, (ty_min, ty_max) in enumerate(phases_y): + for ax, (tx_min, tx_max) in enumerate(phases_x): + # taps of phase (ay, ax) as a stride-1 correlation kernel: + # W[q] = w[s * (t_max - q) + a + p] + iy = torch.tensor( + [ + s * (ty_max - q) + ay + padding[0] + for q in range(ty_max - ty_min + 1) + ] + ) + ix = torch.tensor( + [ + s * (tx_max - q) + ax + padding[1] + for q in range(tx_max - tx_min + 1) + ] + ) + phase_weight = weight[:, :, iy][:, :, :, ix].contiguous() + # The phase taps depend on s and padding too, so a weight + # shared by differently configured deconvs must not reuse + # the placeholders create_constant_placeholder finds by name. + phase_name = ( + f"{weight_node.name}_s{s}_p{padding[0]}x{padding[1]}" + f"_phase_{ay}_{ax}" + ) + with graph.inserting_before(weight_node): + phase_weight_node = create_constant_placeholder( + exp_program=self.exported_program, + graph=graph, + kind=weight_spec.kind, + persistent_buffer=weight_spec.persistent, + name=phase_name, + data=phase_weight, + ) + # create_constant_placeholder only sets meta["val"]; keep the + # external-constant tag so the phase weights follow the + # original weight into its .ptd file. + if "custom" in weight_node.meta: + phase_weight_node.meta["custom"] = dict(weight_node.meta["custom"]) + # (left, right, top, bottom) + pad = [tx_max, -tx_min, ty_max, -ty_min] + with graph.inserting_before(node): + conv_input = x + if any(pad): + conv_input = graph.create_node( + "call_function", + exir_ops.edge.aten.constant_pad_nd.default, + (x, pad, 0.0), + ) + phase_convs.append( + graph.create_node( + "call_function", + exir_ops.edge.aten.convolution.default, + ( + conv_input, + phase_weight_node, + bias_node, + [1, 1], + [0, 0], + [1, 1], + False, + [0, 0], + groups, + ), + ) + ) + + with graph.inserting_before(node): + cat = graph.create_node( + "call_function", exir_ops.edge.aten.cat.default, (phase_convs, 1) + ) + d2s = graph.create_node( + "call_function", + exir_ops.edge.et_xnnpack.depth_to_space_2d.default, + (cat, s), + ) + node.replace_all_uses_with(d2s) + graph.erase_node(node) + if len(weight_node.users) == 0: + delete_constant_placeholder(self.exported_program, weight_node) diff --git a/backends/xnnpack/custom_ops_lib.py b/backends/xnnpack/custom_ops_lib.py new file mode 100644 index 00000000000..290adfa6d84 --- /dev/null +++ b/backends/xnnpack/custom_ops_lib.py @@ -0,0 +1,43 @@ +# Copyright (c) 2026 Software Mansion +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +""" +Custom ops that XNNPACK passes introduce after partitioning. They never reach +the partitioner: a pass creates them inside an already lowered subgraph and a +node visitor serializes them, the reference implementation below only serves +tracing and tests. +""" + +import torch.library + +namespace = "et_xnnpack" +lib = torch.library.Library(namespace, "DEF") + +####################### +## depth_to_space_2d ## +####################### + + +def depth_to_space_2d_impl(x: torch.Tensor, block_size: int) -> torch.Tensor: + """ + Block-major depth to space, the channel order XNNPACK's (and TFLite's) + DEPTH_TO_SPACE uses: input channel (by * block_size + bx) * C + c lands at + output pixel (h * block_size + by, w * block_size + bx), channel c. + aten.pixel_shuffle is the channel-major variant, c * block_size**2 + by * + block_size + bx, and does not map onto XNNPACK's node directly. + """ + n, c_in, h, w = x.shape + bs = block_size + c = c_in // (bs * bs) + y = x.reshape(n, bs, bs, c, h, w) + y = y.permute(0, 3, 4, 1, 5, 2) # n, c, h, by, w, bx + return y.reshape(n, c, h * bs, w * bs) + + +name = "depth_to_space_2d" +lib.define(f"{name}(Tensor x, int block_size) -> Tensor") +lib.impl(name, depth_to_space_2d_impl, "CompositeExplicitAutograd") +depth_to_space_2d_op = torch.ops.et_xnnpack.depth_to_space_2d diff --git a/backends/xnnpack/operators/BUCK b/backends/xnnpack/operators/BUCK index fdc2ed1db49..a27c5d14026 100644 --- a/backends/xnnpack/operators/BUCK +++ b/backends/xnnpack/operators/BUCK @@ -8,6 +8,7 @@ fbcode_target(_kind = runtime.python_library, srcs = glob(["*.py"]), visibility = ["PUBLIC"], deps = [ + "//executorch/backends/xnnpack:custom_ops_lib", "//executorch/backends/xnnpack/utils:xnnpack_utils", "//executorch/exir:graph_module", "//executorch/exir/backend:backend_details", diff --git a/backends/xnnpack/operators/__init__.py b/backends/xnnpack/operators/__init__.py index 4b7b93a8174..3ee54158f7b 100644 --- a/backends/xnnpack/operators/__init__.py +++ b/backends/xnnpack/operators/__init__.py @@ -19,6 +19,7 @@ op_clone, op_conv2d, op_cos, + op_depth_to_space, op_div, op_dynamic_dequantize_ops, op_dynamic_quantize_ops, diff --git a/backends/xnnpack/operators/op_depth_to_space.py b/backends/xnnpack/operators/op_depth_to_space.py new file mode 100644 index 00000000000..0c7a2ae2baa --- /dev/null +++ b/backends/xnnpack/operators/op_depth_to_space.py @@ -0,0 +1,70 @@ +# Copyright (c) 2026 Software Mansion +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from typing import cast, Dict + +import executorch.backends.xnnpack.custom_ops_lib # noqa: F401 registers et_xnnpack ops +import torch +from executorch.backends.transforms import get_shape +from executorch.backends.xnnpack.operators.node_visitor import ( + NodeVisitor, + register_node_visitor, +) +from executorch.backends.xnnpack.serialization.xnnpack_graph_schema import ( + XNNDepthToSpace2d, + XNNGraph, + XNode, +) +from executorch.backends.xnnpack.utils.utils import check_or_raise, get_input_node + + +@register_node_visitor +class DepthToSpace2dVisitor(NodeVisitor): + """ + et_xnnpack.depth_to_space_2d, the block-major depth to space that XNNPACK + provides natively. Introduced by DecomposeDepthwiseConvTransposePass, see + custom_ops_lib.py for the channel order. + """ + + target = "et_xnnpack.depth_to_space_2d.default" + + def __init__(self, *args) -> None: + super().__init__(*args) + + def define_node( + self, + node: torch.fx.Node, + xnn_graph: XNNGraph, + vals_to_ids: Dict[torch.fx.Node, int], + debug_handle: int, + ) -> None: + input_node = get_input_node(node, 0) + self.define_tensor(input_node, xnn_graph, vals_to_ids, convert_to_nhwc=True) + self.define_tensor(node, xnn_graph, vals_to_ids, convert_to_nhwc=True) + + block_size = cast(int, node.args[1]) + in_shape = get_shape(input_node) + out_shape = get_shape(node) + check_or_raise( + len(in_shape) == 4 + and block_size >= 1 + and in_shape[1] % (block_size * block_size) == 0 + and out_shape[2] == in_shape[2] * block_size + and out_shape[3] == in_shape[3] * block_size, + f"depth_to_space_2d(block_size={block_size}) cannot map {in_shape} -> {out_shape}", + ) + + xnn_graph.xnodes.append( + XNode( + xnode_union=XNNDepthToSpace2d( + block_size=block_size, + input_id=vals_to_ids[input_node], + output_id=vals_to_ids[node], + flags=0, + ), + debug_handle=debug_handle, + ) + ) diff --git a/backends/xnnpack/test/passes/test_decompose_depthwise_conv_transpose.py b/backends/xnnpack/test/passes/test_decompose_depthwise_conv_transpose.py new file mode 100644 index 00000000000..5bba9cc4479 --- /dev/null +++ b/backends/xnnpack/test/passes/test_decompose_depthwise_conv_transpose.py @@ -0,0 +1,478 @@ +# Copyright (c) 2026 Software Mansion +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import json +import math +import os +import tempfile +import unittest +from collections import Counter +from unittest import mock + +import executorch.backends.test.harness.stages as BaseStages +import torch +from executorch.backends.test.harness.stages import StageType +from executorch.backends.xnnpack._passes import XNNPACKPassManager +from executorch.backends.xnnpack._passes.decompose_depthwise_conv_transpose import ( + DecomposeDepthwiseConvTransposePass, +) +from executorch.backends.xnnpack._passes.fuse_activation_pass import FuseActivationPass +from executorch.backends.xnnpack._passes.fuse_pad_into_conv_pass import ( + FusePadIntoConvPass, +) +from executorch.backends.xnnpack.serialization.xnnpack_graph_serialize import ( + _get_schema_bytes, + XNNHeader, +) +from executorch.backends.xnnpack.test.tester import RunPasses, Tester +from executorch.exir._serialize import _deserialize_pte_binary +from executorch.exir._serialize._flatbuffer import _flatc_decompile +from executorch.exir.dialects._ops import ops as exir_ops +from executorch.exir.passes.external_constants_pass import ( + delegate_external_constants_pass_unlifted, +) +from executorch.extension.flat_tensor.serialize.serialize import ( + _deserialize_to_flat_tensor, +) + + +class TestDecomposeDepthwiseConvTransposePass(unittest.TestCase): + PassStage = RunPasses([DecomposeDepthwiseConvTransposePass]) + + conv = "executorch_exir_dialects_edge__ops_aten_convolution_default" + pad = "executorch_exir_dialects_edge__ops_aten_constant_pad_nd_default" + cat = "executorch_exir_dialects_edge__ops_aten_cat_default" + d2s = "executorch_exir_dialects_edge__ops_et_xnnpack_depth_to_space_2d_default" + + def setUp(self): + torch._dynamo.reset() + + class DwDeconv(torch.nn.Module): + def __init__( + self, kernel, stride, padding, output_padding=0, bias=True, channels=8 + ): + super().__init__() + self.deconv = torch.nn.ConvTranspose2d( + channels, + channels, + kernel, + stride=stride, + padding=padding, + output_padding=output_padding, + groups=channels, + bias=bias, + ) + with torch.no_grad(): + self.deconv.weight.normal_() + if bias: + self.deconv.bias.normal_() + + def forward(self, x): + return self.deconv(x) + + class QuantizedDwDeconv(torch.nn.Module): + # The XNNPACK quantizer never annotates grouped transposed convs, so + # build the dequantized weight by hand. + def __init__(self): + super().__init__() + self.register_buffer( + "weight", torch.randint(-128, 127, (8, 1, 4, 4), dtype=torch.int8) + ) + + def forward(self, x): + weight = torch.ops.quantized_decomposed.dequantize_per_tensor( + self.weight, 0.02, 0, -128, 127, torch.int8 + ) + return torch.nn.functional.conv_transpose2d( + x, weight, stride=2, padding=1, groups=8 + ) + + def _check_pass(self, module, inputs, counts, tol=1e-3, dynamic_shapes=None): + return ( + Tester(module.eval(), inputs, dynamic_shapes=dynamic_shapes) + .export() + .to_edge() + .run_passes(self.PassStage) + .check_count(counts) + .run_method_and_compare_outputs(atol=tol, rtol=tol) + ) + + @staticmethod + def _xnnpack_node_types(tester) -> Counter: + """Node types in the serialized XNNPACK graph of the single delegate + produced by to_edge_transform_and_lower.""" + graph_module = ( + tester.get_artifact(StageType.TO_EDGE_TRANSFORM_AND_LOWER) + .exported_program() + .graph_module + ) + (delegate,) = [ + node + for node in graph_module.graph.nodes + if node.target == torch.ops.higher_order.executorch_call_delegate + ] + payload = getattr(graph_module, delegate.args[0].target).processed_bytes + header = XNNHeader.from_bytes(payload[: XNNHeader.EXPECTED_LENGTH]) + flatbuffer = payload[ + header.flatbuffer_offset : header.flatbuffer_offset + header.flatbuffer_size + ] + with tempfile.TemporaryDirectory() as d: + schema_path = os.path.join(d, "schema.fbs") + with open(schema_path, "wb") as f: + f.write(_get_schema_bytes()) + bin_path = os.path.join(d, "graph.bin") + with open(bin_path, "wb") as f: + f.write(flatbuffer) + _flatc_decompile(d, schema_path, bin_path) + with open(os.path.join(d, "graph.json")) as f: + # flatc writes unbounded clamp limits as a bare inf + text = ( + f.read().replace("-inf", "-Infinity").replace(" inf", " Infinity") + ) + graph = json.loads(text) + return Counter(node["xnode_union_type"] for node in graph["xnodes"]) + + def _check_lowered( + self, module, inputs, tol=1e-3, dynamic_shapes=None, num_deconvs=1 + ): + # XNNPACK delegates an fp32 depthwise transposed conv on its own, so + # one delegate and matching outputs do not prove the pass ran during + # lowering: count its calls and check the serialized graph. + tester = Tester(module.eval(), inputs, dynamic_shapes=dynamic_shapes).export() + with mock.patch.object( + DecomposeDepthwiseConvTransposePass, + "_decompose", + autospec=True, + side_effect=DecomposeDepthwiseConvTransposePass._decompose, + ) as decompose: + tester.to_edge_transform_and_lower() + self.assertEqual(decompose.call_count, num_deconvs) + tester.check_count( + {"torch.ops.higher_order.executorch_call_delegate": 1} + ).check_not([self.conv]) + node_types = self._xnnpack_node_types(tester) + self.assertEqual(node_types["XNNConvTranspose2d"], 0, node_types) + self.assertEqual(node_types["XNNDepthToSpace2d"], num_deconvs, node_types) + ( + tester.to_executorch() + .serialize() + .run_method_and_compare_outputs(num_runs=3, atol=tol, rtol=tol) + ) + return tester + + def test_upfirdn_4x4_stride2(self): + # StyleGAN-style x2 upsampling filter: 4 phases of 2x2 taps, each with + # a one-sided pad. + module = self.DwDeconv(kernel=4, stride=2, padding=1) + inputs = (torch.randn(1, 8, 6, 10),) + self._check_pass( + module, inputs, {self.conv: 4, self.pad: 4, self.cat: 1, self.d2s: 1} + ) + self._check_lowered(module, inputs) + + def test_no_bias_fp16(self): + module = self.DwDeconv(kernel=4, stride=2, padding=1, bias=False).half() + inputs = (torch.randn(1, 8, 5, 7).half(),) + # fp16 accumulation order differs from the reference: 1 ulp at |y| ~ 10 + self._check_pass(module, inputs, {self.conv: 4, self.d2s: 1}, tol=1e-2) + self._check_lowered(module, inputs, tol=1e-2) + + def test_3x3_stride2_output_padding(self): + # k=3, p=1, output_padding=1 keeps the x2 scaling; the phases have 1 + # and 2 taps per axis. + module = self.DwDeconv(kernel=3, stride=2, padding=1, output_padding=1) + inputs = (torch.randn(1, 8, 6, 6),) + self._check_pass(module, inputs, {self.conv: 4, self.cat: 1, self.d2s: 1}) + self._check_lowered(module, inputs) + + def test_2x2_stride2_no_padding(self): + # Every phase is a single tap, no pads at all. + module = self.DwDeconv(kernel=2, stride=2, padding=0) + inputs = (torch.randn(1, 8, 4, 6),) + self._check_pass( + module, inputs, {self.conv: 4, self.pad: 0, self.cat: 1, self.d2s: 1} + ) + self._check_lowered(module, inputs) + + def test_stride3(self): + module = self.DwDeconv(kernel=3, stride=3, padding=0, channels=4) + inputs = (torch.randn(1, 4, 4, 5),) + self._check_pass(module, inputs, {self.conv: 9, self.cat: 1, self.d2s: 1}) + self._check_lowered(module, inputs) + + def test_conv_before_deconv(self): + # The phase pads read the conv's output rather than a graph input. + module = torch.nn.Sequential( + torch.nn.Conv2d(8, 8, 3, padding=1), + self.DwDeconv(kernel=4, stride=2, padding=1), + ) + inputs = (torch.randn(1, 8, 6, 10),) + tester = self._check_pass( + module, inputs, {self.conv: 5, self.pad: 4, self.cat: 1, self.d2s: 1} + ) + graph = ( + tester.get_artifact(StageType.RUN_PASSES) + .exported_program() + .graph_module.graph + ) + for node in graph.nodes: + if node.target == exir_ops.edge.aten.constant_pad_nd.default: + self.assertEqual( + node.args[0].target, exir_ops.edge.aten.convolution.default + ) + self._check_lowered(module, inputs) + + def test_two_deconvs_in_a_row(self): + # The second deconv's input is the first one's depth_to_space_2d. + module = torch.nn.Sequential( + self.DwDeconv(kernel=4, stride=2, padding=1), + self.DwDeconv(kernel=4, stride=2, padding=1), + ) + inputs = (torch.randn(1, 8, 4, 6),) + self._check_pass( + module, inputs, {self.conv: 8, self.pad: 8, self.cat: 2, self.d2s: 2} + ) + self._check_lowered(module, inputs, num_deconvs=2) + + class SharedWeightDeconvs(torch.nn.Module): + def __init__(self, kernel, stride, paddings, output_paddings): + super().__init__() + self.weight = torch.nn.Parameter(torch.randn(8, 1, kernel, kernel)) + self.stride = stride + self.configs = list(zip(paddings, output_paddings)) + + def forward(self, x): + return sum( + torch.nn.functional.conv_transpose2d( + x, + self.weight, + stride=self.stride, + padding=padding, + output_padding=output_padding, + groups=8, + ) + for padding, output_padding in self.configs + ) + + def _count_phase_placeholders(self, tester): + graph = ( + tester.get_artifact(StageType.RUN_PASSES) + .exported_program() + .graph_module.graph + ) + return sum( + 1 + for node in graph.nodes + if node.op == "placeholder" and "_phase_" in node.name + ) + + def test_shared_weight_different_padding(self): + # Both deconvs are clean x3 upscales of the same weight, but their + # phase taps differ, so they must not share the phase placeholders. + module = self.SharedWeightDeconvs( + kernel=9, stride=3, paddings=(3, 4), output_paddings=(0, 2) + ) + inputs = (torch.randn(1, 8, 5, 6),) + tester = self._check_pass( + module, inputs, {self.conv: 18, self.cat: 2, self.d2s: 2} + ) + self.assertEqual(self._count_phase_placeholders(tester), 18) + self._check_lowered(module, inputs, num_deconvs=2) + + def test_shared_weight_same_params(self): + # Identical deconvs on one weight share one set of phase placeholders. + module = self.SharedWeightDeconvs( + kernel=9, stride=3, paddings=(3, 3), output_paddings=(0, 0) + ) + inputs = (torch.randn(1, 8, 5, 6),) + tester = self._check_pass( + module, inputs, {self.conv: 18, self.cat: 2, self.d2s: 2} + ) + self.assertEqual(self._count_phase_placeholders(tester), 9) + self._check_lowered(module, inputs, num_deconvs=2) + + def test_batch_gt_1(self): + module = self.DwDeconv(kernel=4, stride=2, padding=1) + inputs = (torch.randn(3, 8, 6, 10),) + self._check_pass( + module, inputs, {self.conv: 4, self.pad: 4, self.cat: 1, self.d2s: 1} + ) + self._check_lowered(module, inputs) + + def test_single_element_spatial_params(self): + class Deconv(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.nn.Parameter(torch.randn(8, 1, 4, 4)) + + def forward(self, x): + return torch.ops.aten.convolution.default( + x, self.weight, None, [2], [1], [1], True, [0], 8 + ) + + inputs = (torch.randn(1, 8, 6, 10),) + self._check_pass(Deconv(), inputs, {self.conv: 4, self.pad: 4, self.d2s: 1}) + self._check_lowered(Deconv(), inputs) + + def test_deconv_relu(self): + # The activation follows depth_to_space_2d instead of fusing into the + # removed transposed conv, which relies on the pass running before + # FuseActivationPass in the default pass list. The phase pads read the + # NHWC output of the leading conv and fold into the phase convs through + # FusePadIntoConvPass at the end of that list. + module = torch.nn.Sequential( + torch.nn.Conv2d(8, 8, 3, padding=1), + self.DwDeconv(kernel=4, stride=2, padding=1), + torch.nn.ReLU(), + ) + inputs = (torch.randn(1, 8, 6, 10),) + default_passes = XNNPACKPassManager( + Tester(module.eval(), inputs).export().get_artifact() + ).passes + self.assertLess( + default_passes.index(DecomposeDepthwiseConvTransposePass), + default_passes.index(FuseActivationPass), + ) + tester = ( + Tester(module.eval(), inputs) + .export() + .to_edge() + .run_passes(RunPasses(default_passes)) + .check_count({self.conv: 5, self.pad: 4, self.d2s: 1}) + ) + graph = ( + tester.get_artifact(StageType.RUN_PASSES) + .exported_program() + .graph_module.graph + ) + (relu,) = [ + n for n in graph.nodes if n.target == exir_ops.edge.aten.relu.default + ] + self.assertEqual( + relu.args[0].target, exir_ops.edge.et_xnnpack.depth_to_space_2d.default + ) + convs = [ + n for n in graph.nodes if n.target == exir_ops.edge.aten.convolution.default + ] + self.assertEqual( + [FuseActivationPass.get_fused_activation(n) for n in convs], [None] * 5 + ) + self.assertEqual( + sum(FusePadIntoConvPass.get_fused_padding(n) is not None for n in convs), 4 + ) + self.assertEqual( + sum( + FusePadIntoConvPass.is_folded_pad(n) + for n in graph.nodes + if n.target == exir_ops.edge.aten.constant_pad_nd.default + ), + 4, + ) + self._check_lowered(module, inputs) + + class ExportWithExternalConstants(BaseStages.Export): + def run(self, artifact, inputs): + tagged = torch.export.export(artifact, inputs, strict=True).module() + delegate_external_constants_pass_unlifted( + module=tagged, gen_tag_fn=lambda _: "model" + ) + self.exported_program = torch.export.export(tagged, inputs, strict=True) + + def test_external_constants(self): + # The phase weights inherit the original weight's external-constant + # tag, so every constant ends up in model.ptd and none in the .pte. + module = self.DwDeconv(kernel=4, stride=2, padding=1) + inputs = (torch.randn(1, 8, 6, 10),) + tester = Tester(module.eval(), inputs) + tester.export(self.ExportWithExternalConstants()) + tester.to_edge_transform_and_lower().check_count( + {"torch.ops.higher_order.executorch_call_delegate": 1} + ).to_executorch() + exec = tester.get_artifact() + self.assertIsNone(_deserialize_pte_binary(exec.buffer).named_data) + self.assertEqual(list(exec._tensor_data), ["model"]) + ptd = _deserialize_to_flat_tensor(bytes(exec._tensor_data["model"])) + self.assertEqual(len(ptd.named_data), 5) # 4 phase weights + bias + + from executorch.extension.pybindings import portable_lib as runtime + + loaded = runtime._load_for_executorch_from_buffer( + exec.buffer, bytes(exec._tensor_data["model"]) + ) + self.assertTrue( + torch.allclose(loaded.forward(inputs)[0], module(*inputs), atol=1e-3) + ) + + def test_dynamic_shape(self): + module = self.DwDeconv(kernel=4, stride=2, padding=1) + inputs = (torch.randn(1, 8, 6, 10),) + dynamic_shapes = ( + { + 2: torch.export.Dim("height", min=2, max=16), + 3: torch.export.Dim("width", min=2, max=16), + }, + ) + self._check_pass( + module, + inputs, + {self.conv: 4, self.pad: 4, self.cat: 1, self.d2s: 1}, + dynamic_shapes=dynamic_shapes, + ) + self._check_lowered(module, inputs, dynamic_shapes=dynamic_shapes) + + def test_quantized_is_left_alone(self): + inputs = (torch.randn(1, 8, 6, 6),) + self._check_pass(self.QuantizedDwDeconv(), inputs, {self.conv: 1, self.d2s: 0}) + + def test_not_a_clean_upscale_is_left_alone(self): + # k=4, p=0: output is 2H + 2, not a x2 upscale. + module = self.DwDeconv(kernel=4, stride=2, padding=0) + inputs = (torch.randn(1, 8, 6, 6),) + self._check_pass(module, inputs, {self.conv: 1, self.d2s: 0}) + + def test_grouped_but_not_depthwise_is_left_alone(self): + module = torch.nn.ConvTranspose2d(8, 8, 4, stride=2, padding=1, groups=4) + inputs = (torch.randn(1, 8, 6, 6),) + self._check_pass(module, inputs, {self.conv: 1, self.d2s: 0}) + + def test_phase_math_matches_conv_transpose2d(self): + # The decomposition in plain torch, independent of the lowering. + import executorch.backends.xnnpack.custom_ops_lib as lib + + for k, s, p, o in [ + (4, 2, 1, 0), + (3, 2, 1, 1), + (2, 2, 0, 0), + (3, 3, 0, 0), + (5, 2, 2, 1), + ]: + c, h, w = 3, 5, 6 + x = torch.randn(1, c, h, w) + weight = torch.randn(c, 1, k, k) + ref = torch.nn.functional.conv_transpose2d( + x, weight, stride=s, padding=p, output_padding=o, groups=c + ) + phases = [] + for ay in range(s): + for ax in range(s): + taps = [] + for a in (ay, ax): + t_min = math.ceil(-(a + p) / s) + t_max = math.floor((k - 1 - a - p) / s) + taps.append((t_min, t_max)) + (ty_min, ty_max), (tx_min, tx_max) = taps + iy = torch.tensor( + [s * (ty_max - q) + ay + p for q in range(ty_max - ty_min + 1)] + ) + ix = torch.tensor( + [s * (tx_max - q) + ax + p for q in range(tx_max - tx_min + 1)] + ) + wp = weight[:, :, iy][:, :, :, ix] + xp = torch.nn.functional.pad(x, (tx_max, -tx_min, ty_max, -ty_min)) + phases.append(torch.nn.functional.conv2d(xp, wp, groups=c)) + out = lib.depth_to_space_2d_op(torch.cat(phases, 1), s) + self.assertEqual(out.shape, ref.shape, (k, s, p, o)) + self.assertTrue(torch.allclose(out, ref, atol=1e-5), (k, s, p, o))