Skip to content

Commit 7490bd4

Browse files
Add pass to replace input dim order clones with permutations. (pytorch#21241)
### Summary This pass replaces `dim_order_clone` and `to_dim_order_copy` nodes that consume model inputs by a sequence of permute nodes according to the proposal in pytorch#19299. Fixes pytorch#20095. ### Test plan `pytest backends/transforms/test/test_replace_channels_last_input_clones.py`
1 parent ddaabb9 commit 7490bd4

3 files changed

Lines changed: 564 additions & 0 deletions

File tree

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
# Copyright 2026 NXP
2+
#
3+
# This source code is licensed under the BSD-style license found in the
4+
# LICENSE file in the root directory of this source tree.
5+
6+
from typing import Sequence
7+
8+
import executorch.backends.transforms.channels_last_ops # noqa: F401
9+
10+
import torch
11+
12+
from executorch.exir.dialects._ops import ops as exir_ops
13+
14+
from executorch.exir.pass_base import ExportPass, NodeMetadata, ProxyValue
15+
from torch.fx.passes.infra.pass_base import PassResult
16+
17+
_DIM_ORDER_CHANGING_OPS: frozenset = frozenset(
18+
{
19+
exir_ops.edge.dim_order_ops._to_dim_order_copy.default,
20+
exir_ops.edge.dim_order_ops._clone_dim_order.default,
21+
}
22+
)
23+
24+
# kwargs that carry no tensor semantics and require no value inspection.
25+
_PASS_THROUGH_KWARGS: frozenset = frozenset({"dim_order", "non_blocking"})
26+
27+
# TensorOptions that the exporter may annotate explicitly on `_to_dim_order_copy` nodes even
28+
# when no actual change occurs. These are allowed only when their value is identical to the
29+
# source tensor's property (i.e. truly a no-op). Any value that would actually change the
30+
# tensor (e.g. a different dtype) blocks replacement. Any kwarg not in either set is unknown
31+
# and is also rejected.
32+
_TENSOR_OPTION_KWARGS: frozenset = frozenset(
33+
{"dtype", "layout", "device", "pin_memory"}
34+
)
35+
36+
_ALLOWED_KWARGS: frozenset = _PASS_THROUGH_KWARGS | _TENSOR_OPTION_KWARGS
37+
38+
_TO_NHWC_PERMUTATION: list[int] = [0, 2, 3, 1]
39+
_TO_NCHW_PERMUTATION: list[int] = [0, 3, 1, 2]
40+
41+
42+
def _is_4d_contiguous(dim_order: Sequence[int]) -> bool:
43+
return list(dim_order) == [0, 1, 2, 3]
44+
45+
46+
def _is_4d_channels_last(dim_order: Sequence[int]) -> bool:
47+
return list(dim_order) == [0, 2, 3, 1]
48+
49+
50+
def _is_replaceable_input_boundary_clone(op, args, kwargs) -> bool:
51+
"""Return True if the op/args/kwargs describe a 4D channels-last-to-contiguous clone of a model input.
52+
53+
These are the input boundary clones inserted by `EnforceContiguousDimOrder`: they consume a
54+
channels-last placeholder and produce a contiguous tensor. Replacing them with a permute pair
55+
allows them to be optimized out by other passes, leaving only the no-op `aten.permute_copy`.
56+
"""
57+
if op not in _DIM_ORDER_CHANGING_OPS:
58+
return False
59+
if not args or not hasattr(args[0], "node"):
60+
return False
61+
src = args[0].node
62+
if not isinstance(src, torch.fx.Node) or src.op != "placeholder":
63+
return False
64+
val = src.meta.get("val")
65+
if not isinstance(val, torch.Tensor):
66+
return False
67+
# Primary guard: reject any kwarg outside the known set. Unknown kwargs may carry
68+
# semantics we cannot reason about, so we conservatively block replacement.
69+
if not set(kwargs.keys()) <= _ALLOWED_KWARGS:
70+
return False
71+
# Secondary guard: each TensorOption that is present must be a no-op relative to the
72+
# source tensor. The exporter annotates these even when no actual change occurs, but a
73+
# differing value (e.g. a dtype cast) must not be silently dropped by the replacement.
74+
if kwargs.get("dtype") not in (None, val.dtype):
75+
return False
76+
if kwargs.get("layout") not in (None, val.layout):
77+
return False
78+
if kwargs.get("device") not in (None, val.device):
79+
return False
80+
if kwargs.get("pin_memory") not in (None, False):
81+
return False
82+
return _is_4d_channels_last(val.dim_order()) and _is_4d_contiguous(
83+
kwargs.get("dim_order", [])
84+
)
85+
86+
87+
class ReplaceChannelsLastInputClones(ExportPass):
88+
"""Replace `_to_dim_order_copy` and `_clone_dim_order` with an equivalent sequence in the following pattern. This
89+
approach allows the `channels_last.permute_copy` to be optimized out if there are subsequent channels last
90+
operators in the model, leaving only the `aten.permute_copy`, which is effectively a no-op. As a result, the
91+
input data doesn't have to be permuted in memory.
92+
<model input>
93+
│ [N, C, H, W] shape, (0, 2, 3, 1) dim order
94+
<model input> │ data is stored channels last
95+
│ [N, C, H, W] shape, (0, 2, 3, 1) dim order ┌─────────▼─────────┐
96+
│ data is stored channels last │ aten.permute_copy ◄──── [0, 2, 3, 1] permutation
97+
┌───────────▼────────────┐ └─────────┬─────────┘
98+
│ <dim order clone/copy> │ ────────────────► │ [N, H, W, C] shape, (0, 1, 2, 3) dim order
99+
└───────────┬────────────┘ │ data is stored channels last
100+
│ [N, C, H, W] shape, (0, 1, 2, 3) dim order ┌──────────────▼─────────────┐
101+
▼ data is stored channels first │ channels_last.permute_copy ◄──── [0, 3, 1, 2] permutation
102+
└──────────────┬─────────────┘
103+
│ [N, C, H, W] shape, (0, 1, 2, 3) dim order
104+
▼ data is stored channels first
105+
"""
106+
107+
_modified: bool
108+
109+
def call(self, graph_module: torch.fx.GraphModule) -> PassResult:
110+
self._modified = False
111+
result = super().call(graph_module)
112+
return PassResult(result.graph_module, self._modified)
113+
114+
def call_operator(self, op, args, kwargs, meta: NodeMetadata) -> ProxyValue:
115+
if not _is_replaceable_input_boundary_clone(op, args, kwargs):
116+
return super().call_operator(op, args, kwargs, meta)
117+
118+
x = super().call_operator(
119+
exir_ops.edge.aten.permute_copy.default,
120+
(args[0], _TO_NHWC_PERMUTATION),
121+
{},
122+
meta,
123+
)
124+
x = super().call_operator(
125+
exir_ops.edge.channels_last.permute_copy.default,
126+
(x, _TO_NCHW_PERMUTATION),
127+
{},
128+
meta,
129+
)
130+
131+
self._modified = True
132+
133+
return x

‎backends/transforms/targets.bzl‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -716,3 +716,32 @@ def define_common_targets():
716716
":enforce_contiguous_dim_order",
717717
],
718718
)
719+
720+
runtime.python_library(
721+
name = "replace_channels_last_input_clones",
722+
srcs = ["replace_channels_last_input_clones.py"],
723+
visibility = [
724+
"//executorch/backends/...",
725+
],
726+
deps = [
727+
"//caffe2:torch",
728+
":channels_last_ops",
729+
"//executorch/exir:pass_base",
730+
"//executorch/exir/dialects:lib",
731+
],
732+
)
733+
734+
runtime.python_test(
735+
name = "test_replace_channels_last_input_clones",
736+
srcs = [
737+
"test/test_replace_channels_last_input_clones.py",
738+
],
739+
deps = [
740+
"//caffe2:torch",
741+
"//executorch/exir:lib",
742+
":channels_last_ops",
743+
":enforce_contiguous_dim_order",
744+
":replace_channels_last_input_clones",
745+
"fbsource//third-party/pypi/pytest:pytest",
746+
],
747+
)

0 commit comments

Comments
 (0)