Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion hyper_parallel/compile/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ def compile(self, **inputs: Any) -> None:
joint_graph = trace_model_graph(
self.model, self.train_fn, inputs, **trace_kwargs
)
self._validate_gradient_usage(joint_graph.graph_module)

pipeline = PassPipeline.from_config(self.pass_config, self.parallel_plan)

Expand All @@ -262,6 +263,34 @@ def compile(self, **inputs: Any) -> None:
joint_graph.input_guards.refresh()
self._joint_graph = joint_graph

def _validate_gradient_usage(self, graph_module: torch.fx.GraphModule) -> None:
"""Admit a fixed parameter-usage schema before parallel graph lowering.

Every participating rank must capture the same state names and unused
slots. This conservative contract rejects rank-dependent usage instead
of allowing different collective sequences; it does not merge gradients
between distinct EP owners. Validation communicates once at compilation.
"""
output = next(node for node in graph_module.graph.nodes if node.op == "output")
returned = output.args[0]
end = len(returned) - len(getattr(graph_module, "loss_dict_keys", []))
gradients = returned[1:end]
parameters = dict(self.model.named_parameters(remove_duplicate=False))
names = tuple(name for name in graph_module.state_fqns
if name in parameters and parameters[name].requires_grad)
if len(names) != len(gradients):
raise ValueError("Graph gradient slots must match the trainable parameter schema")
schema = (tuple(graph_module.state_fqns), tuple(zip(names, (grad is None for grad in gradients))))
if (self.pass_config.fsdp_enabled or self.pass_config.pp_enabled) and dist.is_initialized():
world_size = dist.get_world_size()
if world_size > 1:
schemas = [None] * world_size
dist.all_gather_object(schemas, schema)
if any(other != schema for other in schemas):
raise ValueError("Graph training requires the same parameter usage schema on every rank")
if self.pass_config.pp_enabled and any(unused for _, unused in schema[1]):
raise ValueError("Graph pipeline parallel does not support unused parameter gradients yet")

def forward_backward(self, **inputs: Any) -> Any:
"""
Execute one compiled forward+backward step.
Expand Down Expand Up @@ -893,7 +922,7 @@ def _run_graph(self, **inputs):
inputs,
)

def _accumulate_grads(self, grads: List[torch.Tensor]) -> None:
def _accumulate_grads(self, grads: List[Optional[torch.Tensor]]) -> None:
"""Accumulate graph-computed gradients into the live model's parameters.

The graph emits gradients in ``state_fqns`` order (trainable
Expand Down Expand Up @@ -924,6 +953,8 @@ def _accumulate_grads(self, grads: List[torch.Tensor]) -> None:
)

for param, grad in zip(trainable, grads):
if grad is None:
continue
if param.grad is None:
# Match eager's gradient layout so optimizer kernels see the
# parameter's strides rather than a backward matmul's view.
Expand Down
12 changes: 3 additions & 9 deletions hyper_parallel/compile/tracer/graph_tracer.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,15 +494,9 @@ def _fwd_bwd_fn(*plain_args):

grads = torch.autograd.grad(loss, params, allow_unused=True)

# Replace None grads with zeros so the output arity is stable.
processed_grads = []
for grad, param in zip(grads, params):
if grad is None:
processed_grads.append(torch.zeros_like(param))
else:
processed_grads.append(grad)

return [loss] + processed_grads + loss_dict_values
# None occupies its parameter's slot so optimizers can distinguish
# unused parameters from used parameters with a zero gradient.
return [loss] + list(grads) + loss_dict_values

return _fwd_bwd_fn, loss_dict_keys_box

Expand Down
1 change: 1 addition & 0 deletions tests/ut/compile/test_ragged_ep.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ def test_compile_resolves_expert_parameter_aliases(self):
patch("hyper_parallel.compile.compiler.dist.is_initialized", return_value=True),
patch("hyper_parallel.compile.compiler._register_process_group") as register,
patch("hyper_parallel.compile.compiler.trace_model_graph", return_value=joint),
patch.object(compiler, "_validate_gradient_usage"),
patch("hyper_parallel.compile.compiler.PassPipeline.from_config") as pipeline,
):
compiler.compile()
Expand Down
165 changes: 165 additions & 0 deletions tests/ut/compile/test_unused_gradients.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
# Copyright 2026 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Preserve unused-gradient slots without confusing them with tensor zeros."""

import copy
import unittest
from unittest.mock import MagicMock, patch

import torch
from torch import nn

from hyper_parallel.compile import GraphCompiler, PassConfig
from hyper_parallel.compile.passes.parallel.fsdp_pass import FSDPPass
from hyper_parallel.compile.tracer import trace_model_graph


class _Model(nn.Module):
"""Keep a never-used parameter between two distinct trainable paths."""

def __init__(self, use_all: bool = False) -> None:
"""Use zero as a real autograd input and optionally do the same for unused."""
super().__init__()
self.left = nn.Parameter(torch.randn(4, 4))
self.unused = nn.Parameter(torch.ones(4, 4))
self.zero = nn.Parameter(torch.ones(4, 4))
self.right = nn.Parameter(torch.randn(4, 4))
self.use_all = use_all

def forward(self, hidden: torch.Tensor) -> torch.Tensor:
"""Give zero a tensor gradient while unused has no loss dependency."""
value = hidden @ self.left + hidden.square() @ self.right.tanh() + self.zero.sum() * 0
return value + self.unused.sum() * 0 if self.use_all else value


def _loss(model: nn.Module, *, hidden: torch.Tensor) -> tuple:
"""Retain a trailing metric to check the gradient-output boundary."""
loss = model(hidden).square().mean()
return loss, {"mean": loss.detach()}


class TestUnusedGradients(unittest.TestCase):
"""Match eager optimizer skipping, output ordering and parallel admission."""

def test_adamw_skips_unused_but_decays_used_zero_gradient(self) -> None:
"""Ten updates retain None and correct moments without shifting later grads."""
torch.manual_seed(81)
reference = _Model()
candidate = copy.deepcopy(reference)
initial = candidate.unused.detach().clone()
compiler = GraphCompiler(candidate, _loss, PassConfig(fsdp_enabled=False), device=torch.device("cpu"))
optimizers = [torch.optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.1, foreach=False)
for model in (reference, candidate)]
for _ in range(10):
for optimizer in optimizers:
optimizer.zero_grad(set_to_none=True)
for _ in range(2):
hidden = torch.randn(2, 4)
expected, _ = _loss(reference, hidden=hidden)
expected.backward()
actual, metrics = compiler.forward_backward(hidden=hidden)
torch.testing.assert_close(actual, expected)
torch.testing.assert_close(metrics["mean"], expected.detach())
for eager, compiled in zip(reference.parameters(), candidate.parameters()):
if eager.grad is None:
self.assertIsNone(compiled.grad)
else:
torch.testing.assert_close(compiled.grad, eager.grad)
for optimizer in optimizers:
optimizer.step()
for eager, compiled in zip(reference.parameters(), candidate.parameters()):
torch.testing.assert_close(compiled, eager)
self.assertTrue(torch.equal(candidate.unused, initial))
self.assertNotIn(candidate.unused, optimizers[1].state)
self.assertIn(candidate.zero, optimizers[1].state)
self.assertTrue(torch.all(candidate.zero < initial))
for eager, compiled in zip(reference.parameters(), candidate.parameters()):
for key, expected in optimizers[0].state.get(eager, {}).items():
torch.testing.assert_close(optimizers[1].state[compiled][key], expected)

def test_none_does_not_clear_an_earlier_microbatch_gradient(self) -> None:
"""A skipped output must leave an already accumulated gradient intact."""
model = _Model()
compiler = GraphCompiler(model, _loss, PassConfig(fsdp_enabled=False), device=torch.device("cpu"))
previous = torch.full_like(model.unused, 2.)
model.unused.grad = previous
compiler.forward_backward(hidden=torch.randn(2, 4))
self.assertIs(model.unused.grad, previous)
self.assertTrue(torch.equal(model.unused.grad, torch.full_like(previous, 2.)))

def test_fsdp_reduces_used_slots_without_compacting_unused(self) -> None:
"""The original parameter index survives None and the trailing metric."""
model = _Model()
joint = trace_model_graph(model, _loss, {"hidden": torch.randn(2, 4)})
mocked = MagicMock()
mocked.is_initialized.return_value = True
mocked.get_world_size.return_value = 2
mocked.get_rank.return_value = 0
with (patch("hyper_parallel.compile.passes.parallel.fsdp_pass.dist", mocked),
patch("hyper_parallel.compile.passes.parallel.fsdp_pass._resolve_process_group")):
FSDPPass().run(joint.graph_module, PassConfig(fsdp_degree=2), model=model)
outputs = next(node for node in joint.graph_module.graph.nodes if node.op == "output").args[0]
self.assertEqual(len(outputs), 6)
self.assertIsNone(outputs[2])
self.assertEqual([outputs[index].meta["param_name"] for index in (1, 3, 4)], ["left", "zero", "right"])
reductions = [node for node in joint.graph_module.graph.nodes
if "reduce_scatter_tensor" in str(node.target)]
self.assertEqual(len(reductions), 3)

def test_rank_dependent_usage_and_pipeline_unused_are_rejected(self) -> None:
"""All-used ranks also participate before any parallel lowering can run."""
for use_all in (False, True):
with self.subTest(use_all=use_all):
model = _Model(use_all)
compiler = GraphCompiler(model, _loss, PassConfig(fsdp_enabled=True))
graph = trace_model_graph(model, _loss, {"hidden": torch.randn(2, 4)}).graph_module

def gather(schemas, schema):
peer_mask = list(schema[1])
name, unused = peer_mask[1]
peer_mask[1] = (name, not unused)
schemas[:] = [schema, (schema[0], tuple(peer_mask))]

with (patch("hyper_parallel.compile.compiler.dist.is_initialized", return_value=True),
patch("hyper_parallel.compile.compiler.dist.get_world_size", return_value=2),
patch("hyper_parallel.compile.compiler.dist.all_gather_object", side_effect=gather) as collect,
self.assertRaisesRegex(ValueError, "same parameter usage schema")):
compiler._validate_gradient_usage(graph)
collect.assert_called_once()
compiler = GraphCompiler(_Model(), _loss, PassConfig(fsdp_enabled=False, pp_enabled=True))
with (patch("hyper_parallel.compile.compiler.PassPipeline.from_config") as pipeline,
self.assertRaisesRegex(ValueError, "pipeline parallel does not support unused")):
compiler.compile(hidden=torch.randn(2, 4))
pipeline.assert_not_called()

def test_rank_dependent_trainable_identity_is_rejected(self) -> None:
"""Identical state names and all-used flags may still belong to different parameters."""
model, peer = _Model(use_all=True), _Model(use_all=True)
model.left.requires_grad_(False)
peer.right.requires_grad_(False)
inputs = {"hidden": torch.randn(2, 4)}
graph = trace_model_graph(model, _loss, inputs).graph_module
peer_joint = trace_model_graph(peer, _loss, inputs)
peer_schema = (tuple(peer_joint.state_fqns), tuple((name, False) for name in peer_joint.param_names))
compiler = GraphCompiler(model, _loss, PassConfig(fsdp_enabled=True))

def gather(schemas, schema):
schemas[:] = [schema, peer_schema]

with (patch("hyper_parallel.compile.compiler.dist.is_initialized", return_value=True),
patch("hyper_parallel.compile.compiler.dist.get_world_size", return_value=2),
patch("hyper_parallel.compile.compiler.dist.all_gather_object", side_effect=gather),
self.assertRaisesRegex(ValueError, "same parameter usage schema")):
compiler._validate_gradient_usage(graph)
Loading