Skip to content

Commit c19a0f0

Browse files
committed
feat(compile): lower shared FSDP mixed precision policy
1 parent c7d02fe commit c19a0f0

6 files changed

Lines changed: 285 additions & 13 deletions

File tree

‎hyper_parallel/compile/compiler.py‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,12 @@ class and owns them.
4545
from hyper_parallel.core.dtensor.layout import Layout
4646
from hyper_parallel.core.dtensor.placement_types import Replicate, Shard, StridedShard
4747
from hyper_parallel.distributed._builder.fsdp_adapter import FSDP2Manager
48+
from hyper_parallel.distributed.mesh import MeshContext
4849
from hyper_parallel.models.build_options import FSDP2Config
4950
from hyper_parallel.models.external_state import get_model_external_state
5051

5152
from .graph_parallel_plan import GraphParallelPlan
53+
from .mixed_precision import GraphMixedPrecision
5254
from .pass_config import (
5355
GradientReduction,
5456
PassConfig,
@@ -141,6 +143,8 @@ def __init__(
141143
self._storage_signature = None
142144
self._sharded_parameter_names = frozenset()
143145
self._prepared_pass_kwargs = None
146+
self._mixed_precision = None
147+
self._fp32_main_params = getattr(getattr(trainer_config, "optimizer", None), "fp32_main_params", False)
144148
self.device = device or (
145149
torch.device("npu")
146150
if (hasattr(torch, "npu") and torch.npu.is_available())
@@ -242,6 +246,12 @@ def _prepare_model_storage(self) -> None:
242246
if self.pass_config.fsdp_enabled and dist.is_initialized():
243247
self._init_device_mesh(self._mesh_context)
244248
self._bind_parameter_meshes()
249+
precision = getattr(self._fsdp_config, "mix_precision", None)
250+
if precision is not None and any((precision.param_dtype, precision.reduce_dtype, precision.output_dtype,
251+
precision.custom_params, self._fp32_main_params)):
252+
manager = FSDP2Manager(self._fsdp_config, self._mesh_context or MeshContext(), self._fp32_main_params)
253+
policy, modules = manager.mixed_precision_bindings(self.model, self.source_shard_info)
254+
self._mixed_precision = GraphMixedPrecision(self.model, policy, modules)
245255
self._parameter_shapes = {
246256
name: tuple(param.shape) for name, param in self.model.named_parameters(remove_duplicate=False)
247257
}
@@ -284,7 +294,7 @@ def compile(self, **inputs: Any) -> None:
284294
if self.pass_config.ac_mode != "off":
285295
ac_scope = prepare_activation_checkpoint(self.model, self.pass_config)
286296

287-
trace_kwargs = {"state_shapes": self._parameter_shapes}
297+
trace_kwargs = {"state_shapes": self._parameter_shapes, "mixed_precision": self._mixed_precision}
288298
if self.dynamic or self.dynamic_arg_dims is not None:
289299
trace_kwargs.update({
290300
"dynamic": self.dynamic,
@@ -870,6 +880,8 @@ def _build_pass_kwargs(self) -> dict:
870880
# Live model: partitioning passes (FSDPPass) physically shard
871881
# parameters in place, keeping the compiler FSDP-agnostic.
872882
kwargs["model"] = self.model
883+
if self._mixed_precision is not None:
884+
kwargs["parameter_dtypes"] = self._mixed_precision.parameter_dtypes
873885

874886
if self.pass_config.fsdp_enabled:
875887
kwargs["fsdp_group_name"] = "fsdp"
@@ -977,13 +989,18 @@ def _accumulate_grads(self, grads: List[torch.Tensor]) -> None:
977989
)
978990

979991
for param, grad in zip(trainable, grads):
980-
if param.grad is None:
992+
main_grad = self._mixed_precision is not None and self._mixed_precision.policy.apply_grad_on_fp32_main_grad
993+
attribute = "main_grad" if main_grad else "grad"
994+
accumulated = getattr(param, attribute, None)
995+
if accumulated is None:
981996
# Match eager's gradient layout so optimizer kernels see the
982997
# parameter's strides rather than a backward matmul's view.
983-
param.grad = torch.empty_like(param)
984-
param.grad.copy_(grad)
998+
dtype = torch.float32 if main_grad else param.dtype
999+
accumulated = torch.empty_like(param, dtype=dtype)
1000+
accumulated.copy_(grad)
1001+
setattr(param, attribute, accumulated)
9851002
else:
986-
param.grad += grad
1003+
accumulated += grad
9871004

9881005
@staticmethod
9891006
def _trainable_params_in_state_order(
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# Copyright 2026 Huawei Technologies Co., Ltd
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
# ============================================================================
15+
"""Lower the shared FSDP precision policy into the joint forward/backward graph."""
16+
17+
from contextlib import ExitStack, contextmanager
18+
from functools import partial
19+
from typing import Any, Iterator
20+
21+
import torch
22+
from torch import nn
23+
24+
from hyper_parallel.core.fully_shard.utils import MixedPrecisionPolicy, apply_to_tensors, cast_fp_tensor
25+
26+
27+
class GraphMixedPrecision:
28+
"""Keep storage, computation, communication and accumulation dtypes distinct."""
29+
30+
def __init__(self, model: nn.Module, policy: MixedPrecisionPolicy, modules: tuple[nn.Module, ...]) -> None:
31+
"""Bind the shared policy to source parameter identities and unit boundaries.
32+
33+
Args:
34+
model: Final model before graph FSDP slices local storage.
35+
policy: Resolved FSDP policy, including Parameter-specific overrides.
36+
modules: The same module boundaries selected by eager FSDP.
37+
"""
38+
self.policy = policy
39+
self.modules = modules
40+
self.parameter_dtypes = {}
41+
for name, parameter in model.named_parameters(remove_duplicate=False):
42+
custom = (policy.custom_params or {}).get(id(parameter), {})
43+
compute = custom.get("param_dtype", policy.param_dtype) or parameter.dtype
44+
reduce = custom.get("reduce_dtype", policy.reduce_dtype) or compute
45+
self.parameter_dtypes[name] = (compute, reduce)
46+
47+
@contextmanager
48+
def module_casts(self) -> Iterator[None]:
49+
"""Record eager-compatible module input/output casts during capture only."""
50+
with ExitStack() as stack:
51+
for module in self.modules:
52+
if self.policy.cast_forward_inputs and self.policy.param_dtype is not None:
53+
handle = module.register_forward_pre_hook(self._cast_inputs, with_kwargs=True)
54+
stack.callback(handle.remove)
55+
if self.policy.output_dtype is not None:
56+
handle = module.register_forward_hook(self._cast_outputs)
57+
stack.callback(handle.remove)
58+
yield
59+
60+
def cast_state(self, state: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
61+
"""Cast compute views while preserving autograd links to storage inputs."""
62+
return {
63+
name: cast_fp_tensor(self.parameter_dtypes[name][0], value)
64+
if name in self.parameter_dtypes else value
65+
for name, value in state.items()
66+
}
67+
68+
def _cast_inputs(self, module: nn.Module, args: tuple, kwargs: dict) -> tuple:
69+
"""Mirror the eager FSDP pre-forward cast for floating input leaves."""
70+
del module
71+
cast = partial(cast_fp_tensor, self.policy.param_dtype)
72+
return apply_to_tensors(cast, args), apply_to_tensors(cast, kwargs)
73+
74+
def _cast_outputs(self, module: nn.Module, args: tuple, outputs: Any) -> Any:
75+
"""Mirror the eager FSDP post-forward output cast."""
76+
del module, args
77+
return apply_to_tensors(partial(cast_fp_tensor, self.policy.output_dtype), outputs)

‎hyper_parallel/compile/passes/parallel/fsdp_pass.py‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ def _annotate_ag_spec(node: fx.Node, degree: int, params: dict) -> None:
138138
node.meta["fsdp_bucket_spec"] = {
139139
"input_shape": shape,
140140
"output_shape": (shape[0] * degree, *shape[1:]),
141-
"dtype": param.dtype,
141+
"dtype": node.meta.get("param_dtype", param.dtype),
142142
"device": str(param.device),
143143
}
144144

@@ -168,6 +168,18 @@ def annotate_fsdp_bucket_specs(graph_module: fx.GraphModule, model: nn.Module) -
168168
_annotate_ag_spec(node, degree, params)
169169

170170

171+
def _cast_node(graph: fx.Graph, node: fx.Node, dtype: Optional[torch.dtype]) -> fx.Node:
172+
"""Insert a precision conversion while retaining fake dtype/shape metadata."""
173+
value = node.meta.get("val")
174+
if dtype is None or (value is not None and value.dtype == dtype):
175+
return node
176+
cast = graph.call_method("to", args=(node,), kwargs={"dtype": dtype})
177+
cast.meta["autograd_backward"] = node.meta.get("autograd_backward", False)
178+
if isinstance(value, FakeTensor):
179+
cast.meta["val"] = value.to(dtype)
180+
return cast
181+
182+
171183
def _free_tensor_storage(tensor: torch.Tensor) -> None:
172184
"""Release ``tensor``'s storage in place (FSDP reshard free step).
173185
@@ -338,6 +350,7 @@ def __init__(
338350
self._parameter_groups: Dict[str, tuple[str, int]] = {}
339351
self._storage_prepared = False
340352
self._logical_parameter_shapes: Dict[str, tuple] = {}
353+
self._parameter_dtypes: Dict[str, tuple] = {}
341354
# Data-parallel mode ("fsdp" | "ddp" | "hsdp") and the replicate-axis
342355
# group/degree used by DDP / HSDP gradient reduction.
343356
self._dp_mode = "fsdp"
@@ -496,6 +509,7 @@ def _setup_dp_config(
496509
self._parameter_groups = dict(kwargs.get("parameter_groups", {}))
497510
self._storage_prepared = kwargs.get("storage_prepared", False)
498511
self._logical_parameter_shapes = kwargs.get("logical_parameter_shapes", {})
512+
self._parameter_dtypes = kwargs.get("parameter_dtypes", {})
499513
if not self._expert_param_names.issubset(self._gradient_reductions):
500514
raise ValueError("Expert parameters require explicit gradient reduction policies")
501515
self._expert_fsdp_group_name = kwargs.get(
@@ -729,7 +743,9 @@ def _mark_sharded_param(
729743
# Newly inserted collective nodes have no shape-propagated ``val``.
730744
value = param if param is not None else node.meta.get("val")
731745
if isinstance(value, torch.Tensor) and isinstance(value.numel(), int):
732-
node.meta["fsdp_unsharded_nbytes"] = value.numel() * value.element_size()
746+
dtype = self._parameter_dtypes.get(fqn, (value.dtype, None))[0]
747+
element_size = torch.empty((), device="meta", dtype=dtype).element_size()
748+
node.meta["fsdp_unsharded_nbytes"] = value.numel() * element_size
733749
self._fsdp_modules.add(self._get_parent_module_fqn(fqn))
734750
return node
735751

@@ -984,10 +1000,14 @@ def _insert_all_gather_for_params(
9841000
# AllGather + immediate wait for correctness; FsdpOverlapPass may
9851001
# sink the wait past independent compute later.
9861002
with _insertion_point(graph, anchor, param_node):
1003+
dtype = self._parameter_dtypes.get(param_node.meta.get("param_name"), (None, None))[0]
1004+
gather_input = _cast_node(graph, param_node, dtype)
9871005
ag_node = graph.call_function(
9881006
_c10d_functional.all_gather_into_tensor,
989-
args=(param_node, degree, group_name),
1007+
args=(gather_input, degree, group_name),
9901008
)
1009+
if dtype is not None:
1010+
ag_node.meta["param_dtype"] = dtype
9911011
ag_node.meta["comm_type"] = "fsdp_all_gather"
9921012
ag_node.meta["comm_group"] = group_name
9931013
ag_node.meta["param_node"] = param_node.name
@@ -1036,7 +1056,14 @@ def _insert_all_gather_for_params(
10361056
replacement.meta["state_idx"] = param_node.meta.get("state_idx")
10371057

10381058
for user in list(param_node.users.keys()):
1039-
if user not in (ag_node, wait_node):
1059+
if user in (gather_input, ag_node, wait_node):
1060+
continue
1061+
if str(user.target) == "aten._to_copy.default" and user.kwargs == {"dtype": dtype}:
1062+
# The shard was cast before gather; do not retain a second
1063+
# full-size compute copy that defeats resharding.
1064+
user.replace_all_uses_with(replacement)
1065+
graph.erase_node(user)
1066+
else:
10401067
user.replace_input_with(param_node, replacement)
10411068

10421069
self._processed_params.add(param_node.name)
@@ -1149,6 +1176,8 @@ def _insert_grad_reduction( # pylint: disable=too-many-locals
11491176
group_name, degree, self._gradient_reductions.get(fqn, _DEFAULT_GRADIENT_REDUCTION)
11501177
)
11511178
with graph.inserting_before(output_node):
1179+
dtype = self._parameter_dtypes.get(fqn, (None, None))[1]
1180+
grad_node = _cast_node(graph, grad_node, dtype)
11521181
new_returned[i] = self._build_grad_reduction_nodes(
11531182
graph,
11541183
grad_node,
@@ -1449,10 +1478,14 @@ def _insert_backward_gather(
14491478
record.param_node.meta.get("param_name", "")
14501479
)
14511480
with graph.inserting_before(anchor):
1481+
dtype = self._parameter_dtypes.get(record.param_node.meta.get("param_name"), (None, None))[0]
1482+
gather_input = _cast_node(graph, record.param_node, dtype)
14521483
ag_node = graph.call_function(
14531484
_c10d_functional.all_gather_into_tensor,
1454-
args=(record.param_node, degree, group_name),
1485+
args=(gather_input, degree, group_name),
14551486
)
1487+
if dtype is not None:
1488+
ag_node.meta["param_dtype"] = dtype
14561489
ag_node.meta["comm_type"] = "fsdp_all_gather_backward"
14571490
ag_node.meta["comm_group"] = group_name
14581491
ag_node.meta["param_node"] = record.param_node.name

‎hyper_parallel/compile/tracer/graph_tracer.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
import warnings
6060
from collections.abc import Callable, Generator
6161
from collections import deque
62-
from contextlib import contextmanager
62+
from contextlib import contextmanager, nullcontext
6363
from dataclasses import dataclass
6464
from typing import Any, Dict, List
6565

@@ -443,6 +443,7 @@ def _make_fwd_bwd_fn(
443443
state_is_param: list[bool],
444444
state_spec: Any,
445445
user_inputs_spec: Any,
446+
mixed_precision: Any = None,
446447
) -> tuple[Callable, list[str]]:
447448
"""Build the joint callable and its named-loss output metadata."""
448449
num_state_inputs = len(state_fqns)
@@ -460,7 +461,9 @@ def _fwd_bwd_fn(*plain_args):
460461
)
461462

462463
with (
463-
_reparametrize_train_state(model, state_t["model"]),
464+
_reparametrize_train_state(
465+
model, mixed_precision.cast_state(state_t["model"]) if mixed_precision else state_t["model"]
466+
),
464467
_patch_engine_backward(),
465468
):
466469
out = train_fn(model, **user_args)
@@ -515,6 +518,7 @@ def trace_model_graph( # pylint: disable=too-many-locals
515518
dynamic: bool = False,
516519
dynamic_arg_dims: dict[str, int | list[int]] | None = None,
517520
state_shapes: dict[str, tuple] | None = None,
521+
mixed_precision: Any = None,
518522
) -> JointGraph:
519523
"""
520524
Trace model to generate complete forward + backward graph
@@ -529,6 +533,7 @@ def trace_model_graph( # pylint: disable=too-many-locals
529533
Providing a mapping enables selective symbolic tracing, even when dynamic is False.
530534
Parameters and buffers always remain static.
531535
state_shapes: Source-local logical parameter shapes before live FSDP slicing.
536+
mixed_precision: Resolved shared FSDP precision policy and module cast boundaries.
532537
533538
Returns:
534539
JointGraph: Joint forward-backward computation graph
@@ -577,7 +582,7 @@ def trace_model_graph( # pylint: disable=too-many-locals
577582
) + fake_args[num_state_inputs:]
578583

579584
_fwd_bwd_fn, loss_dict_keys_box = _make_fwd_bwd_fn( # pylint: disable=C0103
580-
model, train_fn, state_fqns, state_is_param, state_spec, user_inputs_spec
585+
model, train_fn, state_fqns, state_is_param, state_spec, user_inputs_spec, mixed_precision
581586
)
582587

583588
# make_fx only records ``nn_module_stack`` when the traced callable
@@ -592,6 +597,7 @@ def trace_model_graph( # pylint: disable=too-many-locals
592597
ctx = TracingContext(fake_mode)
593598
with (
594599
fake_mode,
600+
mixed_precision.module_casts() if mixed_precision else nullcontext(),
595601
tracing(ctx),
596602
preserve_node_meta(),
597603
_skip_nested_compile(),

‎hyper_parallel/distributed/_builder/fsdp_adapter.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,22 @@ def parameter_mesh_bindings(
202202
claimed.update(parameters)
203203
return bindings
204204

205+
def mixed_precision_bindings(
206+
self, model: ModuleClass, source_shard_info: SourceShardInfoByFQN | None,
207+
) -> tuple[fully_shard_utils.MixedPrecisionPolicy, tuple[ModuleClass, ...]]:
208+
"""Resolve the eager precision policy and unit boundaries without hooks.
209+
210+
Args:
211+
model: Final model before parameter storage is partitioned.
212+
source_shard_info: Source TP/EP layout returned by the planner.
213+
"""
214+
parameters = dict(model.named_parameters(remove_duplicate=False))
215+
custom = self._resolve_custom_params(parameters) if self.config.mix_precision.custom_params else None
216+
policy = self._build_mixed_precision_policy(custom)
217+
metadata = _build_source_shard_info_by_param(self, model, source_shard_info)
218+
units = self._find_wrap_modules(model, metadata)
219+
return policy, tuple(dict.fromkeys([model, *(unit.module for unit in units)]))
220+
205221
def _resolve_custom_params(
206222
self, parameters_by_fqn: Mapping[str, ParameterClass]
207223
) -> dict[ParameterClass, dict[str, Any]]:

0 commit comments

Comments
 (0)