diff --git a/hyper_parallel/compile/compiler.py b/hyper_parallel/compile/compiler.py index a624ff4bdb..96ec03ee7d 100644 --- a/hyper_parallel/compile/compiler.py +++ b/hyper_parallel/compile/compiler.py @@ -45,10 +45,12 @@ class and owns them. from hyper_parallel.core.dtensor.layout import Layout from hyper_parallel.core.dtensor.placement_types import Replicate, Shard, StridedShard from hyper_parallel.distributed._builder.fsdp_adapter import FSDP2Manager +from hyper_parallel.distributed.mesh import MeshContext from hyper_parallel.models.build_options import FSDP2Config from hyper_parallel.models.external_state import get_model_external_state from .graph_parallel_plan import GraphParallelPlan +from .mixed_precision import GraphMixedPrecision from .pass_config import ( GradientReduction, PassConfig, @@ -57,6 +59,7 @@ class and owns them. ) from .passes.pipeline import PassPipeline from .passes.activation_checkpoint import prepare_activation_checkpoint +from .passes.parallel.fsdp_pass import FSDPPass from .tracer.dynamic_shapes import normalize_dynamic_arg_dims from .tracer.graph_tracer import run_traced_graph, trace_model_graph @@ -135,6 +138,13 @@ def __init__( self._parameter_shapes = {} self._parameter_layouts = None self._layout_signature = None + self._storage_ready = False + self._storage_failed = False + self._storage_signature = None + self._sharded_parameter_names = frozenset() + self._prepared_pass_kwargs = None + self._mixed_precision = None + self._fp32_main_params = getattr(getattr(trainer_config, "optimizer", None), "fp32_main_params", False) self.device = device or ( torch.device("npu") if (hasattr(torch, "npu") and torch.npu.is_available()) @@ -170,21 +180,87 @@ def is_compiled(self) -> bool: @property def parameter_layouts(self) -> Optional[dict]: - """Final parameter-identity layouts, available after lazy compilation. + """Final parameter-identity layouts, available after storage preparation. Cache metadata only: gradients may be replaced on every optimizer step. Replacing a parameter invalidates the mapping, as does compiling again. """ - if self._joint_graph is None: + if not self._storage_ready: return None parameters = dict(self.model.named_parameters()) signature = tuple((id(param), tuple(param.shape), param.dtype, param.device) for param in parameters.values()) if signature != self._layout_signature: self._parameter_layouts = self._build_parameter_layouts(parameters) self._layout_signature = signature - self._joint_graph.graph_module.parameter_layouts = self._parameter_layouts + if self._joint_graph is not None: + self._joint_graph.graph_module.parameter_layouts = self._parameter_layouts return self._parameter_layouts + @property + def parameter_shapes(self) -> dict: + """Source-local logical shapes before FSDP padding and slicing.""" + return dict(self._parameter_shapes) + + @property + def sharded_parameter_names(self) -> frozenset: + """Canonical names of parameters physically partitioned by FSDP.""" + return self._sharded_parameter_names + + def prepare_model(self) -> None: + """Prepare local parameter storage once, before optimizer construction. + + Values may subsequently be loaded or updated in place. Changing a + parameter identity, shape, dtype or device requires a new model and + compiler. A failed preparation is terminal because storage may have + already been partitioned. + """ + external = get_model_external_state(self.model) + if external is not None and external.parameters: + raise ValueError("GraphCompiler does not support externally managed parameters yet") + if self._storage_failed: + raise RuntimeError("Graph storage preparation failed; rebuild the model and compiler") + signature = self._model_storage_signature() + if self._storage_ready: + if signature != self._storage_signature: + raise ValueError("Prepared graph parameter storage changed; rebuild the model and compiler") + return + try: + self._prepare_model_storage() + except Exception: + self._storage_failed = True + raise + self._storage_signature = self._model_storage_signature() + self._storage_ready = True + + def _model_storage_signature(self) -> tuple: + """Describe storage identity without retaining any parameter values.""" + return tuple((name, id(param), tuple(param.shape), param.dtype, param.device, param.requires_grad) + for name, param in self.model.named_parameters(remove_duplicate=False)) + + def _prepare_model_storage(self) -> None: + """Reuse FSDP ownership and slicing before creating optimizer state.""" + self._validate_ep_topology() + tp_size = getattr(self._mesh_context, "tp_size", 1) + if isinstance(tp_size, int) and tp_size > 1 and not self.source_shard_info: + raise ValueError("Graph TP requires source_shard_info from the sharding plan") + if self.pass_config.fsdp_enabled and dist.is_initialized(): + self._init_device_mesh(self._mesh_context) + self._bind_parameter_meshes() + precision = getattr(self._fsdp_config, "mix_precision", None) + if precision is not None and any((precision.param_dtype, precision.reduce_dtype, precision.output_dtype, + precision.custom_params, self._fp32_main_params)): + manager = FSDP2Manager(self._fsdp_config, self._mesh_context or MeshContext(), self._fp32_main_params) + policy, modules = manager.mixed_precision_bindings(self.model, self.source_shard_info) + self._mixed_precision = GraphMixedPrecision(self.model, policy, modules) + self._parameter_shapes = { + name: tuple(param.shape) for name, param in self.model.named_parameters(remove_duplicate=False) + } + self._prepared_pass_kwargs = self._build_pass_kwargs() + if self.pass_config.fsdp_enabled: + fsdp = FSDPPass(parallel_plan=self.parallel_plan) + self._sharded_parameter_names = fsdp.prepare_model(self.pass_config, **self._prepared_pass_kwargs) + self._layout_signature = None + def compile(self, **inputs: Any) -> None: """ Compile model into parallel graph @@ -209,20 +285,7 @@ def compile(self, **inputs: Any) -> None: if self._pytree_pre_hook is not None: self._pytree_pre_hook() - self._validate_ep_topology() - if ( - isinstance(getattr(self._mesh_context, "tp_size", None), int) - and self._mesh_context.tp_size > 1 - and not self.source_shard_info - ): - raise ValueError("Graph TP requires source_shard_info from the sharding plan") - - if self.pass_config.fsdp_enabled and dist.is_initialized(): - # Only build the FSDP mesh when distributed is actually up. - # ``FSDPPass`` early-returns when ``world_size == 1``, so a - # single-process run (no dist, or a single rank) compiles and - # runs as plain graph mode without sharding. - self._init_device_mesh(self._mesh_context) + self.prepare_model() # Activation checkpointing must resolve its scope and run the adapter's # runtime hook BEFORE tracing: the hook configures state the model's @@ -231,23 +294,20 @@ def compile(self, **inputs: Any) -> None: if self.pass_config.ac_mode != "off": ac_scope = prepare_activation_checkpoint(self.model, self.pass_config) - self._bind_parameter_meshes() - self._parameter_shapes = {name: tuple(param.shape) for name, param in self.model.named_parameters()} - self._layout_signature = None - - trace_kwargs = {} + trace_kwargs = {"state_shapes": self._parameter_shapes, "mixed_precision": self._mixed_precision} if self.dynamic or self.dynamic_arg_dims is not None: - trace_kwargs = { + trace_kwargs.update({ "dynamic": self.dynamic, "dynamic_arg_dims": self.dynamic_arg_dims, - } + }) joint_graph = trace_model_graph( self.model, self.train_fn, inputs, **trace_kwargs ) pipeline = PassPipeline.from_config(self.pass_config, self.parallel_plan) - pass_kwargs = self._build_pass_kwargs() + pass_kwargs = dict(self._prepared_pass_kwargs) + pass_kwargs["storage_prepared"] = True # Terminal backend compilation (InductorPass) sizes its regions from # the tracer's fake example inputs; every other pass ignores them. pass_kwargs["example_inputs"] = joint_graph.example_inputs @@ -334,7 +394,7 @@ def _build_parameter_layouts(self, parameters: dict) -> Optional[dict]: if self._fsdp_mesh is None and context is not None: # Plain graph mode still needs metadata, without creating collectives. self._init_mesh_from_context(context, self.pass_config.dp_mode, register_groups=False) - sharded = getattr(self._joint_graph.graph_module, "fsdp_sharded_param_names", frozenset()) + sharded = self._sharded_parameter_names layouts = {} for name, parameter in parameters.items(): if not parameter.requires_grad: @@ -812,11 +872,16 @@ def _build_pass_kwargs(self) -> dict: """ Build kwargs to pass to passes. """ - kwargs = {} + kwargs = { + "storage_prepared": self._storage_ready, + "logical_parameter_shapes": self._parameter_shapes, + } # Live model: partitioning passes (FSDPPass) physically shard # parameters in place, keeping the compiler FSDP-agnostic. kwargs["model"] = self.model + if self._mixed_precision is not None: + kwargs["parameter_dtypes"] = self._mixed_precision.parameter_dtypes if self.pass_config.fsdp_enabled: kwargs["fsdp_group_name"] = "fsdp" @@ -924,13 +989,18 @@ def _accumulate_grads(self, grads: List[torch.Tensor]) -> None: ) for param, grad in zip(trainable, grads): - if param.grad is None: + main_grad = self._mixed_precision is not None and self._mixed_precision.policy.apply_grad_on_fp32_main_grad + attribute = "main_grad" if main_grad else "grad" + accumulated = getattr(param, attribute, None) + if accumulated is None: # Match eager's gradient layout so optimizer kernels see the # parameter's strides rather than a backward matmul's view. - param.grad = torch.empty_like(param) - param.grad.copy_(grad) + dtype = torch.float32 if main_grad else param.dtype + accumulated = torch.empty_like(param, dtype=dtype) + accumulated.copy_(grad) + setattr(param, attribute, accumulated) else: - param.grad += grad + accumulated += grad @staticmethod def _trainable_params_in_state_order( diff --git a/hyper_parallel/compile/mixed_precision.py b/hyper_parallel/compile/mixed_precision.py new file mode 100644 index 0000000000..37dca06f3d --- /dev/null +++ b/hyper_parallel/compile/mixed_precision.py @@ -0,0 +1,77 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Lower the shared FSDP precision policy into the joint forward/backward graph.""" + +from contextlib import ExitStack, contextmanager +from functools import partial +from typing import Any, Iterator + +import torch +from torch import nn + +from hyper_parallel.core.fully_shard.utils import MixedPrecisionPolicy, apply_to_tensors, cast_fp_tensor + + +class GraphMixedPrecision: + """Keep storage, computation, communication and accumulation dtypes distinct.""" + + def __init__(self, model: nn.Module, policy: MixedPrecisionPolicy, modules: tuple[nn.Module, ...]) -> None: + """Bind the shared policy to source parameter identities and unit boundaries. + + Args: + model: Final model before graph FSDP slices local storage. + policy: Resolved FSDP policy, including Parameter-specific overrides. + modules: The same module boundaries selected by eager FSDP. + """ + self.policy = policy + self.modules = modules + self.parameter_dtypes = {} + for name, parameter in model.named_parameters(remove_duplicate=False): + custom = (policy.custom_params or {}).get(id(parameter), {}) + compute = custom.get("param_dtype", policy.param_dtype) or parameter.dtype + reduce = custom.get("reduce_dtype", policy.reduce_dtype) or compute + self.parameter_dtypes[name] = (compute, reduce) + + @contextmanager + def module_casts(self) -> Iterator[None]: + """Record eager-compatible module input/output casts during capture only.""" + with ExitStack() as stack: + for module in self.modules: + if self.policy.cast_forward_inputs and self.policy.param_dtype is not None: + handle = module.register_forward_pre_hook(self._cast_inputs, with_kwargs=True) + stack.callback(handle.remove) + if self.policy.output_dtype is not None: + handle = module.register_forward_hook(self._cast_outputs) + stack.callback(handle.remove) + yield + + def cast_state(self, state: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + """Create compute views whose gradients stay in compute dtype until reduction.""" + return { + name: cast_fp_tensor(self.parameter_dtypes[name][0], value) + if name in self.parameter_dtypes else value + for name, value in state.items() + } + + def _cast_inputs(self, module: nn.Module, args: tuple, kwargs: dict) -> tuple: + """Mirror the eager FSDP pre-forward cast for floating input leaves.""" + del module + cast = partial(cast_fp_tensor, self.policy.param_dtype) + return apply_to_tensors(cast, args), apply_to_tensors(cast, kwargs) + + def _cast_outputs(self, module: nn.Module, args: tuple, outputs: Any) -> Any: + """Mirror the eager FSDP post-forward output cast.""" + del module, args + return apply_to_tensors(partial(cast_fp_tensor, self.policy.output_dtype), outputs) diff --git a/hyper_parallel/compile/passes/parallel/fsdp_pass.py b/hyper_parallel/compile/passes/parallel/fsdp_pass.py index 9fd08182e8..a97bfc64cc 100644 --- a/hyper_parallel/compile/passes/parallel/fsdp_pass.py +++ b/hyper_parallel/compile/passes/parallel/fsdp_pass.py @@ -138,7 +138,7 @@ def _annotate_ag_spec(node: fx.Node, degree: int, params: dict) -> None: node.meta["fsdp_bucket_spec"] = { "input_shape": shape, "output_shape": (shape[0] * degree, *shape[1:]), - "dtype": param.dtype, + "dtype": node.meta.get("param_dtype", param.dtype), "device": str(param.device), } @@ -168,6 +168,18 @@ def annotate_fsdp_bucket_specs(graph_module: fx.GraphModule, model: nn.Module) - _annotate_ag_spec(node, degree, params) +def _cast_node(graph: fx.Graph, node: fx.Node, dtype: Optional[torch.dtype]) -> fx.Node: + """Insert a precision conversion while retaining fake dtype/shape metadata.""" + value = node.meta.get("val") + if dtype is None or (value is not None and value.dtype == dtype): + return node + cast = graph.call_method("to", args=(node,), kwargs={"dtype": dtype}) + cast.meta["autograd_backward"] = node.meta.get("autograd_backward", False) + if isinstance(value, FakeTensor): + cast.meta["val"] = value.to(dtype) + return cast + + def _free_tensor_storage(tensor: torch.Tensor) -> None: """Release ``tensor``'s storage in place (FSDP reshard free step). @@ -336,6 +348,9 @@ def __init__( self._gradient_reductions: Dict[str, GradientReduction] = {} self._source_gradient_groups: Dict[str, str] = {} self._parameter_groups: Dict[str, tuple[str, int]] = {} + self._storage_prepared = False + self._logical_parameter_shapes: Dict[str, tuple] = {} + self._parameter_dtypes: Dict[str, tuple] = {} # Data-parallel mode ("fsdp" | "ddp" | "hsdp") and the replicate-axis # group/degree used by DDP / HSDP gradient reduction. self._dp_mode = "fsdp" @@ -352,6 +367,29 @@ def __init__( # live-shard steps so they are pad-sharded like eager ``fully_shard``. self._padded_params: Dict[str, _PaddedParam] = {} + def prepare_model(self, pass_config: PassConfig, **kwargs: Any) -> frozenset[str]: + """Partition live storage using the same classification as graph lowering. + + This metadata-only graph does not execute model code or collectives. + Call once before building optimizer parameters or loading a checkpoint. + """ + if not dist.is_initialized() or dist.get_world_size() == 1: + return frozenset() + model = self._setup_dp_config(pass_config, kwargs) + self._reset_run_state() + graph = fx.Graph() + parameters = list(model.named_parameters()) + for index, _ in enumerate(parameters): + graph.placeholder(f"param_{index}") + graph.output(()) + module = fx.GraphModule({}, graph) + names = [name for name, _ in parameters] + classification = self._classify_dp_params(module, names, len(names), [True] * len(names), model) + if self._dp_mode == "ddp": + return frozenset() + self._shard_live_model_params(model) + return frozenset(node.meta["param_name"] for node in classification.sharded_nodes) + def run( self, graph_module: fx.GraphModule, @@ -469,6 +507,9 @@ def _setup_dp_config( self._gradient_reductions = dict(kwargs.get("gradient_reductions", {})) self._source_gradient_groups = dict(kwargs.get("source_gradient_groups", {})) self._parameter_groups = dict(kwargs.get("parameter_groups", {})) + self._storage_prepared = kwargs.get("storage_prepared", False) + self._logical_parameter_shapes = kwargs.get("logical_parameter_shapes", {}) + self._parameter_dtypes = kwargs.get("parameter_dtypes", {}) if not self._expert_param_names.issubset(self._gradient_reductions): raise ValueError("Expert parameters require explicit gradient reduction policies") self._expert_fsdp_group_name = kwargs.get( @@ -565,7 +606,8 @@ def _apply_dp_transforms( # index in the FSDP group). ``model.parameters()`` then yields the # shards, so the trainer's optimizer / grad accumulation stay # DP-agnostic; the graph re-gathers each step. - self._shard_live_model_params(model) + if not self._storage_prepared: + self._shard_live_model_params(model) annotate_fsdp_bucket_specs(graph_module, model) return graph_module @@ -631,6 +673,9 @@ def _classify_dp_params( if skip: continue + if fqn in self._logical_parameter_shapes: + param = torch.empty(self._logical_parameter_shapes[fqn], dtype=param.dtype, device="meta") + _, degree = self._param_group(fqn) if fqn in self._expert_param_names and degree == 1: continue @@ -698,7 +743,9 @@ def _mark_sharded_param( # Newly inserted collective nodes have no shape-propagated ``val``. value = param if param is not None else node.meta.get("val") if isinstance(value, torch.Tensor) and isinstance(value.numel(), int): - node.meta["fsdp_unsharded_nbytes"] = value.numel() * value.element_size() + dtype = self._parameter_dtypes.get(fqn, (value.dtype, None))[0] + element_size = torch.empty((), device="meta", dtype=dtype).element_size() + node.meta["fsdp_unsharded_nbytes"] = value.numel() * element_size self._fsdp_modules.add(self._get_parent_module_fqn(fqn)) return node @@ -953,10 +1000,14 @@ def _insert_all_gather_for_params( # AllGather + immediate wait for correctness; FsdpOverlapPass may # sink the wait past independent compute later. with _insertion_point(graph, anchor, param_node): + dtype = self._parameter_dtypes.get(param_node.meta.get("param_name"), (None, None))[0] + gather_input = _cast_node(graph, param_node, dtype) ag_node = graph.call_function( _c10d_functional.all_gather_into_tensor, - args=(param_node, degree, group_name), + args=(gather_input, degree, group_name), ) + if dtype is not None: + ag_node.meta["param_dtype"] = dtype ag_node.meta["comm_type"] = "fsdp_all_gather" ag_node.meta["comm_group"] = group_name ag_node.meta["param_node"] = param_node.name @@ -1005,7 +1056,14 @@ def _insert_all_gather_for_params( replacement.meta["state_idx"] = param_node.meta.get("state_idx") for user in list(param_node.users.keys()): - if user not in (ag_node, wait_node): + if user in (gather_input, ag_node, wait_node): + continue + if str(user.target) == "aten._to_copy.default" and user.kwargs == {"dtype": dtype}: + # The shard was cast before gather; do not retain a second + # full-size compute copy that defeats resharding. + user.replace_all_uses_with(replacement) + graph.erase_node(user) + else: user.replace_input_with(param_node, replacement) self._processed_params.add(param_node.name) @@ -1118,6 +1176,8 @@ def _insert_grad_reduction( # pylint: disable=too-many-locals group_name, degree, self._gradient_reductions.get(fqn, _DEFAULT_GRADIENT_REDUCTION) ) with graph.inserting_before(output_node): + dtype = self._parameter_dtypes.get(fqn, (None, None))[1] + grad_node = _cast_node(graph, grad_node, dtype) new_returned[i] = self._build_grad_reduction_nodes( graph, grad_node, @@ -1418,10 +1478,14 @@ def _insert_backward_gather( record.param_node.meta.get("param_name", "") ) with graph.inserting_before(anchor): + dtype = self._parameter_dtypes.get(record.param_node.meta.get("param_name"), (None, None))[0] + gather_input = _cast_node(graph, record.param_node, dtype) ag_node = graph.call_function( _c10d_functional.all_gather_into_tensor, - args=(record.param_node, degree, group_name), + args=(gather_input, degree, group_name), ) + if dtype is not None: + ag_node.meta["param_dtype"] = dtype ag_node.meta["comm_type"] = "fsdp_all_gather_backward" ag_node.meta["comm_group"] = group_name ag_node.meta["param_node"] = record.param_node.name diff --git a/hyper_parallel/compile/tracer/graph_tracer.py b/hyper_parallel/compile/tracer/graph_tracer.py index ea9a8f4627..fe29269121 100644 --- a/hyper_parallel/compile/tracer/graph_tracer.py +++ b/hyper_parallel/compile/tracer/graph_tracer.py @@ -59,7 +59,7 @@ import warnings from collections.abc import Callable, Generator from collections import deque -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from dataclasses import dataclass from typing import Any, Dict, List @@ -443,6 +443,7 @@ def _make_fwd_bwd_fn( state_is_param: list[bool], state_spec: Any, user_inputs_spec: Any, + mixed_precision: Any = None, ) -> tuple[Callable, list[str]]: """Build the joint callable and its named-loss output metadata.""" num_state_inputs = len(state_fqns) @@ -458,9 +459,10 @@ def _fwd_bwd_fn(*plain_args): user_args = torch.utils._pytree.tree_unflatten( list(user_wrapped), user_inputs_spec ) + model_state = mixed_precision.cast_state(state_t["model"]) if mixed_precision else state_t["model"] with ( - _reparametrize_train_state(model, state_t["model"]), + _reparametrize_train_state(model, model_state), _patch_engine_backward(), ): out = train_fn(model, **user_args) @@ -483,12 +485,12 @@ def _fwd_bwd_fn(*plain_args): else: loss, loss_dict_keys, loss_dict_values = out, [], [] - # Read params from state_t["model"] so the exact trace-time - # tensors (not the live module's) drive autograd.grad. + # Like eager FSDP, differentiate compute parameters. Casting back + # to storage here could round gradients before their reduction. params = [] for idx, fqn in enumerate(state_fqns): if state_is_param[idx]: - param = state_t["model"][fqn] + param = model_state[fqn] if param.requires_grad: params.append(param) @@ -514,6 +516,8 @@ def trace_model_graph( # pylint: disable=too-many-locals *, dynamic: bool = False, dynamic_arg_dims: dict[str, int | list[int]] | None = None, + state_shapes: dict[str, tuple] | None = None, + mixed_precision: Any = None, ) -> JointGraph: """ Trace model to generate complete forward + backward graph @@ -527,6 +531,8 @@ def trace_model_graph( # pylint: disable=too-many-locals dynamic_arg_dims: Dotted input paths to dynamic dimensions (supports negative indices). Providing a mapping enables selective symbolic tracing, even when dynamic is False. Parameters and buffers always remain static. + state_shapes: Source-local logical parameter shapes before live FSDP slicing. + mixed_precision: Resolved shared FSDP precision policy and module cast boundaries. Returns: JointGraph: Joint forward-backward computation graph @@ -566,9 +572,16 @@ def trace_model_graph( # pylint: disable=too-many-locals fake_mode = FakeTensorMode(allow_non_fake_inputs=True, shape_env=ShapeEnv(), static_shapes=True) fake_args = tuple(_fakeify_input(fake_mode, value) for value in full_args) num_state_inputs = len(state_flat) + if state_shapes: + with fake_mode: + fake_args = tuple( + value.new_empty(state_shapes[name], requires_grad=value.requires_grad) + if name in state_shapes and tuple(value.shape) != state_shapes[name] else value + for name, value in zip(state_fqns, fake_args[:num_state_inputs]) + ) + fake_args[num_state_inputs:] _fwd_bwd_fn, loss_dict_keys_box = _make_fwd_bwd_fn( # pylint: disable=C0103 - model, train_fn, state_fqns, state_is_param, state_spec, user_inputs_spec + model, train_fn, state_fqns, state_is_param, state_spec, user_inputs_spec, mixed_precision ) # make_fx only records ``nn_module_stack`` when the traced callable @@ -583,6 +596,7 @@ def trace_model_graph( # pylint: disable=too-many-locals ctx = TracingContext(fake_mode) with ( fake_mode, + mixed_precision.module_casts() if mixed_precision else nullcontext(), tracing(ctx), preserve_node_meta(), _skip_nested_compile(), @@ -631,7 +645,7 @@ def trace_model_graph( # pylint: disable=too-many-locals param_names = [name for name, _ in model.named_parameters() if _.requires_grad] param_shapes = { - name: tuple(param.shape) for name, param in model.named_parameters() + name: (state_shapes or {}).get(name, tuple(param.shape)) for name, param in model.named_parameters() } num_layers = 0 diff --git a/hyper_parallel/distributed/_builder/fsdp_adapter.py b/hyper_parallel/distributed/_builder/fsdp_adapter.py index 05ff1aec52..0c3f288846 100644 --- a/hyper_parallel/distributed/_builder/fsdp_adapter.py +++ b/hyper_parallel/distributed/_builder/fsdp_adapter.py @@ -202,6 +202,22 @@ def parameter_mesh_bindings( claimed.update(parameters) return bindings + def mixed_precision_bindings( + self, model: ModuleClass, source_shard_info: SourceShardInfoByFQN | None, + ) -> tuple[fully_shard_utils.MixedPrecisionPolicy, tuple[ModuleClass, ...]]: + """Resolve the eager precision policy and unit boundaries without hooks. + + Args: + model: Final model before parameter storage is partitioned. + source_shard_info: Source TP/EP layout returned by the planner. + """ + parameters = dict(model.named_parameters(remove_duplicate=False)) + custom = self._resolve_custom_params(parameters) if self.config.mix_precision.custom_params else None + policy = self._build_mixed_precision_policy(custom) + metadata = _build_source_shard_info_by_param(self, model, source_shard_info) + units = self._find_wrap_modules(model, metadata) + return policy, tuple(dict.fromkeys([model, *(unit.module for unit in units)])) + def _resolve_custom_params( self, parameters_by_fqn: Mapping[str, ParameterClass] ) -> dict[ParameterClass, dict[str, Any]]: diff --git a/hyper_parallel/trainer/base.py b/hyper_parallel/trainer/base.py index 7c945d42aa..010b4cdfb7 100644 --- a/hyper_parallel/trainer/base.py +++ b/hyper_parallel/trainer/base.py @@ -615,6 +615,7 @@ def _build_graph_compiler(self) -> None: mesh_context=self.mesh, source_shard_info=getattr(self.model, "_source_shard_info", None), ) + self.graph_compiler.prepare_model() def _graph_train_fn( self, diff --git a/hyper_parallel/trainer/text_trainer.py b/hyper_parallel/trainer/text_trainer.py index da0be2830e..de304e0586 100644 --- a/hyper_parallel/trainer/text_trainer.py +++ b/hyper_parallel/trainer/text_trainer.py @@ -70,11 +70,11 @@ def __init__(self, config: TrainerConfig) -> None: self.base.attach_model_integration_data_pipeline() self.base._compute_train_iters() + self.base._build_training_context() + self.base._build_graph_compiler() self.base._build_optimizer() self.base.model_integration.attach_optimizer(self.base.optimizer) self.base._build_lr_scheduler() - self.base._build_training_context() - self.base._build_graph_compiler() self.base._init_callbacks() def _build_model_assets(self) -> None: diff --git a/tests/ut/compile/test_compiler.py b/tests/ut/compile/test_compiler.py index ea5ae69b50..39cbd7122b 100644 --- a/tests/ut/compile/test_compiler.py +++ b/tests/ut/compile/test_compiler.py @@ -830,12 +830,12 @@ def test_dense_and_expert_layouts_without_fsdp_config(self): model.experts.local_expert_count = 2 context = MeshContext(fsdp_non_moe_mesh=dense, fsdp_moe_mesh=expert, dp_size=8, dp_shard_size=8, ep_size=ep, edp_shard_size=8 // ep) - compiler = GraphCompiler(model, _mse_train_fn, mesh_context=context) + compiler = GraphCompiler(model, _mse_train_fn, PassConfig(enable_overlap=False), mesh_context=context) with patch.object(compiler, "_register_mesh_group"): compiler._init_device_mesh(context) compiler._bind_parameter_meshes() self.assertEqual(compiler._parameter_meshes, {}) - compiler._joint_graph = SimpleNamespace(graph_module=SimpleNamespace()) + compiler.prepare_model() layouts = compiler.parameter_layouts self.assertEqual(layouts[model.dense].mesh.mesh_dim_names, ("fsdp_shard",)) expert_layout = layouts[model.experts.weight] @@ -853,7 +853,7 @@ def test_legacy_named_world_metadata_flattens_dp_and_cp(self): context = MeshContext(device_mesh=mesh, dp_replicate_size=replicate, dp_shard_size=2) model = _make_model() compiler = GraphCompiler(model, _mse_train_fn, PassConfig(fsdp_enabled=False), mesh_context=context) - compiler._joint_graph = SimpleNamespace(graph_module=SimpleNamespace()) + compiler.prepare_model() with patch.object(compiler, "_register_mesh_group") as register: layout = compiler.parameter_layouts[model.weight] register.assert_not_called() @@ -876,7 +876,7 @@ def test_hsdp_and_named_world_layout_meshes(self): compiler._init_device_mesh(context) expected = ("dp",) if world_only else ("fsdp_replicate", "fsdp_shard") self.assertEqual(compiler._fsdp_mesh.mesh_dim_names, expected) - compiler._joint_graph = SimpleNamespace(graph_module=SimpleNamespace()) + compiler.prepare_model() layout_names = expected if world_only else (*expected, "tp") self.assertEqual(compiler.parameter_layouts[model.weight].mesh.mesh_dim_names, layout_names) diff --git a/tests/ut/compile/test_mixed_precision.py b/tests/ut/compile/test_mixed_precision.py new file mode 100644 index 0000000000..6c204755ee --- /dev/null +++ b/tests/ut/compile/test_mixed_precision.py @@ -0,0 +1,145 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Precision policy preserves storage and the complete optimizer update chain.""" + +from copy import deepcopy +from types import SimpleNamespace +from unittest import TestCase + +import torch +from torch import nn + +from hyper_parallel.compile.compiler import GraphCompiler +from hyper_parallel.compile.pass_config import PassConfig +from hyper_parallel.models.build_options import FSDP2Config, FSDP2MixedPrecisionConfig + + +def _loss(model, *, x): + """Keep loss accumulation in FP32 while linear computation is BF16.""" + return model(x).float().square().mean() + + +def _compiler(model, *, fp32_main_params=False, **policy): + """Use the same YAML policy resolver as TextTrainer.""" + config = SimpleNamespace(fsdp_config=FSDP2Config(wrapper_class=["_Block"], + mix_precision=FSDP2MixedPrecisionConfig(**policy)), + optimizer=SimpleNamespace(fp32_main_params=fp32_main_params)) + return GraphCompiler(model, _loss, PassConfig(fsdp_enabled=False), trainer_config=config, + device=torch.device("cpu")) + + +class _Block(nn.Linear): + """Explicitly selected FSDP child unit.""" + + +class _Float32(nn.Module): + """Model-owned explicit FP32 region, as used by routers and normalization.""" + + def forward(self, value: torch.Tensor) -> torch.Tensor: + """Convert the activation before the FP32 projection.""" + return value.float() + + +class TestGraphMixedPrecision(TestCase): + """The default input cast and per-parameter precision are graph operations.""" + + def test_compute_gradients_do_not_round_through_storage_dtype(self): + """Match eager's compute-parameter gradients before communication and accumulation.""" + for storage, compute in ((torch.float32, torch.bfloat16), (torch.bfloat16, torch.float32)): + with self.subTest(storage=storage, compute=compute): + model = nn.Sequential(_Block(2, 2, bias=False, dtype=storage)) + with torch.no_grad(): + model[0].weight.fill_(1) + reference = deepcopy(model).to(compute) + x = torch.full((2, 2), 1 + 2 ** -9) + expected_loss = _loss(reference, x=x.to(compute)) + expected_loss.backward() + compiler = _compiler(model, fp32_main_params=True, + param_dtype=str(compute).removeprefix("torch."), reduce_dtype="float32") + actual_loss, _ = compiler.forward_backward(x=x) + torch.testing.assert_close(actual_loss, expected_loss, rtol=0, atol=0) + output = next(node for node in compiler._joint_graph.graph_module.graph.nodes if node.op == "output") + gradient = output.args[0][1] + self.assertEqual(gradient.meta["val"].dtype, compute) + self.assertEqual(model[0].weight.dtype, storage) + self.assertIsNone(model[0].weight.grad) + torch.testing.assert_close(model[0].weight.main_grad, reference[0].weight.grad.float(), rtol=0, atol=0) + + def test_fp32_storage_bf16_compute_matches_ten_adamw_updates(self): + """Compare losses, gradients, weights and moments against explicit eager casts.""" + torch.manual_seed(17) + model = nn.Sequential(_Block(4, 4)) + reference = deepcopy(model) + compiler = _compiler(model, param_dtype="bfloat16", reduce_dtype="float32") + compiler.prepare_model() + optimizer = torch.optim.AdamW(model.parameters(), lr=0.001, foreach=False) + expected_optimizer = torch.optim.AdamW(reference.parameters(), lr=0.001, foreach=False) + for step in range(10): + x = torch.arange(12, dtype=torch.float32).reshape(3, 4) / 16 + step / 64 + computation = deepcopy(reference).bfloat16() + expected = computation(x.bfloat16()) + expected_loss = expected.float().square().mean() + expected_loss.backward() + for master, parameter in zip(reference.parameters(), computation.parameters()): + master.grad = parameter.grad.float() + loss, _ = compiler.forward_backward(x=x) + torch.testing.assert_close(loss, expected_loss, rtol=0, atol=0) + for actual, expected_parameter in zip(model.parameters(), reference.parameters()): + self.assertEqual(actual.dtype, torch.float32) + torch.testing.assert_close(actual.grad, expected_parameter.grad, rtol=0, atol=0) + optimizer.step() + expected_optimizer.step() + for actual, expected_parameter in zip(model.parameters(), reference.parameters()): + torch.testing.assert_close(actual, expected_parameter, rtol=0, atol=0) + for key in ("exp_avg", "exp_avg_sq", "step"): + torch.testing.assert_close(optimizer.state[actual][key], + expected_optimizer.state[expected_parameter][key], rtol=0, atol=0) + optimizer.zero_grad() + expected_optimizer.zero_grad() + self.assertFalse(model._forward_pre_hooks) + self.assertFalse(model._forward_hooks) + self.assertTrue(any(node.target == torch.ops.aten._to_copy.default + for node in compiler._joint_graph.graph_module.graph.nodes)) + + def test_custom_parameter_dtype_and_module_output_cast(self): + """Pattern overrides preserve an explicit FP32 compute region.""" + model = nn.Sequential(_Block(4, 4), _Float32(), nn.Linear(4, 4)) + compiler = _compiler(model, param_dtype="bfloat16", reduce_dtype="float32", output_dtype="float32", + custom_params={"2.*": {"param_dtype": "float32"}}) + compiler.prepare_model() + precision = compiler._mixed_precision + self.assertEqual(precision.parameter_dtypes["0.weight"], (torch.bfloat16, torch.float32)) + self.assertEqual(precision.parameter_dtypes["2.weight"], (torch.float32, torch.float32)) + state = dict(model.named_parameters()) + cast = precision.cast_state(state) + self.assertEqual(cast["0.weight"].dtype, torch.bfloat16) + self.assertIs(cast["2.weight"], model[2].weight) + with precision.module_casts(): + # Module output casts use the same floating-leaf helper as eager. + result = torch.func.functional_call(model, cast, (torch.ones(2, 4),)) + self.assertEqual(result.dtype, torch.float32) + + def test_fp32_main_gradient_keeps_reduction_precision(self): + """A BF16 model can accumulate FP32 gradients without assigning them to .grad.""" + model = nn.Sequential(_Block(4, 4, dtype=torch.bfloat16)) + compiler = _compiler(model, fp32_main_params=True, param_dtype="bfloat16", reduce_dtype="float32") + compiler.prepare_model() + compiler.forward_backward(x=torch.ones(2, 4)) + first = [parameter.main_grad.clone() for parameter in model.parameters()] + compiler.forward_backward(x=torch.ones(2, 4)) + for parameter, gradient in zip(model.parameters(), first): + self.assertIsNone(parameter.grad) + self.assertEqual(parameter.main_grad.dtype, torch.float32) + torch.testing.assert_close(parameter.main_grad, gradient * 2, rtol=0, atol=0) diff --git a/tests/ut/compile/test_ragged_ep.py b/tests/ut/compile/test_ragged_ep.py index b1ef529a59..067f611f64 100644 --- a/tests/ut/compile/test_ragged_ep.py +++ b/tests/ut/compile/test_ragged_ep.py @@ -76,6 +76,7 @@ def test_compile_resolves_expert_parameter_aliases(self): with ( patch("hyper_parallel.compile.compiler.dist.is_initialized", return_value=True), patch("hyper_parallel.compile.compiler._register_process_group") as register, + patch("hyper_parallel.compile.compiler.FSDPPass.prepare_model", return_value=frozenset()), patch("hyper_parallel.compile.compiler.trace_model_graph", return_value=joint), patch("hyper_parallel.compile.compiler.PassPipeline.from_config") as pipeline, ): diff --git a/tests/ut/compile/test_storage_preparation.py b/tests/ut/compile/test_storage_preparation.py new file mode 100644 index 0000000000..64598490eb --- /dev/null +++ b/tests/ut/compile/test_storage_preparation.py @@ -0,0 +1,156 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Storage preparation is independent of lazy graph capture and optimizer state.""" + +from contextlib import contextmanager +from unittest import TestCase +from unittest.mock import MagicMock, patch + +import torch +from torch import nn + +from hyper_parallel.compile.compiler import GraphCompiler +from hyper_parallel.compile.pass_config import PassConfig +from hyper_parallel.compile.passes.parallel.fsdp_pass import FSDPPass +from hyper_parallel.compile.tracer.graph_tracer import trace_model_graph +from hyper_parallel.components.optim.mixed_precision_optimizer import Float16OptimizerWithFloat16Params +from hyper_parallel.core.optimizer.optimizer import ChainedOptimizer + + +class _Model(nn.Module): + """Include aliases, padding, a scalar, frozen state and a buffer.""" + + def __init__(self, dtype: torch.dtype = torch.float32) -> None: + """Build the state categories that preparation must distinguish.""" + super().__init__() + self.weight = nn.Parameter(torch.arange(15, dtype=dtype).reshape(5, 3) / 16) + self.alias = self.weight + self.scale = nn.Parameter(torch.tensor(0.5, dtype=dtype)) + self.frozen = nn.Parameter(torch.ones(5, dtype=dtype), requires_grad=False) + self.register_buffer("offset", torch.ones(5, dtype=dtype)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Read full logical tensors during forward.""" + return (x @ self.weight.T + self.offset) * self.scale + self.frozen + + +def _loss(model, *, x): + """Use a scalar loss without labels.""" + return model(x).square().mean() + + +@contextmanager +def _distributed(compiler): + """Mock transport setup; slicing, tracing and graph lowering remain real.""" + transport = MagicMock() + transport.is_initialized.return_value = True + transport.get_world_size.return_value = 2 + transport.get_rank.return_value = 0 + with ( + patch("hyper_parallel.compile.compiler.dist", transport), + patch("hyper_parallel.compile.passes.parallel.fsdp_pass.dist", transport), + patch("hyper_parallel.compile.passes.parallel.fsdp_pass._resolve_process_group"), + patch.object(compiler, "_init_device_mesh"), + ): + yield + + +class TestStoragePreparation(TestCase): + """Preparation owns physical slicing; the graph only lowers logical state.""" + + def test_prepare_once_preserves_parameter_identity_and_nonsharded_state(self): + """A second preparation neither re-shards nor changes loaded values.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + original = torch.detach(model.weight).clone() + identity = id(model.weight) + with _distributed(compiler): + compiler.prepare_model() + self.assertEqual(compiler.parameter_shapes["weight"], (5, 3)) + self.assertEqual(compiler.sharded_parameter_names, {"weight"}) + self.assertEqual(model.weight.shape, (3, 3)) + torch.testing.assert_close(model.weight, original[:3], rtol=0, atol=0) + with torch.no_grad(): + model.weight.add_(1) + compiler.prepare_model() + self.assertEqual(id(model.weight), identity) + self.assertIs(model.alias, model.weight) + self.assertEqual(model.scale.shape, ()) + self.assertEqual(model.frozen.shape, (5,)) + self.assertEqual(model.offset.shape, (5,)) + self.assertFalse(compiler.is_compiled) + torch.testing.assert_close(model.weight, original[:3] + 1, rtol=0, atol=0) + + def test_logical_trace_and_pass_do_not_cut_prepared_storage_again(self): + """Trace sees the original five rows while FSDP accepts the three-row shard.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler): + compiler.prepare_model() + shard = torch.detach(model.weight).clone() + for dynamic in (False, True): + with self.subTest(dynamic=dynamic): + joint = trace_model_graph(model, _loss, {"x": torch.ones(2, 3)}, + state_shapes=compiler.parameter_shapes, dynamic=dynamic) + self.assertEqual(joint.param_shapes["weight"], (5, 3)) + self.assertEqual(joint.example_inputs[0].shape, (5, 3)) + FSDPPass().run(joint.graph_module, compiler.pass_config, **compiler._build_pass_kwargs()) + self.assertEqual(joint.graph_module.fsdp_sharded_param_names, {"weight"}) + torch.testing.assert_close(model.weight, shard, rtol=0, atol=0) + + def test_fp32_main_parameters_are_built_from_prepared_shards(self): + """BF16 storage does not leave full-size FP32 optimizer copies behind.""" + model = _Model(torch.bfloat16) + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler): + compiler.prepare_model() + raw = torch.optim.AdamW(model.parameters(), lr=0.01) + optimizer = Float16OptimizerWithFloat16Params(ChainedOptimizer(model, {"adamw": raw}), model) + for parameter, main in optimizer.optimizer_param_by_model_param.items(): + self.assertEqual(parameter.shape, main.shape) + self.assertEqual(main.dtype, torch.float32) + self.assertEqual(model.weight.main_param.shape, (3, 3)) + + def test_failed_preparation_cannot_repeat_partial_slicing(self): + """A partially prepared model must be rebuilt rather than silently reused.""" + compiler = GraphCompiler(_Model(), _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler), patch.object(FSDPPass, "prepare_model", side_effect=ValueError("slice failed")): + with self.assertRaisesRegex(ValueError, "slice failed"): + compiler.prepare_model() + with self.assertRaisesRegex(RuntimeError, "rebuild the model"): + compiler.prepare_model() + + def test_trace_failure_keeps_prepared_storage_and_logical_shapes(self): + """A retry after capture failure cannot re-shard the live model.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler): + compiler.prepare_model() + shard = torch.detach(model.weight).clone() + with patch("hyper_parallel.compile.compiler.trace_model_graph", side_effect=ValueError("trace failed")): + for _ in range(2): + with self.assertRaisesRegex(ValueError, "trace failed"): + compiler.compile(x=torch.ones(2, 3)) + torch.testing.assert_close(model.weight, shard, rtol=0, atol=0) + self.assertEqual(compiler.parameter_shapes["weight"], (5, 3)) + + def test_replacing_prepared_parameter_requires_a_new_model(self): + """Parameter identity is part of the prepared-storage contract.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_enabled=False)) + compiler.prepare_model() + model.weight = nn.Parameter(torch.detach(model.weight).clone()) + with self.assertRaisesRegex(ValueError, "storage changed"): + compiler.prepare_model()