diff --git a/hyper_parallel/compile/compiler.py b/hyper_parallel/compile/compiler.py index a624ff4bdb..690bebb706 100644 --- a/hyper_parallel/compile/compiler.py +++ b/hyper_parallel/compile/compiler.py @@ -45,10 +45,12 @@ class and owns them. from hyper_parallel.core.dtensor.layout import Layout from hyper_parallel.core.dtensor.placement_types import Replicate, Shard, StridedShard from hyper_parallel.distributed._builder.fsdp_adapter import FSDP2Manager +from hyper_parallel.distributed.mesh import MeshContext from hyper_parallel.models.build_options import FSDP2Config from hyper_parallel.models.external_state import get_model_external_state from .graph_parallel_plan import GraphParallelPlan +from .mixed_precision import GraphMixedPrecision from .pass_config import ( GradientReduction, PassConfig, @@ -57,6 +59,7 @@ class and owns them. ) from .passes.pipeline import PassPipeline from .passes.activation_checkpoint import prepare_activation_checkpoint +from .passes.parallel.fsdp_pass import FSDPPass from .tracer.dynamic_shapes import normalize_dynamic_arg_dims from .tracer.graph_tracer import run_traced_graph, trace_model_graph @@ -124,6 +127,7 @@ def __init__( self.pass_config = self._resolve_pass_config( pass_config=pass_config, trainer_config=trainer_config, + mesh_context=mesh_context, ) self._mesh_context = mesh_context self.source_shard_info = source_shard_info @@ -132,9 +136,17 @@ def __init__( self._fsdp_config = getattr(trainer_config, 'fsdp_config', None) self._parameter_meshes = {} self._parameter_groups = {} + self._replicate_param_names = frozenset() self._parameter_shapes = {} self._parameter_layouts = None self._layout_signature = None + self._storage_ready = False + self._storage_failed = False + self._storage_signature = None + self._sharded_parameter_names = frozenset() + self._prepared_pass_kwargs = None + self._mixed_precision = None + self._fp32_main_params = getattr(getattr(trainer_config, "optimizer", None), "fp32_main_params", False) self.device = device or ( torch.device("npu") if (hasattr(torch, "npu") and torch.npu.is_available()) @@ -155,12 +167,13 @@ def _resolve_pass_config( *, pass_config: Optional[PassConfig], trainer_config: Optional[Any], + mesh_context: Optional[Any] = None, ) -> PassConfig: """Resolve pass configuration from explicit or trainer-level input.""" if pass_config is not None: return pass_config if trainer_config is not None: - return build_pass_config_from_trainer_config(trainer_config) + return build_pass_config_from_trainer_config(trainer_config, mesh_context=mesh_context) return PassConfig() @property @@ -168,23 +181,96 @@ def is_compiled(self) -> bool: """Whether a joint graph has already been compiled.""" return self._joint_graph is not None + @property + def execution_backend(self) -> Optional[str]: + """Actual terminal backend, or None before the graph is ready.""" + if self._joint_graph is None: + return None + return getattr(self._joint_graph.graph_module, "meta", {}).get("inductor_compiled", "fx") + @property def parameter_layouts(self) -> Optional[dict]: - """Final parameter-identity layouts, available after lazy compilation. + """Final parameter-identity layouts, available after storage preparation. Cache metadata only: gradients may be replaced on every optimizer step. Replacing a parameter invalidates the mapping, as does compiling again. """ - if self._joint_graph is None: + if not self._storage_ready: return None parameters = dict(self.model.named_parameters()) signature = tuple((id(param), tuple(param.shape), param.dtype, param.device) for param in parameters.values()) if signature != self._layout_signature: self._parameter_layouts = self._build_parameter_layouts(parameters) self._layout_signature = signature - self._joint_graph.graph_module.parameter_layouts = self._parameter_layouts + if self._joint_graph is not None: + self._joint_graph.graph_module.parameter_layouts = self._parameter_layouts return self._parameter_layouts + @property + def parameter_shapes(self) -> dict: + """Source-local logical shapes before FSDP padding and slicing.""" + return dict(self._parameter_shapes) + + @property + def sharded_parameter_names(self) -> frozenset: + """Canonical names of parameters physically partitioned by FSDP.""" + return self._sharded_parameter_names + + def prepare_model(self) -> None: + """Prepare local parameter storage once, before optimizer construction. + + Values may subsequently be loaded or updated in place. Changing a + parameter identity, shape, dtype or device requires a new model and + compiler. A failed preparation is terminal because storage may have + already been partitioned. + """ + external = get_model_external_state(self.model) + if external is not None and external.parameters: + raise ValueError("GraphCompiler does not support externally managed parameters yet") + if self._storage_failed: + raise RuntimeError("Graph storage preparation failed; rebuild the model and compiler") + signature = self._model_storage_signature() + if self._storage_ready: + if signature != self._storage_signature: + raise ValueError("Prepared graph parameter storage changed; rebuild the model and compiler") + return + try: + self._prepare_model_storage() + except Exception: + self._storage_failed = True + raise + self._storage_signature = self._model_storage_signature() + self._storage_ready = True + + def _model_storage_signature(self) -> tuple: + """Describe storage identity without retaining any parameter values.""" + return tuple((name, id(param), tuple(param.shape), param.dtype, param.device, param.requires_grad) + for name, param in self.model.named_parameters(remove_duplicate=False)) + + def _prepare_model_storage(self) -> None: + """Reuse FSDP ownership and slicing before creating optimizer state.""" + self._validate_ep_topology() + tp_size = getattr(self._mesh_context, "tp_size", 1) + if isinstance(tp_size, int) and tp_size > 1 and not self.source_shard_info: + raise ValueError("Graph TP requires source_shard_info from the sharding plan") + if self.pass_config.fsdp_enabled and dist.is_initialized(): + self._init_device_mesh(self._mesh_context) + self._bind_parameter_meshes() + precision = getattr(self._fsdp_config, "mix_precision", None) + if precision is not None and any((precision.param_dtype, precision.reduce_dtype, precision.output_dtype, + precision.custom_params, self._fp32_main_params)): + manager = FSDP2Manager(self._fsdp_config, self._mesh_context or MeshContext(), self._fp32_main_params) + policy, modules = manager.mixed_precision_bindings(self.model, self.source_shard_info) + self._mixed_precision = GraphMixedPrecision(self.model, policy, modules) + self._parameter_shapes = { + name: tuple(param.shape) for name, param in self.model.named_parameters(remove_duplicate=False) + } + self._prepared_pass_kwargs = self._build_pass_kwargs() + if self.pass_config.fsdp_enabled: + fsdp = FSDPPass(parallel_plan=self.parallel_plan) + self._sharded_parameter_names = fsdp.prepare_model(self.pass_config, **self._prepared_pass_kwargs) + self._layout_signature = None + def compile(self, **inputs: Any) -> None: """ Compile model into parallel graph @@ -209,20 +295,7 @@ def compile(self, **inputs: Any) -> None: if self._pytree_pre_hook is not None: self._pytree_pre_hook() - self._validate_ep_topology() - if ( - isinstance(getattr(self._mesh_context, "tp_size", None), int) - and self._mesh_context.tp_size > 1 - and not self.source_shard_info - ): - raise ValueError("Graph TP requires source_shard_info from the sharding plan") - - if self.pass_config.fsdp_enabled and dist.is_initialized(): - # Only build the FSDP mesh when distributed is actually up. - # ``FSDPPass`` early-returns when ``world_size == 1``, so a - # single-process run (no dist, or a single rank) compiles and - # runs as plain graph mode without sharding. - self._init_device_mesh(self._mesh_context) + self.prepare_model() # Activation checkpointing must resolve its scope and run the adapter's # runtime hook BEFORE tracing: the hook configures state the model's @@ -231,32 +304,37 @@ def compile(self, **inputs: Any) -> None: if self.pass_config.ac_mode != "off": ac_scope = prepare_activation_checkpoint(self.model, self.pass_config) - self._bind_parameter_meshes() - self._parameter_shapes = {name: tuple(param.shape) for name, param in self.model.named_parameters()} - self._layout_signature = None - - trace_kwargs = {} + trace_kwargs = {"state_shapes": self._parameter_shapes, "mixed_precision": self._mixed_precision} if self.dynamic or self.dynamic_arg_dims is not None: - trace_kwargs = { + trace_kwargs.update({ "dynamic": self.dynamic, "dynamic_arg_dims": self.dynamic_arg_dims, - } + }) joint_graph = trace_model_graph( self.model, self.train_fn, inputs, **trace_kwargs ) pipeline = PassPipeline.from_config(self.pass_config, self.parallel_plan) - pass_kwargs = self._build_pass_kwargs() + pass_kwargs = dict(self._prepared_pass_kwargs) + pass_kwargs["storage_prepared"] = True # Terminal backend compilation (InductorPass) sizes its regions from # the tracer's fake example inputs; every other pass ignores them. pass_kwargs["example_inputs"] = joint_graph.example_inputs if ac_scope is not None: pass_kwargs["ac_scope"] = ac_scope - # Passes mutate ``graph_module`` in place and return it, so the - # transformed graph lives on ``joint_graph`` for ``forward_backward``. - pipeline.run(joint_graph.graph_module, **pass_kwargs) + # The terminal backend may return a wrapper around the transformed graph. + runtime_graph = pipeline.run(joint_graph.graph_module, **pass_kwargs) + if getattr(runtime_graph, "meta", {}).get("inductor_compiled") == "unavailable": + raise RuntimeError("The requested joint-graph Inductor backend is unavailable in this torch build") + for name in ( + "state_fqns", "state_is_param", "num_state_inputs", "user_inputs_spec", + "loss_dict_keys", "gradient_usage", "fsdp_sharded_param_names", "meta", + ): + if not hasattr(runtime_graph, name) and hasattr(joint_graph.graph_module, name): + setattr(runtime_graph, name, getattr(joint_graph.graph_module, name)) + joint_graph.graph_module = runtime_graph if joint_graph.input_guards is not None: joint_graph.input_guards.refresh() @@ -334,7 +412,7 @@ def _build_parameter_layouts(self, parameters: dict) -> Optional[dict]: if self._fsdp_mesh is None and context is not None: # Plain graph mode still needs metadata, without creating collectives. self._init_mesh_from_context(context, self.pass_config.dp_mode, register_groups=False) - sharded = getattr(self._joint_graph.graph_module, "fsdp_sharded_param_names", frozenset()) + sharded = self._sharded_parameter_names layouts = {} for name, parameter in parameters.items(): if not parameter.requires_grad: @@ -396,9 +474,13 @@ def _bind_parameter_meshes(self) -> None: """Reuse eager unit discovery and managed identities without eager hooks.""" self._parameter_meshes = {} self._parameter_groups = {} - if not isinstance(self._fsdp_config, FSDP2Config) or self._mesh_context is None: + self._replicate_param_names = frozenset() + if not isinstance(self._fsdp_config, FSDP2Config): + return + manager = FSDP2Manager(self._fsdp_config, self._mesh_context or MeshContext()) + self._replicate_param_names = manager.replicate_parameter_names(self.model) + if self._mesh_context is None: return - manager = FSDP2Manager(self._fsdp_config, self._mesh_context) bindings = manager.parameter_mesh_bindings(self.model, self.source_shard_info) registered = {} for name, parameter in self.model.named_parameters(remove_duplicate=False): @@ -812,11 +894,17 @@ def _build_pass_kwargs(self) -> dict: """ Build kwargs to pass to passes. """ - kwargs = {} + kwargs = { + "storage_prepared": self._storage_ready, + "logical_parameter_shapes": self._parameter_shapes, + "replicate_param_names": self._replicate_param_names, + } # Live model: partitioning passes (FSDPPass) physically shard # parameters in place, keeping the compiler FSDP-agnostic. kwargs["model"] = self.model + if self._mixed_precision is not None: + kwargs["parameter_dtypes"] = self._mixed_precision.parameter_dtypes if self.pass_config.fsdp_enabled: kwargs["fsdp_group_name"] = "fsdp" @@ -896,17 +984,9 @@ def _run_graph(self, **inputs): def _accumulate_grads(self, grads: List[torch.Tensor]) -> None: """Accumulate graph-computed gradients into the live model's parameters. - The graph emits gradients in ``state_fqns`` order (trainable - parameters only, shared parameters included once per FQN), which - diverges from ``model.parameters()`` (deduplicated) when the model - ties weights. Mapping by FQN keeps every gradient on the right - parameter; the count check refuses to assign on mismatch instead of - letting ``zip`` silently truncate. - - Accumulation (not overwrite) keeps ``forward_backward`` composable: - several micro-batch steps may run before the caller's optimizer step - (which ends with ``zero_grad``), so per-step grads must sum into - ``param.grad``. + Gradients follow the tracer's trainable state order, with tied + parameters deduplicated. Keep absent edges distinct from zero-valued + gradients and accumulate across calls until the optimizer clears them. """ # ``state_is_param`` is attached to the traced GraphModule by # ``trace_model_graph``, not to the JointGraph dataclass itself. @@ -923,14 +1003,28 @@ def _accumulate_grads(self, grads: List[torch.Tensor]) -> None: f"refusing to assign gradients to avoid silent misalignment." ) - for param, grad in zip(trainable, grads): - if param.grad is None: + usage = getattr(self._joint_graph.graph_module, "gradient_usage", (True,) * len(grads)) + if len(usage) != len(grads): + raise ValueError("Gradient usage does not match the traced gradient count") + main_grad = self._mixed_precision is not None and self._mixed_precision.policy.apply_grad_on_fp32_main_grad + for param, grad, used in zip(trainable, grads, usage): + if not used: + continue + attribute = "main_grad" if main_grad else "grad" + accumulated = getattr(param, attribute, None) + # Eager keeps reduction precision for main gradients and casts each + # ordinary gradient to storage precision before microbatch addition. + if not main_grad: + grad = grad.to(param.dtype) + if accumulated is None: # Match eager's gradient layout so optimizer kernels see the # parameter's strides rather than a backward matmul's view. - param.grad = torch.empty_like(param) - param.grad.copy_(grad) + dtype = grad.dtype + accumulated = torch.empty_like(param, dtype=dtype) + accumulated.copy_(grad) + setattr(param, attribute, accumulated) else: - param.grad += grad + accumulated += grad @staticmethod def _trainable_params_in_state_order( diff --git a/hyper_parallel/compile/mixed_precision.py b/hyper_parallel/compile/mixed_precision.py new file mode 100644 index 0000000000..afcbd4c421 --- /dev/null +++ b/hyper_parallel/compile/mixed_precision.py @@ -0,0 +1,90 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Lower the shared FSDP precision policy into the joint forward/backward graph.""" + +from contextlib import ExitStack, contextmanager +from functools import partial +from typing import Any, Iterator + +import torch +from torch import nn + +from hyper_parallel.core.fully_shard.utils import ( + MixedPrecisionPolicy, + apply_to_tensors, + cast_fp_tensor, + resolve_param_dtypes, +) + + +class GraphMixedPrecision: + """Keep storage, computation, communication and accumulation dtypes distinct.""" + + def __init__(self, model: nn.Module, policy: MixedPrecisionPolicy, modules: tuple[nn.Module, ...]) -> None: + """Bind the shared policy to source parameter identities and unit boundaries. + + Args: + model: Final model before graph FSDP slices local storage. + policy: Resolved FSDP policy, including Parameter-specific overrides. + modules: The same module boundaries selected by eager FSDP. + """ + self.policy = policy + self.modules = modules + self.parameter_dtypes = {} + for name, parameter in model.named_parameters(remove_duplicate=False): + compute, reduce = resolve_param_dtypes(policy, id(parameter), parameter.dtype) + self.parameter_dtypes[name] = (compute or parameter.dtype, reduce) + + @contextmanager + def module_casts(self) -> Iterator[None]: + """Record eager-compatible module input/output casts during capture only.""" + with ExitStack() as stack: + for module in self.modules: + if self.policy.cast_forward_inputs and self.policy.param_dtype is not None: + handle = module.register_forward_pre_hook(self._cast_inputs, with_kwargs=True) + stack.callback(handle.remove) + if self.policy.output_dtype is not None: + handle = module.register_forward_hook(self._cast_outputs) + stack.callback(handle.remove) + yield + + def cast_state(self, state: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + """Create compute views whose gradients stay in compute dtype until reduction.""" + return { + name: cast_fp_tensor(self.parameter_dtypes[name][0], value) + if name in self.parameter_dtypes else value + for name, value in state.items() + } + + def cast_gradient(self, name: str, gradient: torch.Tensor) -> torch.Tensor: + """Apply the reduction dtype even when no collective is needed. + + Args: + name: Source parameter FQN in the joint graph's trainable-state order. + gradient: Compute gradient, or its zero substitute for an unused parameter. + """ + dtype = self.parameter_dtypes[name][1] + return cast_fp_tensor(dtype, gradient) if dtype is not None else gradient + + def _cast_inputs(self, module: nn.Module, args: tuple, kwargs: dict) -> tuple: + """Mirror the eager FSDP pre-forward cast for floating input leaves.""" + del module + cast = partial(cast_fp_tensor, self.policy.param_dtype) + return apply_to_tensors(cast, args), apply_to_tensors(cast, kwargs) + + def _cast_outputs(self, module: nn.Module, args: tuple, outputs: Any) -> Any: + """Mirror the eager FSDP post-forward output cast.""" + del module, args + return apply_to_tensors(partial(cast_fp_tensor, self.policy.output_dtype), outputs) diff --git a/hyper_parallel/compile/pass_config.py b/hyper_parallel/compile/pass_config.py index d1def58abf..dc86909335 100644 --- a/hyper_parallel/compile/pass_config.py +++ b/hyper_parallel/compile/pass_config.py @@ -495,8 +495,9 @@ def build_pass_config_from_trainer_config( *, fsdp_enabled: Optional[bool] = None, enable_overlap: bool = False, + mesh_context: Optional[Any] = None, ) -> PassConfig: - """Project trainer topology intent onto graph-mode pass config. + """Project trainer topology and joint-backend intent onto graph-mode passes. This helper stays in the pass-config module because it computes the graph-pass surface from the trainer topology without altering runtime @@ -520,17 +521,23 @@ def build_pass_config_from_trainer_config( fsdp_enabled: Overrides the sharded-DP intent derived from ``config.fsdp_config`` when provided. enable_overlap: Value for the legacy ``AutoOverlapPass`` switch. + mesh_context: Resolved topology, including the DP replicate axis. Returns: The graph-mode ``PassConfig`` for this trainer topology. """ compile_config = getattr(config, "compile", None) + replicate_degree = getattr(mesh_context, "dp_replicate_size", 1) if fsdp_enabled is None: fsdp_enabled = ( config.fsdp_config.dp_shard_size > 1 or config.fsdp_config.edp_shard_size > 1 + or replicate_degree > 1 ) fsdp_degree = config.fsdp_config.dp_shard_size if fsdp_enabled else None + dp_mode = "fsdp" + if replicate_degree > 1: + dp_mode = "hsdp" if config.fsdp_config.dp_shard_size > 1 else "ddp" ac_mode = "off" ac_layer_count = None @@ -559,6 +566,8 @@ def build_pass_config_from_trainer_config( enable_overlap=enable_overlap, fsdp_enabled=fsdp_enabled, fsdp_degree=fsdp_degree, + dp_mode=dp_mode, + dp_replicate_degree=replicate_degree, fsdp_reshard_after_forward=getattr( config.fsdp_config, "reshard_after_forward", True ), @@ -569,6 +578,8 @@ def build_pass_config_from_trainer_config( compile_config, "fsdp_overlap_grad_reduce", True ), max_in_flight_gb=getattr(compile_config, "max_in_flight_gb", 0.25), + inductor_mode=getattr(compile_config, "inductor_mode", "off"), + inductor_configs=getattr(compile_config, "inductor_configs", None), ac_mode=ac_mode, ac_layer_count=ac_layer_count, ac_layer_indices=ac_layer_indices, diff --git a/hyper_parallel/compile/passes/inductor_pass.py b/hyper_parallel/compile/passes/inductor_pass.py index af5f71a820..9ea6a26920 100644 --- a/hyper_parallel/compile/passes/inductor_pass.py +++ b/hyper_parallel/compile/passes/inductor_pass.py @@ -311,7 +311,7 @@ def run( placeholder metas. Returns: - The same graph module with compiled regions spliced in and the + The runtime module with compiled regions spliced in and the positional calling convention restored. On torch builds without regional Inductor support it is returned untouched, logging a warning and recording ``"unavailable"`` in its meta. @@ -335,7 +335,7 @@ def run( "JointGraph.example_inputs) to size compiled regions" ) if _regional_inductor_available(): - self._compile_graph(graph_module, pass_config, example_inputs) + graph_module = self._compile_graph(graph_module, pass_config, example_inputs) compiled = True else: self._warn_unavailable(mode) @@ -417,6 +417,11 @@ def _compile_graph( _propagate_fake_vals(graph_module) if example_inputs is None: example_inputs = _placeholder_fake_args(graph_module) + else: + # FSDP partitions state storage after capture; user symbols stay unchanged. + state_count = getattr(graph_module, "num_state_inputs", 0) + state_inputs = tuple(_placeholder_fake_args(graph_module)[:state_count]) + example_inputs = state_inputs + tuple(example_inputs[state_count:]) _LOG.info( "InductorPass: compiling graph in %r mode (%d placeholders, %d nodes)", mode, diff --git a/hyper_parallel/compile/passes/parallel/fsdp_pass.py b/hyper_parallel/compile/passes/parallel/fsdp_pass.py index 9fd08182e8..777ab44097 100644 --- a/hyper_parallel/compile/passes/parallel/fsdp_pass.py +++ b/hyper_parallel/compile/passes/parallel/fsdp_pass.py @@ -65,6 +65,8 @@ from torch.fx.node import _side_effectful_functions from torch.ops import _c10d_functional +from hyper_parallel.core.fully_shard.hsdp_utils import get_shard_geometry + from ...graph_parallel_plan import GraphParallelPlan from ...pass_config import GradientReduction, PassConfig, normalize_dp_mode from ..base import GraphPass @@ -138,7 +140,7 @@ def _annotate_ag_spec(node: fx.Node, degree: int, params: dict) -> None: node.meta["fsdp_bucket_spec"] = { "input_shape": shape, "output_shape": (shape[0] * degree, *shape[1:]), - "dtype": param.dtype, + "dtype": node.meta.get("param_dtype", param.dtype), "device": str(param.device), } @@ -168,6 +170,18 @@ def annotate_fsdp_bucket_specs(graph_module: fx.GraphModule, model: nn.Module) - _annotate_ag_spec(node, degree, params) +def _cast_node(graph: fx.Graph, node: fx.Node, dtype: Optional[torch.dtype]) -> fx.Node: + """Insert a precision conversion while retaining fake dtype/shape metadata.""" + value = node.meta.get("val") + if dtype is None or (value is not None and value.dtype == dtype): + return node + cast = graph.call_method("to", args=(node,), kwargs={"dtype": dtype}) + cast.meta["autograd_backward"] = node.meta.get("autograd_backward", False) + if isinstance(value, FakeTensor): + cast.meta["val"] = value.to(dtype) + return cast + + def _free_tensor_storage(tensor: torch.Tensor) -> None: """Release ``tensor``'s storage in place (FSDP reshard free step). @@ -336,6 +350,10 @@ def __init__( self._gradient_reductions: Dict[str, GradientReduction] = {} self._source_gradient_groups: Dict[str, str] = {} self._parameter_groups: Dict[str, tuple[str, int]] = {} + self._replicate_param_names: Set[str] = set() + self._storage_prepared = False + self._logical_parameter_shapes: Dict[str, tuple] = {} + self._parameter_dtypes: Dict[str, tuple] = {} # Data-parallel mode ("fsdp" | "ddp" | "hsdp") and the replicate-axis # group/degree used by DDP / HSDP gradient reduction. self._dp_mode = "fsdp" @@ -352,6 +370,29 @@ def __init__( # live-shard steps so they are pad-sharded like eager ``fully_shard``. self._padded_params: Dict[str, _PaddedParam] = {} + def prepare_model(self, pass_config: PassConfig, **kwargs: Any) -> frozenset[str]: + """Partition live storage using the same classification as graph lowering. + + This metadata-only graph does not execute model code or collectives. + Call once before building optimizer parameters or loading a checkpoint. + """ + if not dist.is_initialized() or dist.get_world_size() == 1: + return frozenset() + model = self._setup_dp_config(pass_config, kwargs) + self._reset_run_state() + graph = fx.Graph() + parameters = list(model.named_parameters()) + for index, _ in enumerate(parameters): + graph.placeholder(f"param_{index}") + graph.output(()) + module = fx.GraphModule({}, graph) + names = [name for name, _ in parameters] + classification = self._classify_dp_params(module, names, len(names), [True] * len(names), model) + if self._dp_mode == "ddp": + return frozenset() + self._shard_live_model_params(model) + return frozenset(node.meta["param_name"] for node in classification.sharded_nodes) + def run( self, graph_module: fx.GraphModule, @@ -439,6 +480,16 @@ def run( len(param_nodes), ) + # Backend propagation starts from physical shards, not captured logical weights. + parameters = dict(model.named_parameters()) + for node in param_nodes: + value = node.meta.get("val") + parameter = parameters.get(node.meta["param_name"]) + if isinstance(value, torch.Tensor) and parameter is not None: + node.meta["val"] = value.new_empty_strided( + parameter.shape, parameter.stride(), requires_grad=value.requires_grad + ) + graph_module.recompile() return graph_module @@ -469,6 +520,10 @@ def _setup_dp_config( self._gradient_reductions = dict(kwargs.get("gradient_reductions", {})) self._source_gradient_groups = dict(kwargs.get("source_gradient_groups", {})) self._parameter_groups = dict(kwargs.get("parameter_groups", {})) + self._replicate_param_names = set(kwargs.get("replicate_param_names", ())) + self._storage_prepared = kwargs.get("storage_prepared", False) + self._logical_parameter_shapes = kwargs.get("logical_parameter_shapes", {}) + self._parameter_dtypes = kwargs.get("parameter_dtypes", {}) if not self._expert_param_names.issubset(self._gradient_reductions): raise ValueError("Expert parameters require explicit gradient reduction policies") self._expert_fsdp_group_name = kwargs.get( @@ -500,6 +555,9 @@ def _setup_dp_config( "can physically shard parameters; the trainer passes it through " "in compile()" ) + unknown = self._replicate_param_names.difference(dict(model.named_parameters(remove_duplicate=False))) + if unknown: + raise ValueError(f"replicate_param_names contains unknown parameter FQNs: {sorted(unknown)}") _LOG.info( "Running dp_mode=%s fsdp_degree=%s dp_replicate_degree=%s world_size=%s", @@ -565,7 +623,8 @@ def _apply_dp_transforms( # index in the FSDP group). ``model.parameters()`` then yields the # shards, so the trainer's optimizer / grad accumulation stay # DP-agnostic; the graph re-gathers each step. - self._shard_live_model_params(model) + if not self._storage_prepared: + self._shard_live_model_params(model) annotate_fsdp_bucket_specs(graph_module, model) return graph_module @@ -631,6 +690,9 @@ def _classify_dp_params( if skip: continue + if fqn in self._logical_parameter_shapes: + param = torch.empty(self._logical_parameter_shapes[fqn], dtype=param.dtype, device="meta") + _, degree = self._param_group(fqn) if fqn in self._expert_param_names and degree == 1: continue @@ -698,7 +760,9 @@ def _mark_sharded_param( # Newly inserted collective nodes have no shape-propagated ``val``. value = param if param is not None else node.meta.get("val") if isinstance(value, torch.Tensor) and isinstance(value.numel(), int): - node.meta["fsdp_unsharded_nbytes"] = value.numel() * value.element_size() + dtype = self._parameter_dtypes.get(fqn, (value.dtype, None))[0] + element_size = torch.empty((), device="meta", dtype=dtype).element_size() + node.meta["fsdp_unsharded_nbytes"] = value.numel() * element_size self._fsdp_modules.add(self._get_parent_module_fqn(fqn)) return node @@ -737,6 +801,8 @@ def _marked_param_stays_replicated( it is still sharded and gradient-reduced. DDP has no shard axis, so every marked param is handled the same way regardless of divisibility. """ + if fqn in self._replicate_param_names: + return True if self._dp_mode == "ddp" or param is None: return False _, degree = self._param_group(fqn) @@ -756,7 +822,7 @@ def _marked_param_stays_replicated( ) return True n = int(shape[0]) - n_pad = -(-n // degree) * degree + n_pad = get_shard_geometry(shape, 0, degree, 0).padded_shape[0] * degree self._padded_params[fqn] = _PaddedParam( n=n, n_pad=n_pad, dim0_pad=n_pad - n, ndim=param.ndim ) @@ -826,6 +892,9 @@ def _shard_live_model_params(self, model: nn.Module) -> None: ) continue + if self._marked_param_stays_replicated(name, param): + continue + if group_name not in group_ranks: group_ranks[group_name] = dist.get_rank( group=_resolve_process_group(group_name) @@ -834,8 +903,8 @@ def _shard_live_model_params(self, model: nn.Module) -> None: # Pad-shard a non-divisible param (recorded by # ``_classify_dp_params``): zero-pad dim 0 to a multiple of the - # degree, then take this rank's equal-size chunk — exactly what - # eager ``fully_shard`` does. A 0-dim scalar cannot be padded and + # degree, then retain this rank's padded communication shard. + # Eager retains only its valid rows. A 0-dim scalar cannot be padded and # stays replicated. pad_info = self._padded_params.get(name) if pad_info is not None: @@ -846,7 +915,9 @@ def _shard_live_model_params(self, model: nn.Module) -> None: [full, full.new_zeros((pad_info.dim0_pad, *full.shape[1:]))], dim=0, ) - param.data = full.chunk(degree, dim=0)[rank].clone() + geometry = get_shard_geometry(original_shape, 0, degree, rank) + shard_size = geometry.padded_shape[0] + param.data = full.narrow(0, rank * shard_size, shard_size).clone() sharded_count += 1 _LOG.info( "Pad-sharded %s: %s -> %s", @@ -856,18 +927,9 @@ def _shard_live_model_params(self, model: nn.Module) -> None: ) continue - # Mirror ``_classify_dp_params``: a 0-dim scalar stays replicated. - if not param.shape or param.shape[0] % degree != 0: - _LOG.info( - "Skip %s: dim 0 (%s) not divisible by fsdp_degree (%s)", - name, - param.shape[0] if param.shape else "scalar", - degree, - ) - continue - original_shape = param.shape - param.data = param.detach().chunk(degree, dim=0)[rank].clone() + geometry = get_shard_geometry(original_shape, 0, degree, rank) + param.data = param.detach().narrow(0, geometry.start, geometry.local_shape[0]).clone() sharded_count += 1 _LOG.info( "Sharded %s: %s -> %s", @@ -953,10 +1015,14 @@ def _insert_all_gather_for_params( # AllGather + immediate wait for correctness; FsdpOverlapPass may # sink the wait past independent compute later. with _insertion_point(graph, anchor, param_node): + dtype = self._parameter_dtypes.get(param_node.meta.get("param_name"), (None, None))[0] + gather_input = _cast_node(graph, param_node, dtype) ag_node = graph.call_function( _c10d_functional.all_gather_into_tensor, - args=(param_node, degree, group_name), + args=(gather_input, degree, group_name), ) + if dtype is not None: + ag_node.meta["param_dtype"] = dtype ag_node.meta["comm_type"] = "fsdp_all_gather" ag_node.meta["comm_group"] = group_name ag_node.meta["param_node"] = param_node.name @@ -1005,7 +1071,14 @@ def _insert_all_gather_for_params( replacement.meta["state_idx"] = param_node.meta.get("state_idx") for user in list(param_node.users.keys()): - if user not in (ag_node, wait_node): + if user in (gather_input, ag_node, wait_node): + continue + if str(user.target) == "aten._to_copy.default" and user.kwargs == {"dtype": dtype}: + # The shard was cast before gather; do not retain a second + # full-size compute copy that defeats resharding. + user.replace_all_uses_with(replacement) + graph.erase_node(user) + else: user.replace_input_with(param_node, replacement) self._processed_params.add(param_node.name) @@ -1076,6 +1149,7 @@ def _insert_grad_reduction( # pylint: disable=too-many-locals ) new_returned = list(returned) + usage_paths = {} # ``train_fn`` may append the traced ``loss_dict`` outputs AFTER the # gradients (see ``trace_model_graph``); those are not gradients and # must stay outside the reduce_scatter range. @@ -1089,6 +1163,9 @@ def _insert_grad_reduction( # pylint: disable=too-many-locals f"parameters are trainable; refusing to insert gradient " f"reduction to avoid silent gradient/state misalignment." ) + usage = getattr(graph_module, "gradient_usage", None) + if usage is not None and len(usage) != num_grads: + raise ValueError("Gradient usage metadata does not match the trainable gradient outputs") # Index 0 is the loss; gradients start at index 1. Gradient i+1 # corresponds to trainable parameter i, whose state_idx is @@ -1118,6 +1195,8 @@ def _insert_grad_reduction( # pylint: disable=too-many-locals group_name, degree, self._gradient_reductions.get(fqn, _DEFAULT_GRADIENT_REDUCTION) ) with graph.inserting_before(output_node): + dtype = self._parameter_dtypes.get(fqn, (None, None))[1] + grad_node = _cast_node(graph, grad_node, dtype) new_returned[i] = self._build_grad_reduction_nodes( graph, grad_node, @@ -1132,11 +1211,56 @@ def _insert_grad_reduction( # pylint: disable=too-many-locals graph, new_returned[i], source_group ) tag_gradient_identity(new_returned[i], state_fqns[state_idx], state_idx) + usage_paths[i - 1] = self._gradient_sync_path(new_returned[i], grad_node) output_node.args = (type(returned)(new_returned),) + tuple(output_node.args[1:]) + self._synchronize_gradient_usage(graph_module, usage_paths, model) return graph_module + @staticmethod + def _gradient_sync_path(result: fx.Node, original: fx.Node) -> list[tuple[int, str]]: + """Read only the newly inserted reduction chain, in execution order.""" + stages = {"fsdp_reduce_scatter": 0, "fsdp_all_reduce": 0, + "dp_all_reduce": 1, "source_all_reduce": 2} + path = [] + node = result + while node is not original and isinstance(node, fx.Node): + comm_type = node.meta.get("comm_type") + if comm_type in stages: + path.append((stages[comm_type], node.meta["comm_group"])) + if not node.args: + break + node = node.args[0] + return list(reversed(path)) + + @staticmethod + def _synchronize_gradient_usage( + graph_module: fx.GraphModule, paths: Dict[int, list[tuple[int, str]]], model: Optional[nn.Module] + ) -> None: + """Resolve unused gradients once at compile time over their actual reduction groups.""" + usage = getattr(graph_module, "gradient_usage", None) + if usage is None or not paths: + return + groups = {} + for index, path in paths.items(): + for stage_group in path: + groups.setdefault(stage_group, []).append(index) + if not groups: + return + if model is None: + raise ValueError("Gradient usage synchronization requires the live model") + parameter = next(model.parameters()) + mask = torch.tensor(usage, dtype=torch.int32, device=parameter.device) + for stage_group, indices in sorted(groups.items()): + group = _resolve_process_group(stage_group[1]) + if dist.get_world_size(group=group) <= 1: + continue + packed = mask[indices].contiguous() + dist.all_reduce(packed, op=dist.ReduceOp.MAX, group=group) + mask[indices] = packed + graph_module.gradient_usage = tuple(bool(value) for value in mask.cpu().tolist()) + @staticmethod def _insert_source_gradient_reduction( graph: fx.Graph, gradient: fx.Node, group_name: str @@ -1418,10 +1542,14 @@ def _insert_backward_gather( record.param_node.meta.get("param_name", "") ) with graph.inserting_before(anchor): + dtype = self._parameter_dtypes.get(record.param_node.meta.get("param_name"), (None, None))[0] + gather_input = _cast_node(graph, record.param_node, dtype) ag_node = graph.call_function( _c10d_functional.all_gather_into_tensor, - args=(record.param_node, degree, group_name), + args=(gather_input, degree, group_name), ) + if dtype is not None: + ag_node.meta["param_dtype"] = dtype ag_node.meta["comm_type"] = "fsdp_all_gather_backward" ag_node.meta["comm_group"] = group_name ag_node.meta["param_node"] = record.param_node.name diff --git a/hyper_parallel/compile/tracer/graph_tracer.py b/hyper_parallel/compile/tracer/graph_tracer.py index ea9a8f4627..f90fd12576 100644 --- a/hyper_parallel/compile/tracer/graph_tracer.py +++ b/hyper_parallel/compile/tracer/graph_tracer.py @@ -59,7 +59,7 @@ import warnings from collections.abc import Callable, Generator from collections import deque -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from dataclasses import dataclass from typing import Any, Dict, List @@ -443,13 +443,15 @@ def _make_fwd_bwd_fn( state_is_param: list[bool], state_spec: Any, user_inputs_spec: Any, -) -> tuple[Callable, list[str]]: - """Build the joint callable and its named-loss output metadata.""" + mixed_precision: Any = None, +) -> tuple[Callable, list[str], list[bool]]: + """Build the joint callable with named-loss and trainable-gradient metadata.""" num_state_inputs = len(state_fqns) # ``_fwd_bwd_fn`` communicates the traced ``loss_dict`` key order to the # enclosing scope through this box (plain closure writes cannot escape # the nested ``def``); the keys are attached to the GraphModule below. loss_dict_keys_box: list[str] = [] + gradient_usage_box: list[bool] = [] def _fwd_bwd_fn(*plain_args): state_wrapped = plain_args[:num_state_inputs] @@ -458,9 +460,10 @@ def _fwd_bwd_fn(*plain_args): user_args = torch.utils._pytree.tree_unflatten( list(user_wrapped), user_inputs_spec ) + model_state = mixed_precision.cast_state(state_t["model"]) if mixed_precision else state_t["model"] with ( - _reparametrize_train_state(model, state_t["model"]), + _reparametrize_train_state(model, model_state), _patch_engine_backward(), ): out = train_fn(model, **user_args) @@ -483,28 +486,32 @@ def _fwd_bwd_fn(*plain_args): else: loss, loss_dict_keys, loss_dict_values = out, [], [] - # Read params from state_t["model"] so the exact trace-time - # tensors (not the live module's) drive autograd.grad. + # Like eager FSDP, differentiate compute parameters. Casting back + # to storage here could round gradients before their reduction. params = [] + param_names = [] for idx, fqn in enumerate(state_fqns): if state_is_param[idx]: - param = state_t["model"][fqn] + param = model_state[fqn] if param.requires_grad: params.append(param) + param_names.append(fqn) grads = torch.autograd.grad(loss, params, allow_unused=True) + gradient_usage_box[:] = [grad is not None for grad in grads] # Replace None grads with zeros so the output arity is stable. processed_grads = [] - for grad, param in zip(grads, params): + for name, grad, param in zip(param_names, grads, params): if grad is None: - processed_grads.append(torch.zeros_like(param)) - else: - processed_grads.append(grad) + grad = torch.zeros_like(param) + if mixed_precision: + grad = mixed_precision.cast_gradient(name, grad) + processed_grads.append(grad) return [loss] + processed_grads + loss_dict_values - return _fwd_bwd_fn, loss_dict_keys_box + return _fwd_bwd_fn, loss_dict_keys_box, gradient_usage_box def trace_model_graph( # pylint: disable=too-many-locals @@ -514,6 +521,8 @@ def trace_model_graph( # pylint: disable=too-many-locals *, dynamic: bool = False, dynamic_arg_dims: dict[str, int | list[int]] | None = None, + state_shapes: dict[str, tuple] | None = None, + mixed_precision: Any = None, ) -> JointGraph: """ Trace model to generate complete forward + backward graph @@ -527,6 +536,8 @@ def trace_model_graph( # pylint: disable=too-many-locals dynamic_arg_dims: Dotted input paths to dynamic dimensions (supports negative indices). Providing a mapping enables selective symbolic tracing, even when dynamic is False. Parameters and buffers always remain static. + state_shapes: Source-local logical parameter shapes before live FSDP slicing. + mixed_precision: Resolved shared FSDP precision policy and module cast boundaries. Returns: JointGraph: Joint forward-backward computation graph @@ -566,9 +577,16 @@ def trace_model_graph( # pylint: disable=too-many-locals fake_mode = FakeTensorMode(allow_non_fake_inputs=True, shape_env=ShapeEnv(), static_shapes=True) fake_args = tuple(_fakeify_input(fake_mode, value) for value in full_args) num_state_inputs = len(state_flat) - - _fwd_bwd_fn, loss_dict_keys_box = _make_fwd_bwd_fn( # pylint: disable=C0103 - model, train_fn, state_fqns, state_is_param, state_spec, user_inputs_spec + if state_shapes: + with fake_mode: + fake_args = tuple( + value.new_empty(state_shapes[name], requires_grad=value.requires_grad) + if name in state_shapes and tuple(value.shape) != state_shapes[name] else value + for name, value in zip(state_fqns, fake_args[:num_state_inputs]) + ) + fake_args[num_state_inputs:] + + _fwd_bwd_fn, loss_dict_keys_box, gradient_usage_box = _make_fwd_bwd_fn( # pylint: disable=C0103 + model, train_fn, state_fqns, state_is_param, state_spec, user_inputs_spec, mixed_precision ) # make_fx only records ``nn_module_stack`` when the traced callable @@ -583,6 +601,7 @@ def trace_model_graph( # pylint: disable=too-many-locals ctx = TracingContext(fake_mode) with ( fake_mode, + mixed_precision.module_casts() if mixed_precision else nullcontext(), tracing(ctx), preserve_node_meta(), _skip_nested_compile(), @@ -628,10 +647,12 @@ def trace_model_graph( # pylint: disable=too-many-locals # (loss, grads, loss_dict) and passes can trim the trailing loss_dict # outputs off the gradient range. traced_graph.loss_dict_keys = list(loss_dict_keys_box) + # None denotes absence from autograd, while a real zero gradient remains used. + traced_graph.gradient_usage = tuple(gradient_usage_box) param_names = [name for name, _ in model.named_parameters() if _.requires_grad] param_shapes = { - name: tuple(param.shape) for name, param in model.named_parameters() + name: (state_shapes or {}).get(name, tuple(param.shape)) for name, param in model.named_parameters() } num_layers = 0 diff --git a/hyper_parallel/core/fully_shard/hsdp_param.py b/hyper_parallel/core/fully_shard/hsdp_param.py index 0c29313c85..eb68d85253 100755 --- a/hyper_parallel/core/fully_shard/hsdp_param.py +++ b/hyper_parallel/core/fully_shard/hsdp_param.py @@ -33,6 +33,7 @@ ParamModuleInfo, ShardedState, apply_gradient_scaling_factor, + get_shard_geometry, ) from hyper_parallel.core.fully_shard.utils import ( CPUOffloadPolicy, @@ -43,6 +44,7 @@ MixedPrecisionPolicy, OffloadPolicy, SourceShardMetaInfo, + resolve_param_dtypes, ) @@ -615,17 +617,12 @@ def _init_sharded_param( base_placements, ) - chunks = torch.chunk(param_data, self.shard_world_size, dim=shard_dim) - if self.shard_rank < len(chunks): - local_shard = chunks[self.shard_rank] - else: - empty_slice = [slice(None)] * param_data.ndim - empty_slice[shard_dim] = slice(param_data.size(shard_dim), None) - local_shard = param_data[tuple(empty_slice)] + geometry = get_shard_geometry(param_data.shape, shard_dim, self.shard_world_size, self.shard_rank) + local_shard = param_data.narrow(shard_dim, geometry.start, geometry.local_shape[shard_dim]) sharded_param = local_shard.clone().contiguous() self.sharded_size = sharded_param.size() self.contiguous_sharded_stride = make_contiguous_strides_for(self.sharded_size) - self.padded_sharded_param_size = chunks[0].size() + self.padded_sharded_param_size = geometry.padded_shape if self.offload_to_cpu and not sharded_param.is_meta: sharded_param = sharded_param.cpu() if self.pin_memory: @@ -665,16 +662,8 @@ def _init_sharded_param( def init_dtype_attrs(self, mp_policy: MixedPrecisionPolicy) -> None: """Initialize dtype attributes, applying source-parameter overrides before policy defaults.""" - custom_dtypes = (mp_policy.custom_params or {}).get(self._src_parameter_id, {}) - param_dtype = custom_dtypes.get("param_dtype", mp_policy.param_dtype) - reduce_dtype = custom_dtypes.get("reduce_dtype", mp_policy.reduce_dtype) self.orig_dtype = self.sharded_param.dtype - if reduce_dtype == param_dtype: - reduce_dtype = None - if param_dtype == self.orig_dtype: - param_dtype = None - self.param_dtype = param_dtype - self.reduce_dtype = reduce_dtype + self.param_dtype, self.reduce_dtype = resolve_param_dtypes(mp_policy, self._src_parameter_id, self.orig_dtype) self.apply_grad_on_fp32_main_grad = mp_policy.apply_grad_on_fp32_main_grad def init_unsharded_param_buffers( diff --git a/hyper_parallel/core/fully_shard/hsdp_utils.py b/hyper_parallel/core/fully_shard/hsdp_utils.py index 5e21b60fd3..9f5c7cab13 100644 --- a/hyper_parallel/core/fully_shard/hsdp_utils.py +++ b/hyper_parallel/core/fully_shard/hsdp_utils.py @@ -25,6 +25,47 @@ from hyper_parallel.core.dtensor.dtensor import DTensor +@dataclass(frozen=True) +class ShardGeometry: + """Valid local shard and uniform communication shape within a source-local tensor.""" + + start: int + local_shape: torch.Size + padded_shape: torch.Size + + +def get_shard_geometry( + shape: Sequence[int], shard_dim: int, shard_world_size: int, shard_rank: int +) -> ShardGeometry: + """Plan chunk-based FSDP sharding without allocating or modifying tensors. + + Args: + shape: Parameter shape after any source parallelism, such as TP or EP. + shard_dim: Non-negative dimension to shard. + shard_world_size: Size of the parameter's FSDP shard group. + shard_rank: Rank within that group. + + Returns: + Valid slice start and shape, plus the uniform padded communication shape. + Ranks beyond the chunks returned by ``torch.chunk`` own an empty slice. + + Raises: + ValueError: If the shard dimension, group size or group-local rank is invalid. + """ + if shard_world_size < 1 or not 0 <= shard_rank < shard_world_size: + raise ValueError(f"Invalid shard rank {shard_rank} for shard degree {shard_world_size}") + if not 0 <= shard_dim < len(shape): + raise ValueError(f"Invalid shard dimension {shard_dim} for shape {tuple(shape)}") + size = shape[shard_dim] + chunk_size = (size + shard_world_size - 1) // shard_world_size + start = min(shard_rank * chunk_size, size) + local_shape = list(shape) + padded_shape = list(shape) + local_shape[shard_dim] = min(chunk_size, size - start) + padded_shape[shard_dim] = chunk_size + return ShardGeometry(start, torch.Size(local_shape), torch.Size(padded_shape)) + + class ShardedState(Enum): """ Parameter shard state diff --git a/hyper_parallel/core/fully_shard/utils.py b/hyper_parallel/core/fully_shard/utils.py index 58c954a6ae..0d7475e41d 100644 --- a/hyper_parallel/core/fully_shard/utils.py +++ b/hyper_parallel/core/fully_shard/utils.py @@ -255,7 +255,7 @@ class MixedPrecisionPolicy: Attributes: param_dtype: Data type for parameter computation. If None, uses original dtype. - reduce_dtype: Data type for gradient reduction. If None, uses param_dtype. + reduce_dtype: Data type for gradient reduction. If None, uses the gradient dtype. output_dtype: Data type for module outputs. If None, no casting applied. custom_params: Per-parameter overrides that take precedence over this policy's ``param_dtype`` and ``reduce_dtype`` defaults. @@ -293,6 +293,30 @@ def _check_custom_param(self) -> None: raise ValueError(f"custom_params {dtype_name} must be torch.dtype or None, got {dtype!r}") +def resolve_param_dtypes( + policy: MixedPrecisionPolicy, source_parameter_id: int, storage_dtype: DType +) -> tuple[Optional[DType], Optional[DType]]: + """Resolve parameter casts without retaining source parameters or choosing a runtime gradient dtype. + + Args: + policy: Shared precision policy with source-ID-indexed parameter overrides. + source_parameter_id: Original parameter identity, before FSDP replaces its storage. + storage_dtype: Local parameter storage dtype. + + Returns: + Compute and reduction cast targets. None preserves storage or gradient dtype, + respectively; explicit per-field None overrides policy defaults. + """ + custom_dtypes = (policy.custom_params or {}).get(source_parameter_id, {}) + param_dtype = custom_dtypes.get("param_dtype", policy.param_dtype) + reduce_dtype = custom_dtypes.get("reduce_dtype", policy.reduce_dtype) + if reduce_dtype == param_dtype: + reduce_dtype = None + if param_dtype == storage_dtype: + param_dtype = None + return param_dtype, reduce_dtype + + @dataclass class OffloadPolicy: """ diff --git a/hyper_parallel/distributed/_builder/fsdp_adapter.py b/hyper_parallel/distributed/_builder/fsdp_adapter.py index 05ff1aec52..21d73bdf03 100644 --- a/hyper_parallel/distributed/_builder/fsdp_adapter.py +++ b/hyper_parallel/distributed/_builder/fsdp_adapter.py @@ -131,6 +131,16 @@ def _build_expert_fsdp_mesh(self) -> DeviceMesh: return expert_mesh[("edp_replicate", "edp_shard")] return expert_mesh["edp_shard"] + def replicate_parameter_names(self, model: ModuleClass) -> frozenset[str]: + """Resolve configured replicated parameters, including all tied aliases. + + Args: + model: Final model before FSDP storage is prepared. + """ + parameters = self._resolve_replicate_params(model) or set() + return frozenset(name for name, parameter in model.named_parameters(remove_duplicate=False) + if parameter in parameters) + def _build_dense_fsdp_mesh(self) -> DeviceMesh | None: """Select the prebuilt dense FSDP child mesh when available.""" dense_mesh = self.mesh_context.fsdp_non_moe_mesh @@ -202,6 +212,22 @@ def parameter_mesh_bindings( claimed.update(parameters) return bindings + def mixed_precision_bindings( + self, model: ModuleClass, source_shard_info: SourceShardInfoByFQN | None, + ) -> tuple[fully_shard_utils.MixedPrecisionPolicy, tuple[ModuleClass, ...]]: + """Resolve the eager precision policy and unit boundaries without hooks. + + Args: + model: Final model before parameter storage is partitioned. + source_shard_info: Source TP/EP layout returned by the planner. + """ + parameters = dict(model.named_parameters(remove_duplicate=False)) + custom = self._resolve_custom_params(parameters) if self.config.mix_precision.custom_params else None + policy = self._build_mixed_precision_policy(custom) + metadata = _build_source_shard_info_by_param(self, model, source_shard_info) + units = self._find_wrap_modules(model, metadata) + return policy, tuple(dict.fromkeys([model, *(unit.module for unit in units)])) + def _resolve_custom_params( self, parameters_by_fqn: Mapping[str, ParameterClass] ) -> dict[ParameterClass, dict[str, Any]]: diff --git a/hyper_parallel/models/build_options.py b/hyper_parallel/models/build_options.py index 58d8e0c400..d6c377fb5d 100644 --- a/hyper_parallel/models/build_options.py +++ b/hyper_parallel/models/build_options.py @@ -123,6 +123,8 @@ class CompileConfig: ``max_in_flight_gb`` follows PyTorch's overlap scheduler naming and binary units; here it bounds early all-gather replicas until first use, per phase. Eager FSDP2 module prefetch depths do not configure these graph passes. + ``inductor_mode`` and ``inductor_configs`` select the terminal joint-graph + backend independently of decoder compilation; ``off`` executes FX directly. """ enabled: bool = False @@ -138,6 +140,9 @@ class CompileConfig: fsdp_allgather_prefetch: bool = True fsdp_overlap_grad_reduce: bool = True max_in_flight_gb: float = 0.25 + # Joint-graph backend; backend/mode/options above configure decoder compilation. + inductor_mode: Literal["off", "regional", "full"] = "off" + inductor_configs: Optional[dict[str, Any]] = None def selects_graph_compiler(self) -> bool: """Return whether this config selects the graph-compiler trainer path.""" @@ -153,6 +158,7 @@ def __post_init__(self) -> None: if not isinstance(getattr(self, name), bool): raise TypeError(f"compile.{name} must be a bool") self._validate_max_in_flight_gb() + self._validate_joint_graph_backend() if self.dynamic_arg_dims is not None and not isinstance(self.dynamic_arg_dims, dict): raise ValueError("compile.dynamic_arg_dims must be a mapping or None") if not isinstance(self.mode, str) or not self.mode.strip(): @@ -185,6 +191,18 @@ def _validate_max_in_flight_gb(self) -> None: ): raise ValueError("compile.max_in_flight_gb must be a finite non-negative number") + def _validate_joint_graph_backend(self) -> None: + """Validate the terminal backend independently of decoder options.""" + if self.inductor_mode not in ("off", "regional", "full"): + raise ValueError("compile.inductor_mode must be 'off', 'regional' or 'full'") + if self.inductor_configs is not None and ( + not isinstance(self.inductor_configs, dict) + or any(not isinstance(key, str) for key in self.inductor_configs) + ): + raise ValueError("compile.inductor_configs must be a mapping with string keys or None") + if self.inductor_configs and self.inductor_mode == "off": + raise ValueError("compile.inductor_configs requires an enabled compile.inductor_mode") + @dataclass class FSDP2MixedPrecisionConfig: diff --git a/hyper_parallel/trainer/base.py b/hyper_parallel/trainer/base.py index 7c945d42aa..010b4cdfb7 100644 --- a/hyper_parallel/trainer/base.py +++ b/hyper_parallel/trainer/base.py @@ -615,6 +615,7 @@ def _build_graph_compiler(self) -> None: mesh_context=self.mesh, source_shard_info=getattr(self.model, "_source_shard_info", None), ) + self.graph_compiler.prepare_model() def _graph_train_fn( self, diff --git a/hyper_parallel/trainer/text_trainer.py b/hyper_parallel/trainer/text_trainer.py index da0be2830e..de304e0586 100644 --- a/hyper_parallel/trainer/text_trainer.py +++ b/hyper_parallel/trainer/text_trainer.py @@ -70,11 +70,11 @@ def __init__(self, config: TrainerConfig) -> None: self.base.attach_model_integration_data_pipeline() self.base._compute_train_iters() + self.base._build_training_context() + self.base._build_graph_compiler() self.base._build_optimizer() self.base.model_integration.attach_optimizer(self.base.optimizer) self.base._build_lr_scheduler() - self.base._build_training_context() - self.base._build_graph_compiler() self.base._init_callbacks() def _build_model_assets(self) -> None: diff --git a/tests/ut/auto_models/distributed/fsdp/test_fsdp_adapter.py b/tests/ut/auto_models/distributed/fsdp/test_fsdp_adapter.py index 875412c9f9..70b34cd50b 100644 --- a/tests/ut/auto_models/distributed/fsdp/test_fsdp_adapter.py +++ b/tests/ut/auto_models/distributed/fsdp/test_fsdp_adapter.py @@ -73,6 +73,16 @@ def test_graph_unit_bindings_follow_actual_inner_parameter_claims(self): self.assertEqual(len(bindings), len(list(model.parameters())) - 1) self.assertTrue(all(not getattr(param, "_hsdp_param_initialized", False) for param in model.parameters())) + def test_replicate_parameter_names_validate_and_expand_tied_aliases(self): + """Both execution paths select configured replicas by actual parameter identity.""" + model = _DecoderModel() + model.head.bias = model.layers[0].bias + manager = FSDP2Manager(FSDP2Config(replicate_params=["layers.0.bias"]), MeshContext()) + self.assertEqual(manager.replicate_parameter_names(model), frozenset(("layers.0.bias", "head.bias"))) + manager.config.replicate_params = ["missing"] + with self.assertRaisesRegex(ValueError, "unknown parameter FQN"): + manager.replicate_parameter_names(model) + def test_preinitialized_parameters_are_absent_from_graph_bindings(self): """An already managed nested parameter is not claimed a second time.""" model = _DecoderModel() diff --git a/tests/ut/auto_models/test_build_options.py b/tests/ut/auto_models/test_build_options.py index 9acee13238..c400247bbc 100644 --- a/tests/ut/auto_models/test_build_options.py +++ b/tests/ut/auto_models/test_build_options.py @@ -60,6 +60,8 @@ def test_compile_config_fields(self): ("fsdp_allgather_prefetch", "True"), ("fsdp_overlap_grad_reduce", "True"), ("max_in_flight_gb", "0.25"), + ("inductor_mode", "'off'"), + ("inductor_configs", "None"), ], ) @@ -108,6 +110,19 @@ def test_compile_config_validation(self): with self.assertRaisesRegex(ValueError, "dynamo_cache_size_limit"): CompileConfig(dynamo_cache_size_limit=0) + def test_joint_backend_validation(self): + """Reject malformed backend options and configurations disabled by their mode.""" + cases = ( + {"inductor_mode": "eager"}, + {"inductor_mode": "full", "inductor_configs": ["debug"]}, + {"inductor_mode": "full", "inductor_configs": {1: True}}, + {"inductor_configs": {"debug": True}}, + ) + for options in cases: + with self.subTest(options=options): + with self.assertRaisesRegex(ValueError, "compile.inductor_"): + CompileConfig(**options) + def test_use_joint_graph_selects_graph_compiler(self): """Graph-mode trainer selection is driven by compile.enabled + flag.""" config = CompileConfig( diff --git a/tests/ut/compile/test_compiler.py b/tests/ut/compile/test_compiler.py index ea5ae69b50..03617cc441 100644 --- a/tests/ut/compile/test_compiler.py +++ b/tests/ut/compile/test_compiler.py @@ -33,6 +33,7 @@ tracer tests validate; only the compiler wiring is asserted here. """ +from copy import deepcopy import os from types import SimpleNamespace import unittest @@ -70,6 +71,45 @@ def _fixed_batch(): class TestGraphCompilerForwardBackward(unittest.TestCase): """``compile`` / lazy ``forward_backward`` without distributed.""" + def test_terminal_wrapper_preserves_training_metadata(self) -> None: + """Execute the returned module while preserving loss and gradient output layout.""" + model = _make_model() + x, y = _fixed_batch() + expected_loss = _mse_train_fn(model, x=x, y=y) + expected_grads = torch.autograd.grad(expected_loss, tuple(model.parameters())) + compiler = GraphCompiler(model, _mse_train_fn, PassConfig(fsdp_enabled=False)) + wrappers = [] + + class RuntimeWrapper(nn.Module): + """Mirror the backend's positional state-and-input calling convention.""" + + def __init__(self, inner): + """Keep the executable without its tracer metadata.""" + super().__init__() + self.inner = inner + + def forward(self, *args): + """Forward every flat state and user input positionally.""" + return self.inner(*args) + + def wrap(graph_module, **_kwargs): + """Return a terminal executable without copying tracer metadata.""" + wrappers.append(RuntimeWrapper(graph_module)) + return wrappers[-1] + + self.assertIsNone(compiler.execution_backend) + with patch("hyper_parallel.compile.compiler.PassPipeline.run", side_effect=wrap): + loss, _ = compiler.forward_backward(x=x, y=y) + + self.assertIs(compiler._joint_graph.graph_module, wrappers[0]) + self.assertEqual(compiler.execution_backend, "fx") + self.assertEqual(wrappers[0].num_state_inputs, len(tuple(model.parameters()))) + self.assertEqual(wrappers[0].state_fqns, compiler._joint_graph.state_fqns) + self.assertEqual(wrappers[0].gradient_usage, (True, True)) + torch.testing.assert_close(loss, expected_loss, rtol=0, atol=0) + for parameter, expected in zip(model.parameters(), expected_grads): + torch.testing.assert_close(parameter.grad, expected, rtol=0, atol=0) + def test_unit_bindings_reach_passes_with_tied_expert(self) -> None: """Claim nested experts before the root and keep every alias on EDP.""" model = nn.Module() @@ -278,6 +318,39 @@ def test_grad_count_mismatch_raises(self): with self.assertRaises(ValueError): comp.forward_backward(x=torch.randn(2, 4), y=torch.randn(2, 4)) + def test_unused_parameters_preserve_adamw_skip_semantics(self): + """An absent autograd edge must not create weight decay or optimizer state.""" + model = _make_model() + model.register_parameter("unused", nn.Parameter(torch.ones(4))) + model.register_parameter("zero_gradient", nn.Parameter(torch.ones(4))) + reference = deepcopy(model) + + def loss_with_zero_gradient(current, *, x, y): + """Keep a real zero gradient distinct from an absent autograd edge.""" + return _mse_train_fn(current, x=x, y=y) + current.zero_gradient.sum() * 0 + + compiler = GraphCompiler(model, loss_with_zero_gradient, PassConfig(fsdp_enabled=False), + device=torch.device("cpu")) + optimizer = torch.optim.AdamW(model.parameters(), lr=0.01, weight_decay=0.1, foreach=False) + reference_optimizer = torch.optim.AdamW(reference.parameters(), lr=0.01, weight_decay=0.1, foreach=False) + x, y = _fixed_batch() + for _ in range(10): + expected_loss = loss_with_zero_gradient(reference, x=x, y=y) + expected_loss.backward() + loss, _ = compiler.forward_backward(x=x, y=y) + torch.testing.assert_close(loss, expected_loss, rtol=0, atol=0) + self.assertIsNone(model.unused.grad) + torch.testing.assert_close(model.zero_gradient.grad, torch.zeros(4), rtol=0, atol=0) + optimizer.step() + reference_optimizer.step() + for actual, expected in zip(model.parameters(), reference.parameters()): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + self.assertNotIn(model.unused, optimizer.state) + self.assertNotIn(reference.unused, reference_optimizer.state) + self.assertIn(model.zero_gradient, optimizer.state) + optimizer.zero_grad(set_to_none=True) + reference_optimizer.zero_grad(set_to_none=True) + class TestGraphCompilerDevice(unittest.TestCase): """``to`` (device move).""" @@ -830,12 +903,12 @@ def test_dense_and_expert_layouts_without_fsdp_config(self): model.experts.local_expert_count = 2 context = MeshContext(fsdp_non_moe_mesh=dense, fsdp_moe_mesh=expert, dp_size=8, dp_shard_size=8, ep_size=ep, edp_shard_size=8 // ep) - compiler = GraphCompiler(model, _mse_train_fn, mesh_context=context) + compiler = GraphCompiler(model, _mse_train_fn, PassConfig(enable_overlap=False), mesh_context=context) with patch.object(compiler, "_register_mesh_group"): compiler._init_device_mesh(context) compiler._bind_parameter_meshes() self.assertEqual(compiler._parameter_meshes, {}) - compiler._joint_graph = SimpleNamespace(graph_module=SimpleNamespace()) + compiler.prepare_model() layouts = compiler.parameter_layouts self.assertEqual(layouts[model.dense].mesh.mesh_dim_names, ("fsdp_shard",)) expert_layout = layouts[model.experts.weight] @@ -853,7 +926,7 @@ def test_legacy_named_world_metadata_flattens_dp_and_cp(self): context = MeshContext(device_mesh=mesh, dp_replicate_size=replicate, dp_shard_size=2) model = _make_model() compiler = GraphCompiler(model, _mse_train_fn, PassConfig(fsdp_enabled=False), mesh_context=context) - compiler._joint_graph = SimpleNamespace(graph_module=SimpleNamespace()) + compiler.prepare_model() with patch.object(compiler, "_register_mesh_group") as register: layout = compiler.parameter_layouts[model.weight] register.assert_not_called() @@ -876,7 +949,7 @@ def test_hsdp_and_named_world_layout_meshes(self): compiler._init_device_mesh(context) expected = ("dp",) if world_only else ("fsdp_replicate", "fsdp_shard") self.assertEqual(compiler._fsdp_mesh.mesh_dim_names, expected) - compiler._joint_graph = SimpleNamespace(graph_module=SimpleNamespace()) + compiler.prepare_model() layout_names = expected if world_only else (*expected, "tp") self.assertEqual(compiler.parameter_layouts[model.weight].mesh.mesh_dim_names, layout_names) diff --git a/tests/ut/compile/test_fsdp_pass.py b/tests/ut/compile/test_fsdp_pass.py index 0714e76480..a7e9860030 100644 --- a/tests/ut/compile/test_fsdp_pass.py +++ b/tests/ut/compile/test_fsdp_pass.py @@ -515,6 +515,92 @@ def test_run_pad_shards_non_divisible_params(self): "the [3, 4] gradient must be padded back to [4, 4] before reduction", ) + def test_shard_storage_preserves_padded_and_empty_tails(self): + """Every graph rank keeps uniform storage without changing parameter identity.""" + degree = 4 + for rows in (0, 2, 5): + source = torch.arange(rows * 4, dtype=torch.float32).reshape(rows, 4) + padded_rows = -(-rows // degree) * degree + padded = torch.cat((source, source.new_zeros(padded_rows - rows, 4))) + for rank in range(degree): + with self.subTest(rows=rows, rank=rank): + model = _OddParam() + model.odd = nn.Parameter(source.clone()) + param = model.odd + gm = _odd_joint_graph() + with _patch_dist(world_size=degree, rank=rank): + FSDPPass().run(gm, PassConfig(fsdp_degree=degree), model=model) + expected = padded.chunk(degree)[rank] + self.assertIs(model.odd, param) + torch.testing.assert_close(model.odd, expected, atol=0, rtol=0) + gm.graph.lint() + + def test_configured_replica_preserves_storage_and_uses_all_reduce(self): + """Explicit replicas bypass padding and gathers while retaining DP synchronization.""" + for mode in ("fsdp", "hsdp"): + with self.subTest(mode=mode): + model = _OddParam() + original = model.odd + gm = _odd_joint_graph() + with _patch_dist(world_size=4): + FSDPPass().run(gm, PassConfig(dp_mode=mode, fsdp_degree=2, dp_replicate_degree=2), + model=model, replicate_param_names={"odd"}) + self.assertIs(model.odd, original) + self.assertEqual(tuple(model.odd.shape), (3, 4)) + self.assertEqual(_count_targets(gm, "all_gather_into_tensor"), 0) + self.assertEqual(_count_targets(gm, "reduce_scatter_tensor"), 0) + self.assertEqual(_count_targets(gm, "all_reduce"), 2 if mode == "hsdp" else 1) + self.assertFalse(gm.fsdp_sharded_param_names) + with _patch_dist(world_size=2), self.assertRaisesRegex(ValueError, "unknown parameter FQNs"): + FSDPPass().run(_odd_joint_graph(), PassConfig(fsdp_degree=2), + model=_OddParam(), replicate_param_names={"missing"}) + + def test_unused_gradient_usage_uses_edp_and_source_groups_without_world_or(self): + """All-false ranks still participate, without mixing different expert shards.""" + gm = _linear_joint_graph() + gm.gradient_usage = (False, False) + calls = [] + + def reduce_usage(tensor, *, op, group): + self.assertEqual(op, mocked.ReduceOp.MAX) + calls.append((group, tensor.clone())) + if group == "dense": + tensor.fill_(1) + + with _patch_dist(world_size=4) as mocked, patch(_RESOLVE_PG_PATH, side_effect=lambda name: name): + mocked.get_world_size.side_effect = lambda group=None: 4 if group is None else 2 + mocked.all_reduce.side_effect = reduce_usage + FSDPPass().run(gm, PassConfig(fsdp_degree=2), model=nn.Linear(4, 4), fsdp_group_name="dense", + expert_param_names={"weight"}, expert_fsdp_group_name="edp", + gradient_reductions={"weight": GradientReduction("sum", 4)}, + source_gradient_groups={"bias": "tp"}) + self.assertEqual([group for group, _ in calls], ["dense", "edp", "tp"]) + self.assertEqual([tensor.tolist() for _, tensor in calls], [[0], [0], [1]]) + self.assertEqual(gm.gradient_usage, (False, True)) + + def test_usage_propagates_in_hsdp_then_source_execution_order(self): + """Compile-time usage follows shard, replicate, and source reductions exactly once.""" + gm = _linear_joint_graph() + gm.gradient_usage = (False, False) + calls = [] + + def reduce_usage(tensor, *, op, group): + self.assertEqual(op, mocked.ReduceOp.MAX) + calls.append((group, tensor.tolist())) + if group == "fsdp": + tensor[0] = 1 + elif group == "replicas": + tensor[1] = 1 + + with _patch_dist(world_size=8) as mocked, patch(_RESOLVE_PG_PATH, side_effect=lambda name: name): + mocked.get_world_size.side_effect = lambda group=None: 8 if group is None else 2 + mocked.all_reduce.side_effect = reduce_usage + FSDPPass().run(gm, PassConfig(dp_mode="hsdp", fsdp_degree=2, dp_replicate_degree=2), + model=nn.Linear(4, 4), dp_replicate_group_name="replicas", + source_gradient_groups={"bias": "tp"}) + self.assertEqual(calls, [("fsdp", [0, 0]), ("replicas", [1, 0]), ("tp", [1])]) + self.assertEqual(gm.gradient_usage, (True, True)) + def test_source_replicate_sum_follows_waited_fsdp_reduction(self): """TP-replicated gradients sum after DP shards, retaining both identities.""" gm = _linear_joint_graph() diff --git a/tests/ut/compile/test_graphtrainer_config.py b/tests/ut/compile/test_graphtrainer_config.py index 042d4f92ca..790dfa047b 100644 --- a/tests/ut/compile/test_graphtrainer_config.py +++ b/tests/ut/compile/test_graphtrainer_config.py @@ -400,6 +400,22 @@ def test_graph_mode_allows_pipeline_parallel_topology(self): self.assertTrue(config.compile.selects_graph_compiler()) + def test_dp_replicas_enable_ddp_or_hsdp_without_changing_single_rank_defaults(self): + """Resolved DP replicas select reductions even when the shard degree is one.""" + for shard, replicas, mode in ((1, 4, "ddp"), (2, 2, "hsdp"), (4, 1, "fsdp"), (1, 1, "fsdp")): + with self.subTest(shard=shard, replicas=replicas): + config = SimpleNamespace(fsdp_config=SimpleNamespace(dp_shard_size=shard, edp_shard_size=1)) + result = build_pass_config_from_trainer_config( + config, mesh_context=MeshContext(dp_replicate_size=replicas)) + self.assertEqual(result.fsdp_enabled, shard > 1 or replicas > 1) + self.assertEqual(result.dp_mode, mode) + self.assertEqual(result.dp_replicate_degree, replicas) + disabled = build_pass_config_from_trainer_config( + config, mesh_context=MeshContext(dp_replicate_size=replicas), fsdp_enabled=False) + self.assertFalse(disabled.fsdp_enabled) + self.assertFalse(build_pass_config_from_trainer_config( + SimpleNamespace(fsdp_config=SimpleNamespace(dp_shard_size=1, edp_shard_size=1))).fsdp_enabled) + def test_build_pass_config_projects_parallel_topology(self): """Trainer topology and inferred FSDP intent are copied to PassConfig.""" config = SimpleNamespace( diff --git a/tests/ut/compile/test_inductor_pass.py b/tests/ut/compile/test_inductor_pass.py index 10c206adb9..9077072295 100644 --- a/tests/ut/compile/test_inductor_pass.py +++ b/tests/ut/compile/test_inductor_pass.py @@ -27,9 +27,8 @@ itself; every per-stage forward/backward subgraph is compiled instead (both ``regional`` and ``full``), while unknown ``call_module`` stubs are still rejected. -4. Guards: missing ``example_inputs``, mode canonicalization, pipeline - placement (terminal), and that inductor settings stay PassConfig-owned - (PP-style) instead of being projected from ``CompileConfig``. +4. Guards: missing ``example_inputs``, mode canonicalization, terminal + placement, and explicit joint-backend settings projected from ``CompileConfig``. """ # The backend env var must be set before torch / hyper_parallel are imported, @@ -62,6 +61,7 @@ InductorPass, _propagate_fake_vals, ) +from hyper_parallel.compile.passes.parallel.fsdp_pass import FSDPPass from hyper_parallel.compile.passes.parallel.pp_schedule import PipelineScheduleBase from hyper_parallel.compile.passes.pipeline import PassPipeline from hyper_parallel.compile.tracer.graph_tracer import trace_model_graph @@ -447,7 +447,7 @@ def test_pipeline_allows_pp_with_inductor(self): self.assertEqual(joint.graph_module.meta.get("inductor_compiled"), "regional") def test_trainer_config_projection_strategy(self): - """Test inductor config is PassConfig-owned (PP-style), not projected.""" + """Project joint backend options independently of decoder compilation options.""" trainer_config = SimpleNamespace( accelerator=SimpleNamespace( tp_size=1, sequence_parallel=False, loss_parallel=False @@ -458,13 +458,65 @@ def test_trainer_config_projection_strategy(self): compile=SimpleNamespace( inductor_mode="regional", inductor_configs={"reorder_for_peak_memory": True}, + backend="decoder_backend", + options={"decoder_only": True}, ), ) config = build_pass_config_from_trainer_config(trainer_config) - # Like PP, inductor settings live on PassConfig only; the projected - # config keeps the defaults and ignores CompileConfig-level overrides. - self.assertEqual(config.inductor_mode, "off") - self.assertIsNone(config.inductor_configs) + self.assertEqual(config.inductor_mode, "regional") + self.assertEqual(config.inductor_configs, {"reorder_for_peak_memory": True}) + + def test_retains_backend_replacement(self): + """The pass returns the executable wrapper produced by backend lowering.""" + joint = self._traced() + wrapper = nn.Sequential(joint.graph_module) + wrapper.meta = {} + with mock.patch.object(InductorPass, "_compile_graph", return_value=wrapper): + result = InductorPass().run( + joint.graph_module, PassConfig(inductor_mode="full"), example_inputs=joint.example_inputs + ) + self.assertIs(result, wrapper) + self.assertEqual(result.meta["inductor_compiled"], "full") + + def test_compiler_rejects_unavailable_backend(self): + """An unavailable backend cannot be mistaken for successful compilation.""" + compiler = GraphCompiler( + _make_model(), _train_fn, pass_config=PassConfig(fsdp_enabled=False, inductor_mode="full") + ) + with mock.patch( + "hyper_parallel.compile.passes.inductor_pass._regional_inductor_available", return_value=False + ): + with self.assertRaisesRegex(RuntimeError, "unavailable"): + x, y = _fixed_batch() + compiler.compile(x=x, y=y) + self.assertFalse(compiler.is_compiled) + self.assertIsNone(compiler.execution_backend) + + def test_fsdp_backend_inputs_describe_physical_storage(self): + """Backend sizing uses padded shards while retaining the traced user inputs.""" + model = nn.Linear(4, 5) + joint = trace_model_graph(model, _train_fn, {"x": torch.randn(2, 4), "y": torch.randn(2, 5)}) + transport = mock.MagicMock() + transport.is_initialized.return_value = True + transport.get_world_size.return_value = 2 + transport.get_rank.return_value = 1 + with mock.patch("hyper_parallel.compile.passes.parallel.fsdp_pass.dist", transport), \ + mock.patch("hyper_parallel.compile.passes.parallel.fsdp_pass._resolve_process_group"): + graph_module = FSDPPass().run(joint.graph_module, PassConfig(fsdp_degree=2), model=model) + parameters = dict(model.named_parameters()) + for node in graph_module.graph.find_nodes(op="placeholder"): + name = node.meta.get("param_name") + if name in parameters: + self.assertEqual(tuple(node.meta["val"].shape), tuple(parameters[name].shape)) + self.assertEqual(node.meta["val"].stride(), parameters[name].stride()) + with mock.patch("hyper_parallel.compile.passes.inductor_pass._propagate_fake_vals"), \ + mock.patch("torch.fx.passes.regional_inductor.regional_inductor") as backend: + InductorPass().run(graph_module, PassConfig(inductor_mode="full"), example_inputs=joint.example_inputs) + state_count = graph_module.num_state_inputs + runtime_inputs = backend.call_args.args[1] + self.assertEqual(tuple(runtime_inputs[0].shape), tuple(model.weight.shape)) + for actual, expected in zip(runtime_inputs[state_count:], joint.example_inputs[state_count:]): + self.assertIs(actual, expected) if __name__ == "__main__": diff --git a/tests/ut/compile/test_mixed_precision.py b/tests/ut/compile/test_mixed_precision.py new file mode 100644 index 0000000000..7cb87264c0 --- /dev/null +++ b/tests/ut/compile/test_mixed_precision.py @@ -0,0 +1,271 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Precision policy preserves storage and the complete optimizer update chain.""" + +from copy import deepcopy +from itertools import product +from types import SimpleNamespace +from unittest import TestCase +from unittest.mock import MagicMock, patch + +import torch +from torch import nn + +from hyper_parallel.compile.compiler import GraphCompiler +from hyper_parallel.compile.mixed_precision import GraphMixedPrecision +from hyper_parallel.compile.pass_config import PassConfig +from hyper_parallel.core.fully_shard.utils import MixedPrecisionPolicy +from hyper_parallel.models.build_options import FSDP2Config, FSDP2MixedPrecisionConfig + + +def _loss(model, *, x): + """Keep loss accumulation in FP32 while linear computation is BF16.""" + return model(x).float().square().mean() + + +def _compiler(model, *, fp32_main_params=False, **policy): + """Use the same YAML policy resolver as TextTrainer.""" + config = SimpleNamespace(fsdp_config=FSDP2Config(wrapper_class=["_Block"], + mix_precision=FSDP2MixedPrecisionConfig(**policy)), + optimizer=SimpleNamespace(fp32_main_params=fp32_main_params)) + return GraphCompiler(model, _loss, PassConfig(fsdp_enabled=False), trainer_config=config, + device=torch.device("cpu")) + + +class _Block(nn.Linear): + """Explicitly selected FSDP child unit.""" + + +class _Float32(nn.Module): + """Model-owned explicit FP32 region, as used by routers and normalization.""" + + def forward(self, value: torch.Tensor) -> torch.Tensor: + """Convert the activation before the FP32 projection.""" + return value.float() + + +class TestGraphMixedPrecision(TestCase): + """The default input cast and per-parameter precision are graph operations.""" + + def test_compute_gradients_do_not_round_through_storage_dtype(self): + """Match eager's compute-parameter gradients before communication and accumulation.""" + for (storage, compute), reduce in product( + ((torch.float32, torch.bfloat16), (torch.bfloat16, torch.float32)), (None, "float32") + ): + with self.subTest(storage=storage, compute=compute, reduce=reduce): + model = nn.Sequential(_Block(2, 2, bias=False, dtype=storage)) + with torch.no_grad(): + model[0].weight.fill_(1) + reference = deepcopy(model).to(compute) + x = torch.full((2, 2), 1 + 2 ** -9) + expected_loss = _loss(reference, x=x.to(compute)) + expected_loss.backward() + compiler = _compiler(model, fp32_main_params=True, + param_dtype=str(compute).removeprefix("torch."), reduce_dtype=reduce) + actual_loss, _ = compiler.forward_backward(x=x) + torch.testing.assert_close(actual_loss, expected_loss, rtol=0, atol=0) + output = next(node for node in compiler._joint_graph.graph_module.graph.nodes if node.op == "output") + gradient = output.args[0][1] + self.assertEqual(gradient.meta["val"].dtype, torch.float32 if reduce else compute) + self.assertEqual(model[0].weight.dtype, storage) + self.assertIsNone(model[0].weight.grad) + expected_dtype = torch.float32 if reduce else compute + self.assertEqual(model[0].weight.main_grad.dtype, expected_dtype) + torch.testing.assert_close(model[0].weight.main_grad, reference[0].weight.grad.to(expected_dtype), + rtol=0, atol=0) + + def test_single_rank_explicit_reduction_dtype_is_captured(self): + """Single-rank shortcuts cannot bypass reduction rounding or main-gradient storage.""" + for initialized, fsdp_enabled, main_grad in product((False, True), repeat=3): + with self.subTest(initialized=initialized, fsdp_enabled=fsdp_enabled, main_grad=main_grad): + model = nn.Sequential(_Block(2, 2, bias=False)) + with torch.no_grad(): + model[0].weight.fill_(1) + reference = deepcopy(model) + x = torch.full((2, 2), 1 + 2 ** -9) + expected_loss = _loss(reference, x=x) + expected_loss.backward() + expected = reference[0].weight.grad.bfloat16() + self.assertFalse(torch.equal(expected.float(), reference[0].weight.grad)) + if not main_grad: + expected = expected.float() + compiler = _compiler(model, fp32_main_params=main_grad, reduce_dtype="bfloat16") + compiler.pass_config.fsdp_enabled = fsdp_enabled + compiler.pass_config.fsdp_degree = 1 + distributed = MagicMock() + distributed.is_initialized.return_value = initialized + distributed.get_world_size.return_value = 1 + with ( + patch("hyper_parallel.compile.compiler.dist", distributed), + patch("hyper_parallel.compile.passes.parallel.fsdp_pass.dist", distributed), + patch.object(compiler, "_init_device_mesh"), + ): + loss, _ = compiler.forward_backward(x=x) + torch.testing.assert_close(loss, expected_loss, rtol=0, atol=0) + gradient = model[0].weight.main_grad if main_grad else model[0].weight.grad + torch.testing.assert_close(gradient, expected, rtol=0, atol=0) + graph = compiler._joint_graph.graph_module + output = next(node for node in graph.graph.nodes if node.op == "output") + self.assertEqual(output.args[0][1].meta["val"].dtype, torch.bfloat16) + self.assertFalse(any("_c10d_functional" in str(node.target) for node in graph.graph.nodes)) + + def test_reduction_cast_preserves_trainable_state_and_named_losses(self): + """Frozen state and buffers cannot shift gradient dtype assignments or include named losses.""" + model = nn.Sequential(_Block(2, 2)) + model[0].bias.requires_grad_(False) + model[0].register_parameter("unused", nn.Parameter(torch.ones(2, 2))) + model[0].register_buffer("offset", torch.ones(2)) + model[0].alias_weight = model[0].weight + reference = deepcopy(model) + x = torch.full((2, 2), 1 + 2 ** -9) + expected_loss = _loss(reference, x=x) + expected_loss.backward() + compiler = _compiler(model, reduce_dtype="bfloat16") + + def named_loss(current, **inputs): + """Expose a forward loss separately from trainable gradient outputs.""" + value = _loss(current, **inputs) + return value, {"aux": value} + + compiler.train_fn = named_loss + loss, losses = compiler.forward_backward(x=x) + torch.testing.assert_close(loss, expected_loss, rtol=0, atol=0) + torch.testing.assert_close(losses["aux"], expected_loss, rtol=0, atol=0) + torch.testing.assert_close(model[0].weight.grad, reference[0].weight.grad.bfloat16().float(), rtol=0, atol=0) + self.assertIsNone(model[0].unused.grad) + self.assertIsNone(model[0].bias.grad) + self.assertIs(model[0].alias_weight, model[0].weight) + previous = torch.ones_like(model[0].unused) + model[0].unused.grad = previous + compiler.forward_backward(x=x) + self.assertIs(model[0].unused.grad, previous) + torch.testing.assert_close(previous, torch.ones_like(previous), rtol=0, atol=0) + + def test_parameter_overrides_preserve_implicit_reduction_dtype(self): + """Missing fields inherit defaults while None and redundant casts preserve gradient dtype.""" + cases = ( + (None, torch.bfloat16, torch.float32), + ({"param_dtype": None}, torch.float32, torch.float32), + ({"reduce_dtype": None}, torch.bfloat16, None), + ({"param_dtype": None, "reduce_dtype": None}, torch.float32, None), + ({"param_dtype": torch.float32}, torch.float32, None), + ({"reduce_dtype": torch.bfloat16}, torch.bfloat16, None), + ) + for custom, compute, reduce in cases: + with self.subTest(custom=custom): + model = nn.Linear(2, 2) + policy = MixedPrecisionPolicy( + param_dtype=torch.bfloat16, + reduce_dtype=torch.float32, + custom_params={model.weight: custom} if custom is not None else None, + ) + precision = GraphMixedPrecision(model, policy, ()) + self.assertEqual(precision.parameter_dtypes["weight"], (compute, reduce)) + self.assertEqual(precision.parameter_dtypes["bias"], (torch.bfloat16, torch.float32)) + cast = precision.cast_state(dict(model.named_parameters())) + self.assertEqual(cast["weight"].dtype, compute) + + def test_fp32_storage_bf16_compute_matches_ten_adamw_updates(self): + """Compare losses, gradients, weights and moments against explicit eager casts.""" + torch.manual_seed(17) + model = nn.Sequential(_Block(4, 4)) + reference = deepcopy(model) + compiler = _compiler(model, param_dtype="bfloat16", reduce_dtype="float32") + compiler.prepare_model() + optimizer = torch.optim.AdamW(model.parameters(), lr=0.001, foreach=False) + expected_optimizer = torch.optim.AdamW(reference.parameters(), lr=0.001, foreach=False) + for step in range(10): + x = torch.arange(12, dtype=torch.float32).reshape(3, 4) / 16 + step / 64 + computation = deepcopy(reference).bfloat16() + expected = computation(x.bfloat16()) + expected_loss = expected.float().square().mean() + expected_loss.backward() + for master, parameter in zip(reference.parameters(), computation.parameters()): + master.grad = parameter.grad.float() + loss, _ = compiler.forward_backward(x=x) + torch.testing.assert_close(loss, expected_loss, rtol=0, atol=0) + for actual, expected_parameter in zip(model.parameters(), reference.parameters()): + self.assertEqual(actual.dtype, torch.float32) + torch.testing.assert_close(actual.grad, expected_parameter.grad, rtol=0, atol=0) + optimizer.step() + expected_optimizer.step() + for actual, expected_parameter in zip(model.parameters(), reference.parameters()): + torch.testing.assert_close(actual, expected_parameter, rtol=0, atol=0) + for key in ("exp_avg", "exp_avg_sq", "step"): + torch.testing.assert_close(optimizer.state[actual][key], + expected_optimizer.state[expected_parameter][key], rtol=0, atol=0) + optimizer.zero_grad() + expected_optimizer.zero_grad() + self.assertFalse(model._forward_pre_hooks) + self.assertFalse(model._forward_hooks) + self.assertTrue(any(node.target == torch.ops.aten._to_copy.default + for node in compiler._joint_graph.graph_module.graph.nodes)) + + def test_custom_parameter_dtype_and_module_output_cast(self): + """Pattern overrides preserve an explicit FP32 compute region.""" + model = nn.Sequential(_Block(4, 4), _Float32(), nn.Linear(4, 4)) + compiler = _compiler(model, param_dtype="bfloat16", reduce_dtype="float32", output_dtype="float32", + custom_params={"2.*": {"param_dtype": "float32"}}) + compiler.prepare_model() + precision = compiler._mixed_precision + self.assertEqual(precision.parameter_dtypes["0.weight"], (torch.bfloat16, torch.float32)) + self.assertEqual(precision.parameter_dtypes["2.weight"], (torch.float32, None)) + state = dict(model.named_parameters()) + cast = precision.cast_state(state) + self.assertEqual(cast["0.weight"].dtype, torch.bfloat16) + self.assertIs(cast["2.weight"], model[2].weight) + with precision.module_casts(): + # Module output casts use the same floating-leaf helper as eager. + result = torch.func.functional_call(model, cast, (torch.ones(2, 4),)) + self.assertEqual(result.dtype, torch.float32) + + def test_fp32_main_gradient_keeps_reduction_precision(self): + """A BF16 model can accumulate FP32 gradients without assigning them to .grad.""" + model = nn.Sequential(_Block(4, 4, dtype=torch.bfloat16)) + compiler = _compiler(model, fp32_main_params=True, param_dtype="bfloat16", reduce_dtype="float32") + compiler.prepare_model() + compiler.forward_backward(x=torch.ones(2, 4)) + first = [parameter.main_grad.clone() for parameter in model.parameters()] + compiler.forward_backward(x=torch.ones(2, 4)) + for parameter, gradient in zip(model.parameters(), first): + self.assertIsNone(parameter.grad) + self.assertEqual(parameter.main_grad.dtype, torch.float32) + torch.testing.assert_close(parameter.main_grad, gradient * 2, rtol=0, atol=0) + + def test_microbatch_accumulation_preserves_eager_rounding(self): + """Reduction precision and per-microbatch storage casts precede accumulation.""" + cases = ( + (torch.float32, "bfloat16", True, (1.0, 2 ** -8), 1.0, torch.bfloat16), + (torch.bfloat16, "float32", False, (1.0, -0.998046875), 0.0, torch.bfloat16), + ) + for storage, reduction, main, inputs, expected, dtype in cases: + with self.subTest(storage=storage, reduction=reduction, main=main): + model = nn.Sequential(_Block(1, 1, bias=False, dtype=storage)) + compiler = _compiler(model, fp32_main_params=main, param_dtype="float32", + reduce_dtype=reduction) + + def summed_loss(current, *, x): + """Make the weight gradient equal the microbatch input.""" + return current(x).sum() + + compiler.train_fn = summed_loss + for value in inputs: + compiler.forward_backward(x=torch.tensor([[value]])) + gradient = model[0].weight.main_grad if main else model[0].weight.grad + self.assertEqual(gradient.dtype, dtype) + torch.testing.assert_close(gradient, torch.tensor([[expected]], dtype=dtype), rtol=0, atol=0) + if main: + gradient.div_(3) + torch.testing.assert_close(gradient, torch.tensor([[1 / 3]], dtype=dtype), rtol=0, atol=0) diff --git a/tests/ut/compile/test_ragged_ep.py b/tests/ut/compile/test_ragged_ep.py index b1ef529a59..067f611f64 100644 --- a/tests/ut/compile/test_ragged_ep.py +++ b/tests/ut/compile/test_ragged_ep.py @@ -76,6 +76,7 @@ def test_compile_resolves_expert_parameter_aliases(self): with ( patch("hyper_parallel.compile.compiler.dist.is_initialized", return_value=True), patch("hyper_parallel.compile.compiler._register_process_group") as register, + patch("hyper_parallel.compile.compiler.FSDPPass.prepare_model", return_value=frozenset()), patch("hyper_parallel.compile.compiler.trace_model_graph", return_value=joint), patch("hyper_parallel.compile.compiler.PassPipeline.from_config") as pipeline, ): diff --git a/tests/ut/compile/test_storage_preparation.py b/tests/ut/compile/test_storage_preparation.py new file mode 100644 index 0000000000..64598490eb --- /dev/null +++ b/tests/ut/compile/test_storage_preparation.py @@ -0,0 +1,156 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Storage preparation is independent of lazy graph capture and optimizer state.""" + +from contextlib import contextmanager +from unittest import TestCase +from unittest.mock import MagicMock, patch + +import torch +from torch import nn + +from hyper_parallel.compile.compiler import GraphCompiler +from hyper_parallel.compile.pass_config import PassConfig +from hyper_parallel.compile.passes.parallel.fsdp_pass import FSDPPass +from hyper_parallel.compile.tracer.graph_tracer import trace_model_graph +from hyper_parallel.components.optim.mixed_precision_optimizer import Float16OptimizerWithFloat16Params +from hyper_parallel.core.optimizer.optimizer import ChainedOptimizer + + +class _Model(nn.Module): + """Include aliases, padding, a scalar, frozen state and a buffer.""" + + def __init__(self, dtype: torch.dtype = torch.float32) -> None: + """Build the state categories that preparation must distinguish.""" + super().__init__() + self.weight = nn.Parameter(torch.arange(15, dtype=dtype).reshape(5, 3) / 16) + self.alias = self.weight + self.scale = nn.Parameter(torch.tensor(0.5, dtype=dtype)) + self.frozen = nn.Parameter(torch.ones(5, dtype=dtype), requires_grad=False) + self.register_buffer("offset", torch.ones(5, dtype=dtype)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Read full logical tensors during forward.""" + return (x @ self.weight.T + self.offset) * self.scale + self.frozen + + +def _loss(model, *, x): + """Use a scalar loss without labels.""" + return model(x).square().mean() + + +@contextmanager +def _distributed(compiler): + """Mock transport setup; slicing, tracing and graph lowering remain real.""" + transport = MagicMock() + transport.is_initialized.return_value = True + transport.get_world_size.return_value = 2 + transport.get_rank.return_value = 0 + with ( + patch("hyper_parallel.compile.compiler.dist", transport), + patch("hyper_parallel.compile.passes.parallel.fsdp_pass.dist", transport), + patch("hyper_parallel.compile.passes.parallel.fsdp_pass._resolve_process_group"), + patch.object(compiler, "_init_device_mesh"), + ): + yield + + +class TestStoragePreparation(TestCase): + """Preparation owns physical slicing; the graph only lowers logical state.""" + + def test_prepare_once_preserves_parameter_identity_and_nonsharded_state(self): + """A second preparation neither re-shards nor changes loaded values.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + original = torch.detach(model.weight).clone() + identity = id(model.weight) + with _distributed(compiler): + compiler.prepare_model() + self.assertEqual(compiler.parameter_shapes["weight"], (5, 3)) + self.assertEqual(compiler.sharded_parameter_names, {"weight"}) + self.assertEqual(model.weight.shape, (3, 3)) + torch.testing.assert_close(model.weight, original[:3], rtol=0, atol=0) + with torch.no_grad(): + model.weight.add_(1) + compiler.prepare_model() + self.assertEqual(id(model.weight), identity) + self.assertIs(model.alias, model.weight) + self.assertEqual(model.scale.shape, ()) + self.assertEqual(model.frozen.shape, (5,)) + self.assertEqual(model.offset.shape, (5,)) + self.assertFalse(compiler.is_compiled) + torch.testing.assert_close(model.weight, original[:3] + 1, rtol=0, atol=0) + + def test_logical_trace_and_pass_do_not_cut_prepared_storage_again(self): + """Trace sees the original five rows while FSDP accepts the three-row shard.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler): + compiler.prepare_model() + shard = torch.detach(model.weight).clone() + for dynamic in (False, True): + with self.subTest(dynamic=dynamic): + joint = trace_model_graph(model, _loss, {"x": torch.ones(2, 3)}, + state_shapes=compiler.parameter_shapes, dynamic=dynamic) + self.assertEqual(joint.param_shapes["weight"], (5, 3)) + self.assertEqual(joint.example_inputs[0].shape, (5, 3)) + FSDPPass().run(joint.graph_module, compiler.pass_config, **compiler._build_pass_kwargs()) + self.assertEqual(joint.graph_module.fsdp_sharded_param_names, {"weight"}) + torch.testing.assert_close(model.weight, shard, rtol=0, atol=0) + + def test_fp32_main_parameters_are_built_from_prepared_shards(self): + """BF16 storage does not leave full-size FP32 optimizer copies behind.""" + model = _Model(torch.bfloat16) + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler): + compiler.prepare_model() + raw = torch.optim.AdamW(model.parameters(), lr=0.01) + optimizer = Float16OptimizerWithFloat16Params(ChainedOptimizer(model, {"adamw": raw}), model) + for parameter, main in optimizer.optimizer_param_by_model_param.items(): + self.assertEqual(parameter.shape, main.shape) + self.assertEqual(main.dtype, torch.float32) + self.assertEqual(model.weight.main_param.shape, (3, 3)) + + def test_failed_preparation_cannot_repeat_partial_slicing(self): + """A partially prepared model must be rebuilt rather than silently reused.""" + compiler = GraphCompiler(_Model(), _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler), patch.object(FSDPPass, "prepare_model", side_effect=ValueError("slice failed")): + with self.assertRaisesRegex(ValueError, "slice failed"): + compiler.prepare_model() + with self.assertRaisesRegex(RuntimeError, "rebuild the model"): + compiler.prepare_model() + + def test_trace_failure_keeps_prepared_storage_and_logical_shapes(self): + """A retry after capture failure cannot re-shard the live model.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler): + compiler.prepare_model() + shard = torch.detach(model.weight).clone() + with patch("hyper_parallel.compile.compiler.trace_model_graph", side_effect=ValueError("trace failed")): + for _ in range(2): + with self.assertRaisesRegex(ValueError, "trace failed"): + compiler.compile(x=torch.ones(2, 3)) + torch.testing.assert_close(model.weight, shard, rtol=0, atol=0) + self.assertEqual(compiler.parameter_shapes["weight"], (5, 3)) + + def test_replacing_prepared_parameter_requires_a_new_model(self): + """Parameter identity is part of the prepared-storage contract.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_enabled=False)) + compiler.prepare_model() + model.weight = nn.Parameter(torch.detach(model.weight).clone()) + with self.assertRaisesRegex(ValueError, "storage changed"): + compiler.prepare_model() diff --git a/tests/ut/compile/test_tracer.py b/tests/ut/compile/test_tracer.py index bdb2ed2c35..814ebeba3e 100644 --- a/tests/ut/compile/test_tracer.py +++ b/tests/ut/compile/test_tracer.py @@ -109,6 +109,31 @@ def test_state_fqns_and_param_flag(self): self.assertEqual(joint.state_fqns, ["lin.weight", "lin.bias", "buf"]) self.assertEqual(gm.state_is_param, [True, True, False]) self.assertEqual(gm.num_state_inputs, 3) + self.assertEqual(gm.gradient_usage, (True, True)) + + def test_gradient_usage_distinguishes_unused_zero_and_empty_gradients(self): + """Keep a used mask in gradient order without shifting aliases, frozen state or named losses.""" + model = nn.Linear(2, 2, bias=False) + model.register_parameter("unused", nn.Parameter(torch.ones(2, 2))) + model.register_parameter("empty", nn.Parameter(torch.empty(0, 2))) + model.register_parameter("frozen", nn.Parameter(torch.ones(2, 2), requires_grad=False)) + model.alias = model.weight + model.register_buffer("offset", torch.ones(2)) + + def train_fn(current, x, y): + """Produce real zero gradients and a truly absent parameter gradient.""" + loss = (current(x) - y).square().sum() + current.empty.sum() + current.frozen.sum() + return loss, {"aux": loss + current.offset.sum()} + + joint, _, grads, losses = _trace_and_run(model, train_fn, torch.zeros(2, 2), torch.zeros(2, 2)) + self.assertEqual(joint.state_fqns, ["weight", "unused", "empty", "frozen", "offset"]) + self.assertEqual(joint.graph_module.gradient_usage, (True, False, True)) + self.assertEqual(len(grads), 3) + torch.testing.assert_close(grads[0], torch.zeros_like(model.weight), rtol=0, atol=0) + torch.testing.assert_close(grads[1], torch.zeros_like(model.unused), rtol=0, atol=0) + self.assertEqual(grads[2].shape, model.empty.shape) + self.assertEqual(joint.graph_module.loss_dict_keys, ["aux"]) + self.assertEqual(list(losses), ["aux"]) def test_param_names_only_trainable(self): """Test ``param_names`` lists trainable parameter names.""" diff --git a/tests/ut/core/fully_shard/test_hsdp_utils.py b/tests/ut/core/fully_shard/test_hsdp_utils.py index 90582e491a..f0c11c204f 100644 --- a/tests/ut/core/fully_shard/test_hsdp_utils.py +++ b/tests/ut/core/fully_shard/test_hsdp_utils.py @@ -37,6 +37,7 @@ get_hsdp_state, get_managed_modules_parameters, get_rank_list_for_axes, + get_shard_geometry, get_split_rank_lists_for_axes, infer_fully_shard_param_mode, is_dtensor_managed_param, @@ -44,6 +45,41 @@ ) +class TestShardGeometry(unittest.TestCase): + """Shared geometry follows Torch chunks, including absent and empty shards.""" + + def test_matches_torch_chunk(self): + """Plan valid views and padded shapes on source-local parameter dimensions.""" + cases = ( + ((8, 3), 0, 4), ((10, 3), 0, 3), ((5, 3), 0, 4), ((2, 3), 0, 4), + ((0, 3), 0, 4), ((5, 0), 0, 4), ((3, 8), 1, 4), ((3, 5), 1, 4), + ((3, 0), 1, 4), ((5, 3), 0, 1), + ) + for shape, dim, degree in cases: + source = torch.arange(torch.Size(shape).numel()).reshape(shape) + chunks = source.chunk(degree, dim) + for rank in range(degree): + with self.subTest(shape=shape, dim=dim, degree=degree, rank=rank): + geometry = get_shard_geometry(shape, dim, degree, rank) + expected = chunks[rank] if rank < len(chunks) else source.narrow(dim, shape[dim], 0) + actual = source.narrow(dim, geometry.start, geometry.local_shape[dim]) + self.assertEqual(geometry.local_shape, expected.shape, + f"local shape={geometry.local_shape}, expected={expected.shape}") + self.assertEqual(geometry.padded_shape, chunks[0].shape, + f"padded shape={geometry.padded_shape}, expected={chunks[0].shape}") + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + def test_rejects_invalid_shard_coordinates(self): + """Invalid dimensions and group coordinates fail before tensor slicing.""" + for shape, dim, degree, rank in ( + ((), 0, 2, 0), ((4,), -1, 2, 0), ((4,), 1, 2, 0), + ((4,), 0, 0, 0), ((4,), 0, 2, -1), ((4,), 0, 2, 2), + ): + with self.subTest(shape=shape, dim=dim, degree=degree, rank=rank): + with self.assertRaises(ValueError): + get_shard_geometry(shape, dim, degree, rank) + + class SimpleLinear(nn.Module): """Simple linear module for testing.""" diff --git a/tests/ut/core/fully_shard/test_param.py b/tests/ut/core/fully_shard/test_param.py index b325e3bac8..b4df9198b2 100644 --- a/tests/ut/core/fully_shard/test_param.py +++ b/tests/ut/core/fully_shard/test_param.py @@ -127,9 +127,10 @@ class TestTorchHSDPParamHelpers(unittest.TestCase): def test_reduce_comm_dtype_prefers_parameter_policy_and_falls_back_to_grad(self): """Effective reduction dtype should be resolved entirely by the parameter.""" hsdp_param = _new_param() - grad = torch.ones(2, dtype=torch.bfloat16) - - self.assertEqual(hsdp_param.reduce_comm_dtype(grad), torch.bfloat16) + for dtype in (torch.bfloat16, torch.float32, torch.float64): + with self.subTest(dtype=dtype): + grad = torch.ones(2, dtype=dtype) + self.assertEqual(hsdp_param.reduce_comm_dtype(grad), dtype) hsdp_param.reduce_dtype = torch.float16 self.assertEqual(hsdp_param.reduce_comm_dtype(grad), torch.float16)