Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 101 additions & 31 deletions hyper_parallel/compile/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,12 @@ class and owns them.
from hyper_parallel.core.dtensor.layout import Layout
from hyper_parallel.core.dtensor.placement_types import Replicate, Shard, StridedShard
from hyper_parallel.distributed._builder.fsdp_adapter import FSDP2Manager
from hyper_parallel.distributed.mesh import MeshContext
from hyper_parallel.models.build_options import FSDP2Config
from hyper_parallel.models.external_state import get_model_external_state

from .graph_parallel_plan import GraphParallelPlan
from .mixed_precision import GraphMixedPrecision
from .pass_config import (
GradientReduction,
PassConfig,
Expand All @@ -57,6 +59,7 @@ class and owns them.
)
from .passes.pipeline import PassPipeline
from .passes.activation_checkpoint import prepare_activation_checkpoint
from .passes.parallel.fsdp_pass import FSDPPass
from .tracer.dynamic_shapes import normalize_dynamic_arg_dims
from .tracer.graph_tracer import run_traced_graph, trace_model_graph

Expand Down Expand Up @@ -135,6 +138,13 @@ def __init__(
self._parameter_shapes = {}
self._parameter_layouts = None
self._layout_signature = None
self._storage_ready = False
self._storage_failed = False
self._storage_signature = None
self._sharded_parameter_names = frozenset()
self._prepared_pass_kwargs = None
self._mixed_precision = None
self._fp32_main_params = getattr(getattr(trainer_config, "optimizer", None), "fp32_main_params", False)
self.device = device or (
torch.device("npu")
if (hasattr(torch, "npu") and torch.npu.is_available())
Expand Down Expand Up @@ -170,21 +180,87 @@ def is_compiled(self) -> bool:

@property
def parameter_layouts(self) -> Optional[dict]:
"""Final parameter-identity layouts, available after lazy compilation.
"""Final parameter-identity layouts, available after storage preparation.

Cache metadata only: gradients may be replaced on every optimizer step.
Replacing a parameter invalidates the mapping, as does compiling again.
"""
if self._joint_graph is None:
if not self._storage_ready:
return None
parameters = dict(self.model.named_parameters())
signature = tuple((id(param), tuple(param.shape), param.dtype, param.device) for param in parameters.values())
if signature != self._layout_signature:
self._parameter_layouts = self._build_parameter_layouts(parameters)
self._layout_signature = signature
self._joint_graph.graph_module.parameter_layouts = self._parameter_layouts
if self._joint_graph is not None:
self._joint_graph.graph_module.parameter_layouts = self._parameter_layouts
return self._parameter_layouts

@property
def parameter_shapes(self) -> dict:
"""Source-local logical shapes before FSDP padding and slicing."""
return dict(self._parameter_shapes)

@property
def sharded_parameter_names(self) -> frozenset:
"""Canonical names of parameters physically partitioned by FSDP."""
return self._sharded_parameter_names

def prepare_model(self) -> None:
"""Prepare local parameter storage once, before optimizer construction.

Values may subsequently be loaded or updated in place. Changing a
parameter identity, shape, dtype or device requires a new model and
compiler. A failed preparation is terminal because storage may have
already been partitioned.
"""
external = get_model_external_state(self.model)
if external is not None and external.parameters:
raise ValueError("GraphCompiler does not support externally managed parameters yet")
if self._storage_failed:
raise RuntimeError("Graph storage preparation failed; rebuild the model and compiler")
signature = self._model_storage_signature()
if self._storage_ready:
if signature != self._storage_signature:
raise ValueError("Prepared graph parameter storage changed; rebuild the model and compiler")
return
try:
self._prepare_model_storage()
except Exception:
self._storage_failed = True
raise
self._storage_signature = self._model_storage_signature()
self._storage_ready = True

def _model_storage_signature(self) -> tuple:
"""Describe storage identity without retaining any parameter values."""
return tuple((name, id(param), tuple(param.shape), param.dtype, param.device, param.requires_grad)
for name, param in self.model.named_parameters(remove_duplicate=False))

def _prepare_model_storage(self) -> None:
"""Reuse FSDP ownership and slicing before creating optimizer state."""
self._validate_ep_topology()
tp_size = getattr(self._mesh_context, "tp_size", 1)
if isinstance(tp_size, int) and tp_size > 1 and not self.source_shard_info:
raise ValueError("Graph TP requires source_shard_info from the sharding plan")
if self.pass_config.fsdp_enabled and dist.is_initialized():
self._init_device_mesh(self._mesh_context)
self._bind_parameter_meshes()
precision = getattr(self._fsdp_config, "mix_precision", None)
if precision is not None and any((precision.param_dtype, precision.reduce_dtype, precision.output_dtype,
precision.custom_params, self._fp32_main_params)):
manager = FSDP2Manager(self._fsdp_config, self._mesh_context or MeshContext(), self._fp32_main_params)
policy, modules = manager.mixed_precision_bindings(self.model, self.source_shard_info)
self._mixed_precision = GraphMixedPrecision(self.model, policy, modules)
self._parameter_shapes = {
name: tuple(param.shape) for name, param in self.model.named_parameters(remove_duplicate=False)
}
self._prepared_pass_kwargs = self._build_pass_kwargs()
if self.pass_config.fsdp_enabled:
fsdp = FSDPPass(parallel_plan=self.parallel_plan)
self._sharded_parameter_names = fsdp.prepare_model(self.pass_config, **self._prepared_pass_kwargs)
self._layout_signature = None

def compile(self, **inputs: Any) -> None:
"""
Compile model into parallel graph
Expand All @@ -209,20 +285,7 @@ def compile(self, **inputs: Any) -> None:
if self._pytree_pre_hook is not None:
self._pytree_pre_hook()

self._validate_ep_topology()
if (
isinstance(getattr(self._mesh_context, "tp_size", None), int)
and self._mesh_context.tp_size > 1
and not self.source_shard_info
):
raise ValueError("Graph TP requires source_shard_info from the sharding plan")

if self.pass_config.fsdp_enabled and dist.is_initialized():
# Only build the FSDP mesh when distributed is actually up.
# ``FSDPPass`` early-returns when ``world_size == 1``, so a
# single-process run (no dist, or a single rank) compiles and
# runs as plain graph mode without sharding.
self._init_device_mesh(self._mesh_context)
self.prepare_model()

# Activation checkpointing must resolve its scope and run the adapter's
# runtime hook BEFORE tracing: the hook configures state the model's
Expand All @@ -231,23 +294,20 @@ def compile(self, **inputs: Any) -> None:
if self.pass_config.ac_mode != "off":
ac_scope = prepare_activation_checkpoint(self.model, self.pass_config)

self._bind_parameter_meshes()
self._parameter_shapes = {name: tuple(param.shape) for name, param in self.model.named_parameters()}
self._layout_signature = None

trace_kwargs = {}
trace_kwargs = {"state_shapes": self._parameter_shapes, "mixed_precision": self._mixed_precision}
if self.dynamic or self.dynamic_arg_dims is not None:
trace_kwargs = {
trace_kwargs.update({
"dynamic": self.dynamic,
"dynamic_arg_dims": self.dynamic_arg_dims,
}
})
joint_graph = trace_model_graph(
self.model, self.train_fn, inputs, **trace_kwargs
)

pipeline = PassPipeline.from_config(self.pass_config, self.parallel_plan)

pass_kwargs = self._build_pass_kwargs()
pass_kwargs = dict(self._prepared_pass_kwargs)
pass_kwargs["storage_prepared"] = True
# Terminal backend compilation (InductorPass) sizes its regions from
# the tracer's fake example inputs; every other pass ignores them.
pass_kwargs["example_inputs"] = joint_graph.example_inputs
Expand Down Expand Up @@ -334,7 +394,7 @@ def _build_parameter_layouts(self, parameters: dict) -> Optional[dict]:
if self._fsdp_mesh is None and context is not None:
# Plain graph mode still needs metadata, without creating collectives.
self._init_mesh_from_context(context, self.pass_config.dp_mode, register_groups=False)
sharded = getattr(self._joint_graph.graph_module, "fsdp_sharded_param_names", frozenset())
sharded = self._sharded_parameter_names
layouts = {}
for name, parameter in parameters.items():
if not parameter.requires_grad:
Expand Down Expand Up @@ -812,11 +872,16 @@ def _build_pass_kwargs(self) -> dict:
"""
Build kwargs to pass to passes.
"""
kwargs = {}
kwargs = {
"storage_prepared": self._storage_ready,
"logical_parameter_shapes": self._parameter_shapes,
}

# Live model: partitioning passes (FSDPPass) physically shard
# parameters in place, keeping the compiler FSDP-agnostic.
kwargs["model"] = self.model
if self._mixed_precision is not None:
kwargs["parameter_dtypes"] = self._mixed_precision.parameter_dtypes

if self.pass_config.fsdp_enabled:
kwargs["fsdp_group_name"] = "fsdp"
Expand Down Expand Up @@ -924,13 +989,18 @@ def _accumulate_grads(self, grads: List[torch.Tensor]) -> None:
)

for param, grad in zip(trainable, grads):
if param.grad is None:
main_grad = self._mixed_precision is not None and self._mixed_precision.policy.apply_grad_on_fp32_main_grad
attribute = "main_grad" if main_grad else "grad"
accumulated = getattr(param, attribute, None)
if accumulated is None:
# Match eager's gradient layout so optimizer kernels see the
# parameter's strides rather than a backward matmul's view.
param.grad = torch.empty_like(param)
param.grad.copy_(grad)
dtype = torch.float32 if main_grad else param.dtype
accumulated = torch.empty_like(param, dtype=dtype)
accumulated.copy_(grad)
setattr(param, attribute, accumulated)
else:
param.grad += grad
accumulated += grad

@staticmethod
def _trainable_params_in_state_order(
Expand Down
77 changes: 77 additions & 0 deletions hyper_parallel/compile/mixed_precision.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# Copyright 2026 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Lower the shared FSDP precision policy into the joint forward/backward graph."""

from contextlib import ExitStack, contextmanager
from functools import partial
from typing import Any, Iterator

import torch
from torch import nn

from hyper_parallel.core.fully_shard.utils import MixedPrecisionPolicy, apply_to_tensors, cast_fp_tensor


class GraphMixedPrecision:
"""Keep storage, computation, communication and accumulation dtypes distinct."""

def __init__(self, model: nn.Module, policy: MixedPrecisionPolicy, modules: tuple[nn.Module, ...]) -> None:
"""Bind the shared policy to source parameter identities and unit boundaries.

Args:
model: Final model before graph FSDP slices local storage.
policy: Resolved FSDP policy, including Parameter-specific overrides.
modules: The same module boundaries selected by eager FSDP.
"""
self.policy = policy
self.modules = modules
self.parameter_dtypes = {}
for name, parameter in model.named_parameters(remove_duplicate=False):
custom = (policy.custom_params or {}).get(id(parameter), {})
compute = custom.get("param_dtype", policy.param_dtype) or parameter.dtype
reduce = custom.get("reduce_dtype", policy.reduce_dtype) or compute
self.parameter_dtypes[name] = (compute, reduce)

@contextmanager
def module_casts(self) -> Iterator[None]:
"""Record eager-compatible module input/output casts during capture only."""
with ExitStack() as stack:
for module in self.modules:
if self.policy.cast_forward_inputs and self.policy.param_dtype is not None:
handle = module.register_forward_pre_hook(self._cast_inputs, with_kwargs=True)
stack.callback(handle.remove)
if self.policy.output_dtype is not None:
handle = module.register_forward_hook(self._cast_outputs)
stack.callback(handle.remove)
yield

def cast_state(self, state: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
"""Create compute views whose gradients stay in compute dtype until reduction."""
return {
name: cast_fp_tensor(self.parameter_dtypes[name][0], value)
if name in self.parameter_dtypes else value
for name, value in state.items()
}

def _cast_inputs(self, module: nn.Module, args: tuple, kwargs: dict) -> tuple:
"""Mirror the eager FSDP pre-forward cast for floating input leaves."""
del module
cast = partial(cast_fp_tensor, self.policy.param_dtype)
return apply_to_tensors(cast, args), apply_to_tensors(cast, kwargs)

def _cast_outputs(self, module: nn.Module, args: tuple, outputs: Any) -> Any:
"""Mirror the eager FSDP post-forward output cast."""
del module, args
return apply_to_tensors(partial(cast_fp_tensor, self.policy.output_dtype), outputs)
Loading
Loading