Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 143 additions & 49 deletions hyper_parallel/compile/compiler.py

Large diffs are not rendered by default.

90 changes: 90 additions & 0 deletions hyper_parallel/compile/mixed_precision.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Copyright 2026 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Lower the shared FSDP precision policy into the joint forward/backward graph."""

from contextlib import ExitStack, contextmanager
from functools import partial
from typing import Any, Iterator

import torch
from torch import nn

from hyper_parallel.core.fully_shard.utils import (
MixedPrecisionPolicy,
apply_to_tensors,
cast_fp_tensor,
resolve_param_dtypes,
)


class GraphMixedPrecision:
"""Keep storage, computation, communication and accumulation dtypes distinct."""

def __init__(self, model: nn.Module, policy: MixedPrecisionPolicy, modules: tuple[nn.Module, ...]) -> None:
"""Bind the shared policy to source parameter identities and unit boundaries.

Args:
model: Final model before graph FSDP slices local storage.
policy: Resolved FSDP policy, including Parameter-specific overrides.
modules: The same module boundaries selected by eager FSDP.
"""
self.policy = policy
self.modules = modules
self.parameter_dtypes = {}
for name, parameter in model.named_parameters(remove_duplicate=False):
compute, reduce = resolve_param_dtypes(policy, id(parameter), parameter.dtype)
self.parameter_dtypes[name] = (compute or parameter.dtype, reduce)

@contextmanager
def module_casts(self) -> Iterator[None]:
"""Record eager-compatible module input/output casts during capture only."""
with ExitStack() as stack:
for module in self.modules:
if self.policy.cast_forward_inputs and self.policy.param_dtype is not None:
handle = module.register_forward_pre_hook(self._cast_inputs, with_kwargs=True)
stack.callback(handle.remove)
if self.policy.output_dtype is not None:
handle = module.register_forward_hook(self._cast_outputs)
stack.callback(handle.remove)
yield

def cast_state(self, state: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
"""Create compute views whose gradients stay in compute dtype until reduction."""
return {
name: cast_fp_tensor(self.parameter_dtypes[name][0], value)
if name in self.parameter_dtypes else value
for name, value in state.items()
}

def cast_gradient(self, name: str, gradient: torch.Tensor) -> torch.Tensor:
"""Apply the reduction dtype even when no collective is needed.

Args:
name: Source parameter FQN in the joint graph's trainable-state order.
gradient: Compute gradient, or its zero substitute for an unused parameter.
"""
dtype = self.parameter_dtypes[name][1]
return cast_fp_tensor(dtype, gradient) if dtype is not None else gradient

def _cast_inputs(self, module: nn.Module, args: tuple, kwargs: dict) -> tuple:
"""Mirror the eager FSDP pre-forward cast for floating input leaves."""
del module
cast = partial(cast_fp_tensor, self.policy.param_dtype)
return apply_to_tensors(cast, args), apply_to_tensors(cast, kwargs)

def _cast_outputs(self, module: nn.Module, args: tuple, outputs: Any) -> Any:
"""Mirror the eager FSDP post-forward output cast."""
del module, args
return apply_to_tensors(partial(cast_fp_tensor, self.policy.output_dtype), outputs)
13 changes: 12 additions & 1 deletion hyper_parallel/compile/pass_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -495,8 +495,9 @@ def build_pass_config_from_trainer_config(
*,
fsdp_enabled: Optional[bool] = None,
enable_overlap: bool = False,
mesh_context: Optional[Any] = None,
) -> PassConfig:
"""Project trainer topology intent onto graph-mode pass config.
"""Project trainer topology and joint-backend intent onto graph-mode passes.

This helper stays in the pass-config module because it computes the
graph-pass surface from the trainer topology without altering runtime
Expand All @@ -520,17 +521,23 @@ def build_pass_config_from_trainer_config(
fsdp_enabled: Overrides the sharded-DP intent derived from
``config.fsdp_config`` when provided.
enable_overlap: Value for the legacy ``AutoOverlapPass`` switch.
mesh_context: Resolved topology, including the DP replicate axis.

Returns:
The graph-mode ``PassConfig`` for this trainer topology.
"""
compile_config = getattr(config, "compile", None)
replicate_degree = getattr(mesh_context, "dp_replicate_size", 1)
if fsdp_enabled is None:
fsdp_enabled = (
config.fsdp_config.dp_shard_size > 1
or config.fsdp_config.edp_shard_size > 1
or replicate_degree > 1
)
fsdp_degree = config.fsdp_config.dp_shard_size if fsdp_enabled else None
dp_mode = "fsdp"
if replicate_degree > 1:
dp_mode = "hsdp" if config.fsdp_config.dp_shard_size > 1 else "ddp"

ac_mode = "off"
ac_layer_count = None
Expand Down Expand Up @@ -559,6 +566,8 @@ def build_pass_config_from_trainer_config(
enable_overlap=enable_overlap,
fsdp_enabled=fsdp_enabled,
fsdp_degree=fsdp_degree,
dp_mode=dp_mode,
dp_replicate_degree=replicate_degree,
fsdp_reshard_after_forward=getattr(
config.fsdp_config, "reshard_after_forward", True
),
Expand All @@ -569,6 +578,8 @@ def build_pass_config_from_trainer_config(
compile_config, "fsdp_overlap_grad_reduce", True
),
max_in_flight_gb=getattr(compile_config, "max_in_flight_gb", 0.25),
inductor_mode=getattr(compile_config, "inductor_mode", "off"),
inductor_configs=getattr(compile_config, "inductor_configs", None),
ac_mode=ac_mode,
ac_layer_count=ac_layer_count,
ac_layer_indices=ac_layer_indices,
Expand Down
9 changes: 7 additions & 2 deletions hyper_parallel/compile/passes/inductor_pass.py
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ def run(
placeholder metas.

Returns:
The same graph module with compiled regions spliced in and the
The runtime module with compiled regions spliced in and the
positional calling convention restored. On torch builds without
regional Inductor support it is returned untouched, logging a
warning and recording ``"unavailable"`` in its meta.
Expand All @@ -335,7 +335,7 @@ def run(
"JointGraph.example_inputs) to size compiled regions"
)
if _regional_inductor_available():
self._compile_graph(graph_module, pass_config, example_inputs)
graph_module = self._compile_graph(graph_module, pass_config, example_inputs)
compiled = True
else:
self._warn_unavailable(mode)
Expand Down Expand Up @@ -417,6 +417,11 @@ def _compile_graph(
_propagate_fake_vals(graph_module)
if example_inputs is None:
example_inputs = _placeholder_fake_args(graph_module)
else:
# FSDP partitions state storage after capture; user symbols stay unchanged.
state_count = getattr(graph_module, "num_state_inputs", 0)
state_inputs = tuple(_placeholder_fake_args(graph_module)[:state_count])
example_inputs = state_inputs + tuple(example_inputs[state_count:])
_LOG.info(
"InductorPass: compiling graph in %r mode (%d placeholders, %d nodes)",
mode,
Expand Down
Loading
Loading