Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
204 changes: 204 additions & 0 deletions hyper_parallel/compile/checkpoint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
# Copyright 2026 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Bridge graph-owned parameter storage to the existing DCP chunk contract."""

from __future__ import annotations

from copy import copy
from typing import Any, TYPE_CHECKING

import torch
import torch.distributed as dist

from hyper_parallel.core.distributed_checkpoint import FileSystemReader
from hyper_parallel.core.distributed_checkpoint.metadata import CHUNK_INFO, ChunkInfo, ChunkStorageMetadata
from hyper_parallel.core.dtensor.dtensor import DTensor
from hyper_parallel.core.dtensor.layout import Layout, infer_slice_area_by_layout

if TYPE_CHECKING:
from .compiler import GraphCompiler


class GraphCheckpointAdapter:
"""Describe local graph tensors without replacing live Parameters or the DCP planner.

Checkpoints are taken at optimizer-step boundaries after storage preparation.
Model tensors, FQN-keyed optimizer moments and FP32 main weights share one
chunk description. Scalar optimizer counters remain replicated.
"""

def __init__(self, compiler: GraphCompiler) -> None:
"""Bind prepared storage and its canonical parameter chunk descriptions."""
compiler.prepare_model()
self.compiler = compiler
self.parameters = dict(compiler.model.named_parameters(remove_duplicate=False))
self.parameter_shapes = compiler.parameter_shapes
self.parameter_layouts = compiler.parameter_layouts
self.canonical_names = {id(parameter): name for name, parameter in compiler.model.named_parameters()}
self.chunks = {
name: self._parameter_chunk(self.canonical_names[id(parameter)], parameter)
for name, parameter in self.parameters.items()
}

def _parameter_chunk(self, name: str, parameter: torch.nn.Parameter) -> ChunkInfo:
"""Locate the valid rectangle within source-first shard geometry."""
logical_shape = self.parameter_shapes[name]
source = self.compiler.source_layout_for_parameter(name)
global_shape = logical_shape
offsets = [0] * len(logical_shape)
if source is not None and source[1] is not None:
source_layout = Layout.from_device_mesh(source[1])
source_layout.set_placements(tuple(source[0]))
source_layout.placement_to_tensor_map(len(logical_shape))
global_shape = tuple(source_layout.get_global_shape(logical_shape))
rank = source_layout.rank_list.index(source_layout.mesh.rank)
area = infer_slice_area_by_layout(source_layout, rank, global_shape)
if tuple(end - start for start, end in area) != logical_shape:
raise ValueError(f"Graph checkpoint requires a rectangular source shard for {name}")
offsets = [start for start, _ in area]

sizes = list(logical_shape)
if name in self.compiler.sharded_parameter_names:
mesh, axis = self._shard_axis(parameter)
local_start = min(mesh.get_local_rank(axis) * parameter.shape[0], logical_shape[0])
offsets[0] += local_start
sizes[0] = min(parameter.shape[0], logical_shape[0] - local_start)
return ChunkInfo(ChunkStorageMetadata(tuple(offsets), tuple(sizes)), tuple(global_shape))

def _shard_axis(self, parameter: torch.nn.Parameter) -> tuple:
"""Resolve the physical FSDP axis from the compiler's final Layout."""
layout = self.parameter_layouts[parameter]
mesh = layout.mesh
for axis, name in enumerate(mesh.mesh_dim_names or ()):
if name in ("fsdp_shard", "edp_shard", "fsdp", "dp", "dp_shard"):
return mesh, axis
raise ValueError("Graph checkpoint cannot resolve the parameter's FSDP shard axis")

def prepare(self, payload: dict[str, Any]) -> dict[str, Any]:
"""Return detached, unpadded DCP views carrying global chunk metadata."""
return self._map(payload, restoring=False)

def restore(self, payload: dict[str, Any]) -> dict[str, Any]:
"""Restore physical padding before the existing model/optimizer loaders run."""
return self._map(payload, restoring=True)

def restore_optimizer_state(self, optimizers: list[Any], *, checkpoint_id: str) -> None:
"""Remove priming-only slots after the standard strict optimizer load.

Args:
optimizers: Loaded optimizer runtimes, in checkpoint payload order.
checkpoint_id: Actual DCP directory whose metadata records persisted state.
"""
metadata = FileSystemReader(checkpoint_id).load_metadata()
persisted = set()
for name in metadata.state_dict_metadata:
if name.startswith("optimizer."):
while name:
persisted.add(name)
name = name.rpartition(".")[0]
for index, optimizer in enumerate(optimizers):
prefix = f"optimizer.{index}" if len(optimizers) > 1 else "optimizer"
owners = getattr(optimizer, "model_param_by_optimizer_param", {})
for leaf in getattr(optimizer, "chained_optimizers", (optimizer,)):
for parameter in list(leaf.state):
name = self.canonical_names.get(id(owners.get(parameter, parameter)))
if name in self.parameters and f"{prefix}.state.{name}" not in persisted:
del leaf.state[parameter]

def _owner(self, key: Any, current: str | None) -> str | None:
"""Resolve ordinary, flattened optimizer and main-weight FQN keys."""
if isinstance(key, str):
if key in self.parameters:
return key
if key.startswith("state."):
candidate = key[len("state."):].rsplit(".", 1)[0]
if candidate in self.parameters:
return candidate
return current

def _tensor_view(self, value: torch.Tensor, owner: str, path: tuple, restoring: bool) -> torch.Tensor:
"""Add a chunk to an unpadded view, or reconstruct physical padding on load."""
parameter = self.parameters[owner]
if value.ndim == 0 and parameter.ndim != 0:
return value
chunk = self.chunks[owner]
expected = chunk.chunk.sizes if restoring else tuple(parameter.shape)
if tuple(value.shape) != expected:
raise ValueError(f"Graph checkpoint tensor {path} has shape {tuple(value.shape)}; expected {expected}")
if restoring:
if tuple(value.shape) == tuple(parameter.shape):
return value
result = value.new_zeros(parameter.shape)
result[tuple(slice(0, size) for size in value.shape)].copy_(value)
return result
result = value.detach()[tuple(slice(0, size) for size in chunk.chunk.sizes)]
setattr(result, CHUNK_INFO, chunk)
return result

def _map(self, payload: dict[str, Any], *, restoring: bool) -> dict[str, Any]:
"""Transform only tensor-owner payloads, preserving other training state."""
def visit(value: Any, owner: str | None = None, path: tuple = ()) -> Any:
"""Copy the payload containers and visit their tensor leaves."""
if isinstance(value, dict):
result = copy(value)
for key, item in value.items():
result[key] = visit(item, self._owner(key, owner), (*path, key))
return result
if isinstance(value, (list, tuple)):
return type(value)(visit(item, owner, (*path, index)) for index, item in enumerate(value))
if not isinstance(value, torch.Tensor):
return value
if owner is None:
if path and path[0] == "optimizer" and value.ndim:
raise ValueError("Graph checkpoint requires FQN-keyed optimizer state")
return value
return self._tensor_view(value, owner, path, restoring)

result = copy(payload)
for key in ("model", "optimizer"):
if key in payload:
result[key] = visit(payload[key], path=(key,))
return result

@torch.no_grad()
def full_model_state(self, *, keep: bool) -> dict[str, Any]:
"""Gather FSDP first, remove its padding, then gather the source TP/EP layout."""
result = {}
for name, value in self.compiler.model.state_dict(keep_vars=True).items():
parameter = self.parameters.get(name)
if isinstance(value, DTensor):
value = value.full_tensor()
elif parameter is not None:
parameter_name = self.canonical_names[id(parameter)]
value = value.detach()
if parameter_name in self.compiler.sharded_parameter_names:
mesh, axis = self._shard_axis(parameter)
group = mesh.get_group(axis)
gathered = value.new_empty((value.shape[0] * dist.get_world_size(group), *value.shape[1:]))
dist.all_gather_into_tensor(gathered, value.contiguous(), group=group)
value = gathered[:self.parameter_shapes[parameter_name][0]]
source = self.compiler.source_layout_for_parameter(parameter_name)
if source is not None and source[1] is not None:
layout = Layout.from_device_mesh(source[1])
layout.set_placements(tuple(source[0]))
layout.placement_to_tensor_map(value.ndim)
value = DTensor.from_local_with_layout(value, layout).full_tensor()
else:
layout = getattr(value, "_sharding_spec", None)
if isinstance(value, torch.Tensor) and layout is not None:
value = DTensor.from_local_with_layout(value.detach(), layout).full_tensor()
if keep:
result[name] = value.detach().cpu() if isinstance(value, torch.Tensor) else value
return result
107 changes: 80 additions & 27 deletions hyper_parallel/compile/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ class and owns them.
)
from .passes.pipeline import PassPipeline
from .passes.activation_checkpoint import prepare_activation_checkpoint
from .passes.parallel.fsdp_pass import FSDPPass
from .tracer.dynamic_shapes import normalize_dynamic_arg_dims
from .tracer.graph_tracer import run_traced_graph, trace_model_graph

Expand Down Expand Up @@ -135,6 +136,11 @@ def __init__(
self._parameter_shapes = {}
self._parameter_layouts = None
self._layout_signature = None
self._storage_ready = False
self._storage_failed = False
self._storage_signature = None
self._sharded_parameter_names = frozenset()
self._prepared_pass_kwargs = None
self.device = device or (
torch.device("npu")
if (hasattr(torch, "npu") and torch.npu.is_available())
Expand Down Expand Up @@ -170,21 +176,81 @@ def is_compiled(self) -> bool:

@property
def parameter_layouts(self) -> Optional[dict]:
"""Final parameter-identity layouts, available after lazy compilation.
"""Final parameter-identity layouts, available after storage preparation.

Cache metadata only: gradients may be replaced on every optimizer step.
Replacing a parameter invalidates the mapping, as does compiling again.
"""
if self._joint_graph is None:
if not self._storage_ready:
return None
parameters = dict(self.model.named_parameters())
signature = tuple((id(param), tuple(param.shape), param.dtype, param.device) for param in parameters.values())
if signature != self._layout_signature:
self._parameter_layouts = self._build_parameter_layouts(parameters)
self._layout_signature = signature
self._joint_graph.graph_module.parameter_layouts = self._parameter_layouts
if self._joint_graph is not None:
self._joint_graph.graph_module.parameter_layouts = self._parameter_layouts
return self._parameter_layouts

@property
def parameter_shapes(self) -> dict:
"""Source-local logical shapes before FSDP padding and slicing."""
return dict(self._parameter_shapes)

@property
def sharded_parameter_names(self) -> frozenset:
"""Canonical names of parameters physically partitioned by FSDP."""
return self._sharded_parameter_names

def prepare_model(self) -> None:
"""Prepare local parameter storage once, before optimizer construction.

Values may subsequently be loaded or updated in place. Changing a
parameter identity, shape, dtype or device requires a new model and
compiler. A failed preparation is terminal because storage may have
already been partitioned.
"""
external = get_model_external_state(self.model)
if external is not None and external.parameters:
raise ValueError("GraphCompiler does not support externally managed parameters yet")
if self._storage_failed:
raise RuntimeError("Graph storage preparation failed; rebuild the model and compiler")
signature = self._model_storage_signature()
if self._storage_ready:
if signature != self._storage_signature:
raise ValueError("Prepared graph parameter storage changed; rebuild the model and compiler")
return
try:
self._prepare_model_storage()
except Exception:
self._storage_failed = True
raise
self._storage_signature = self._model_storage_signature()
self._storage_ready = True

def _model_storage_signature(self) -> tuple:
"""Describe storage identity without retaining any parameter values."""
return tuple((name, id(param), tuple(param.shape), param.dtype, param.device, param.requires_grad)
for name, param in self.model.named_parameters(remove_duplicate=False))

def _prepare_model_storage(self) -> None:
"""Reuse FSDP ownership and slicing before creating optimizer state."""
self._validate_ep_topology()
tp_size = getattr(self._mesh_context, "tp_size", 1)
if isinstance(tp_size, int) and tp_size > 1 and not self.source_shard_info:
raise ValueError("Graph TP requires source_shard_info from the sharding plan")
if self.pass_config.fsdp_enabled and dist.is_initialized():
self._init_device_mesh(self._mesh_context)
self._bind_parameter_meshes()
self._parameter_shapes = {
name: tuple(param.shape) for name, param in self.model.named_parameters(remove_duplicate=False)
}
self._prepared_pass_kwargs = self._build_pass_kwargs()
if self.pass_config.fsdp_enabled:
fsdp = FSDPPass(parallel_plan=self.parallel_plan)
self._sharded_parameter_names = fsdp.prepare_model(self.pass_config, **self._prepared_pass_kwargs)
self._layout_signature = None

def compile(self, **inputs: Any) -> None:
"""
Compile model into parallel graph
Expand All @@ -209,20 +275,7 @@ def compile(self, **inputs: Any) -> None:
if self._pytree_pre_hook is not None:
self._pytree_pre_hook()

self._validate_ep_topology()
if (
isinstance(getattr(self._mesh_context, "tp_size", None), int)
and self._mesh_context.tp_size > 1
and not self.source_shard_info
):
raise ValueError("Graph TP requires source_shard_info from the sharding plan")

if self.pass_config.fsdp_enabled and dist.is_initialized():
# Only build the FSDP mesh when distributed is actually up.
# ``FSDPPass`` early-returns when ``world_size == 1``, so a
# single-process run (no dist, or a single rank) compiles and
# runs as plain graph mode without sharding.
self._init_device_mesh(self._mesh_context)
self.prepare_model()

# Activation checkpointing must resolve its scope and run the adapter's
# runtime hook BEFORE tracing: the hook configures state the model's
Expand All @@ -231,23 +284,20 @@ def compile(self, **inputs: Any) -> None:
if self.pass_config.ac_mode != "off":
ac_scope = prepare_activation_checkpoint(self.model, self.pass_config)

self._bind_parameter_meshes()
self._parameter_shapes = {name: tuple(param.shape) for name, param in self.model.named_parameters()}
self._layout_signature = None

trace_kwargs = {}
trace_kwargs = {"state_shapes": self._parameter_shapes}
if self.dynamic or self.dynamic_arg_dims is not None:
trace_kwargs = {
trace_kwargs.update({
"dynamic": self.dynamic,
"dynamic_arg_dims": self.dynamic_arg_dims,
}
})
joint_graph = trace_model_graph(
self.model, self.train_fn, inputs, **trace_kwargs
)

pipeline = PassPipeline.from_config(self.pass_config, self.parallel_plan)

pass_kwargs = self._build_pass_kwargs()
pass_kwargs = dict(self._prepared_pass_kwargs)
pass_kwargs["storage_prepared"] = True
# Terminal backend compilation (InductorPass) sizes its regions from
# the tracer's fake example inputs; every other pass ignores them.
pass_kwargs["example_inputs"] = joint_graph.example_inputs
Expand Down Expand Up @@ -334,7 +384,7 @@ def _build_parameter_layouts(self, parameters: dict) -> Optional[dict]:
if self._fsdp_mesh is None and context is not None:
# Plain graph mode still needs metadata, without creating collectives.
self._init_mesh_from_context(context, self.pass_config.dp_mode, register_groups=False)
sharded = getattr(self._joint_graph.graph_module, "fsdp_sharded_param_names", frozenset())
sharded = self._sharded_parameter_names
layouts = {}
for name, parameter in parameters.items():
if not parameter.requires_grad:
Expand Down Expand Up @@ -812,7 +862,10 @@ def _build_pass_kwargs(self) -> dict:
"""
Build kwargs to pass to passes.
"""
kwargs = {}
kwargs = {
"storage_prepared": self._storage_ready,
"logical_parameter_shapes": self._parameter_shapes,
}

# Live model: partitioning passes (FSDPPass) physically shard
# parameters in place, keeping the compiler FSDP-agnostic.
Expand Down
Loading
Loading