From c7d02feadaeffb90ce492e56d5f2084a2fd02054 Mon Sep 17 00:00:00 2001 From: yu-chen-code Date: Fri, 6 Nov 2026 22:56:16 +0800 Subject: [PATCH 1/3] feat(compile): prepare FSDP storage before optimizer construction --- hyper_parallel/compile/compiler.py | 107 +++++++++--- .../compile/passes/parallel/fsdp_pass.py | 33 +++- hyper_parallel/compile/tracer/graph_tracer.py | 11 +- hyper_parallel/trainer/base.py | 1 + hyper_parallel/trainer/text_trainer.py | 4 +- tests/ut/compile/test_compiler.py | 8 +- tests/ut/compile/test_ragged_ep.py | 1 + tests/ut/compile/test_storage_preparation.py | 156 ++++++++++++++++++ 8 files changed, 286 insertions(+), 35 deletions(-) create mode 100644 tests/ut/compile/test_storage_preparation.py diff --git a/hyper_parallel/compile/compiler.py b/hyper_parallel/compile/compiler.py index a624ff4bdb..594bdadaf7 100644 --- a/hyper_parallel/compile/compiler.py +++ b/hyper_parallel/compile/compiler.py @@ -57,6 +57,7 @@ class and owns them. ) from .passes.pipeline import PassPipeline from .passes.activation_checkpoint import prepare_activation_checkpoint +from .passes.parallel.fsdp_pass import FSDPPass from .tracer.dynamic_shapes import normalize_dynamic_arg_dims from .tracer.graph_tracer import run_traced_graph, trace_model_graph @@ -135,6 +136,11 @@ def __init__( self._parameter_shapes = {} self._parameter_layouts = None self._layout_signature = None + self._storage_ready = False + self._storage_failed = False + self._storage_signature = None + self._sharded_parameter_names = frozenset() + self._prepared_pass_kwargs = None self.device = device or ( torch.device("npu") if (hasattr(torch, "npu") and torch.npu.is_available()) @@ -170,21 +176,81 @@ def is_compiled(self) -> bool: @property def parameter_layouts(self) -> Optional[dict]: - """Final parameter-identity layouts, available after lazy compilation. + """Final parameter-identity layouts, available after storage preparation. Cache metadata only: gradients may be replaced on every optimizer step. Replacing a parameter invalidates the mapping, as does compiling again. """ - if self._joint_graph is None: + if not self._storage_ready: return None parameters = dict(self.model.named_parameters()) signature = tuple((id(param), tuple(param.shape), param.dtype, param.device) for param in parameters.values()) if signature != self._layout_signature: self._parameter_layouts = self._build_parameter_layouts(parameters) self._layout_signature = signature - self._joint_graph.graph_module.parameter_layouts = self._parameter_layouts + if self._joint_graph is not None: + self._joint_graph.graph_module.parameter_layouts = self._parameter_layouts return self._parameter_layouts + @property + def parameter_shapes(self) -> dict: + """Source-local logical shapes before FSDP padding and slicing.""" + return dict(self._parameter_shapes) + + @property + def sharded_parameter_names(self) -> frozenset: + """Canonical names of parameters physically partitioned by FSDP.""" + return self._sharded_parameter_names + + def prepare_model(self) -> None: + """Prepare local parameter storage once, before optimizer construction. + + Values may subsequently be loaded or updated in place. Changing a + parameter identity, shape, dtype or device requires a new model and + compiler. A failed preparation is terminal because storage may have + already been partitioned. + """ + external = get_model_external_state(self.model) + if external is not None and external.parameters: + raise ValueError("GraphCompiler does not support externally managed parameters yet") + if self._storage_failed: + raise RuntimeError("Graph storage preparation failed; rebuild the model and compiler") + signature = self._model_storage_signature() + if self._storage_ready: + if signature != self._storage_signature: + raise ValueError("Prepared graph parameter storage changed; rebuild the model and compiler") + return + try: + self._prepare_model_storage() + except Exception: + self._storage_failed = True + raise + self._storage_signature = self._model_storage_signature() + self._storage_ready = True + + def _model_storage_signature(self) -> tuple: + """Describe storage identity without retaining any parameter values.""" + return tuple((name, id(param), tuple(param.shape), param.dtype, param.device, param.requires_grad) + for name, param in self.model.named_parameters(remove_duplicate=False)) + + def _prepare_model_storage(self) -> None: + """Reuse FSDP ownership and slicing before creating optimizer state.""" + self._validate_ep_topology() + tp_size = getattr(self._mesh_context, "tp_size", 1) + if isinstance(tp_size, int) and tp_size > 1 and not self.source_shard_info: + raise ValueError("Graph TP requires source_shard_info from the sharding plan") + if self.pass_config.fsdp_enabled and dist.is_initialized(): + self._init_device_mesh(self._mesh_context) + self._bind_parameter_meshes() + self._parameter_shapes = { + name: tuple(param.shape) for name, param in self.model.named_parameters(remove_duplicate=False) + } + self._prepared_pass_kwargs = self._build_pass_kwargs() + if self.pass_config.fsdp_enabled: + fsdp = FSDPPass(parallel_plan=self.parallel_plan) + self._sharded_parameter_names = fsdp.prepare_model(self.pass_config, **self._prepared_pass_kwargs) + self._layout_signature = None + def compile(self, **inputs: Any) -> None: """ Compile model into parallel graph @@ -209,20 +275,7 @@ def compile(self, **inputs: Any) -> None: if self._pytree_pre_hook is not None: self._pytree_pre_hook() - self._validate_ep_topology() - if ( - isinstance(getattr(self._mesh_context, "tp_size", None), int) - and self._mesh_context.tp_size > 1 - and not self.source_shard_info - ): - raise ValueError("Graph TP requires source_shard_info from the sharding plan") - - if self.pass_config.fsdp_enabled and dist.is_initialized(): - # Only build the FSDP mesh when distributed is actually up. - # ``FSDPPass`` early-returns when ``world_size == 1``, so a - # single-process run (no dist, or a single rank) compiles and - # runs as plain graph mode without sharding. - self._init_device_mesh(self._mesh_context) + self.prepare_model() # Activation checkpointing must resolve its scope and run the adapter's # runtime hook BEFORE tracing: the hook configures state the model's @@ -231,23 +284,20 @@ def compile(self, **inputs: Any) -> None: if self.pass_config.ac_mode != "off": ac_scope = prepare_activation_checkpoint(self.model, self.pass_config) - self._bind_parameter_meshes() - self._parameter_shapes = {name: tuple(param.shape) for name, param in self.model.named_parameters()} - self._layout_signature = None - - trace_kwargs = {} + trace_kwargs = {"state_shapes": self._parameter_shapes} if self.dynamic or self.dynamic_arg_dims is not None: - trace_kwargs = { + trace_kwargs.update({ "dynamic": self.dynamic, "dynamic_arg_dims": self.dynamic_arg_dims, - } + }) joint_graph = trace_model_graph( self.model, self.train_fn, inputs, **trace_kwargs ) pipeline = PassPipeline.from_config(self.pass_config, self.parallel_plan) - pass_kwargs = self._build_pass_kwargs() + pass_kwargs = dict(self._prepared_pass_kwargs) + pass_kwargs["storage_prepared"] = True # Terminal backend compilation (InductorPass) sizes its regions from # the tracer's fake example inputs; every other pass ignores them. pass_kwargs["example_inputs"] = joint_graph.example_inputs @@ -334,7 +384,7 @@ def _build_parameter_layouts(self, parameters: dict) -> Optional[dict]: if self._fsdp_mesh is None and context is not None: # Plain graph mode still needs metadata, without creating collectives. self._init_mesh_from_context(context, self.pass_config.dp_mode, register_groups=False) - sharded = getattr(self._joint_graph.graph_module, "fsdp_sharded_param_names", frozenset()) + sharded = self._sharded_parameter_names layouts = {} for name, parameter in parameters.items(): if not parameter.requires_grad: @@ -812,7 +862,10 @@ def _build_pass_kwargs(self) -> dict: """ Build kwargs to pass to passes. """ - kwargs = {} + kwargs = { + "storage_prepared": self._storage_ready, + "logical_parameter_shapes": self._parameter_shapes, + } # Live model: partitioning passes (FSDPPass) physically shard # parameters in place, keeping the compiler FSDP-agnostic. diff --git a/hyper_parallel/compile/passes/parallel/fsdp_pass.py b/hyper_parallel/compile/passes/parallel/fsdp_pass.py index 9fd08182e8..2bb6c72829 100644 --- a/hyper_parallel/compile/passes/parallel/fsdp_pass.py +++ b/hyper_parallel/compile/passes/parallel/fsdp_pass.py @@ -336,6 +336,8 @@ def __init__( self._gradient_reductions: Dict[str, GradientReduction] = {} self._source_gradient_groups: Dict[str, str] = {} self._parameter_groups: Dict[str, tuple[str, int]] = {} + self._storage_prepared = False + self._logical_parameter_shapes: Dict[str, tuple] = {} # Data-parallel mode ("fsdp" | "ddp" | "hsdp") and the replicate-axis # group/degree used by DDP / HSDP gradient reduction. self._dp_mode = "fsdp" @@ -352,6 +354,29 @@ def __init__( # live-shard steps so they are pad-sharded like eager ``fully_shard``. self._padded_params: Dict[str, _PaddedParam] = {} + def prepare_model(self, pass_config: PassConfig, **kwargs: Any) -> frozenset[str]: + """Partition live storage using the same classification as graph lowering. + + This metadata-only graph does not execute model code or collectives. + Call once before building optimizer parameters or loading a checkpoint. + """ + if not dist.is_initialized() or dist.get_world_size() == 1: + return frozenset() + model = self._setup_dp_config(pass_config, kwargs) + self._reset_run_state() + graph = fx.Graph() + parameters = list(model.named_parameters()) + for index, _ in enumerate(parameters): + graph.placeholder(f"param_{index}") + graph.output(()) + module = fx.GraphModule({}, graph) + names = [name for name, _ in parameters] + classification = self._classify_dp_params(module, names, len(names), [True] * len(names), model) + if self._dp_mode == "ddp": + return frozenset() + self._shard_live_model_params(model) + return frozenset(node.meta["param_name"] for node in classification.sharded_nodes) + def run( self, graph_module: fx.GraphModule, @@ -469,6 +494,8 @@ def _setup_dp_config( self._gradient_reductions = dict(kwargs.get("gradient_reductions", {})) self._source_gradient_groups = dict(kwargs.get("source_gradient_groups", {})) self._parameter_groups = dict(kwargs.get("parameter_groups", {})) + self._storage_prepared = kwargs.get("storage_prepared", False) + self._logical_parameter_shapes = kwargs.get("logical_parameter_shapes", {}) if not self._expert_param_names.issubset(self._gradient_reductions): raise ValueError("Expert parameters require explicit gradient reduction policies") self._expert_fsdp_group_name = kwargs.get( @@ -565,7 +592,8 @@ def _apply_dp_transforms( # index in the FSDP group). ``model.parameters()`` then yields the # shards, so the trainer's optimizer / grad accumulation stay # DP-agnostic; the graph re-gathers each step. - self._shard_live_model_params(model) + if not self._storage_prepared: + self._shard_live_model_params(model) annotate_fsdp_bucket_specs(graph_module, model) return graph_module @@ -631,6 +659,9 @@ def _classify_dp_params( if skip: continue + if fqn in self._logical_parameter_shapes: + param = torch.empty(self._logical_parameter_shapes[fqn], dtype=param.dtype, device="meta") + _, degree = self._param_group(fqn) if fqn in self._expert_param_names and degree == 1: continue diff --git a/hyper_parallel/compile/tracer/graph_tracer.py b/hyper_parallel/compile/tracer/graph_tracer.py index ea9a8f4627..06be84774a 100644 --- a/hyper_parallel/compile/tracer/graph_tracer.py +++ b/hyper_parallel/compile/tracer/graph_tracer.py @@ -514,6 +514,7 @@ def trace_model_graph( # pylint: disable=too-many-locals *, dynamic: bool = False, dynamic_arg_dims: dict[str, int | list[int]] | None = None, + state_shapes: dict[str, tuple] | None = None, ) -> JointGraph: """ Trace model to generate complete forward + backward graph @@ -527,6 +528,7 @@ def trace_model_graph( # pylint: disable=too-many-locals dynamic_arg_dims: Dotted input paths to dynamic dimensions (supports negative indices). Providing a mapping enables selective symbolic tracing, even when dynamic is False. Parameters and buffers always remain static. + state_shapes: Source-local logical parameter shapes before live FSDP slicing. Returns: JointGraph: Joint forward-backward computation graph @@ -566,6 +568,13 @@ def trace_model_graph( # pylint: disable=too-many-locals fake_mode = FakeTensorMode(allow_non_fake_inputs=True, shape_env=ShapeEnv(), static_shapes=True) fake_args = tuple(_fakeify_input(fake_mode, value) for value in full_args) num_state_inputs = len(state_flat) + if state_shapes: + with fake_mode: + fake_args = tuple( + value.new_empty(state_shapes[name], requires_grad=value.requires_grad) + if name in state_shapes and tuple(value.shape) != state_shapes[name] else value + for name, value in zip(state_fqns, fake_args[:num_state_inputs]) + ) + fake_args[num_state_inputs:] _fwd_bwd_fn, loss_dict_keys_box = _make_fwd_bwd_fn( # pylint: disable=C0103 model, train_fn, state_fqns, state_is_param, state_spec, user_inputs_spec @@ -631,7 +640,7 @@ def trace_model_graph( # pylint: disable=too-many-locals param_names = [name for name, _ in model.named_parameters() if _.requires_grad] param_shapes = { - name: tuple(param.shape) for name, param in model.named_parameters() + name: (state_shapes or {}).get(name, tuple(param.shape)) for name, param in model.named_parameters() } num_layers = 0 diff --git a/hyper_parallel/trainer/base.py b/hyper_parallel/trainer/base.py index 410f1b4303..7ebaa596e5 100644 --- a/hyper_parallel/trainer/base.py +++ b/hyper_parallel/trainer/base.py @@ -616,6 +616,7 @@ def _build_graph_compiler(self) -> None: mesh_context=self.mesh, source_shard_info=getattr(self.model, "_source_shard_info", None), ) + self.graph_compiler.prepare_model() def _graph_train_fn( self, diff --git a/hyper_parallel/trainer/text_trainer.py b/hyper_parallel/trainer/text_trainer.py index ce9ace63af..bb052b3a24 100644 --- a/hyper_parallel/trainer/text_trainer.py +++ b/hyper_parallel/trainer/text_trainer.py @@ -70,11 +70,11 @@ def __init__(self, config: TrainerConfig) -> None: self.base.attach_model_integration_data_pipeline() self.base._compute_train_iters() + self.base._build_training_context() + self.base._build_graph_compiler() self.base._build_optimizer() self.base.model_integration.attach_optimizer(self.base.optimizer) self.base._build_lr_scheduler() - self.base._build_training_context() - self.base._build_graph_compiler() self.base._init_callbacks() def _build_model_assets(self) -> None: diff --git a/tests/ut/compile/test_compiler.py b/tests/ut/compile/test_compiler.py index ea5ae69b50..39cbd7122b 100644 --- a/tests/ut/compile/test_compiler.py +++ b/tests/ut/compile/test_compiler.py @@ -830,12 +830,12 @@ def test_dense_and_expert_layouts_without_fsdp_config(self): model.experts.local_expert_count = 2 context = MeshContext(fsdp_non_moe_mesh=dense, fsdp_moe_mesh=expert, dp_size=8, dp_shard_size=8, ep_size=ep, edp_shard_size=8 // ep) - compiler = GraphCompiler(model, _mse_train_fn, mesh_context=context) + compiler = GraphCompiler(model, _mse_train_fn, PassConfig(enable_overlap=False), mesh_context=context) with patch.object(compiler, "_register_mesh_group"): compiler._init_device_mesh(context) compiler._bind_parameter_meshes() self.assertEqual(compiler._parameter_meshes, {}) - compiler._joint_graph = SimpleNamespace(graph_module=SimpleNamespace()) + compiler.prepare_model() layouts = compiler.parameter_layouts self.assertEqual(layouts[model.dense].mesh.mesh_dim_names, ("fsdp_shard",)) expert_layout = layouts[model.experts.weight] @@ -853,7 +853,7 @@ def test_legacy_named_world_metadata_flattens_dp_and_cp(self): context = MeshContext(device_mesh=mesh, dp_replicate_size=replicate, dp_shard_size=2) model = _make_model() compiler = GraphCompiler(model, _mse_train_fn, PassConfig(fsdp_enabled=False), mesh_context=context) - compiler._joint_graph = SimpleNamespace(graph_module=SimpleNamespace()) + compiler.prepare_model() with patch.object(compiler, "_register_mesh_group") as register: layout = compiler.parameter_layouts[model.weight] register.assert_not_called() @@ -876,7 +876,7 @@ def test_hsdp_and_named_world_layout_meshes(self): compiler._init_device_mesh(context) expected = ("dp",) if world_only else ("fsdp_replicate", "fsdp_shard") self.assertEqual(compiler._fsdp_mesh.mesh_dim_names, expected) - compiler._joint_graph = SimpleNamespace(graph_module=SimpleNamespace()) + compiler.prepare_model() layout_names = expected if world_only else (*expected, "tp") self.assertEqual(compiler.parameter_layouts[model.weight].mesh.mesh_dim_names, layout_names) diff --git a/tests/ut/compile/test_ragged_ep.py b/tests/ut/compile/test_ragged_ep.py index b1ef529a59..067f611f64 100644 --- a/tests/ut/compile/test_ragged_ep.py +++ b/tests/ut/compile/test_ragged_ep.py @@ -76,6 +76,7 @@ def test_compile_resolves_expert_parameter_aliases(self): with ( patch("hyper_parallel.compile.compiler.dist.is_initialized", return_value=True), patch("hyper_parallel.compile.compiler._register_process_group") as register, + patch("hyper_parallel.compile.compiler.FSDPPass.prepare_model", return_value=frozenset()), patch("hyper_parallel.compile.compiler.trace_model_graph", return_value=joint), patch("hyper_parallel.compile.compiler.PassPipeline.from_config") as pipeline, ): diff --git a/tests/ut/compile/test_storage_preparation.py b/tests/ut/compile/test_storage_preparation.py new file mode 100644 index 0000000000..64598490eb --- /dev/null +++ b/tests/ut/compile/test_storage_preparation.py @@ -0,0 +1,156 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Storage preparation is independent of lazy graph capture and optimizer state.""" + +from contextlib import contextmanager +from unittest import TestCase +from unittest.mock import MagicMock, patch + +import torch +from torch import nn + +from hyper_parallel.compile.compiler import GraphCompiler +from hyper_parallel.compile.pass_config import PassConfig +from hyper_parallel.compile.passes.parallel.fsdp_pass import FSDPPass +from hyper_parallel.compile.tracer.graph_tracer import trace_model_graph +from hyper_parallel.components.optim.mixed_precision_optimizer import Float16OptimizerWithFloat16Params +from hyper_parallel.core.optimizer.optimizer import ChainedOptimizer + + +class _Model(nn.Module): + """Include aliases, padding, a scalar, frozen state and a buffer.""" + + def __init__(self, dtype: torch.dtype = torch.float32) -> None: + """Build the state categories that preparation must distinguish.""" + super().__init__() + self.weight = nn.Parameter(torch.arange(15, dtype=dtype).reshape(5, 3) / 16) + self.alias = self.weight + self.scale = nn.Parameter(torch.tensor(0.5, dtype=dtype)) + self.frozen = nn.Parameter(torch.ones(5, dtype=dtype), requires_grad=False) + self.register_buffer("offset", torch.ones(5, dtype=dtype)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Read full logical tensors during forward.""" + return (x @ self.weight.T + self.offset) * self.scale + self.frozen + + +def _loss(model, *, x): + """Use a scalar loss without labels.""" + return model(x).square().mean() + + +@contextmanager +def _distributed(compiler): + """Mock transport setup; slicing, tracing and graph lowering remain real.""" + transport = MagicMock() + transport.is_initialized.return_value = True + transport.get_world_size.return_value = 2 + transport.get_rank.return_value = 0 + with ( + patch("hyper_parallel.compile.compiler.dist", transport), + patch("hyper_parallel.compile.passes.parallel.fsdp_pass.dist", transport), + patch("hyper_parallel.compile.passes.parallel.fsdp_pass._resolve_process_group"), + patch.object(compiler, "_init_device_mesh"), + ): + yield + + +class TestStoragePreparation(TestCase): + """Preparation owns physical slicing; the graph only lowers logical state.""" + + def test_prepare_once_preserves_parameter_identity_and_nonsharded_state(self): + """A second preparation neither re-shards nor changes loaded values.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + original = torch.detach(model.weight).clone() + identity = id(model.weight) + with _distributed(compiler): + compiler.prepare_model() + self.assertEqual(compiler.parameter_shapes["weight"], (5, 3)) + self.assertEqual(compiler.sharded_parameter_names, {"weight"}) + self.assertEqual(model.weight.shape, (3, 3)) + torch.testing.assert_close(model.weight, original[:3], rtol=0, atol=0) + with torch.no_grad(): + model.weight.add_(1) + compiler.prepare_model() + self.assertEqual(id(model.weight), identity) + self.assertIs(model.alias, model.weight) + self.assertEqual(model.scale.shape, ()) + self.assertEqual(model.frozen.shape, (5,)) + self.assertEqual(model.offset.shape, (5,)) + self.assertFalse(compiler.is_compiled) + torch.testing.assert_close(model.weight, original[:3] + 1, rtol=0, atol=0) + + def test_logical_trace_and_pass_do_not_cut_prepared_storage_again(self): + """Trace sees the original five rows while FSDP accepts the three-row shard.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler): + compiler.prepare_model() + shard = torch.detach(model.weight).clone() + for dynamic in (False, True): + with self.subTest(dynamic=dynamic): + joint = trace_model_graph(model, _loss, {"x": torch.ones(2, 3)}, + state_shapes=compiler.parameter_shapes, dynamic=dynamic) + self.assertEqual(joint.param_shapes["weight"], (5, 3)) + self.assertEqual(joint.example_inputs[0].shape, (5, 3)) + FSDPPass().run(joint.graph_module, compiler.pass_config, **compiler._build_pass_kwargs()) + self.assertEqual(joint.graph_module.fsdp_sharded_param_names, {"weight"}) + torch.testing.assert_close(model.weight, shard, rtol=0, atol=0) + + def test_fp32_main_parameters_are_built_from_prepared_shards(self): + """BF16 storage does not leave full-size FP32 optimizer copies behind.""" + model = _Model(torch.bfloat16) + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler): + compiler.prepare_model() + raw = torch.optim.AdamW(model.parameters(), lr=0.01) + optimizer = Float16OptimizerWithFloat16Params(ChainedOptimizer(model, {"adamw": raw}), model) + for parameter, main in optimizer.optimizer_param_by_model_param.items(): + self.assertEqual(parameter.shape, main.shape) + self.assertEqual(main.dtype, torch.float32) + self.assertEqual(model.weight.main_param.shape, (3, 3)) + + def test_failed_preparation_cannot_repeat_partial_slicing(self): + """A partially prepared model must be rebuilt rather than silently reused.""" + compiler = GraphCompiler(_Model(), _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler), patch.object(FSDPPass, "prepare_model", side_effect=ValueError("slice failed")): + with self.assertRaisesRegex(ValueError, "slice failed"): + compiler.prepare_model() + with self.assertRaisesRegex(RuntimeError, "rebuild the model"): + compiler.prepare_model() + + def test_trace_failure_keeps_prepared_storage_and_logical_shapes(self): + """A retry after capture failure cannot re-shard the live model.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_degree=2), device=torch.device("cpu")) + with _distributed(compiler): + compiler.prepare_model() + shard = torch.detach(model.weight).clone() + with patch("hyper_parallel.compile.compiler.trace_model_graph", side_effect=ValueError("trace failed")): + for _ in range(2): + with self.assertRaisesRegex(ValueError, "trace failed"): + compiler.compile(x=torch.ones(2, 3)) + torch.testing.assert_close(model.weight, shard, rtol=0, atol=0) + self.assertEqual(compiler.parameter_shapes["weight"], (5, 3)) + + def test_replacing_prepared_parameter_requires_a_new_model(self): + """Parameter identity is part of the prepared-storage contract.""" + model = _Model() + compiler = GraphCompiler(model, _loss, PassConfig(fsdp_enabled=False)) + compiler.prepare_model() + model.weight = nn.Parameter(torch.detach(model.weight).clone()) + with self.assertRaisesRegex(ValueError, "storage changed"): + compiler.prepare_model() From bf0eb74c35a9cfcce13c38b5a013dad7ca35e9de Mon Sep 17 00:00:00 2001 From: yu-chen-code Date: Fri, 6 Nov 2026 23:03:05 +0800 Subject: [PATCH 2/3] feat(compile): bridge graph parameter storage to checkpoint chunks --- hyper_parallel/compile/checkpoint.py | 180 ++++++++++++++++++ .../models/_transformers/checkpoint_loader.py | 6 +- .../trainer/callbacks/checkpoint_callback.py | 22 ++- .../test_checkpoint_manager_save.py | 24 +++ tests/ut/compile/test_checkpoint.py | 99 ++++++++++ 5 files changed, 328 insertions(+), 3 deletions(-) create mode 100644 hyper_parallel/compile/checkpoint.py create mode 100644 tests/ut/compile/test_checkpoint.py diff --git a/hyper_parallel/compile/checkpoint.py b/hyper_parallel/compile/checkpoint.py new file mode 100644 index 0000000000..976bfa78fa --- /dev/null +++ b/hyper_parallel/compile/checkpoint.py @@ -0,0 +1,180 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Bridge graph-owned parameter storage to the existing DCP chunk contract.""" + +from __future__ import annotations + +from copy import copy +from typing import Any, TYPE_CHECKING + +import torch +import torch.distributed as dist + +from hyper_parallel.core.distributed_checkpoint.metadata import CHUNK_INFO, ChunkInfo, ChunkStorageMetadata +from hyper_parallel.core.dtensor.dtensor import DTensor +from hyper_parallel.core.dtensor.layout import Layout, infer_slice_area_by_layout + +if TYPE_CHECKING: + from .compiler import GraphCompiler + + +class GraphCheckpointAdapter: + """Describe local graph tensors without replacing live Parameters or the DCP planner. + + Checkpoints are taken at optimizer-step boundaries after storage preparation. + Model tensors, FQN-keyed optimizer moments and FP32 main weights share one + chunk description. Scalar optimizer counters remain replicated. + """ + + def __init__(self, compiler: GraphCompiler) -> None: + """Bind prepared storage and its canonical parameter chunk descriptions.""" + compiler.prepare_model() + self.compiler = compiler + self.parameters = dict(compiler.model.named_parameters(remove_duplicate=False)) + self.parameter_shapes = compiler.parameter_shapes + self.parameter_layouts = compiler.parameter_layouts + self.canonical_names = {id(parameter): name for name, parameter in compiler.model.named_parameters()} + self.chunks = { + name: self._parameter_chunk(self.canonical_names[id(parameter)], parameter) + for name, parameter in self.parameters.items() + } + + def _parameter_chunk(self, name: str, parameter: torch.nn.Parameter) -> ChunkInfo: + """Locate the valid rectangle within source-first shard geometry.""" + logical_shape = self.parameter_shapes[name] + source = self.compiler.source_layout_for_parameter(name) + global_shape = logical_shape + offsets = [0] * len(logical_shape) + if source is not None and source[1] is not None: + source_layout = Layout.from_device_mesh(source[1]) + source_layout.set_placements(tuple(source[0])) + source_layout.placement_to_tensor_map(len(logical_shape)) + global_shape = tuple(source_layout.get_global_shape(logical_shape)) + rank = source_layout.rank_list.index(source_layout.mesh.rank) + area = infer_slice_area_by_layout(source_layout, rank, global_shape) + if tuple(end - start for start, end in area) != logical_shape: + raise ValueError(f"Graph checkpoint requires a rectangular source shard for {name}") + offsets = [start for start, _ in area] + + sizes = list(logical_shape) + if name in self.compiler.sharded_parameter_names: + mesh, axis = self._shard_axis(parameter) + local_start = min(mesh.get_local_rank(axis) * parameter.shape[0], logical_shape[0]) + offsets[0] += local_start + sizes[0] = min(parameter.shape[0], logical_shape[0] - local_start) + return ChunkInfo(ChunkStorageMetadata(tuple(offsets), tuple(sizes)), tuple(global_shape)) + + def _shard_axis(self, parameter: torch.nn.Parameter) -> tuple: + """Resolve the physical FSDP axis from the compiler's final Layout.""" + layout = self.parameter_layouts[parameter] + mesh = layout.mesh + for axis, name in enumerate(mesh.mesh_dim_names or ()): + if name in ("fsdp_shard", "edp_shard", "fsdp", "dp", "dp_shard"): + return mesh, axis + raise ValueError("Graph checkpoint cannot resolve the parameter's FSDP shard axis") + + def prepare(self, payload: dict[str, Any]) -> dict[str, Any]: + """Return detached, unpadded DCP views carrying global chunk metadata.""" + return self._map(payload, restoring=False) + + def restore(self, payload: dict[str, Any]) -> dict[str, Any]: + """Restore physical padding before the existing model/optimizer loaders run.""" + return self._map(payload, restoring=True) + + def _owner(self, key: Any, current: str | None) -> str | None: + """Resolve ordinary, flattened optimizer and main-weight FQN keys.""" + if isinstance(key, str): + if key in self.parameters: + return key + if key.startswith("state."): + candidate = key[len("state."):].rsplit(".", 1)[0] + if candidate in self.parameters: + return candidate + return current + + def _tensor_view(self, value: torch.Tensor, owner: str, path: tuple, restoring: bool) -> torch.Tensor: + """Add a chunk to an unpadded view, or reconstruct physical padding on load.""" + parameter = self.parameters[owner] + if value.ndim == 0 and parameter.ndim != 0: + return value + chunk = self.chunks[owner] + expected = chunk.chunk.sizes if restoring else tuple(parameter.shape) + if tuple(value.shape) != expected: + raise ValueError(f"Graph checkpoint tensor {path} has shape {tuple(value.shape)}; expected {expected}") + if restoring: + if tuple(value.shape) == tuple(parameter.shape): + return value + result = value.new_zeros(parameter.shape) + result[tuple(slice(0, size) for size in value.shape)].copy_(value) + return result + result = value.detach()[tuple(slice(0, size) for size in chunk.chunk.sizes)] + setattr(result, CHUNK_INFO, chunk) + return result + + def _map(self, payload: dict[str, Any], *, restoring: bool) -> dict[str, Any]: + """Transform only tensor-owner payloads, preserving other training state.""" + def visit(value: Any, owner: str | None = None, path: tuple = ()) -> Any: + """Copy the payload containers and visit their tensor leaves.""" + if isinstance(value, dict): + result = copy(value) + for key, item in value.items(): + result[key] = visit(item, self._owner(key, owner), (*path, key)) + return result + if isinstance(value, (list, tuple)): + return type(value)(visit(item, owner, (*path, index)) for index, item in enumerate(value)) + if not isinstance(value, torch.Tensor): + return value + if owner is None: + if path and path[0] == "optimizer" and value.ndim: + raise ValueError("Graph checkpoint requires FQN-keyed optimizer state") + return value + return self._tensor_view(value, owner, path, restoring) + + result = copy(payload) + for key in ("model", "optimizer"): + if key in payload: + result[key] = visit(payload[key], path=(key,)) + return result + + @torch.no_grad() + def full_model_state(self, *, keep: bool) -> dict[str, Any]: + """Gather FSDP first, remove its padding, then gather the source TP/EP layout.""" + result = {} + for name, value in self.compiler.model.state_dict(keep_vars=True).items(): + parameter = self.parameters.get(name) + if isinstance(value, DTensor): + value = value.full_tensor() + elif parameter is not None: + parameter_name = self.canonical_names[id(parameter)] + value = value.detach() + if parameter_name in self.compiler.sharded_parameter_names: + mesh, axis = self._shard_axis(parameter) + group = mesh.get_group(axis) + gathered = value.new_empty((value.shape[0] * dist.get_world_size(group), *value.shape[1:])) + dist.all_gather_into_tensor(gathered, value.contiguous(), group=group) + value = gathered[:self.parameter_shapes[parameter_name][0]] + source = self.compiler.source_layout_for_parameter(parameter_name) + if source is not None and source[1] is not None: + layout = Layout.from_device_mesh(source[1]) + layout.set_placements(tuple(source[0])) + layout.placement_to_tensor_map(value.ndim) + value = DTensor.from_local_with_layout(value, layout).full_tensor() + else: + layout = getattr(value, "_sharding_spec", None) + if isinstance(value, torch.Tensor) and layout is not None: + value = DTensor.from_local_with_layout(value.detach(), layout).full_tensor() + if keep: + result[name] = value.detach().cpu() if isinstance(value, torch.Tensor) else value + return result diff --git a/hyper_parallel/models/_transformers/checkpoint_loader.py b/hyper_parallel/models/_transformers/checkpoint_loader.py index 52f83c64eb..baaaaab786 100644 --- a/hyper_parallel/models/_transformers/checkpoint_loader.py +++ b/hyper_parallel/models/_transformers/checkpoint_loader.py @@ -75,6 +75,7 @@ def save_pretrained( *, max_shard_size: int | str = "5GB", save_original_format: bool | None = None, + state_dict: dict[str, Any] | None = None, **kwargs: Any, ) -> bool: """Gather model weights and save a Transformers-compatible checkpoint. @@ -87,6 +88,8 @@ def save_pretrained( max_shard_size: Maximum size of each Hugging Face weight shard. save_original_format: Whether to reverse registered weight conversions. None uses the adapter default, or True when no adapter specifies one. + state_dict: Optional full model state gathered by an external storage owner. + All ranks supply a mapping; non-writing ranks may supply an empty mapping. **kwargs: Additional arguments forwarded to ``save_pretrained``. Returns: @@ -96,7 +99,8 @@ def save_pretrained( if not callable(save_method): raise TypeError("CheckpointManager.save_pretrained requires a Transformers model") is_main_process = self._is_main_process() - state_dict = self._gather_full_state_dict(keep_state_dict=is_main_process) + if state_dict is None: + state_dict = self._gather_full_state_dict(keep_state_dict=is_main_process) if not is_main_process: return False export_model = self._hf_export_model() diff --git a/hyper_parallel/trainer/callbacks/checkpoint_callback.py b/hyper_parallel/trainer/callbacks/checkpoint_callback.py index 3a0a134d64..cdf6982f04 100644 --- a/hyper_parallel/trainer/callbacks/checkpoint_callback.py +++ b/hyper_parallel/trainer/callbacks/checkpoint_callback.py @@ -26,6 +26,7 @@ import torch # pylint: disable=forbidden-backend-import from hyper_parallel.components.checkpoint import build_checkpointer +from hyper_parallel.compile.checkpoint import GraphCheckpointAdapter from hyper_parallel.components.checkpoint.dcp_checkpointer import ( STEP_PREFIX, initialize_optimizer_state, @@ -287,6 +288,14 @@ def _model_state_dict(self) -> Dict[str, Any]: } return {name: value for name, value in state_dict.items() if name in trainable} + def _graph_checkpoint_payload(self, payload: Dict[str, Any], *, restoring: bool = False) -> Dict[str, Any]: + """Describe graph-owned storage through DCP's existing tensor chunk interface.""" + compiler = getattr(self.trainer, "graph_compiler", None) + if compiler is None: + return payload + adapter = GraphCheckpointAdapter(compiler) + return adapter.restore(payload) if restoring else adapter.prepare(payload) + def _collect_extra_state(self, state: TrainerState) -> Dict[str, Any]: """Build the extra_state bundle (progress / scheduler / dataloader / RNG).""" # Prefer the iterator snapshot: with background prefetching the loader has @@ -391,6 +400,7 @@ def _save_checkpoint(self, state: TrainerState, force_sync: bool = False) -> Non ) if self._save_train_state: checkpoint_state["extra_state"] = self._collect_extra_state(state) + checkpoint_state = self._graph_checkpoint_payload(checkpoint_state) external = get_model_external_state(self.trainer.model) if external is not None: @@ -444,7 +454,14 @@ def _save_hf_checkpoint(self, save_dir: str, global_step: int) -> None: hf_dir, ) - write_assets = self._hf_checkpoint_manager.save_pretrained(hf_dir) + compiler = getattr(self.trainer, "graph_compiler", None) + if compiler is None: + write_assets = self._hf_checkpoint_manager.save_pretrained(hf_dir) + else: + full_state = GraphCheckpointAdapter(compiler).full_model_state( + keep=not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0 + ) + write_assets = self._hf_checkpoint_manager.save_pretrained(hf_dir, state_dict=full_state) if write_assets: self._save_hf_assets(hf_dir) if torch.distributed.is_available() and torch.distributed.is_initialized(): @@ -643,7 +660,7 @@ def _build_restore_skeleton(self, restore_path: str, external: Any) -> tuple[Dic checkpoint_state["optimizer"] = _unwrap_single( [optimizer.state_dict() for optimizer in optimizers] ) - return checkpoint_state, optimizers + return self._graph_checkpoint_payload(checkpoint_state), optimizers def _load_optimizer_payload(self, checkpoint_state: Dict[str, Any], optimizers: List[Any], persisted_keys: Optional[frozenset[str]]) -> None: @@ -663,6 +680,7 @@ def _apply_restored_payload(self, checkpoint_state: Dict[str, Any], optimizers: external: Any, runtime: Any, restore_path: str, load_plan: Any) -> None: """Install the loaded dense and external model, optimizer, and train state.""" + checkpoint_state = self._graph_checkpoint_payload(checkpoint_state, restoring=True) if external is not None: full_model_keys = set(self.trainer.model.state_dict()) dense_model_keys = set(checkpoint_state["model"]) diff --git a/tests/ut/auto_models/_transformers/test_checkpoint_manager_save.py b/tests/ut/auto_models/_transformers/test_checkpoint_manager_save.py index 084f7e317d..c79c7ffbec 100644 --- a/tests/ut/auto_models/_transformers/test_checkpoint_manager_save.py +++ b/tests/ut/auto_models/_transformers/test_checkpoint_manager_save.py @@ -53,6 +53,30 @@ def save_pretrained(self, save_directory: str, **kwargs: Any) -> None: class TestCheckpointManagerSave(unittest.TestCase): """Verify DCP-backed export does not read live training parameters.""" + @arg_mark(plat_marks=["cpu_linux"], level_mark="level0", + card_mark="onecard", essential_mark="essential") + def test_external_state_mapping_preserves_gather_ownership(self) -> None: + """Explicit graph-owned state skips gathering; omitted state keeps the original collective.""" + for main, provided in ((True, True), (False, True), (True, False), (False, False)): + with self.subTest(main=main, provided=provided): + model = ExportableModel("unregistered_export_test") + manager = CheckpointManager(model) + state = {"weight": torch.full((2,), 7.0)} if main else {} + options = {"state_dict": state} if provided else {} + with patch.object(manager, "_is_main_process", return_value=main), \ + patch.object(manager, "_gather_full_state_dict", return_value=state) as gather: + if provided: + gather.side_effect = RuntimeError("duplicate gather") + self.assertIs(manager.save_pretrained("/unused", **options), main) + if provided: + gather.assert_not_called() + else: + gather.assert_called_once_with(keep_state_dict=main) + self.assertIs(bool(model.exports), main) + if main: + self.assertIs(model.exports[0][2]["state_dict"], state) + self.assertFalse(torch.equal(state["weight"], model.weight)) + @arg_mark(plat_marks=["cpu_linux"], level_mark="level0", card_mark="onecard", essential_mark="essential") def test_hsdp_export_uses_original_class_without_mutating_training_model(self) -> None: diff --git a/tests/ut/compile/test_checkpoint.py b/tests/ut/compile/test_checkpoint.py new file mode 100644 index 0000000000..c6c13c1998 --- /dev/null +++ b/tests/ut/compile/test_checkpoint.py @@ -0,0 +1,99 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Graph checkpoint payloads retain global ownership and discard physical padding.""" + +from types import SimpleNamespace +from unittest import TestCase +from unittest.mock import patch + +import torch + +from hyper_parallel.compile.checkpoint import GraphCheckpointAdapter +from hyper_parallel.core.distributed_checkpoint.metadata import CHUNK_INFO +from hyper_parallel.core.dtensor.device_mesh import DeviceMesh +from hyper_parallel.core.dtensor.placement_types import Shard + + +def _adapter(rank: int, source: tuple | None = None) -> GraphCheckpointAdapter: + """Build a metadata-only two-rank storage owner without distributed initialization.""" + model = torch.nn.Module() + model.weight = torch.nn.Parameter(torch.arange(9, dtype=torch.float32).reshape(3, 3)) + model.alias = model.weight + model.scalar = torch.nn.Parameter(torch.tensor(2.0)) + model.register_buffer("counter", torch.tensor(4)) + mesh = SimpleNamespace(mesh_dim_names=("fsdp_shard",), get_local_rank=lambda axis: rank) + compiler = SimpleNamespace( + model=model, prepare_model=lambda: None, + parameter_shapes={"weight": (5, 3), "alias": (5, 3), "scalar": ()}, + sharded_parameter_names={"weight"}, parameter_layouts={model.weight: SimpleNamespace(mesh=mesh)}, + source_layout_for_parameter=lambda name: source if name == "weight" else None, + ) + return GraphCheckpointAdapter(compiler) + + +class TestGraphCheckpoint(TestCase): + """Use the existing tensor chunk protocol for model, moments and main weights.""" + + def test_padding_alias_and_optimizer_ownership_roundtrip(self): + """A trailing shard stores two valid rows and reloads all physical tensors correctly.""" + adapter = _adapter(rank=1) + state = adapter.compiler.model.state_dict() + moment = torch.ones(3, 3) + payload = { + "model": state, + "optimizer": {"state": {"weight": {"step": torch.tensor(5), "exp_avg": moment}}, + "mixed_precision_optimizer": {"fp32_from_fp16_params": {"weight": moment.clone()}}}, + "extra_state": {"weight": torch.ones(17)}, + } + checkpoint = adapter.prepare(payload) + for value in (checkpoint["model"]["weight"], checkpoint["model"]["alias"], + checkpoint["optimizer"]["state"]["weight"]["exp_avg"], + checkpoint["optimizer"]["mixed_precision_optimizer"]["fp32_from_fp16_params"]["weight"]): + info = getattr(value, CHUNK_INFO) + self.assertEqual(info.global_shape, (5, 3)) + self.assertEqual(info.chunk.offsets, (3, 0)) + self.assertEqual(info.chunk.sizes, (2, 3)) + self.assertEqual(value.shape, (2, 3)) + self.assertFalse(hasattr(moment, CHUNK_INFO)) + self.assertEqual(adapter.compiler.model.weight.shape, (3, 3)) + restored = adapter.restore(checkpoint) + torch.testing.assert_close(restored["model"]["weight"][:2], state["weight"][:2], rtol=0, atol=0) + self.assertEqual(restored["model"]["weight"][2].count_nonzero(), 0) + self.assertEqual(restored["optimizer"]["state"]["weight"]["step"], 5) + self.assertEqual(restored["model"]["counter"], 4) + self.assertIs(checkpoint["extra_state"], payload["extra_state"]) + self.assertEqual(getattr(checkpoint["model"], "_metadata"), getattr(state, "_metadata")) + + def test_flat_optimizer_and_invalid_state(self): + """Flat FQN moments work; unsupported factorization or integer keys fail explicitly.""" + adapter = _adapter(rank=0) + flat = adapter.prepare({"optimizer": {"state.weight.exp_avg": torch.ones(3, 3)}}) + self.assertEqual(getattr(flat["optimizer"]["state.weight.exp_avg"], CHUNK_INFO).chunk.offsets, (0, 0)) + for optimizer in ({"state": {0: {"exp_avg": torch.ones(3, 3)}}}, + {"state": {"weight": {"exp_avg": torch.ones(3)}}}): + with self.subTest(optimizer=optimizer), self.assertRaises(ValueError): + adapter.prepare({"optimizer": optimizer}) + + def test_source_shard_precedes_padded_fsdp_chunk(self): + """FSDP's valid tail stays inside its source TP shard rather than the padded global shape.""" + with patch("hyper_parallel.core.dtensor.device_mesh.dist.is_initialized", return_value=True), \ + patch("hyper_parallel.core.dtensor.device_mesh.dist.get_rank", return_value=3): + source_mesh = DeviceMesh("cpu", [1, 3], mesh_dim_names=("tp",), _init_backend=False) + adapter = _adapter(rank=1, source=((Shard(0),), source_mesh)) + value = adapter.prepare({"model": adapter.compiler.model.state_dict()})["model"]["weight"] + info = getattr(value, CHUNK_INFO) + self.assertEqual(info.global_shape, (10, 3)) + self.assertEqual(info.chunk.offsets, (8, 0)) + self.assertEqual(info.chunk.sizes, (2, 3)) From 813f984f07e466a6549193d5c2b85c539bc4a1bd Mon Sep 17 00:00:00 2001 From: yu-chen-code Date: Sat, 7 Nov 2026 00:30:17 +0800 Subject: [PATCH 3/3] fix(compile): preserve lazy optimizer state during graph restore --- hyper_parallel/compile/checkpoint.py | 24 +++++++++++++++++ .../trainer/callbacks/checkpoint_callback.py | 7 +++-- tests/ut/compile/test_checkpoint.py | 26 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/hyper_parallel/compile/checkpoint.py b/hyper_parallel/compile/checkpoint.py index 976bfa78fa..c69f5c685c 100644 --- a/hyper_parallel/compile/checkpoint.py +++ b/hyper_parallel/compile/checkpoint.py @@ -22,6 +22,7 @@ import torch import torch.distributed as dist +from hyper_parallel.core.distributed_checkpoint import FileSystemReader from hyper_parallel.core.distributed_checkpoint.metadata import CHUNK_INFO, ChunkInfo, ChunkStorageMetadata from hyper_parallel.core.dtensor.dtensor import DTensor from hyper_parallel.core.dtensor.layout import Layout, infer_slice_area_by_layout @@ -93,6 +94,29 @@ def restore(self, payload: dict[str, Any]) -> dict[str, Any]: """Restore physical padding before the existing model/optimizer loaders run.""" return self._map(payload, restoring=True) + def restore_optimizer_state(self, optimizers: list[Any], *, checkpoint_id: str) -> None: + """Remove priming-only slots after the standard strict optimizer load. + + Args: + optimizers: Loaded optimizer runtimes, in checkpoint payload order. + checkpoint_id: Actual DCP directory whose metadata records persisted state. + """ + metadata = FileSystemReader(checkpoint_id).load_metadata() + persisted = set() + for name in metadata.state_dict_metadata: + if name.startswith("optimizer."): + while name: + persisted.add(name) + name = name.rpartition(".")[0] + for index, optimizer in enumerate(optimizers): + prefix = f"optimizer.{index}" if len(optimizers) > 1 else "optimizer" + owners = getattr(optimizer, "model_param_by_optimizer_param", {}) + for leaf in getattr(optimizer, "chained_optimizers", (optimizer,)): + for parameter in list(leaf.state): + name = self.canonical_names.get(id(owners.get(parameter, parameter))) + if name in self.parameters and f"{prefix}.state.{name}" not in persisted: + del leaf.state[parameter] + def _owner(self, key: Any, current: str | None) -> str | None: """Resolve ordinary, flattened optimizer and main-weight FQN keys.""" if isinstance(key, str): diff --git a/hyper_parallel/trainer/callbacks/checkpoint_callback.py b/hyper_parallel/trainer/callbacks/checkpoint_callback.py index cdf6982f04..09927e9082 100644 --- a/hyper_parallel/trainer/callbacks/checkpoint_callback.py +++ b/hyper_parallel/trainer/callbacks/checkpoint_callback.py @@ -663,7 +663,7 @@ def _build_restore_skeleton(self, restore_path: str, external: Any) -> tuple[Dic return self._graph_checkpoint_payload(checkpoint_state), optimizers def _load_optimizer_payload(self, checkpoint_state: Dict[str, Any], optimizers: List[Any], - persisted_keys: Optional[frozenset[str]]) -> None: + persisted_keys: Optional[frozenset[str]], checkpoint_id: Optional[str] = None) -> None: """Restore dense optimizer leaves or reload dense main parameters.""" optimizer_sds = _as_list(checkpoint_state.get("optimizer")) for optimizer, optimizer_sd in zip(optimizers, optimizer_sds): @@ -671,6 +671,9 @@ def _load_optimizer_payload(self, checkpoint_state: Dict[str, Any], optimizers: drop_primed = getattr(optimizer, "drop_unpersisted_dense_state", None) if drop_primed is not None and persisted_keys is not None: drop_primed(persisted_keys) + compiler = getattr(self.trainer, "graph_compiler", None) + if compiler is not None and checkpoint_id is not None and optimizers: + GraphCheckpointAdapter(compiler).restore_optimizer_state(optimizers, checkpoint_id=checkpoint_id) if not optimizers: for optimizer in _as_list(self.trainer.optimizer): if isinstance(optimizer, MixedPrecisionOptimizer): @@ -698,7 +701,7 @@ def _apply_restored_payload(self, checkpoint_state: Dict[str, Any], optimizers: self.trainer.config.model_init_dtype, ) persisted_keys = load_plan.persisted_optimizer_keys if load_plan is not None else None - self._load_optimizer_payload(checkpoint_state, optimizers, persisted_keys) + self._load_optimizer_payload(checkpoint_state, optimizers, persisted_keys, checkpoint_id=restore_path) if self._restore_train_state: self._apply_extra_state(checkpoint_state["extra_state"]) diff --git a/tests/ut/compile/test_checkpoint.py b/tests/ut/compile/test_checkpoint.py index c6c13c1998..6326113cd4 100644 --- a/tests/ut/compile/test_checkpoint.py +++ b/tests/ut/compile/test_checkpoint.py @@ -97,3 +97,29 @@ def test_source_shard_precedes_padded_fsdp_chunk(self): self.assertEqual(info.global_shape, (10, 3)) self.assertEqual(info.chunk.offsets, (8, 0)) self.assertEqual(info.chunk.sizes, (2, 3)) + + def test_restore_preserves_lazy_optimizer_membership(self): + """Drop only primed live state, including independently keyed FP32 main parameters.""" + adapter = _adapter(rank=0) + model = adapter.compiler.model + main_scalar = torch.nn.Parameter(torch.tensor(2.)) + zeros = torch.zeros(3, 3) + saved = {"exp_avg": zeros} + leaf = SimpleNamespace(state={model.weight: saved, main_scalar: {"exp_avg": torch.tensor(0.)}, + "global": {"count": 1}}) + wrapped = SimpleNamespace(chained_optimizers=[leaf], model_param_by_optimizer_param={main_scalar: model.scalar}) + plain = SimpleNamespace(state={model.weight: {"exp_avg": zeros}, model.scalar: {"exp_avg": torch.tensor(0.)}}) + metadata = SimpleNamespace(state_dict_metadata={ + "optimizer.0.state.weight.exp_avg": object(), "optimizer.1.state.weight.exp_avg": object(), + "optimizer.0.mixed_precision_optimizer.fp32_from_fp16_params.scalar": object(), + }) + with patch("hyper_parallel.compile.checkpoint.FileSystemReader") as reader: + reader.return_value.load_metadata.return_value = metadata + adapter.restore_optimizer_state([wrapped, plain], checkpoint_id="/checkpoint") + self.assertIs(leaf.state[model.weight], saved) + self.assertEqual(saved["exp_avg"].count_nonzero(), 0) + self.assertNotIn(main_scalar, leaf.state) + self.assertNotIn(model.scalar, plain.state) + self.assertIn(model.weight, plain.state) + self.assertEqual(leaf.state["global"], {"count": 1}) + self.assertEqual(main_scalar, 2)