From 4877541939d11319658f9c1bdff2799c95d88bfa Mon Sep 17 00:00:00 2001 From: yu-chen-code Date: Wed, 11 Nov 2026 12:32:18 +0800 Subject: [PATCH] refactor(fsdp): share eager and graph shard geometry --- .../compile/passes/parallel/fsdp_pass.py | 15 ++++--- hyper_parallel/core/fully_shard/hsdp_param.py | 12 ++---- hyper_parallel/core/fully_shard/hsdp_utils.py | 41 +++++++++++++++++++ tests/ut/compile/test_fsdp_pass.py | 20 +++++++++ tests/ut/core/fully_shard/test_hsdp_utils.py | 36 ++++++++++++++++ 5 files changed, 111 insertions(+), 13 deletions(-) diff --git a/hyper_parallel/compile/passes/parallel/fsdp_pass.py b/hyper_parallel/compile/passes/parallel/fsdp_pass.py index 0e7a70b3f4..5b4671eb7b 100644 --- a/hyper_parallel/compile/passes/parallel/fsdp_pass.py +++ b/hyper_parallel/compile/passes/parallel/fsdp_pass.py @@ -65,6 +65,8 @@ from torch.fx.node import _side_effectful_functions from torch.ops import _c10d_functional +from hyper_parallel.core.fully_shard.hsdp_utils import get_shard_geometry + from ...graph_parallel_plan import GraphParallelPlan from ...pass_config import GradientReduction, PassConfig, normalize_dp_mode from ..base import GraphPass @@ -810,7 +812,7 @@ def _marked_param_stays_replicated( ) return True n = int(shape[0]) - n_pad = -(-n // degree) * degree + n_pad = get_shard_geometry(shape, 0, degree, 0).padded_shape[0] * degree self._padded_params[fqn] = _PaddedParam( n=n, n_pad=n_pad, dim0_pad=n_pad - n, ndim=param.ndim ) @@ -891,8 +893,8 @@ def _shard_live_model_params(self, model: nn.Module) -> None: # Pad-shard a non-divisible param (recorded by # ``_classify_dp_params``): zero-pad dim 0 to a multiple of the - # degree, then take this rank's equal-size chunk — exactly what - # eager ``fully_shard`` does. A 0-dim scalar cannot be padded and + # degree, then retain this rank's padded communication shard. + # Eager retains only its valid rows. A 0-dim scalar cannot be padded and # stays replicated. pad_info = self._padded_params.get(name) if pad_info is not None: @@ -903,7 +905,9 @@ def _shard_live_model_params(self, model: nn.Module) -> None: [full, full.new_zeros((pad_info.dim0_pad, *full.shape[1:]))], dim=0, ) - param.data = full.chunk(degree, dim=0)[rank].clone() + geometry = get_shard_geometry(original_shape, 0, degree, rank) + shard_size = geometry.padded_shape[0] + param.data = full.narrow(0, rank * shard_size, shard_size).clone() sharded_count += 1 _LOG.info( "Pad-sharded %s: %s -> %s", @@ -914,7 +918,8 @@ def _shard_live_model_params(self, model: nn.Module) -> None: continue original_shape = param.shape - param.data = param.detach().chunk(degree, dim=0)[rank].clone() + geometry = get_shard_geometry(original_shape, 0, degree, rank) + param.data = param.detach().narrow(0, geometry.start, geometry.local_shape[0]).clone() sharded_count += 1 _LOG.info( "Sharded %s: %s -> %s", diff --git a/hyper_parallel/core/fully_shard/hsdp_param.py b/hyper_parallel/core/fully_shard/hsdp_param.py index 06b993a7c3..eb68d85253 100755 --- a/hyper_parallel/core/fully_shard/hsdp_param.py +++ b/hyper_parallel/core/fully_shard/hsdp_param.py @@ -33,6 +33,7 @@ ParamModuleInfo, ShardedState, apply_gradient_scaling_factor, + get_shard_geometry, ) from hyper_parallel.core.fully_shard.utils import ( CPUOffloadPolicy, @@ -616,17 +617,12 @@ def _init_sharded_param( base_placements, ) - chunks = torch.chunk(param_data, self.shard_world_size, dim=shard_dim) - if self.shard_rank < len(chunks): - local_shard = chunks[self.shard_rank] - else: - empty_slice = [slice(None)] * param_data.ndim - empty_slice[shard_dim] = slice(param_data.size(shard_dim), None) - local_shard = param_data[tuple(empty_slice)] + geometry = get_shard_geometry(param_data.shape, shard_dim, self.shard_world_size, self.shard_rank) + local_shard = param_data.narrow(shard_dim, geometry.start, geometry.local_shape[shard_dim]) sharded_param = local_shard.clone().contiguous() self.sharded_size = sharded_param.size() self.contiguous_sharded_stride = make_contiguous_strides_for(self.sharded_size) - self.padded_sharded_param_size = chunks[0].size() + self.padded_sharded_param_size = geometry.padded_shape if self.offload_to_cpu and not sharded_param.is_meta: sharded_param = sharded_param.cpu() if self.pin_memory: diff --git a/hyper_parallel/core/fully_shard/hsdp_utils.py b/hyper_parallel/core/fully_shard/hsdp_utils.py index 5e21b60fd3..9f5c7cab13 100644 --- a/hyper_parallel/core/fully_shard/hsdp_utils.py +++ b/hyper_parallel/core/fully_shard/hsdp_utils.py @@ -25,6 +25,47 @@ from hyper_parallel.core.dtensor.dtensor import DTensor +@dataclass(frozen=True) +class ShardGeometry: + """Valid local shard and uniform communication shape within a source-local tensor.""" + + start: int + local_shape: torch.Size + padded_shape: torch.Size + + +def get_shard_geometry( + shape: Sequence[int], shard_dim: int, shard_world_size: int, shard_rank: int +) -> ShardGeometry: + """Plan chunk-based FSDP sharding without allocating or modifying tensors. + + Args: + shape: Parameter shape after any source parallelism, such as TP or EP. + shard_dim: Non-negative dimension to shard. + shard_world_size: Size of the parameter's FSDP shard group. + shard_rank: Rank within that group. + + Returns: + Valid slice start and shape, plus the uniform padded communication shape. + Ranks beyond the chunks returned by ``torch.chunk`` own an empty slice. + + Raises: + ValueError: If the shard dimension, group size or group-local rank is invalid. + """ + if shard_world_size < 1 or not 0 <= shard_rank < shard_world_size: + raise ValueError(f"Invalid shard rank {shard_rank} for shard degree {shard_world_size}") + if not 0 <= shard_dim < len(shape): + raise ValueError(f"Invalid shard dimension {shard_dim} for shape {tuple(shape)}") + size = shape[shard_dim] + chunk_size = (size + shard_world_size - 1) // shard_world_size + start = min(shard_rank * chunk_size, size) + local_shape = list(shape) + padded_shape = list(shape) + local_shape[shard_dim] = min(chunk_size, size - start) + padded_shape[shard_dim] = chunk_size + return ShardGeometry(start, torch.Size(local_shape), torch.Size(padded_shape)) + + class ShardedState(Enum): """ Parameter shard state diff --git a/tests/ut/compile/test_fsdp_pass.py b/tests/ut/compile/test_fsdp_pass.py index 912b750cc4..a7e9860030 100644 --- a/tests/ut/compile/test_fsdp_pass.py +++ b/tests/ut/compile/test_fsdp_pass.py @@ -515,6 +515,26 @@ def test_run_pad_shards_non_divisible_params(self): "the [3, 4] gradient must be padded back to [4, 4] before reduction", ) + def test_shard_storage_preserves_padded_and_empty_tails(self): + """Every graph rank keeps uniform storage without changing parameter identity.""" + degree = 4 + for rows in (0, 2, 5): + source = torch.arange(rows * 4, dtype=torch.float32).reshape(rows, 4) + padded_rows = -(-rows // degree) * degree + padded = torch.cat((source, source.new_zeros(padded_rows - rows, 4))) + for rank in range(degree): + with self.subTest(rows=rows, rank=rank): + model = _OddParam() + model.odd = nn.Parameter(source.clone()) + param = model.odd + gm = _odd_joint_graph() + with _patch_dist(world_size=degree, rank=rank): + FSDPPass().run(gm, PassConfig(fsdp_degree=degree), model=model) + expected = padded.chunk(degree)[rank] + self.assertIs(model.odd, param) + torch.testing.assert_close(model.odd, expected, atol=0, rtol=0) + gm.graph.lint() + def test_configured_replica_preserves_storage_and_uses_all_reduce(self): """Explicit replicas bypass padding and gathers while retaining DP synchronization.""" for mode in ("fsdp", "hsdp"): diff --git a/tests/ut/core/fully_shard/test_hsdp_utils.py b/tests/ut/core/fully_shard/test_hsdp_utils.py index 90582e491a..f0c11c204f 100644 --- a/tests/ut/core/fully_shard/test_hsdp_utils.py +++ b/tests/ut/core/fully_shard/test_hsdp_utils.py @@ -37,6 +37,7 @@ get_hsdp_state, get_managed_modules_parameters, get_rank_list_for_axes, + get_shard_geometry, get_split_rank_lists_for_axes, infer_fully_shard_param_mode, is_dtensor_managed_param, @@ -44,6 +45,41 @@ ) +class TestShardGeometry(unittest.TestCase): + """Shared geometry follows Torch chunks, including absent and empty shards.""" + + def test_matches_torch_chunk(self): + """Plan valid views and padded shapes on source-local parameter dimensions.""" + cases = ( + ((8, 3), 0, 4), ((10, 3), 0, 3), ((5, 3), 0, 4), ((2, 3), 0, 4), + ((0, 3), 0, 4), ((5, 0), 0, 4), ((3, 8), 1, 4), ((3, 5), 1, 4), + ((3, 0), 1, 4), ((5, 3), 0, 1), + ) + for shape, dim, degree in cases: + source = torch.arange(torch.Size(shape).numel()).reshape(shape) + chunks = source.chunk(degree, dim) + for rank in range(degree): + with self.subTest(shape=shape, dim=dim, degree=degree, rank=rank): + geometry = get_shard_geometry(shape, dim, degree, rank) + expected = chunks[rank] if rank < len(chunks) else source.narrow(dim, shape[dim], 0) + actual = source.narrow(dim, geometry.start, geometry.local_shape[dim]) + self.assertEqual(geometry.local_shape, expected.shape, + f"local shape={geometry.local_shape}, expected={expected.shape}") + self.assertEqual(geometry.padded_shape, chunks[0].shape, + f"padded shape={geometry.padded_shape}, expected={chunks[0].shape}") + torch.testing.assert_close(actual, expected, atol=0, rtol=0) + + def test_rejects_invalid_shard_coordinates(self): + """Invalid dimensions and group coordinates fail before tensor slicing.""" + for shape, dim, degree, rank in ( + ((), 0, 2, 0), ((4,), -1, 2, 0), ((4,), 1, 2, 0), + ((4,), 0, 0, 0), ((4,), 0, 2, -1), ((4,), 0, 2, 2), + ): + with self.subTest(shape=shape, dim=dim, degree=degree, rank=rank): + with self.assertRaises(ValueError): + get_shard_geometry(shape, dim, degree, rank) + + class SimpleLinear(nn.Module): """Simple linear module for testing."""