Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions hyper_parallel/compile/passes/parallel/fsdp_pass.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@
from torch.fx.node import _side_effectful_functions
from torch.ops import _c10d_functional

from hyper_parallel.core.fully_shard.hsdp_utils import get_shard_geometry

from ...graph_parallel_plan import GraphParallelPlan
from ...pass_config import GradientReduction, PassConfig, normalize_dp_mode
from ..base import GraphPass
Expand Down Expand Up @@ -810,7 +812,7 @@ def _marked_param_stays_replicated(
)
return True
n = int(shape[0])
n_pad = -(-n // degree) * degree
n_pad = get_shard_geometry(shape, 0, degree, 0).padded_shape[0] * degree
self._padded_params[fqn] = _PaddedParam(
n=n, n_pad=n_pad, dim0_pad=n_pad - n, ndim=param.ndim
)
Expand Down Expand Up @@ -891,8 +893,8 @@ def _shard_live_model_params(self, model: nn.Module) -> None:

# Pad-shard a non-divisible param (recorded by
# ``_classify_dp_params``): zero-pad dim 0 to a multiple of the
# degree, then take this rank's equal-size chunk — exactly what
# eager ``fully_shard`` does. A 0-dim scalar cannot be padded and
# degree, then retain this rank's padded communication shard.
# Eager retains only its valid rows. A 0-dim scalar cannot be padded and
# stays replicated.
pad_info = self._padded_params.get(name)
if pad_info is not None:
Expand All @@ -903,7 +905,9 @@ def _shard_live_model_params(self, model: nn.Module) -> None:
[full, full.new_zeros((pad_info.dim0_pad, *full.shape[1:]))],
dim=0,
)
param.data = full.chunk(degree, dim=0)[rank].clone()
geometry = get_shard_geometry(original_shape, 0, degree, rank)
shard_size = geometry.padded_shape[0]
param.data = full.narrow(0, rank * shard_size, shard_size).clone()
sharded_count += 1
_LOG.info(
"Pad-sharded %s: %s -> %s",
Expand All @@ -914,7 +918,8 @@ def _shard_live_model_params(self, model: nn.Module) -> None:
continue

original_shape = param.shape
param.data = param.detach().chunk(degree, dim=0)[rank].clone()
geometry = get_shard_geometry(original_shape, 0, degree, rank)
param.data = param.detach().narrow(0, geometry.start, geometry.local_shape[0]).clone()
sharded_count += 1
_LOG.info(
"Sharded %s: %s -> %s",
Expand Down
12 changes: 4 additions & 8 deletions hyper_parallel/core/fully_shard/hsdp_param.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
ParamModuleInfo,
ShardedState,
apply_gradient_scaling_factor,
get_shard_geometry,
)
from hyper_parallel.core.fully_shard.utils import (
CPUOffloadPolicy,
Expand Down Expand Up @@ -616,17 +617,12 @@ def _init_sharded_param(
base_placements,
)

chunks = torch.chunk(param_data, self.shard_world_size, dim=shard_dim)
if self.shard_rank < len(chunks):
local_shard = chunks[self.shard_rank]
else:
empty_slice = [slice(None)] * param_data.ndim
empty_slice[shard_dim] = slice(param_data.size(shard_dim), None)
local_shard = param_data[tuple(empty_slice)]
geometry = get_shard_geometry(param_data.shape, shard_dim, self.shard_world_size, self.shard_rank)
local_shard = param_data.narrow(shard_dim, geometry.start, geometry.local_shape[shard_dim])
sharded_param = local_shard.clone().contiguous()
self.sharded_size = sharded_param.size()
self.contiguous_sharded_stride = make_contiguous_strides_for(self.sharded_size)
self.padded_sharded_param_size = chunks[0].size()
self.padded_sharded_param_size = geometry.padded_shape
if self.offload_to_cpu and not sharded_param.is_meta:
sharded_param = sharded_param.cpu()
if self.pin_memory:
Expand Down
41 changes: 41 additions & 0 deletions hyper_parallel/core/fully_shard/hsdp_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,47 @@
from hyper_parallel.core.dtensor.dtensor import DTensor


@dataclass(frozen=True)
class ShardGeometry:
"""Valid local shard and uniform communication shape within a source-local tensor."""

start: int
local_shape: torch.Size
padded_shape: torch.Size


def get_shard_geometry(
shape: Sequence[int], shard_dim: int, shard_world_size: int, shard_rank: int
) -> ShardGeometry:
"""Plan chunk-based FSDP sharding without allocating or modifying tensors.

Args:
shape: Parameter shape after any source parallelism, such as TP or EP.
shard_dim: Non-negative dimension to shard.
shard_world_size: Size of the parameter's FSDP shard group.
shard_rank: Rank within that group.

Returns:
Valid slice start and shape, plus the uniform padded communication shape.
Ranks beyond the chunks returned by ``torch.chunk`` own an empty slice.

Raises:
ValueError: If the shard dimension, group size or group-local rank is invalid.
"""
if shard_world_size < 1 or not 0 <= shard_rank < shard_world_size:
raise ValueError(f"Invalid shard rank {shard_rank} for shard degree {shard_world_size}")
if not 0 <= shard_dim < len(shape):
raise ValueError(f"Invalid shard dimension {shard_dim} for shape {tuple(shape)}")
size = shape[shard_dim]
chunk_size = (size + shard_world_size - 1) // shard_world_size
start = min(shard_rank * chunk_size, size)
local_shape = list(shape)
padded_shape = list(shape)
local_shape[shard_dim] = min(chunk_size, size - start)
padded_shape[shard_dim] = chunk_size
return ShardGeometry(start, torch.Size(local_shape), torch.Size(padded_shape))


class ShardedState(Enum):
"""
Parameter shard state
Expand Down
20 changes: 20 additions & 0 deletions tests/ut/compile/test_fsdp_pass.py
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,26 @@ def test_run_pad_shards_non_divisible_params(self):
"the [3, 4] gradient must be padded back to [4, 4] before reduction",
)

def test_shard_storage_preserves_padded_and_empty_tails(self):
"""Every graph rank keeps uniform storage without changing parameter identity."""
degree = 4
for rows in (0, 2, 5):
source = torch.arange(rows * 4, dtype=torch.float32).reshape(rows, 4)
padded_rows = -(-rows // degree) * degree
padded = torch.cat((source, source.new_zeros(padded_rows - rows, 4)))
for rank in range(degree):
with self.subTest(rows=rows, rank=rank):
model = _OddParam()
model.odd = nn.Parameter(source.clone())
param = model.odd
gm = _odd_joint_graph()
with _patch_dist(world_size=degree, rank=rank):
FSDPPass().run(gm, PassConfig(fsdp_degree=degree), model=model)
expected = padded.chunk(degree)[rank]
self.assertIs(model.odd, param)
torch.testing.assert_close(model.odd, expected, atol=0, rtol=0)
gm.graph.lint()

def test_configured_replica_preserves_storage_and_uses_all_reduce(self):
"""Explicit replicas bypass padding and gathers while retaining DP synchronization."""
for mode in ("fsdp", "hsdp"):
Expand Down
36 changes: 36 additions & 0 deletions tests/ut/core/fully_shard/test_hsdp_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,49 @@
get_hsdp_state,
get_managed_modules_parameters,
get_rank_list_for_axes,
get_shard_geometry,
get_split_rank_lists_for_axes,
infer_fully_shard_param_mode,
is_dtensor_managed_param,
unwrap_dtensor_param,
)


class TestShardGeometry(unittest.TestCase):
"""Shared geometry follows Torch chunks, including absent and empty shards."""

def test_matches_torch_chunk(self):
"""Plan valid views and padded shapes on source-local parameter dimensions."""
cases = (
((8, 3), 0, 4), ((10, 3), 0, 3), ((5, 3), 0, 4), ((2, 3), 0, 4),
((0, 3), 0, 4), ((5, 0), 0, 4), ((3, 8), 1, 4), ((3, 5), 1, 4),
((3, 0), 1, 4), ((5, 3), 0, 1),
)
for shape, dim, degree in cases:
source = torch.arange(torch.Size(shape).numel()).reshape(shape)
chunks = source.chunk(degree, dim)
for rank in range(degree):
with self.subTest(shape=shape, dim=dim, degree=degree, rank=rank):
geometry = get_shard_geometry(shape, dim, degree, rank)
expected = chunks[rank] if rank < len(chunks) else source.narrow(dim, shape[dim], 0)
actual = source.narrow(dim, geometry.start, geometry.local_shape[dim])
self.assertEqual(geometry.local_shape, expected.shape,
f"local shape={geometry.local_shape}, expected={expected.shape}")
self.assertEqual(geometry.padded_shape, chunks[0].shape,
f"padded shape={geometry.padded_shape}, expected={chunks[0].shape}")
torch.testing.assert_close(actual, expected, atol=0, rtol=0)

def test_rejects_invalid_shard_coordinates(self):
"""Invalid dimensions and group coordinates fail before tensor slicing."""
for shape, dim, degree, rank in (
((), 0, 2, 0), ((4,), -1, 2, 0), ((4,), 1, 2, 0),
((4,), 0, 0, 0), ((4,), 0, 2, -1), ((4,), 0, 2, 2),
):
with self.subTest(shape=shape, dim=dim, degree=degree, rank=rank):
with self.assertRaises(ValueError):
get_shard_geometry(shape, dim, degree, rank)


class SimpleLinear(nn.Module):
"""Simple linear module for testing."""

Expand Down
Loading