Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions hyper_parallel/compile/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -496,14 +496,12 @@ def _init_device_mesh(self, mesh_context: Optional[Any] = None):
self.pass_config.fsdp_degree = world_size

def _validate_ep_topology(self) -> None:
"""Limit graph EP to the synchronous, non-pipelined basic topology."""
"""Limit graph EP to supported non-pipelined topologies."""
context = self._mesh_context
if context is None or getattr(context, "ep_size", 1) <= 1:
return
if any(
getattr(context, axis, 1) != 1 for axis in ("cp_size", "pp_size")
):
raise ValueError("Graph EP currently requires CP=PP=1")
if getattr(context, "pp_size", 1) != 1:
raise ValueError("Graph EP currently requires PP=1")
if self.pass_config.enable_overlap:
raise ValueError("Graph EP currently requires enable_overlap=False")

Expand Down
36 changes: 33 additions & 3 deletions hyper_parallel/distributed/context_parallel/collectives.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,15 @@
# pylint: disable=missing-apache-license-header

from dataclasses import dataclass
from typing import Any, Callable, Optional
from typing import Any, Callable, Optional, Union
import torch # pylint: disable=forbidden-backend-import
import torch.distributed as dist # pylint: disable=forbidden-backend-import
from torch import Tensor # pylint: disable=forbidden-backend-import
from torch.distributed import _functional_collectives
from torch.distributed.nn.functional import all_gather as differentiable_all_gather # pylint: disable=forbidden-backend-import
from hyper_parallel.core.dtensor.device_mesh import DeviceMesh
from hyper_parallel.core.utils import communication
from hyper_parallel.core.utils.communication import _is_tracing_or_compiling


_ULYSSES_WRAPPED_FLAG = "_hyper_ulysses_wrapped"
Expand Down Expand Up @@ -360,18 +362,42 @@ def wait(self) -> Tensor:
raise RuntimeError(f"unknown async CP collective kind {self.kind!r}")


@dataclass
class _FunctionalCPCollective:
"""Keep a captured collective as tensors until its consumer materializes it."""

output: Tensor
dimension: int
all_to_all: bool = False
waited: bool = False

def wait(self) -> Tensor:
"""Wait on the functional result and restore its logical tensor layout."""
if self.waited:
raise RuntimeError("an async CP collective handle cannot be waited twice")
self.waited = True
output = _functional_collectives.wait_tensor(self.output)
if self.all_to_all:
return _reconstruct_all_to_all(output, self.dimension)
return _move_dim_from_front(output, self.dimension)


def async_cp_allgather_launch(
tensor: Tensor,
gather_dim: int,
cp_mesh: DeviceMesh,
) -> AsyncCPCollective:
) -> Union[AsyncCPCollective, _FunctionalCPCollective]:
"""Launch an asynchronous all-gather along gather_dim."""
world_size = cp_mesh.size()
if world_size <= 1:
return AsyncCPCollective(
tensor, None, tensor, None, world_size, "allgather", gather_dim
)
gather_dim = _normalize_dim(gather_dim, tensor.dim())
if _is_tracing_or_compiling():
send = _move_dim_to_front(tensor, gather_dim)
output = _functional_collectives.all_gather_single(send, 0, cp_mesh.get_group())
return _FunctionalCPCollective(output, gather_dim)
send = _move_dim_to_front(tensor.detach(), gather_dim)
output_shape = list(send.shape)
output_shape[0] *= world_size
Expand All @@ -392,7 +418,7 @@ def async_ulysses_seq_to_head_launch(
seq_dim: int,
head_dim: int,
cp_mesh: DeviceMesh,
) -> AsyncCPCollective:
) -> Union[AsyncCPCollective, _FunctionalCPCollective]:
"""Launch an asynchronous Ulysses sequence-to-head A2A."""
world_size = cp_mesh.size()
if world_size <= 1:
Expand All @@ -410,6 +436,10 @@ def async_ulysses_seq_to_head_launch(
head_dim = _normalize_dim(head_dim, tensor.dim())
if seq_dim == head_dim:
raise ValueError("Ulysses seq_dim and head_dim must be different")
if _is_tracing_or_compiling():
send = _prepare_ulysses_send(tensor, scatter_dim=head_dim, world_size=world_size)
output = _functional_collectives.all_to_all_single(send, None, None, cp_mesh.get_group())
return _FunctionalCPCollective(output, seq_dim, all_to_all=True)
send = _prepare_ulysses_send(
tensor.detach(), scatter_dim=head_dim, world_size=world_size
)
Expand Down
11 changes: 5 additions & 6 deletions tests/ut/compile/test_ragged_ep.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,12 +52,11 @@ class TestRaggedExperts(unittest.TestCase):
"""Reject unsupported meshes and retain exact expert ownership."""

def test_compile_rejects_unsupported_ep_topologies(self):
"""Reject extra parallel axes and overlap before tracing or communication."""
for axis in ("cp_size", "pp_size"):
context = MeshContext(ep_size=2, **{axis: 2})
compiler = GraphCompiler(_LocalExperts(), _loss, PassConfig(enable_overlap=False), mesh_context=context)
with self.subTest(axis=axis), self.assertRaisesRegex(ValueError, "CP=PP=1"):
compiler.compile()
"""Reject pipelining and EP overlap before tracing or communication."""
compiler = GraphCompiler(_LocalExperts(), _loss, PassConfig(enable_overlap=False),
mesh_context=MeshContext(ep_size=2, pp_size=2))
with self.assertRaisesRegex(ValueError, "PP=1"):
compiler.compile()
compiler = GraphCompiler(_LocalExperts(), _loss, PassConfig(enable_overlap=True),
mesh_context=MeshContext(ep_size=2))
with self.assertRaisesRegex(ValueError, "enable_overlap=False"):
Expand Down
87 changes: 87 additions & 0 deletions tests/ut/distributed/context_parallel/test_graph_collectives.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
# Copyright 2026 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Tensor-backed launch/wait adapters for captured CP communication."""

from types import SimpleNamespace
import unittest
from unittest.mock import patch

import torch

from hyper_parallel.compile import GraphCompiler
from hyper_parallel.compile.pass_config import PassConfig
from hyper_parallel.distributed.context_parallel import collectives


def _loss(model: torch.nn.Module, *, x: torch.Tensor) -> torch.Tensor:
"""A small traceable objective for compiler topology validation."""
return model(x).sum()


class TestGraphCPCollectives(unittest.TestCase):
"""Mock only the functional transport; retain tensor layouts and autograd."""

def test_allgather_wait_restores_layout_and_keeps_autograd(self) -> None:
"""The graph handle waits at the consumer and retains both replica contributions."""
tensor = torch.randn(1, 3, 4, requires_grad=True)
mesh = SimpleNamespace(size=lambda: 2, get_group=lambda: "cp")
prefix = "hyper_parallel.distributed.context_parallel.collectives"
with (
patch(f"{prefix}._is_tracing_or_compiling", return_value=True),
patch(f"{prefix}._functional_collectives.all_gather_single",
side_effect=lambda value, *unused: torch.cat((value, value))) as gather,
patch(f"{prefix}._functional_collectives.wait_tensor", side_effect=lambda value: value) as wait,
patch(f"{prefix}.dist") as eager,
):
handle = collectives.async_cp_allgather_launch(tensor, 1, mesh)
wait.assert_not_called()
result = handle.wait()
torch.testing.assert_close(result, torch.cat((tensor, tensor), dim=1), rtol=0, atol=0)
result.sum().backward()
torch.testing.assert_close(tensor.grad, torch.full_like(tensor, 2), rtol=0, atol=0)
self.assertEqual(gather.call_args.args[1:], (0, "cp"))
wait.assert_called_once()
self.assertEqual(eager.mock_calls, [])
with self.assertRaisesRegex(RuntimeError, "waited twice"):
handle.wait()

def test_cp_ep_topology_is_supported(self) -> None:
"""CP may compose with EP after its collectives gain a graph-safe path."""
context = SimpleNamespace(ep_size=2, cp_size=2, pp_size=1)
compiler = GraphCompiler(torch.nn.Linear(4, 4), _loss, PassConfig(enable_overlap=False),
mesh_context=context, device=torch.device("cpu"))
compiler._validate_ep_topology()

def test_ulysses_wait_restores_sequence_and_head_layout(self) -> None:
"""Functional A2A retains the same split/reconstruct contract and gradient edge."""
tensor = torch.randn(1, 3, 4, requires_grad=True)
mesh = SimpleNamespace(size=lambda: 2, get_group=lambda: "cp")
prefix = "hyper_parallel.distributed.context_parallel.collectives"
with (
patch(f"{prefix}._is_tracing_or_compiling", return_value=True),
patch(f"{prefix}._functional_collectives.all_to_all_single",
side_effect=lambda value, *unused: value) as exchange,
patch(f"{prefix}._functional_collectives.wait_tensor", side_effect=lambda value: value) as wait,
patch(f"{prefix}.dist") as eager,
):
handle = collectives.async_ulysses_seq_to_head_launch(tensor, 1, -1, mesh)
wait.assert_not_called()
result = handle.wait()
torch.testing.assert_close(result, torch.cat(tensor.chunk(2, dim=-1), dim=1), rtol=0, atol=0)
result.sum().backward()
torch.testing.assert_close(tensor.grad, torch.ones_like(tensor), rtol=0, atol=0)
self.assertEqual(exchange.call_args.args[1:], (None, None, "cp"))
wait.assert_called_once()
self.assertEqual(eager.mock_calls, [])
Loading