diff --git a/hyper_parallel/compile/compiler.py b/hyper_parallel/compile/compiler.py index a624ff4bd..92b9580ba 100644 --- a/hyper_parallel/compile/compiler.py +++ b/hyper_parallel/compile/compiler.py @@ -496,14 +496,12 @@ def _init_device_mesh(self, mesh_context: Optional[Any] = None): self.pass_config.fsdp_degree = world_size def _validate_ep_topology(self) -> None: - """Limit graph EP to the synchronous, non-pipelined basic topology.""" + """Limit graph EP to supported non-pipelined topologies.""" context = self._mesh_context if context is None or getattr(context, "ep_size", 1) <= 1: return - if any( - getattr(context, axis, 1) != 1 for axis in ("cp_size", "pp_size") - ): - raise ValueError("Graph EP currently requires CP=PP=1") + if getattr(context, "pp_size", 1) != 1: + raise ValueError("Graph EP currently requires PP=1") if self.pass_config.enable_overlap: raise ValueError("Graph EP currently requires enable_overlap=False") diff --git a/hyper_parallel/distributed/context_parallel/collectives.py b/hyper_parallel/distributed/context_parallel/collectives.py index d102d5f37..d15f104d5 100644 --- a/hyper_parallel/distributed/context_parallel/collectives.py +++ b/hyper_parallel/distributed/context_parallel/collectives.py @@ -39,13 +39,15 @@ # pylint: disable=missing-apache-license-header from dataclasses import dataclass -from typing import Any, Callable, Optional +from typing import Any, Callable, Optional, Union import torch # pylint: disable=forbidden-backend-import import torch.distributed as dist # pylint: disable=forbidden-backend-import from torch import Tensor # pylint: disable=forbidden-backend-import +from torch.distributed import _functional_collectives from torch.distributed.nn.functional import all_gather as differentiable_all_gather # pylint: disable=forbidden-backend-import from hyper_parallel.core.dtensor.device_mesh import DeviceMesh from hyper_parallel.core.utils import communication +from hyper_parallel.core.utils.communication import _is_tracing_or_compiling _ULYSSES_WRAPPED_FLAG = "_hyper_ulysses_wrapped" @@ -360,11 +362,31 @@ def wait(self) -> Tensor: raise RuntimeError(f"unknown async CP collective kind {self.kind!r}") +@dataclass +class _FunctionalCPCollective: + """Keep a captured collective as tensors until its consumer materializes it.""" + + output: Tensor + dimension: int + all_to_all: bool = False + waited: bool = False + + def wait(self) -> Tensor: + """Wait on the functional result and restore its logical tensor layout.""" + if self.waited: + raise RuntimeError("an async CP collective handle cannot be waited twice") + self.waited = True + output = _functional_collectives.wait_tensor(self.output) + if self.all_to_all: + return _reconstruct_all_to_all(output, self.dimension) + return _move_dim_from_front(output, self.dimension) + + def async_cp_allgather_launch( tensor: Tensor, gather_dim: int, cp_mesh: DeviceMesh, -) -> AsyncCPCollective: +) -> Union[AsyncCPCollective, _FunctionalCPCollective]: """Launch an asynchronous all-gather along gather_dim.""" world_size = cp_mesh.size() if world_size <= 1: @@ -372,6 +394,10 @@ def async_cp_allgather_launch( tensor, None, tensor, None, world_size, "allgather", gather_dim ) gather_dim = _normalize_dim(gather_dim, tensor.dim()) + if _is_tracing_or_compiling(): + send = _move_dim_to_front(tensor, gather_dim) + output = _functional_collectives.all_gather_single(send, 0, cp_mesh.get_group()) + return _FunctionalCPCollective(output, gather_dim) send = _move_dim_to_front(tensor.detach(), gather_dim) output_shape = list(send.shape) output_shape[0] *= world_size @@ -392,7 +418,7 @@ def async_ulysses_seq_to_head_launch( seq_dim: int, head_dim: int, cp_mesh: DeviceMesh, -) -> AsyncCPCollective: +) -> Union[AsyncCPCollective, _FunctionalCPCollective]: """Launch an asynchronous Ulysses sequence-to-head A2A.""" world_size = cp_mesh.size() if world_size <= 1: @@ -410,6 +436,10 @@ def async_ulysses_seq_to_head_launch( head_dim = _normalize_dim(head_dim, tensor.dim()) if seq_dim == head_dim: raise ValueError("Ulysses seq_dim and head_dim must be different") + if _is_tracing_or_compiling(): + send = _prepare_ulysses_send(tensor, scatter_dim=head_dim, world_size=world_size) + output = _functional_collectives.all_to_all_single(send, None, None, cp_mesh.get_group()) + return _FunctionalCPCollective(output, seq_dim, all_to_all=True) send = _prepare_ulysses_send( tensor.detach(), scatter_dim=head_dim, world_size=world_size ) diff --git a/tests/ut/compile/test_ragged_ep.py b/tests/ut/compile/test_ragged_ep.py index b1ef529a5..ac1143083 100644 --- a/tests/ut/compile/test_ragged_ep.py +++ b/tests/ut/compile/test_ragged_ep.py @@ -52,12 +52,11 @@ class TestRaggedExperts(unittest.TestCase): """Reject unsupported meshes and retain exact expert ownership.""" def test_compile_rejects_unsupported_ep_topologies(self): - """Reject extra parallel axes and overlap before tracing or communication.""" - for axis in ("cp_size", "pp_size"): - context = MeshContext(ep_size=2, **{axis: 2}) - compiler = GraphCompiler(_LocalExperts(), _loss, PassConfig(enable_overlap=False), mesh_context=context) - with self.subTest(axis=axis), self.assertRaisesRegex(ValueError, "CP=PP=1"): - compiler.compile() + """Reject pipelining and EP overlap before tracing or communication.""" + compiler = GraphCompiler(_LocalExperts(), _loss, PassConfig(enable_overlap=False), + mesh_context=MeshContext(ep_size=2, pp_size=2)) + with self.assertRaisesRegex(ValueError, "PP=1"): + compiler.compile() compiler = GraphCompiler(_LocalExperts(), _loss, PassConfig(enable_overlap=True), mesh_context=MeshContext(ep_size=2)) with self.assertRaisesRegex(ValueError, "enable_overlap=False"): diff --git a/tests/ut/distributed/context_parallel/test_graph_collectives.py b/tests/ut/distributed/context_parallel/test_graph_collectives.py new file mode 100644 index 000000000..8d40f77d4 --- /dev/null +++ b/tests/ut/distributed/context_parallel/test_graph_collectives.py @@ -0,0 +1,87 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tensor-backed launch/wait adapters for captured CP communication.""" + +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import torch + +from hyper_parallel.compile import GraphCompiler +from hyper_parallel.compile.pass_config import PassConfig +from hyper_parallel.distributed.context_parallel import collectives + + +def _loss(model: torch.nn.Module, *, x: torch.Tensor) -> torch.Tensor: + """A small traceable objective for compiler topology validation.""" + return model(x).sum() + + +class TestGraphCPCollectives(unittest.TestCase): + """Mock only the functional transport; retain tensor layouts and autograd.""" + + def test_allgather_wait_restores_layout_and_keeps_autograd(self) -> None: + """The graph handle waits at the consumer and retains both replica contributions.""" + tensor = torch.randn(1, 3, 4, requires_grad=True) + mesh = SimpleNamespace(size=lambda: 2, get_group=lambda: "cp") + prefix = "hyper_parallel.distributed.context_parallel.collectives" + with ( + patch(f"{prefix}._is_tracing_or_compiling", return_value=True), + patch(f"{prefix}._functional_collectives.all_gather_single", + side_effect=lambda value, *unused: torch.cat((value, value))) as gather, + patch(f"{prefix}._functional_collectives.wait_tensor", side_effect=lambda value: value) as wait, + patch(f"{prefix}.dist") as eager, + ): + handle = collectives.async_cp_allgather_launch(tensor, 1, mesh) + wait.assert_not_called() + result = handle.wait() + torch.testing.assert_close(result, torch.cat((tensor, tensor), dim=1), rtol=0, atol=0) + result.sum().backward() + torch.testing.assert_close(tensor.grad, torch.full_like(tensor, 2), rtol=0, atol=0) + self.assertEqual(gather.call_args.args[1:], (0, "cp")) + wait.assert_called_once() + self.assertEqual(eager.mock_calls, []) + with self.assertRaisesRegex(RuntimeError, "waited twice"): + handle.wait() + + def test_cp_ep_topology_is_supported(self) -> None: + """CP may compose with EP after its collectives gain a graph-safe path.""" + context = SimpleNamespace(ep_size=2, cp_size=2, pp_size=1) + compiler = GraphCompiler(torch.nn.Linear(4, 4), _loss, PassConfig(enable_overlap=False), + mesh_context=context, device=torch.device("cpu")) + compiler._validate_ep_topology() + + def test_ulysses_wait_restores_sequence_and_head_layout(self) -> None: + """Functional A2A retains the same split/reconstruct contract and gradient edge.""" + tensor = torch.randn(1, 3, 4, requires_grad=True) + mesh = SimpleNamespace(size=lambda: 2, get_group=lambda: "cp") + prefix = "hyper_parallel.distributed.context_parallel.collectives" + with ( + patch(f"{prefix}._is_tracing_or_compiling", return_value=True), + patch(f"{prefix}._functional_collectives.all_to_all_single", + side_effect=lambda value, *unused: value) as exchange, + patch(f"{prefix}._functional_collectives.wait_tensor", side_effect=lambda value: value) as wait, + patch(f"{prefix}.dist") as eager, + ): + handle = collectives.async_ulysses_seq_to_head_launch(tensor, 1, -1, mesh) + wait.assert_not_called() + result = handle.wait() + torch.testing.assert_close(result, torch.cat(tensor.chunk(2, dim=-1), dim=1), rtol=0, atol=0) + result.sum().backward() + torch.testing.assert_close(tensor.grad, torch.ones_like(tensor), rtol=0, atol=0) + self.assertEqual(exchange.call_args.args[1:], (None, None, "cp")) + wait.assert_called_once() + self.assertEqual(eager.mock_calls, [])