From cc5e07742b4cebd421cedb340ac3fcaa3b000947 Mon Sep 17 00:00:00 2001 From: yu-chen-code Date: Fri, 6 Nov 2026 23:28:05 +0800 Subject: [PATCH] feat(compile): prepare document compression indices as graph inputs --- .../context_parallel/compressed_document.py | 93 +++++++++++++++++- .../test_graph_compressed_document.py | 97 +++++++++++++++++++ 2 files changed, 185 insertions(+), 5 deletions(-) create mode 100644 tests/ut/distributed/context_parallel/test_graph_compressed_document.py diff --git a/hyper_parallel/distributed/context_parallel/compressed_document.py b/hyper_parallel/distributed/context_parallel/compressed_document.py index c0ba1c537d..661c05921a 100644 --- a/hyper_parallel/distributed/context_parallel/compressed_document.py +++ b/hyper_parallel/distributed/context_parallel/compressed_document.py @@ -24,6 +24,7 @@ import torch # pylint: disable=forbidden-backend-import from torch.nn import functional # pylint: disable=forbidden-backend-import +from torch.utils._pytree import register_pytree_node @dataclass(frozen=True) @@ -93,6 +94,75 @@ def compressed_owner_offsets(boundaries: tuple[int, ...], ratio: int, local_leng return (*offsets, keys[-1]) +def _pool_blocks(projected: torch.Tensor, rows: torch.Tensor, count: int, ratio: int) -> torch.Tensor: + """Apply the original document-local FP32 softmax to prepared block rows.""" + blocks = projected.index_select(1, rows).reshape(projected.shape[0], count, ratio, projected.shape[-1]) + values, scores = blocks.chunk(2, dim=-1) + return (values * scores.float().softmax(dim=2).to(values.dtype)).sum(dim=2) + + +@dataclass(frozen=True) +class PreparedCompressionPlan: + """Read-only CP1 compression indices supplied as runtime graph inputs. + + The ratio is static model configuration. Row values and the number of + complete blocks may change between batches without Python document loops. + """ + + ratio: int + rows: torch.Tensor + block_positions: torch.Tensor + + def __post_init__(self) -> None: + """Validate the tensor schema without reading any index values.""" + if not isinstance(self.ratio, int) or self.ratio <= 1: + raise ValueError("Prepared compression requires an integer ratio greater than one") + if self.rows.ndim != 1 or self.block_positions.ndim != 1: + raise ValueError("Prepared compression indices must be one-dimensional") + if self.rows.dtype != torch.long or self.block_positions.dtype != torch.long: + raise ValueError("Prepared compression indices must use int64") + if self.rows.numel() != self.block_positions.numel() * self.ratio: + raise ValueError("Prepared compression rows do not match the complete block count") + + @staticmethod + def _validate_context(context: Any) -> None: + """Reject CP communication until its tensor routing metadata is supported.""" + if context is not None and getattr(context, "size", 1) != 1: + raise ValueError("Prepared graph compression currently requires CP=1") + + @classmethod + def pytree_unflatten(cls, children: Any, context: tuple) -> PreparedCompressionPlan: + """Restore tensor leaves without interpreting their document-boundary values.""" + return cls(context[0], *children) + + def pool(self, key: torch.Tensor, gate: torch.Tensor, context: Any = None) -> torch.Tensor: + """Pool complete same-document groups with the eager compressor's math. + + Args: + key: Projected values shaped [batch, raw tokens, channels]. + gate: Pooling logits with the same shape as key. + context: None or a single-rank CP context. + + Returns: + Pooled values for this batch's complete document-local blocks. + """ + self._validate_context(context) + return _pool_blocks(torch.cat((key, gate), dim=-1), self.rows, self.block_positions.numel(), self.ratio) + + def positions(self, cos: torch.Tensor, sin: torch.Tensor, context: Any = None) -> tuple: + """Select original block-start rotary coefficients without changing dtype.""" + self._validate_context(context) + return torch.cat((cos, sin), dim=-1).index_select(1, self.block_positions).chunk(2, dim=-1) + + def pytree_flatten(self) -> tuple[list[torch.Tensor], tuple]: + """Keep only the model's ratio static; every batch-dependent index is a tensor.""" + return [self.rows, self.block_positions], (self.ratio,) + + +register_pytree_node(PreparedCompressionPlan, PreparedCompressionPlan.pytree_flatten, + PreparedCompressionPlan.pytree_unflatten) + + @dataclass(frozen=True) class DocumentCompressionPlan: """Immutable block ranges; never enumerate the full sequence on the CPU.""" @@ -104,6 +174,23 @@ class DocumentCompressionPlan: local_blocks: tuple[tuple[int, int], ...] route: CompressionBoundaryRoute + def prepare(self, device: torch.device) -> PreparedCompressionPlan: + """Materialize graph input tensors before capture, using the existing CPU plan. + + Args: + device: Device on which the model will consume the block indices. + + Returns: + Tensor-only indices for complete document-local compression blocks. + + Raises: + ValueError: When the plan includes a multi-rank CP routing contract. + """ + if len(self.counts) != 1 or self.route.needs_exchange: + raise ValueError("Prepared graph compression currently requires CP=1") + rows, positions = self.indices(device) + return PreparedCompressionPlan(self.ratio, rows, positions) + def indices(self, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]: """Reuse device indices for local blocks and their original rotary positions. @@ -144,11 +231,7 @@ def pool(self, key: torch.Tensor, gate: torch.Tensor, context: Any) -> torch.Ten """ rows = self.indices(key.device)[0] projected = self.exchange(torch.cat((key, gate), dim=-1), context) - blocks = projected.index_select(1, rows).reshape( - key.shape[0], self.counts[self.rank], self.ratio, projected.shape[-1], - ) - values, scores = blocks.chunk(2, dim=-1) - return (values * scores.float().softmax(dim=2).to(values.dtype)).sum(dim=2) + return _pool_blocks(projected, rows, self.counts[self.rank], self.ratio) def positions(self, cos: torch.Tensor, sin: torch.Tensor, context: Any) -> tuple: """Take original block-start RoPE; retain FP32 boundary metadata precision. diff --git a/tests/ut/distributed/context_parallel/test_graph_compressed_document.py b/tests/ut/distributed/context_parallel/test_graph_compressed_document.py new file mode 100644 index 0000000000..f49c01b4db --- /dev/null +++ b/tests/ut/distributed/context_parallel/test_graph_compressed_document.py @@ -0,0 +1,97 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Prepared document compression metadata across changing packed boundaries.""" + +import copy +import unittest + +import torch +from torch import nn + +from hyper_parallel.compile import GraphCompiler, PassConfig +from hyper_parallel.distributed.context_parallel.compressed_document import document_compression_plan + +_BOUNDARIES = ((0, 12, 24), (0, 7, 16, 24), (0, 5, 10, 17, 24), (0, 3, 8, 15, 24)) + + +class _CompressionModel(nn.Module): + """Apply trainable key and pooling projections with document-local geometry.""" + + def __init__(self) -> None: + """Create the two projections used by the original pooling formula.""" + super().__init__() + self.key = nn.Linear(8, 8) + self.gate = nn.Linear(8, 8) + + def forward(self, hidden: torch.Tensor, plan) -> torch.Tensor: + """Pool complete blocks and retain their original rotary positions.""" + bank = plan.pool(self.key(hidden), self.gate(hidden), None) + cos, sin = plan.positions(hidden.cos(), hidden.sin(), None) + return bank * cos + sin + + +def _loss(model: nn.Module, *, hidden: torch.Tensor, plan) -> torch.Tensor: + """Exercise both the projection and metadata paths in backward.""" + return model(hidden, plan).square().mean() + + +class TestPreparedDocumentCompression(unittest.TestCase): + """Separate CPU document planning from tensor-only graph replay.""" + + def test_prepared_pool_positions_and_gradients_match_eager_exactly(self) -> None: + """Use the same rows, block starts, dtype and gradients as the eager plan.""" + for dtype in (torch.float32, torch.bfloat16): + for boundaries in _BOUNDARIES: + with self.subTest(dtype=dtype, boundaries=boundaries): + plan = document_compression_plan(boundaries, 4, 24, 1, 0) + prepared = plan.prepare(torch.device("cpu")) + key = torch.randn(1, 24, 8, dtype=dtype, requires_grad=True) + gate = torch.randn_like(key, requires_grad=True) + expected = plan.pool(key, gate, None) + actual = prepared.pool(key, gate) + self.assertTrue(torch.equal(actual, expected)) + expected_grads = torch.autograd.grad(expected.sum(), (key, gate)) + actual_grads = torch.autograd.grad(actual.sum(), (key, gate)) + for left, right in zip(actual_grads, expected_grads): + self.assertTrue(torch.equal(left, right)) + for left, right in zip(prepared.positions(key, gate), plan.positions(key, gate, None)): + self.assertTrue(torch.equal(left, right)) + + def test_dynamic_block_counts_and_boundary_values_reuse_joint_graph(self) -> None: + """Fixed S may change document count, incomplete tails and complete-block count.""" + torch.manual_seed(62) + reference = _CompressionModel() + candidate = copy.deepcopy(reference) + compiler = GraphCompiler(candidate, _loss, PassConfig(fsdp_enabled=False), device=torch.device("cpu"), + dynamic_arg_dims={"plan.rows": [0], "plan.block_positions": [0]}) + graph = None + for boundaries in _BOUNDARIES: + hidden = torch.randn(1, 24, 8) + plan = document_compression_plan(boundaries, 4, 24, 1, 0) + expected = _loss(reference, hidden=hidden, plan=plan) + expected.backward() + actual, _ = compiler.forward_backward(hidden=hidden, plan=plan.prepare(hidden.device)) + torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-6) + for left, right in zip(candidate.parameters(), reference.parameters()): + torch.testing.assert_close(left.grad, right.grad, rtol=1e-5, atol=1e-6) + if graph is None: + graph = compiler._joint_graph + self.assertIs(compiler._joint_graph, graph) + + def test_cp_routing_requires_a_separate_tensor_contract(self) -> None: + """Do not silently discard boundary exchanges or CP bank ownership.""" + plan = document_compression_plan((0, 7, 16, 24), 4, 12, 2, 0) + with self.assertRaisesRegex(ValueError, "requires CP=1"): + plan.prepare(torch.device("cpu"))