Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 88 additions & 5 deletions hyper_parallel/distributed/context_parallel/compressed_document.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

import torch # pylint: disable=forbidden-backend-import
from torch.nn import functional # pylint: disable=forbidden-backend-import
from torch.utils._pytree import register_pytree_node


@dataclass(frozen=True)
Expand Down Expand Up @@ -93,6 +94,75 @@ def compressed_owner_offsets(boundaries: tuple[int, ...], ratio: int, local_leng
return (*offsets, keys[-1])


def _pool_blocks(projected: torch.Tensor, rows: torch.Tensor, count: int, ratio: int) -> torch.Tensor:
"""Apply the original document-local FP32 softmax to prepared block rows."""
blocks = projected.index_select(1, rows).reshape(projected.shape[0], count, ratio, projected.shape[-1])
values, scores = blocks.chunk(2, dim=-1)
return (values * scores.float().softmax(dim=2).to(values.dtype)).sum(dim=2)


@dataclass(frozen=True)
class PreparedCompressionPlan:
"""Read-only CP1 compression indices supplied as runtime graph inputs.

The ratio is static model configuration. Row values and the number of
complete blocks may change between batches without Python document loops.
"""

ratio: int
rows: torch.Tensor
block_positions: torch.Tensor

def __post_init__(self) -> None:
"""Validate the tensor schema without reading any index values."""
if not isinstance(self.ratio, int) or self.ratio <= 1:
raise ValueError("Prepared compression requires an integer ratio greater than one")
if self.rows.ndim != 1 or self.block_positions.ndim != 1:
raise ValueError("Prepared compression indices must be one-dimensional")
if self.rows.dtype != torch.long or self.block_positions.dtype != torch.long:
raise ValueError("Prepared compression indices must use int64")
if self.rows.numel() != self.block_positions.numel() * self.ratio:
raise ValueError("Prepared compression rows do not match the complete block count")

@staticmethod
def _validate_context(context: Any) -> None:
"""Reject CP communication until its tensor routing metadata is supported."""
if context is not None and getattr(context, "size", 1) != 1:
raise ValueError("Prepared graph compression currently requires CP=1")

@classmethod
def pytree_unflatten(cls, children: Any, context: tuple) -> PreparedCompressionPlan:
"""Restore tensor leaves without interpreting their document-boundary values."""
return cls(context[0], *children)

def pool(self, key: torch.Tensor, gate: torch.Tensor, context: Any = None) -> torch.Tensor:
"""Pool complete same-document groups with the eager compressor's math.

Args:
key: Projected values shaped [batch, raw tokens, channels].
gate: Pooling logits with the same shape as key.
context: None or a single-rank CP context.

Returns:
Pooled values for this batch's complete document-local blocks.
"""
self._validate_context(context)
return _pool_blocks(torch.cat((key, gate), dim=-1), self.rows, self.block_positions.numel(), self.ratio)

def positions(self, cos: torch.Tensor, sin: torch.Tensor, context: Any = None) -> tuple:
"""Select original block-start rotary coefficients without changing dtype."""
self._validate_context(context)
return torch.cat((cos, sin), dim=-1).index_select(1, self.block_positions).chunk(2, dim=-1)

def pytree_flatten(self) -> tuple[list[torch.Tensor], tuple]:
"""Keep only the model's ratio static; every batch-dependent index is a tensor."""
return [self.rows, self.block_positions], (self.ratio,)


register_pytree_node(PreparedCompressionPlan, PreparedCompressionPlan.pytree_flatten,
PreparedCompressionPlan.pytree_unflatten)


@dataclass(frozen=True)
class DocumentCompressionPlan:
"""Immutable block ranges; never enumerate the full sequence on the CPU."""
Expand All @@ -104,6 +174,23 @@ class DocumentCompressionPlan:
local_blocks: tuple[tuple[int, int], ...]
route: CompressionBoundaryRoute

def prepare(self, device: torch.device) -> PreparedCompressionPlan:
"""Materialize graph input tensors before capture, using the existing CPU plan.

Args:
device: Device on which the model will consume the block indices.

Returns:
Tensor-only indices for complete document-local compression blocks.

Raises:
ValueError: When the plan includes a multi-rank CP routing contract.
"""
if len(self.counts) != 1 or self.route.needs_exchange:
raise ValueError("Prepared graph compression currently requires CP=1")
rows, positions = self.indices(device)
return PreparedCompressionPlan(self.ratio, rows, positions)

def indices(self, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:
"""Reuse device indices for local blocks and their original rotary positions.

Expand Down Expand Up @@ -144,11 +231,7 @@ def pool(self, key: torch.Tensor, gate: torch.Tensor, context: Any) -> torch.Ten
"""
rows = self.indices(key.device)[0]
projected = self.exchange(torch.cat((key, gate), dim=-1), context)
blocks = projected.index_select(1, rows).reshape(
key.shape[0], self.counts[self.rank], self.ratio, projected.shape[-1],
)
values, scores = blocks.chunk(2, dim=-1)
return (values * scores.float().softmax(dim=2).to(values.dtype)).sum(dim=2)
return _pool_blocks(projected, rows, self.counts[self.rank], self.ratio)

def positions(self, cos: torch.Tensor, sin: torch.Tensor, context: Any) -> tuple:
"""Take original block-start RoPE; retain FP32 boundary metadata precision.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
# Copyright 2026 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Prepared document compression metadata across changing packed boundaries."""

import copy
import unittest

import torch
from torch import nn

from hyper_parallel.compile import GraphCompiler, PassConfig
from hyper_parallel.distributed.context_parallel.compressed_document import document_compression_plan

_BOUNDARIES = ((0, 12, 24), (0, 7, 16, 24), (0, 5, 10, 17, 24), (0, 3, 8, 15, 24))


class _CompressionModel(nn.Module):
"""Apply trainable key and pooling projections with document-local geometry."""

def __init__(self) -> None:
"""Create the two projections used by the original pooling formula."""
super().__init__()
self.key = nn.Linear(8, 8)
self.gate = nn.Linear(8, 8)

def forward(self, hidden: torch.Tensor, plan) -> torch.Tensor:
"""Pool complete blocks and retain their original rotary positions."""
bank = plan.pool(self.key(hidden), self.gate(hidden), None)
cos, sin = plan.positions(hidden.cos(), hidden.sin(), None)
return bank * cos + sin


def _loss(model: nn.Module, *, hidden: torch.Tensor, plan) -> torch.Tensor:
"""Exercise both the projection and metadata paths in backward."""
return model(hidden, plan).square().mean()


class TestPreparedDocumentCompression(unittest.TestCase):
"""Separate CPU document planning from tensor-only graph replay."""

def test_prepared_pool_positions_and_gradients_match_eager_exactly(self) -> None:
"""Use the same rows, block starts, dtype and gradients as the eager plan."""
for dtype in (torch.float32, torch.bfloat16):
for boundaries in _BOUNDARIES:
with self.subTest(dtype=dtype, boundaries=boundaries):
plan = document_compression_plan(boundaries, 4, 24, 1, 0)
prepared = plan.prepare(torch.device("cpu"))
key = torch.randn(1, 24, 8, dtype=dtype, requires_grad=True)
gate = torch.randn_like(key, requires_grad=True)
expected = plan.pool(key, gate, None)
actual = prepared.pool(key, gate)
self.assertTrue(torch.equal(actual, expected))
expected_grads = torch.autograd.grad(expected.sum(), (key, gate))
actual_grads = torch.autograd.grad(actual.sum(), (key, gate))
for left, right in zip(actual_grads, expected_grads):
self.assertTrue(torch.equal(left, right))
for left, right in zip(prepared.positions(key, gate), plan.positions(key, gate, None)):
self.assertTrue(torch.equal(left, right))

def test_dynamic_block_counts_and_boundary_values_reuse_joint_graph(self) -> None:
"""Fixed S may change document count, incomplete tails and complete-block count."""
torch.manual_seed(62)
reference = _CompressionModel()
candidate = copy.deepcopy(reference)
compiler = GraphCompiler(candidate, _loss, PassConfig(fsdp_enabled=False), device=torch.device("cpu"),
dynamic_arg_dims={"plan.rows": [0], "plan.block_positions": [0]})
graph = None
for boundaries in _BOUNDARIES:
hidden = torch.randn(1, 24, 8)
plan = document_compression_plan(boundaries, 4, 24, 1, 0)
expected = _loss(reference, hidden=hidden, plan=plan)
expected.backward()
actual, _ = compiler.forward_backward(hidden=hidden, plan=plan.prepare(hidden.device))
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-6)
for left, right in zip(candidate.parameters(), reference.parameters()):
torch.testing.assert_close(left.grad, right.grad, rtol=1e-5, atol=1e-6)
if graph is None:
graph = compiler._joint_graph
self.assertIs(compiler._joint_graph, graph)

def test_cp_routing_requires_a_separate_tensor_contract(self) -> None:
"""Do not silently discard boundary exchanges or CP bank ownership."""
plan = document_compression_plan((0, 7, 16, 24), 4, 12, 2, 0)
with self.assertRaisesRegex(ValueError, "requires CP=1"):
plan.prepare(torch.device("cpu"))
Loading