diff --git a/hyper_parallel/components/functional/sparse_flash_mla.py b/hyper_parallel/components/functional/sparse_flash_mla.py index 9a6656da85..852aeb43cf 100644 --- a/hyper_parallel/components/functional/sparse_flash_mla.py +++ b/hyper_parallel/components/functional/sparse_flash_mla.py @@ -57,9 +57,11 @@ def query(self, tensor: torch.Tensor) -> torch.Tensor: """Pack a [1,Q,...] tensor in the plan's document order.""" return pack_ranges(tensor.squeeze(0), self.q_ranges) - def key(self, tensor: torch.Tensor) -> torch.Tensor: - """Pack complete visible document prefixes from a [1,K,...] bank.""" - return pack_ranges(tensor.squeeze(0), self.k_ranges) + def key(self, tensor: torch.Tensor, key_offset: int = 0) -> torch.Tensor: + """Pack global document prefixes using the physical origin of a [1,K,...] bank.""" + ranges = self.k_ranges if not key_offset else tuple((start-key_offset, end-key_offset) + for start, end in self.k_ranges) + return pack_ranges(tensor.squeeze(0), ranges) def lengths(self, device: torch.device) -> tuple: """Reuse device length tensors, including across inference and recomputation.""" @@ -227,17 +229,18 @@ def _bsnd_sparse_flash_mla( query: torch.Tensor, raw_key: torch.Tensor, main_key: torch.Tensor | None, indices: torch.Tensor | None, sinks: torch.Tensor, scale: float, ratio: int, window: int, query_offset: int, - segment: tuple[int, int, int, int, int], raw_document_starts: tuple = (), + segment: tuple[int, int, int, int, int], raw_key_offset: int = 0, main_key_offset: int = 0, + raw_document_starts: tuple = (), ) -> torch.Tensor: - """Run one document segment per batch item using the original all-gather banks. + """Run one document segment per batch item with independently routed KV banks. The operator owns Q tiling within this BSND call. Multiple packed documents use the TND entry with cumulative document lengths. Args: query: Local attention Q in [B, H, Q, D]. - raw_key: Global raw KV in [B, 1, K, D]. - main_key: Global compressed KV in [B, Kc, D], or None for SWA only. + raw_key: Full or halo raw KV in [B, 1, K, D]. + main_key: Full or prefix compressed KV in [B, Kc, D], or None for SWA only. indices: Global compressed selection [B, Q, topk], invalid slots -1. sinks: One scalar logit per head. scale: Attention dot-product scale. @@ -256,11 +259,11 @@ def _bsnd_sparse_flash_mla( document_start = dict(raw_document_starts).get(start, key_start * max(ratio, 1)) raw_left = max(document_start, query_offset + start - window + 1) raw_right = query_offset + end - raw = raw_key[:, :, raw_left:raw_right].transpose(1, 2).contiguous() + raw = raw_key[:, :, raw_left-raw_key_offset:raw_right-raw_key_offset].transpose(1, 2).contiguous() main = None selected = None if ratio and key_end > key_start: - main = main_key[:, key_start:key_end].unsqueeze(2).contiguous() + main = main_key[:, key_start-main_key_offset:key_end-main_key_offset].unsqueeze(2).contiguous() original = indices[:, start:end] selected = torch.where(original >= 0, original-key_start, -1).int() # Backward infers the active count from this width and causality; only KL requires 512 slots. @@ -335,14 +338,15 @@ def backward(ctx: Any, grad_output: torch.Tensor) -> tuple: def sparse_flash_mla( query: torch.Tensor, raw_key: torch.Tensor, main_key: torch.Tensor | None, indices: torch.Tensor | None, sinks: torch.Tensor, scale: float, ratio: int, - window: int, query_offset: int, segments: tuple, raw_document_starts: tuple = (), + window: int, query_offset: int, segments: tuple, raw_key_offset: int = 0, main_key_offset: int = 0, + raw_document_starts: tuple = (), ) -> torch.Tensor: - """Batch packed documents in native TND while preserving the all-gather banks. + """Batch packed documents in native TND with independently routed KV banks. Args: query: Local attention Q [batch, heads, queries, 512]. - raw_key: Global raw KV [batch, 1, keys, 512]. - main_key: Global compressed KV [batch, compressed keys, 512], or None. + raw_key: Full or halo raw KV [batch, 1, keys, 512]. + main_key: Full or document-prefix compressed KV [batch, compressed keys, 512], or None. indices: Sorted global compressed key IDs, invalid slots -1. sinks: Per-head scalar sink logits. scale: Attention dot-product scale. @@ -350,6 +354,8 @@ def sparse_flash_mla( window: Raw sliding-window size, including the current token. query_offset: Global position of the first local query. segments: Local Q start/end, compressed document start/prefix end and residual. + raw_key_offset: Global position of the first row in the compact raw KV bank. + main_key_offset: Global compressed ID represented by row zero of main_key. raw_document_starts: Explicit (local Q start, raw document start) pairs for natural lengths. Returns: @@ -360,7 +366,7 @@ def sparse_flash_mla( if len(segments) == 1: return _bsnd_sparse_flash_mla( query, raw_key, main_key, indices, sinks, scale, ratio, window, query_offset, segments[0], - raw_document_starts, + raw_key_offset, main_key_offset, raw_document_starts, ) if query.shape[0] != 1: raise ValueError("Packed Sparse Flash MLA requires batch size one") @@ -373,11 +379,12 @@ def sparse_flash_mla( if plan is None: continue raw_ranges = tuple( - (max(document_starts.get(s, ks * max(ratio, 1)), query_offset+s-window+1), query_offset+e) + (max(document_starts.get(s, ks * max(ratio, 1)), query_offset+s-window+1)-raw_key_offset, + query_offset+e-raw_key_offset) for s, e, ks, _, _ in plan.segments ) raw = pack_ranges(raw_key.squeeze(0).squeeze(0), raw_ranges).unsqueeze(1) - main = plan.key(main_key).unsqueeze(1) if has_keys else None + main = plan.key(main_key, main_key_offset).unsqueeze(1) if has_keys else None selected = plan.local_indices(indices) if has_keys else None part = _SparseFlashMlaTND.apply( plan.query(query.transpose(1, 2)), raw, main, selected, sinks.float(), diff --git a/hyper_parallel/components/functional/sparse_lightning_indexer_kl_loss_grad.py b/hyper_parallel/components/functional/sparse_lightning_indexer_kl_loss_grad.py index 895c53241d..228cb950d8 100644 --- a/hyper_parallel/components/functional/sparse_lightning_indexer_kl_loss_grad.py +++ b/hyper_parallel/components/functional/sparse_lightning_indexer_kl_loss_grad.py @@ -60,6 +60,8 @@ def validate_fused_kl_inputs( key: Indexer K [batch, keys, dimension]. weights: Per-query Indexer head weights. indices: Ordinary causal selections, before optional width padding. + Compact CP banks can retain the global selection width; slots beyond + the physical bank are invalid padding, up to the native width of 512. Raises: ValueError: Tensor relationships or logical selection width are invalid. @@ -68,8 +70,8 @@ def validate_fused_kl_inputs( validate_indexer_tensors(query, key, weights) if indices.ndim != 3 or indices.shape[:2] != query.shape[:2]: raise ValueError("Fused KL indices must have shape [B, Q, TopK] matching Indexer Q") - if key.shape[1] == 0 or indices.shape[-1] != min(512, key.shape[1]): - raise ValueError(f"Fused KL requires ordinary TopK512 selections from a nonempty bank; " + if key.shape[1] == 0 or not min(512, key.shape[1]) <= indices.shape[-1] <= 512: + raise ValueError(f"Fused KL requires ordinary TopK512 selections, allowing padding for a compact bank; " f"got K={key.shape[1]}, TopK={indices.shape[-1]}") if indices.device != query.device: raise ValueError(f"Fused KL indices must share Indexer Q's device, got {indices.device} and {query.device}") diff --git a/hyper_parallel/components/modules/shared_compressed_dsa_attention.py b/hyper_parallel/components/modules/shared_compressed_dsa_attention.py index 5f7b761ec7..fa99bc8285 100644 --- a/hyper_parallel/components/modules/shared_compressed_dsa_attention.py +++ b/hyper_parallel/components/modules/shared_compressed_dsa_attention.py @@ -28,6 +28,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass, field +from functools import partial from typing import Any, Protocol import torch # pylint: disable=forbidden-backend-import @@ -43,16 +44,19 @@ from hyper_parallel.components.functional.sparse_lightning_indexer_kl_loss_grad import ( gather_selected_keys, sparse_lightning_indexer_kl_loss, validate_fused_kl_inputs, ) -from hyper_parallel.core.activation_memory.wrapper import checkpoint_exclude_wrapper from hyper_parallel.core.activation_memory.recompute_state import is_recomputing +from hyper_parallel.core.activation_memory.wrapper import checkpoint_exclude_wrapper from hyper_parallel.distributed.context_parallel.compressed_document import ( CompressionBoundaryRoute, compressed_document_offsets, document_compression_plan, ) +from hyper_parallel.distributed.context_parallel.sequence_dispatch import ( + SequenceDispatchRoute, document_prefix_route, sliding_window_route, +) from hyper_parallel.models.replacement import module_replacement class SequenceGatherHandle(Protocol): - """Handle returned by an asynchronous sequence all-gather.""" + """Handle returned by asynchronous differentiable bank communication.""" def wait(self) -> torch.Tensor: """Wait for communication and return the differentiable result.""" @@ -71,6 +75,25 @@ def wait(self) -> torch.Tensor: return self.gather_sequence(self.tensor, self.sequence_dim) +@dataclass +class _DeferredBankLaunch: + """Keep main K behind Index K in the shared communication queue.""" + + launcher: Callable[[], SequenceGatherHandle] | None + handle: SequenceGatherHandle | None = None + + def start(self) -> SequenceGatherHandle: + """Launch once in the differentiable forward, then release captured inputs.""" + if self.handle is None: + self.handle = self.launcher() + self.launcher = None + return self.handle + + def wait(self) -> torch.Tensor: + """Resolve the same differentiable handle at its first consumer.""" + return self.start().wait() + + @dataclass class SharedCompressedAttentionState: """Autograd-bearing CSA2 state scoped to one training forward. @@ -93,6 +116,7 @@ class SharedCompressedAttentionState: index_key_by_source: dict[int, torch.Tensor] = field(default_factory=dict) topk_indices_by_source: dict[int, torch.Tensor] = field(default_factory=dict) candidate_blocks_by_source: dict[int, torch.Tensor] = field(default_factory=dict) + compressed_kv_offsets: dict[int, int] = field(default_factory=dict) @staticmethod def _require( @@ -109,9 +133,10 @@ def _require( ) return values[source_layer] - def publish_compressed_kv(self, source_layer: int, value: torch.Tensor) -> None: - """Publish compressed K=V while preserving its autograd graph.""" + def publish_compressed_kv(self, source_layer: int, value: torch.Tensor, key_offset: int = 0) -> None: + """Publish compressed K=V and its global origin, preserving the autograd graph.""" self.compressed_kv_by_source[source_layer] = value + self.compressed_kv_offsets[source_layer] = key_offset def require_compressed_kv(self, source_layer: int | None, consumer_layer: int) -> torch.Tensor: """Read compressed K=V from the consumer's configured Full layer.""" @@ -179,19 +204,22 @@ class SharedCompressedIndexerOutput: @dataclass(frozen=True) class SharedCompressedAttentionCPContext: - """KV-all-gather context for shared compressed DSA. + """Communication for local queries and shared compressed banks. - Query-side tensors remain local sequence shards. Raw KV, compressed KV, - and index keys are gathered in CP-rank order. ``launch_sequence`` is - optional so numerical tests and non-overlapped backends can supply only a - differentiable synchronous gather. + ``launch_raw`` exchanges the same-document SWA halo when supplied. + Optional ``launch_main`` exchanges main KV document prefixes. + Index K retains differentiable all-gather. + Contexts without ``launch_raw`` provide the full raw-bank reference. """ size: int rank: int gather_sequence: Callable[[torch.Tensor, int], torch.Tensor] launch_sequence: Callable[[torch.Tensor, int], SequenceGatherHandle] | None = None + launch_raw: Callable[[torch.Tensor, int, SequenceDispatchRoute], SequenceGatherHandle] | None = None + launch_main: Callable[[torch.Tensor, int, SequenceDispatchRoute], SequenceGatherHandle] | None = None exchange_boundary: Callable[[torch.Tensor, CompressionBoundaryRoute], torch.Tensor] | None = None + indexer_priority: bool = False def launch(self, tensor: torch.Tensor, sequence_dim: int) -> SequenceGatherHandle: """Launch an async gather or create a deferred synchronous gather.""" @@ -1106,6 +1134,7 @@ def forward( query_segments: tuple[tuple[int, int, int, int, int], ...] | None = None, block_plan: Any = None, key_position_embeddings: tuple | None = None, + prepare_attention: Callable[[], None] | None = None, ) -> SharedCompressedIndexerOutput: """Project local queries and score them against global compressed keys. @@ -1122,6 +1151,9 @@ def forward( latent, cos, sin, index_key, cp_context, block_plan, key_position_embeddings, ) + if prepare_attention is not None: + prepare_attention() + query = self.q_b_proj(query_residual.detach()) if query.shape[-1] % self.head_dim: raise ValueError( @@ -1334,12 +1366,15 @@ def _project_raw_kv( cos: torch.Tensor, sin: torch.Tensor, cp_context: SharedCompressedAttentionCPContext | None, + raw_route: SequenceDispatchRoute | None = None, ) -> tuple[torch.Tensor | None, SequenceGatherHandle | None]: """Project raw K=V and launch CP communication before query GEMMs.""" key_value = self.kv_norm(self.kv_proj(hidden_states)).unsqueeze(1) key_value = _apply_v41_rope(key_value, cos, sin) if cp_context is None: return key_value, None + if raw_route is not None: + return None, cp_context.launch_raw(key_value, 2, raw_route) return None, cp_context.launch(key_value, 2) def _project_query( @@ -1349,11 +1384,13 @@ def _project_query( sin: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: """Project and normalize the local query heads.""" - batch_size, sequence_length, _ = hidden_states.shape query_residual = self.q_a_norm(self.q_a_proj(hidden_states)) - query = self.q_b_proj(query_residual).view(batch_size, sequence_length, -1, self.head_dim) - query = query.transpose(1, 2) - return query_residual, _apply_v41_rope(query, cos, sin) + return query_residual, self._expand_query(query_residual, cos, sin) + + def _expand_query(self, query_residual: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: + """Expand main heads after low-rank query inputs have been dispatched.""" + query = self.q_b_proj(query_residual).view(*query_residual.shape[:2], -1, self.head_dim) + return _apply_v41_rope(query.transpose(1, 2), cos, sin) @staticmethod def _validate_forward_inputs( @@ -1445,21 +1482,39 @@ def _resolve_packed_segments( query_segments = packed_sequence.indexer_segments(ratio) return segment_starts, minimum_key_indices, query_segments + def _plan_bank_dispatch( + self, packed_sequence: SharedCompressedPackedSequence | None, + cp_context: SharedCompressedAttentionCPContext | None, + sequence_length: int, global_sequence_length: int, device: torch.device, + ) -> tuple[SequenceDispatchRoute | None, SequenceDispatchRoute | None, tuple[int, ...]]: + """Reuse document boundaries to route each bank in its own physical coordinates.""" + boundaries = (0, global_sequence_length) + if packed_sequence is not None: + boundaries = packed_sequence.prepare(device, (self.compress_ratio,)).boundaries() + raw_route, main_route = None, None + if cp_context is not None and cp_context.launch_raw is not None: + raw_route = sliding_window_route( + boundaries, sequence_length, cp_context.size, cp_context.rank, self.sliding_window, + ) + if self.compress_ratio and cp_context is not None and cp_context.launch_main is not None: + main_route = document_prefix_route( + boundaries, sequence_length, cp_context.size, cp_context.rank, self.compress_ratio, + ) + return raw_route, main_route, tuple(end - start for start, end in zip(boundaries, boundaries[1:])) + def _prepare_compression( self, packed: SharedCompressedPackedSequence | None, context: SharedCompressedAttentionCPContext | None, - embeddings: dict, global_length: int, device: torch.device, + embeddings: dict, lengths: tuple, device: torch.device, ) -> tuple: """Prepare block-start positions only for a source that needs document-local grouping.""" if not self.is_kv_source or self.compress_ratio <= 1: return None, None size, rank = (1, 0) if context is None else (context.size, context.rank) if packed is None: - local_length = global_length // size - if local_length % self.compress_ratio == 0: + if (sum(lengths)//size) % self.compress_ratio == 0: return None, None - packed = SharedCompressedPackedSequence( - torch.tensor((0, global_length)), rank * local_length, local_length, global_length, - ) + packed = SharedCompressedPackedSequence(torch.tensor((0, sum(lengths))), + rank*(sum(lengths)//size), sum(lengths)//size, sum(lengths)) packed = packed.prepare(device, (self.compress_ratio,)) plan = packed.block_plan(self.compress_ratio, size, rank) positions = None if plan is None else packed.compressed_positions(plan, embeddings["compress"], context) @@ -1471,8 +1526,10 @@ def _compute_compressed_kv( position_embeddings: dict[str, tuple[torch.Tensor, torch.Tensor]], shared_state: "SharedCompressedAttentionState", cp_context: "SharedCompressedAttentionCPContext | None", + main_route: SequenceDispatchRoute | None = None, block_plan: Any = None, key_position_embeddings: tuple | None = None, + defer_communication: bool = False, ) -> tuple[torch.Tensor | None, "SequenceGatherHandle | None"]: """Compress local hidden states and publish or launch the compressed KV. @@ -1490,12 +1547,21 @@ def _compute_compressed_kv( if cp_context is None: if not is_recomputing(): shared_state.publish_compressed_kv(self.layer_idx, compressed) - elif block_plan is not None: - compressed_handle = block_plan.launch_bank(compressed, cp_context) else: - compressed_handle = cp_context.launch(compressed, 1) + launch = partial(self._launch_compressed_bank, compressed, cp_context, main_route, block_plan) + compressed_handle = _DeferredBankLaunch(launch) if defer_communication else launch() return latent, compressed_handle + @staticmethod + def _launch_compressed_bank(compressed: torch.Tensor, context: SharedCompressedAttentionCPContext, + route: SequenceDispatchRoute | None, block_plan: Any) -> SequenceGatherHandle: + """Use the configured physical bank layout at the scheduled launch point.""" + if route is not None: + return context.launch_main(compressed, 1, route) + if block_plan is not None: + return block_plan.launch_bank(compressed, context) + return context.launch(compressed, 1) + def _run_indexer( self, hidden_states: torch.Tensor, @@ -1510,6 +1576,7 @@ def _run_indexer( query_segments: tuple, block_plan: Any = None, key_position_embeddings: tuple | None = None, + prepare_attention: Callable[[], None] | None = None, ) -> Any | None: """Run the indexer on index-source layers and publish its forward outputs.""" if not self.is_index_source: @@ -1540,6 +1607,7 @@ def _run_indexer( query_segments=query_segments, block_plan=block_plan, key_position_embeddings=key_position_embeddings, + prepare_attention=prepare_attention, ) if is_recomputing(): # A recompute must not publish again: its consumers read the entries @@ -1556,6 +1624,25 @@ def _run_indexer( ) return indexer_output + def _finish_bank_communication( + self, key_value: torch.Tensor | None, raw_kv_handle: SequenceGatherHandle | None, + compressed_handle: SequenceGatherHandle | None, shared_state: SharedCompressedAttentionState, + main_route: SequenceDispatchRoute | None, + ) -> torch.Tensor: + """Resolve differentiable banks only when attention and KL can consume them.""" + if raw_kv_handle is not None: + key_value = raw_kv_handle.wait() + if compressed_handle is not None: + compressed = compressed_handle.wait() + if not is_recomputing(): + shared_state.publish_compressed_kv( + self.layer_idx, compressed, 0 if main_route is None else main_route.global_start, + ) + if key_value is None: + raise RuntimeError("raw KV communication did not produce a tensor") + + return key_value + def _build_sparse_indices( self, key_value: torch.Tensor, @@ -1565,6 +1652,8 @@ def _build_sparse_indices( query_offset: int, global_sequence_length: int, segment_starts: torch.Tensor | None, + raw_key_offset: int = 0, + main_key_offset: int = 0, ) -> tuple[torch.Tensor, torch.Tensor]: """Assemble dense-reference banks only after reference dispatch is selected.""" window = build_sliding_window_indices( @@ -1573,10 +1662,13 @@ def _build_sparse_indices( ) if segment_starts is not None: window.masked_fill_(window < segment_starts.unsqueeze(-1), -1) + window = torch.where(window >= 0, window - raw_key_offset, -1) if compressed_kv is None: return key_value, window combined_key_value = torch.cat((key_value, compressed_kv.unsqueeze(1)), dim=2) - compressed_indices = torch.where(topk_indices >= 0, topk_indices.long() + global_sequence_length, -1) + compressed_indices = torch.where( + topk_indices >= 0, topk_indices.long() - main_key_offset + key_value.shape[2], -1, + ) return combined_key_value, torch.cat((window, compressed_indices), dim=-1) def _apply_indexer_loss( @@ -1586,6 +1678,7 @@ def _apply_indexer_loss( compressed_kv: torch.Tensor | None, tp_context: "SharedCompressedAttentionTPContext | None", query_segments: tuple, + main_key_offset: int = 0, ) -> torch.Tensor: """Auto-scale the query by the indexer KL loss when configured.""" if ( @@ -1594,13 +1687,23 @@ def _apply_indexer_loss( and self.indexer.loss_coeff and compressed_kv is not None ): + # KL uses the original owner's queries. A view aligns its student K + # with the compact teacher bank while preserving global Index K gradients. + index_key = indexer_output.index_key + if main_key_offset or compressed_kv.shape[1] != index_key.shape[1]: + index_key = index_key[:, main_key_offset:main_key_offset+compressed_kv.shape[1]] + selected = indexer_output.topk_indices + if main_key_offset: + selected = torch.where(selected >= 0, selected - main_key_offset, -1) + query_segments = tuple((s, e, ks-main_key_offset, ke-main_key_offset, residual) + for s, e, ks, ke, residual in query_segments) indexer_loss = shared_compressed_indexer_kl_loss( indexer_output.index_query, - indexer_output.index_key, + index_key, indexer_output.merge_weight, query, compressed_kv, - indexer_output.topk_indices, + selected, self.sinks, attention_scale=self.scaling, loss_coeff=self.indexer.loss_coeff, @@ -1616,14 +1719,18 @@ def _apply_indexer_loss( def _run_sparse_attention( self, query: torch.Tensor, key_value: torch.Tensor, compressed_kv: torch.Tensor | None, topk_indices: torch.Tensor | None, query_offset: int, global_sequence_length: int, - segment_starts: torch.Tensor | None, query_segments: tuple, raw_document_starts: tuple = (), + segment_starts: torch.Tensor | None, query_segments: tuple, raw_key_offset: int = 0, + main_key_offset: int = 0, raw_document_starts: tuple = (), global_key_length: int | None = None, ) -> torch.Tensor: """Run the chosen implementation; unsupported explicit fusion always raises.""" if self.use_fused_ops: + expected_topk = min(512, global_key_length if global_key_length is not None + else global_sequence_length // max(1, self.compress_ratio)) if not (self.candidate_source_layer_idx is None - and (topk_indices is None or topk_indices.shape[-1] == min(512, compressed_kv.shape[1])) - and supports_fused_attention_inputs(query, self.compress_ratio, self.sliding_window, - 0 if topk_indices is None else topk_indices.shape[-1])): + and (topk_indices is None or topk_indices.shape[-1] == expected_topk) + and supports_fused_attention_inputs( + query, self.compress_ratio if expected_topk else 0, self.sliding_window, + 0 if topk_indices is None else topk_indices.shape[-1])): raise RuntimeError( "V4.1 fused attention is required but this call is unsupported: " f"layer={self.layer_idx}, device={query.device}, dtype={query.dtype}, Q={tuple(query.shape)}, " @@ -1632,7 +1739,8 @@ def _run_sparse_attention( ) output = sparse_flash_mla( query, key_value, compressed_kv, topk_indices, self.sinks, self.scaling, - self.compress_ratio, self.sliding_window, query_offset, query_segments, raw_document_starts, + self.compress_ratio, self.sliding_window, query_offset, query_segments, raw_key_offset, main_key_offset, + raw_document_starts, ) if compressed_kv is not None and all(end == start for _, _, start, end, _ in query_segments): # Empty-prefix ranks must still participate in the shared bank's collective backward. @@ -1640,7 +1748,7 @@ def _run_sparse_attention( return output combined_key_value, sparse_indices = self._build_sparse_indices( key_value, compressed_kv, topk_indices, query.shape[2], query_offset, - global_sequence_length, segment_starts, + global_sequence_length, segment_starts, raw_key_offset, main_key_offset, ) return reference_sparse_attention(query, combined_key_value, sparse_indices, self.sinks, self.scaling) @@ -1691,6 +1799,8 @@ def forward( position_ids, sequence_length, ) + if packed_sequence is not None: + packed_sequence = packed_sequence.prepare(hidden_states.device, (self.compress_ratio,)) segment_starts, minimum_key_indices, query_segments = self._resolve_packed_segments( packed_sequence, batch_size, @@ -1700,18 +1810,37 @@ def forward( hidden_states.device, ) - if packed_sequence is not None: - packed_sequence = packed_sequence.prepare(hidden_states.device, (self.compress_ratio,)) + raw_route, main_route, document_lengths = self._plan_bank_dispatch( + packed_sequence, cp_context, sequence_length, global_sequence_length, hidden_states.device, + ) block_plan, key_positions = self._prepare_compression( - packed_sequence, cp_context, position_embeddings, global_sequence_length, hidden_states.device, + packed_sequence, cp_context, position_embeddings, document_lengths, hidden_states.device, ) raw_document_starts = () if packed_sequence is None else packed_sequence.raw_document_starts() + raw_key_offset = 0 if raw_route is None else raw_route.global_start rope_type = "compress" if self.compress_ratio else "main" cos, sin = position_embeddings[rope_type] - key_value, raw_kv_handle = self._project_raw_kv(hidden_states, cos, sin, cp_context) - query_residual, query = self._project_query(hidden_states, cos, sin) + key_value, raw_kv_handle = self._project_raw_kv(hidden_states, cos, sin, cp_context, raw_route) + overlap = (self.is_index_source and cp_context is not None + and cp_context.indexer_priority) + prepare_attention = None + if overlap: + query_residual = self.q_a_norm(self.q_a_proj(hidden_states)) + query = None + + def _prepare_attention() -> None: + """Keep differentiable main Q work outside selection checkpoint regions.""" + nonlocal query + if isinstance(compressed_handle, _DeferredBankLaunch): + compressed_handle.start() + query = self._expand_query(query_residual, cos, sin) + + prepare_attention = _prepare_attention + else: + query_residual, query = self._project_query(hidden_states, cos, sin) latent, compressed_handle = self._compute_compressed_kv( - hidden_states, position_embeddings, shared_state, cp_context, block_plan, key_positions, + hidden_states, position_embeddings, shared_state, cp_context, main_route, block_plan, key_positions, + defer_communication=overlap and cp_context.indexer_priority, ) indexer_output = self._run_indexer( hidden_states, @@ -1726,26 +1855,27 @@ def forward( query_segments, block_plan, key_positions, + prepare_attention=prepare_attention, ) - if raw_kv_handle is not None: - key_value = raw_kv_handle.wait() - if compressed_handle is not None: - compressed = compressed_handle.wait() - if not is_recomputing(): - shared_state.publish_compressed_kv(self.layer_idx, compressed) - if key_value is None: - raise RuntimeError("raw KV all-gather did not produce a tensor") + key_value = self._finish_bank_communication( + key_value, raw_kv_handle, compressed_handle, shared_state, main_route, + ) compressed_kv = None + main_key_offset = 0 topk_indices = None if self.compress_ratio: compressed_kv = shared_state.require_compressed_kv(self.kv_source_layer_idx, self.layer_idx) topk_indices = shared_state.require_topk_indices(self.index_source_layer_idx, self.layer_idx) - query = self._apply_indexer_loss(indexer_output, query, compressed_kv, tp_context, query_segments) + main_key_offset = shared_state.compressed_kv_offsets.get(self.kv_source_layer_idx, 0) + query = self._apply_indexer_loss( + indexer_output, query, compressed_kv, tp_context, query_segments, main_key_offset, + ) attention_output = self._run_sparse_attention( query, key_value, compressed_kv, topk_indices, query_offset, global_sequence_length, - segment_starts, query_segments, raw_document_starts, + segment_starts, query_segments, raw_key_offset, main_key_offset, raw_document_starts, + sum(length // max(1, self.compress_ratio) for length in document_lengths), ) output = self._project_output(attention_output, cos, sin, batch_size, sequence_length) return output, None @@ -1762,6 +1892,8 @@ def __init__( context: Mapping[str, Any] | None = None, *, use_fused_ops: bool = False, + swa_comm: str = "all_to_all_v", + main_kv_comm: str = "all_gather", ) -> None: """Transfer source state and select the implementation used for comparisons. @@ -1774,6 +1906,10 @@ def __init__( raising on unsupported calls without automatic fallback. False selects eager PyTorch attention, Indexer and KL on every device, without loading optional operator extensions. + swa_comm: Raw KV communication, 'all_to_all_v' exchanges only the SWA + halo; 'all_gather' keeps the full-bank comparison. + main_kv_comm: 'all_to_all_v' dispatches document-causal main KV prefixes; + 'all_gather' retains the full-bank comparison. Note: All implementations retain the same parameter and CP/TP contracts. @@ -1782,19 +1918,26 @@ def __init__( Fusion supports ordinary causal r1/r2 training on NPU with BF16 main attention, D=512, SWA=128, Indexer D=128, TopK=512 and TP=1. Active fused KL requires 8, 16, 32 or 64 Indexer heads. Candidate - pools are unsupported. Packed inputs retain the existing alignment - constraints and are dispatched as separate BSND document segments. + pools are unsupported. Packed documents are compressed independently; + CP still requires equal contiguous raw query shards and batch one. + Multiple document segments use a native TND call. The ops-transformer training extension is loaded on the first supported fused call; a missing extension raises ImportError. """ if not isinstance(use_fused_ops, bool): raise ValueError(f"use_fused_ops must be a bool, got {use_fused_ops!r}") + if swa_comm not in ("all_gather", "all_to_all_v"): + raise ValueError("swa_comm must be 'all_gather' or 'all_to_all_v'") + if main_kv_comm not in ("all_gather", "all_to_all_v"): + raise ValueError("main_kv_comm must be 'all_gather' or 'all_to_all_v'") del module_fqn, context super().__init__( module, replace_indexer=True, use_fused_ops=use_fused_ops, ) + self.swa_comm = swa_comm + self.main_kv_comm = main_kv_comm __all__ = [ diff --git a/hyper_parallel/distributed/context_parallel/collectives.py b/hyper_parallel/distributed/context_parallel/collectives.py index 362e79bbb2..3f4e0e10b4 100644 --- a/hyper_parallel/distributed/context_parallel/collectives.py +++ b/hyper_parallel/distributed/context_parallel/collectives.py @@ -251,14 +251,14 @@ def backward( ctx: Any, grad_output: Tensor, ) -> tuple[Tensor, None, None, None, None, None]: - """Reduce and select this rank's input-gradient sequence shard.""" - grad = grad_output.contiguous().clone() - work = dist.all_reduce(grad, group=ctx.group, async_op=True) + """SUM consumer gradients directly into each owner's sequence shard.""" + send = _move_dim_to_front(grad_output, ctx.gather_dim) + output = grad_output.new_empty((send.shape[0] // ctx.world_size, *send.shape[1:])) + work = dist.reduce_scatter_tensor( + output, send, op=dist.ReduceOp.SUM, group=ctx.group, async_op=True, + ) work.wait() - rank = dist.get_rank(ctx.group) - local = torch.chunk( - grad, ctx.world_size, dim=ctx.gather_dim - )[rank].contiguous() + local = _move_dim_from_front(output, ctx.gather_dim) return ( local, None, diff --git a/hyper_parallel/distributed/context_parallel/sequence_dispatch.py b/hyper_parallel/distributed/context_parallel/sequence_dispatch.py new file mode 100644 index 0000000000..b7d8b2c85b --- /dev/null +++ b/hyper_parallel/distributed/context_parallel/sequence_dispatch.py @@ -0,0 +1,259 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Differentiable all-to-all-v dispatch of sequence bank intervals.""" + +from __future__ import annotations + +from bisect import bisect_left, bisect_right +from dataclasses import dataclass +from functools import lru_cache +from typing import Any + +import torch # pylint: disable=forbidden-backend-import +import torch.distributed as dist # pylint: disable=forbidden-backend-import + +from hyper_parallel.distributed.context_parallel.compressed_document import ( + compressed_document_offsets, compressed_owner_offsets, +) + + +@dataclass(frozen=True) +class SequenceDispatchRoute: + """Row splits for compact intervals assembled from contiguous owner shards. + + Global start/end bound the requested ranges, including gaps when present. + Self rows stay local. Remote rows are ordered by owner rank, so the local + shard fits between the received prefix and suffix without tensor indices. + Backward reverses the exchange and SUMs overlapping consumer contributions. + """ + + local_length: int + global_start: int + global_end: int + local_output_start: int + send_offsets: tuple[int, ...] + input_splits: tuple[int, ...] + output_splits: tuple[int, ...] + needs_exchange: bool + send_ranges: tuple[tuple[int, int], ...] = () + + def owner_ranges(self) -> tuple[tuple[int, int], ...]: + """Return send slices in destination-rank order, including multi-range peers.""" + return self.send_ranges or tuple(zip(self.send_offsets, self.input_splits)) + + +def interval_route( + local_length: int, rank: int, intervals: tuple[tuple[int, int], ...], +) -> SequenceDispatchRoute: + """Derive owner/consumer intersections; all ranks agree whether to exchange.""" + local_start = rank * local_length + global_start, global_end = intervals[rank] + offsets, sends, receives = [], [], [] + for peer, (start, end) in enumerate(intervals): + first = max(local_start, start) + count = max(0, min(local_start + local_length, end) - first) if peer != rank else 0 + offsets.append(first - local_start if count else 0) + sends.append(count) + peer_start = peer * local_length + receives.append(max(0, min(peer_start + local_length, global_end) - max(peer_start, global_start)) + if peer != rank else 0) + needs_exchange = any(start != peer * local_length or end != (peer + 1) * local_length + for peer, (start, end) in enumerate(intervals)) + return SequenceDispatchRoute(local_length, global_start, global_end, local_start - global_start, + tuple(offsets), tuple(sends), tuple(receives), needs_exchange) + + +def _intersect_ranges(ranges: tuple, start: int, length: int) -> tuple: + """Intersect sorted global ranges with one source shard, retaining row order.""" + first = bisect_right(ranges, start, key=lambda item: item[1]) + last = bisect_left(ranges, start + length, key=lambda item: item[0]) + return tuple((max(begin, start) - start, min(end, start + length) - max(begin, start)) + for begin, end in ranges[first:last]) + + +def _validate_dispatch_intervals(owner_offsets: tuple, rank: int, intervals: tuple) -> None: + """Validate variable owner coverage before any rank enters a collective.""" + size = len(intervals) + if (len(owner_offsets) != size+1 or owner_offsets[0] != 0 or not 0 <= rank < size + or any(end < start for start, end in zip(owner_offsets, owner_offsets[1:]))): + raise ValueError("Dispatch requires ordered owner offsets and a valid rank") + for peer, ranges in enumerate(intervals): + start, end = owner_offsets[peer:peer+2] + if (any(first < 0 or last <= first or last > owner_offsets[-1] for first, last in ranges) + or any(left[1] > right[0] for left, right in zip(ranges, ranges[1:])) + or sum(count for _, count in _intersect_ranges(ranges, start, end-start)) != end-start): + raise ValueError("Dispatch ranges must be sorted, disjoint, in bounds, and cover the complete local shard") + + +def _receive_counts(intervals: tuple, owners: tuple, rank: int) -> tuple[int, ...]: + """Count requested rows by source, excluding the local bypass.""" + return tuple(sum(count for _, count in _intersect_ranges(intervals[rank], owners[peer], + owners[peer+1]-owners[peer])) + if peer != rank else 0 for peer in range(len(intervals))) + + +def multi_interval_route(local_length: int, rank: int, intervals: tuple[tuple, ...], + *, + owner_offsets: tuple[int, ...] | None = None) -> SequenceDispatchRoute: + """Plan compact demands, retaining each owner's complete, possibly empty shard. + + Args: + local_length: Physical K rows on this source rank. + rank: Local rank in the sequence-ordered group. + intervals: Sorted, disjoint global ranges required by each destination. + owner_offsets: Canonical source boundaries; omitted for equal-sized shards. + """ + size = len(intervals) + owners = tuple(peer*local_length for peer in range(size+1)) if owner_offsets is None else owner_offsets + _validate_dispatch_intervals(owners, rank, intervals) + if owners[rank+1]-owners[rank] != local_length: + raise ValueError("Local K length differs from the supplied owner offsets") + local_start = owners[rank] + sends = tuple(_intersect_ranges(ranges, local_start, local_length) if peer != rank else () + for peer, ranges in enumerate(intervals)) + receives = _receive_counts(intervals, owners, rank) + needs_exchange = any(sum(end-start for start, end in ranges) > owners[peer+1]-owners[peer] + for peer, ranges in enumerate(intervals)) + requested = intervals[rank] + return SequenceDispatchRoute( + local_length, requested[0][0] if requested else local_start, + requested[-1][1] if requested else local_start, sum(receives[:rank]), + (0,) * size, tuple(sum(count for _, count in ranges) for ranges in sends), receives, + needs_exchange, tuple(item for ranges in sends for item in ranges), + ) + + +class _SequenceDispatchWait(torch.autograd.Function): + """Attach remote bank rows to their owners without retaining send buffers.""" + + @staticmethod + def forward(ctx: Any, local: torch.Tensor, received: torch.Tensor, work: Any, + route: SequenceDispatchRoute, group: Any) -> torch.Tensor: + """Wait before exposing received data and insert the uncommunicated local rows.""" + ctx.route = route + ctx.group = group + work.wait() + before = route.local_output_start + return torch.cat((received[:before], local, received[before:]), dim=0) + + @staticmethod + def backward(ctx: Any, grad: torch.Tensor) -> tuple: + """Reverse dispatch, SUMming consumers in rank order, including empty ranks.""" + route = ctx.route + first, last = route.local_output_start, route.local_output_start + route.local_length + send = torch.cat((grad[:first], grad[last:]), dim=0) + received = grad.new_empty((sum(route.input_splits), *grad.shape[1:])) + work = dist.all_to_all_single( + received, send, output_split_sizes=list(route.input_splits), + input_split_sizes=list(route.output_splits), group=ctx.group, async_op=True, + ) + local = grad[first:last].clone(memory_format=torch.contiguous_format) + work.wait() + cursor = 0 + for offset, count in route.owner_ranges(): + if count: + local.narrow(0, offset, count).add_(received.narrow(0, cursor, count)) + cursor += count + return local, None, None, None, None + + +@dataclass +class AsyncSequenceDispatch: + """Single-use launch handle owning send storage until the communication completes.""" + + local: torch.Tensor + send: torch.Tensor | None + received: torch.Tensor | None + work: Any + route: SequenceDispatchRoute + group: Any + sequence_dim: int + waited: bool = False + + def wait(self) -> torch.Tensor: + """Finish the exchange and return the compact bank in its original layout.""" + if self.waited: + raise RuntimeError("a sequence dispatch handle cannot be waited twice") + self.waited = True + output = self.local + if self.received is not None: + output = _SequenceDispatchWait.apply( + self.local, self.received, self.work, self.route, self.group, + ) + self.send = None + self.received = None + return output.movedim(0, self.sequence_dim).contiguous() + + +def async_sequence_dispatch(tensor: torch.Tensor, sequence_dim: int, + route: SequenceDispatchRoute, group: Any) -> AsyncSequenceDispatch: + """Launch variable row exchange with an autograd edge attached at wait. + + Args: + tensor: Contiguous local owner shard of a raw, compressed or Index K bank. + sequence_dim: Sequence dimension; feature dimensions are transported unchanged. + route: Immutable per-batch routing agreed by every member of the CP group. + group: Existing framework-owned CP process group, in sequence-shard rank order. + """ + if not -tensor.ndim <= sequence_dim < tensor.ndim: + raise ValueError("sequence dispatch dimension is outside the input tensor dimensions") + sequence_dim %= tensor.ndim + if tensor.shape[sequence_dim] != route.local_length: + raise ValueError("sequence dispatch owner length does not match the bank's physical layout") + local = tensor.movedim(sequence_dim, 0).contiguous() + if not route.needs_exchange: + return AsyncSequenceDispatch(local, None, None, None, route, group, sequence_dim) + parts = [local.detach().narrow(0, offset, count) + for offset, count in route.owner_ranges() if count] + send = torch.cat(parts, dim=0) if parts else local.detach()[:0] + received = local.new_empty((sum(route.output_splits), *local.shape[1:])) + work = dist.all_to_all_single( + received, send, output_split_sizes=list(route.output_splits), + input_split_sizes=list(route.input_splits), group=group, async_op=True, + ) + return AsyncSequenceDispatch(local, send, received, work, route, group, sequence_dim) + + +@lru_cache(maxsize=128) +def sliding_window_route(boundaries: tuple[int, ...], local_length: int, size: int, + rank: int, window: int) -> SequenceDispatchRoute: + """Plan the same-document left halo; every rank agrees whether to exchange.""" + if local_length <= 0 or size <= 0 or not 0 <= rank < size or window <= 0: + raise ValueError("SWA dispatch requires positive dimensions and a valid CP rank") + if (len(boundaries) < 2 or boundaries[0] != 0 or boundaries[-1] != local_length * size + or any(end <= start for start, end in zip(boundaries, boundaries[1:]))): + raise ValueError("SWA document boundaries must span the full contiguous CP sequence") + intervals = tuple( + (max(boundaries[bisect_right(boundaries, peer * local_length) - 1], peer * local_length - window + 1), + (peer + 1) * local_length) + for peer in range(size) + ) + return interval_route(local_length, rank, intervals) + + +@lru_cache(maxsize=128) +def document_prefix_route(boundaries: tuple[int, ...], local_length: int, size: int, + rank: int, ratio: int) -> SequenceDispatchRoute: + """Dispatch document-causal prefixes in canonical, independently compressed IDs.""" + keys = compressed_document_offsets(boundaries, ratio) + owners = compressed_owner_offsets(boundaries, ratio, local_length) + if len(owners) != size+1 or not 0 <= rank < size: + raise ValueError("Document prefix dispatch requires a valid contiguous CP group") + intervals = [] + for peer in range(size): + start = keys[bisect_right(boundaries, peer*local_length)-1] + end = owners[peer+1] + intervals.append(((start, end),) if start < end else ()) + return multi_interval_route(owners[rank+1]-owners[rank], rank, tuple(intervals), owner_offsets=owners) diff --git a/hyper_parallel/models/deepseek_v41/adapter/distributed/shared_attention_context_parallel.py b/hyper_parallel/models/deepseek_v41/adapter/distributed/shared_attention_context_parallel.py index 30b6f6b302..9fdd538d4b 100644 --- a/hyper_parallel/models/deepseek_v41/adapter/distributed/shared_attention_context_parallel.py +++ b/hyper_parallel/models/deepseek_v41/adapter/distributed/shared_attention_context_parallel.py @@ -14,6 +14,8 @@ # ============================================================================ """Build TP/CP contexts for DeepSeek-V4.1 shared attention.""" +from __future__ import annotations + from functools import wraps from typing import Any @@ -33,12 +35,14 @@ async_cp_allgather_launch, ) from hyper_parallel.distributed.context_parallel.compressed_document import CompressionBoundaryRoute +from hyper_parallel.distributed.context_parallel.sequence_dispatch import async_sequence_dispatch from hyper_parallel.distributed.recipe_spec import inner_wrapper -def _build_shared_attention_cp_context(cp_mesh: Any) -> SharedCompressedAttentionCPContext: - """Build V4.1's asynchronous KV-all-gather communication context.""" - +def _build_shared_attention_cp_context( + cp_mesh: Any, swa_comm: str = "all_to_all_v", main_kv_comm: str = "all_gather", +) -> SharedCompressedAttentionCPContext: + """Build raw/main bank communication while keeping Index K globally gathered.""" def _gather_sequence(tensor: torch.Tensor, sequence_dim: int) -> torch.Tensor: """All-gather one sequence dimension in CP-rank order.""" return async_cp_allgather_launch(tensor, sequence_dim, cp_mesh).wait() @@ -59,12 +63,19 @@ def _exchange_boundary(tensor: torch.Tensor, route: CompressionBoundaryRoute) -> # Even empty receives must remain connected so every rank participates in backward. return torch.cat((received, local), dim=0).movedim(0, 1).contiguous() + def _launch_dispatch(tensor: torch.Tensor, sequence_dim: int, route: Any) -> Any: + """Exchange planned bank intervals on the existing CP group.""" + return async_sequence_dispatch(tensor, sequence_dim, route, cp_mesh.get_group()) + return SharedCompressedAttentionCPContext( size=cp_mesh.size(), rank=cp_mesh.get_local_rank(), gather_sequence=_gather_sequence, launch_sequence=_launch_sequence, exchange_boundary=_exchange_boundary, + launch_raw=_launch_dispatch if swa_comm == "all_to_all_v" else None, + launch_main=_launch_dispatch if main_kv_comm == "all_to_all_v" else None, + indexer_priority=True, ) @@ -95,9 +106,9 @@ def deepseek_v41_shared_attention_parallel_wrapper( ) -> _ForwardRewriteRequest: """Install V4.1's explicit TP Indexer and Colossal CP collectives. - Queries and Top-K indices remain sequence-local; raw KV, compressed KV, - and index keys use asynchronous differentiable all-gather. Projection - GEMMs overlap communication. Indexer Q/merge heads remain TP-local and + Raw KV uses SWA halo all-to-all-v, with full all-gather as a comparison. + Main KV selects prefix dispatch or all-gather; Index K remains globally gathered. + Launch/wait separation permits independent projection work. Indexer Q/merge heads remain TP-local and reduce their score and KL-target contributions before global Top-K. """ del mesh, ep_mesh @@ -110,7 +121,9 @@ def deepseek_v41_shared_attention_parallel_wrapper( original_forward = target_module.forward cp_context = ( - _build_shared_attention_cp_context(cp_mesh) + _build_shared_attention_cp_context( + cp_mesh, target_module.swa_comm, target_module.main_kv_comm, + ) if cp_mesh is not None and cp_mesh.size() > 1 else None ) @@ -122,7 +135,7 @@ def deepseek_v41_shared_attention_parallel_wrapper( @wraps(original_forward) def cp_forward(*args: Any, **kwargs: Any) -> Any: - """Run local queries against CP-replicated V4.1 key/value states.""" + """Run local queries against the configured V4.1 bank layouts.""" if "shared_attention_cp_context" in kwargs or "shared_attention_tp_context" in kwargs: raise ValueError("shared attention parallel contexts are owned by the wrapper") call_kwargs = dict(kwargs)