From d0e4b2549bfef2d0c333237f7bb37efcf80e774f Mon Sep 17 00:00:00 2001 From: howardforpr Date: Tue, 3 Nov 2026 20:26:39 +0800 Subject: [PATCH 01/15] feat: add INT4 KV codec and cross-loop residual reconstruction --- NOTICE | 27 ++ tests/test_kv_quantization.py | 403 ++++++++++++++++++++++++++++ vllm_rlt/kernels/kv_quantization.py | 208 ++++++++++++++ 3 files changed, 638 insertions(+) create mode 100644 tests/test_kv_quantization.py create mode 100644 vllm_rlt/kernels/kv_quantization.py diff --git a/NOTICE b/NOTICE index 4e818fb8..8b7cefe2 100644 --- a/NOTICE +++ b/NOTICE @@ -12,3 +12,30 @@ All rights reserved. vllm-rlt modifies the execution structure, configuration handling, model loading, and attention integration for standalone loop-level scheduling and depth-aware paged KV caching. It does not include an upstream CDB runtime implementation. + +The affine INT4 reference codec in vllm_rlt/kernels/kv_quantization.py adapts +FlashLoop's packing.py and cross-loop residual arithmetic in cache.py, revision +aeaadee10d75b7aa8c4a42a809f15c75d6d3bdc6, to vllm-rlt's logical KV tensor layout. +https://github.com/Superone77/FlashLoop + +MIT License + +Copyright (c) 2026 FlashLoop contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tests/test_kv_quantization.py b/tests/test_kv_quantization.py new file mode 100644 index 00000000..c35d85bc --- /dev/null +++ b/tests/test_kv_quantization.py @@ -0,0 +1,403 @@ +import numpy as np +import pytest +import torch + +from vllm_rlt.kernels.kv_quantization import ( + decode_residual, + encode_residual, + pack_kv, + unpack_kv, +) + +_DEVICES = ["cpu", pytest.param("cuda", marks=pytest.mark.gpu)] + + +def scalar_reference(values, kind, group_size): + """Scalar affine equations, independent of the codec's tensor layout operations.""" + source = values.float().cpu().numpy() + tokens, heads, dimensions = source.shape + codes = np.zeros(source.shape, dtype=np.uint8) + decoded = np.zeros(source.shape, dtype=np.float32) + extent = tokens if kind == "key" else dimensions + for head in range(heads): + for fixed in range(dimensions if kind == "key" else tokens): + for start in range(0, extent, group_size): + indices = [ + (index, head, fixed) if kind == "key" else (fixed, head, index) + for index in range(start, min(start + group_size, extent)) + ] + low = min(source[index] for index in indices) + high = max(source[index] for index in indices) + # CPU/CUDA can round division by 15 differently at code boundaries. + step = ( + np.float32((torch.tensor(high - low, device=values.device) / 15).item()) + if high != low + else np.float32(1) + ) + stored_low, stored_step = np.float32(np.float16(low)), np.float32(np.float16(step)) + for index in indices: + code = min(15, max(0, round(float(np.float32((source[index] - low) / step))))) + codes[index] = code + decoded[index] = np.float32(np.float32(code) * stored_step + stored_low) + payload = np.zeros((tokens, heads, (dimensions + 1) // 2), dtype=np.uint8) + for token in range(tokens): + for head in range(heads): + for dimension in range(dimensions): + payload[token, head, dimension // 2] |= int(codes[token, head, dimension]) << ( + 4 * (dimension % 2) + ) + return torch.from_numpy(payload), torch.from_numpy(decoded).to(values.dtype) + + +def test_known_nibble_order_and_ties_to_even(): + values = torch.tensor([0, 0.5, 1.5, 2.5, 14.5, 15]).reshape(1, 1, 6) + packed = pack_kv(values, kind="value", group_size=6) + assert packed.payload.tolist() == [[[0x00, 0x22, 0xFE]]] + torch.testing.assert_close( + unpack_kv(packed), torch.tensor([0, 0, 2, 2, 14, 15.0]).reshape(1, 1, 6) + ) + odd = pack_kv(torch.tensor([[[0.0, 1.0, 15.0]]]), kind="value", group_size=3) + assert odd.payload.tolist() == [[[0x10, 0x0F]]] + + +@pytest.mark.parametrize("device", _DEVICES) +@pytest.mark.parametrize("kind", ["key", "value"]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16, torch.float64]) +@pytest.mark.parametrize("group_size", [1, 3, 64]) +def test_groups_partial_groups_and_noncontiguous_inputs(kind, dtype, group_size, device): + values = torch.randn(5, 2, 14, generator=torch.Generator().manual_seed(714)).to( + device=device, dtype=dtype + )[..., ::2] + assert not values.is_contiguous() + original = values.clone() + expected_payload, expected = scalar_reference(values, kind, group_size) + packed = pack_kv(values, kind=kind, group_size=group_size) + assert packed.original_shape == (5, 2, 7) + assert packed.original_dtype == dtype + assert packed.payload.dtype == torch.uint8 + assert packed.minimum.dtype == packed.scale.dtype == torch.float16 + groups = ( + (5 + group_size - 1) // group_size if kind == "key" else (7 + group_size - 1) // group_size + ) + assert ( + packed.minimum.shape + == packed.scale.shape + == ((groups, 2, 7) if kind == "key" else (5, 2, groups)) + ) + torch.testing.assert_close(packed.payload.cpu(), expected_payload, rtol=0, atol=0) + torch.testing.assert_close(unpack_kv(packed).cpu(), expected, rtol=0, atol=0) + torch.testing.assert_close(values, original, rtol=0, atol=0) + + +@pytest.mark.parametrize("device", _DEVICES) +@pytest.mark.parametrize("kind", ["key", "value"]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("extent", [63, 64, 65, 128, 129]) +def test_default_group_boundaries(kind, dtype, extent, device): + tokens, channels = (extent, 7) if kind == "key" else (5, extent) + values = torch.randn(tokens, 1, channels * 2, generator=torch.Generator().manual_seed(517)).to( + dtype + )[..., ::2] + values = values.to(device) + expected_payload, expected = scalar_reference(values, kind, 64) + packed = pack_kv(values, kind=kind) + torch.testing.assert_close(packed.payload.cpu(), expected_payload, rtol=0, atol=0) + torch.testing.assert_close(unpack_kv(packed).cpu(), expected, rtol=0, atol=0) + metadata_elements = ( + ((tokens + 63) // 64) * channels if kind == "key" else tokens * ((channels + 63) // 64) + ) + assert packed.storage_bytes == tokens * ((channels + 1) // 2) + 4 * metadata_elements + if channels % 2: + assert torch.count_nonzero(packed.payload[..., -1] >> 4) == 0 + + +def test_codes_use_parameters_before_fp16_metadata_rounding(): + values = torch.tensor([[[0.10001, 0.10005, 0.10009, 0.10021]]]) + packed = pack_kv(values, kind="value", group_size=4) + assert packed.payload.tolist() == [[[0x30, 0xF6]]] + assert packed.minimum.item() == torch.tensor(0.10001).half().item() + _, expected = scalar_reference(values, "value", 4) + torch.testing.assert_close(unpack_kv(packed), expected, rtol=0, atol=0) + assert unpack_kv(packed)[0, 0, 0] != values[0, 0, 0] + + +@pytest.mark.parametrize("kind", ["key", "value"]) +def test_constant_groups_and_scale_underflow(kind): + packed = pack_kv(torch.full((3, 2, 5), 1.0001), kind=kind, group_size=2) + assert torch.count_nonzero(packed.payload) == 0 + assert torch.all(packed.scale == 1) + torch.testing.assert_close(unpack_kv(packed), torch.ones(3, 2, 5), rtol=0, atol=0) + values = torch.tensor([0.0, 1e-8]).reshape((2, 1, 1) if kind == "key" else (1, 1, 2)) + packed = pack_kv(values, kind=kind, group_size=2) + assert torch.count_nonzero(packed.payload) > 0 + assert torch.count_nonzero(packed.scale) == 0 + torch.testing.assert_close(unpack_kv(packed), torch.zeros_like(values), rtol=0, atol=0) + + +@pytest.mark.parametrize("kind,metadata_elements", [("key", 2 * 2 * 7), ("value", 5 * 2 * 3)]) +def test_storage_bytes_include_metadata_and_channel_padding(kind, metadata_elements): + packed = pack_kv(torch.ones(5, 2, 7), kind=kind, group_size=3) + assert packed.storage_bytes == 5 * 2 * 4 + 4 * metadata_elements + assert packed.storage_bytes == sum( + tensor.numel() * tensor.element_size() + for tensor in (packed.payload, packed.minimum, packed.scale) + ) + + +@pytest.mark.parametrize("device", _DEVICES) +@pytest.mark.parametrize("kind", ["key", "value"]) +@pytest.mark.parametrize("shape", [(0, 2, 7), (5, 0, 7)]) +def test_empty_token_or_head_dimension(kind, device, shape): + values = torch.empty(shape, dtype=torch.bfloat16, device=device) + packed = pack_kv(values, kind=kind) + assert packed.payload.shape == (*shape[:2], 4) + assert unpack_kv(packed).shape == values.shape + assert unpack_kv(packed).dtype == values.dtype + assert packed.storage_bytes == 0 + + +@pytest.mark.gpu +@pytest.mark.parametrize("kind", ["key", "value"]) +def test_oversized_group_does_not_expand_decode_scratch(kind): + values = torch.randn(5, 2, 7, device="cuda", dtype=torch.bfloat16) + peaks = [] + for group_size in (7, 65536): + packed = pack_kv(values, kind=kind, group_size=group_size) + torch.cuda.synchronize() + allocated = torch.cuda.memory_allocated() + torch.cuda.reset_peak_memory_stats() + decoded = unpack_kv(packed) + torch.cuda.synchronize() + peaks.append(torch.cuda.max_memory_allocated() - allocated) + assert decoded.shape == values.shape + assert peaks[1] <= peaks[0] + + +@pytest.mark.parametrize("kind", ["key", "value"]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16, torch.float64]) +def test_residual_chain_uses_reconstructed_predecessors(kind, dtype): + generator = torch.Generator().manual_seed(429) + raw_previous = torch.randn(5, 2, 7, generator=generator).to(dtype) + reconstructed = unpack_kv(pack_kv(raw_previous, kind=kind, group_size=3)) + _, expected = scalar_reference(raw_previous, kind, 3) + torch.testing.assert_close(reconstructed, expected, rtol=0, atol=0) + distinguishes_raw_predecessor = False + for _ in range(3): + current = (raw_previous + torch.randn(5, 2, 7, generator=generator).to(dtype) * 0.3).to( + dtype + ) + saved_current, saved_previous = current.clone(), reconstructed.clone() + residual = encode_residual(current, reconstructed, kind=kind, group_size=3) + expected_payload, decoded_delta = scalar_reference((current - expected).to(dtype), kind, 3) + expected = (expected + decoded_delta).to(dtype) + _, wrong_delta = scalar_reference((current - raw_previous).to(dtype), kind, 3) + wrong = (reconstructed + wrong_delta).to(dtype) + distinguishes_raw_predecessor |= not torch.equal(expected, wrong) + torch.testing.assert_close(residual.data.payload, expected_payload, rtol=0, atol=0) + previous_input = reconstructed + reconstructed = decode_residual(residual, previous_input) + torch.testing.assert_close(reconstructed, expected, rtol=0, atol=0) + torch.testing.assert_close(current, saved_current, rtol=0, atol=0) + torch.testing.assert_close(previous_input, saved_previous, rtol=0, atol=0) + assert residual.positions is None + assert residual.num_tokens == 5 + assert residual.storage_bytes == residual.data.storage_bytes + raw_previous = current + assert distinguishes_raw_predecessor + + +@pytest.mark.parametrize("device", _DEVICES) +@pytest.mark.parametrize("kind", ["key", "value"]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_quantized_base_nested_sparse_residual_chain(kind, dtype, device): + generator = torch.Generator().manual_seed(937) + raw_previous = torch.randn(129, 1, 65, generator=generator).to(dtype) + reconstructed = unpack_kv(pack_kv(raw_previous.to(device), kind=kind)) + _, expected = scalar_reference(raw_previous.to(device), kind, 64) + torch.testing.assert_close(reconstructed.cpu(), expected, rtol=0, atol=0) + active = torch.arange(0, 129, 2, dtype=torch.int32) # 65 compact K rows cross a group boundary. + for positions in (active, active[::3]): + current = ( + raw_previous[positions] + + torch.randn(len(positions), 1, 65, generator=generator).to(dtype) * 0.3 + ) + expected_payload, decoded_delta = scalar_reference( + (current - expected[positions]).to(device), kind, 64 + ) + saved_previous = reconstructed.clone() + residual = encode_residual( + current.to(device), + reconstructed, + kind=kind, + positions=positions.to(device), + ) + expected[positions] += decoded_delta + raw_previous[positions] = current + previous_input = reconstructed + reconstructed = decode_residual(residual, previous_input) + torch.testing.assert_close(previous_input, saved_previous, rtol=0, atol=0) + torch.testing.assert_close(residual.data.payload.cpu(), expected_payload, rtol=0, atol=0) + torch.testing.assert_close(reconstructed.cpu(), expected, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "index_dtype", [torch.int8, torch.int16, torch.int32, torch.int64, torch.float32] +) +@pytest.mark.parametrize("contiguous", [False, True]) +def test_sparse_rows_use_compact_key_groups_and_clone_position_map(index_dtype, contiguous): + previous = torch.full((8, 1, 2), 10.0) + positions = torch.tensor([1, 0, 6, 0, 7, 0], dtype=index_dtype)[::2] + if contiguous: + positions = positions.contiguous() + current = torch.tensor([10.0, 25.0, 13.0]).reshape(3, 1, 1).expand(3, 1, 2) + residual = encode_residual(current, previous, kind="key", positions=positions, group_size=2) + assert residual.data.payload.tolist() == [[[0x00]], [[0xFF]], [[0x00]]] + assert residual.num_tokens == 8 + assert residual.positions.dtype == torch.int32 + assert residual.positions.device == current.device + assert residual.positions.is_contiguous() + assert residual.storage_bytes == residual.data.storage_bytes + 3 * 4 + assert residual.positions.untyped_storage().nbytes() == 3 * 4 + positions.fill_(0) + expected = previous.clone() + expected[[1, 6, 7]] = current + actual = decode_residual(residual, previous) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + torch.testing.assert_close(previous, torch.full_like(previous, 10), rtol=0, atol=0) + + +def test_sparse_absence_differs_from_present_zero_codes(): + previous = torch.arange(24.0).reshape(6, 1, 4) + positions = torch.tensor([1, 4]) + current = previous[positions] + 2 + residual = encode_residual(current, previous, kind="value", positions=positions, group_size=4) + assert torch.count_nonzero(residual.data.payload) == 0 + expected = previous.clone() + expected[positions] += 2 + torch.testing.assert_close(decode_residual(residual, previous), expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("device", _DEVICES) +@pytest.mark.parametrize("index_dtype", [torch.int32, torch.int64]) +def test_sparse_reconstruction_preserves_subnormal_values(device, index_dtype): + tiny = torch.finfo(torch.float32).tiny / 2 + previous = torch.tensor([tiny, -tiny, 1, -1, -0.0, 0.0, tiny, -tiny], device=device).reshape( + 4, 1, 2 + ) + saved_previous = previous.clone() + positions = torch.tensor([0, 2], device=device, dtype=index_dtype) + residual = encode_residual(previous[positions], previous, kind="value", positions=positions) + expected = previous.clone() + expected[2].zero_() + assert torch.equal( + decode_residual(residual, previous).view(torch.int32), expected.view(torch.int32) + ) + assert torch.equal(previous.view(torch.int32), saved_previous.view(torch.int32)) + + +@pytest.mark.parametrize("kind", ["key", "value"]) +def test_empty_residual_inherits_all_previous_rows(kind): + previous = torch.arange(24.0).reshape(6, 1, 4) + residual = encode_residual( + previous[:0], previous, kind=kind, positions=torch.empty(0, dtype=torch.long) + ) + torch.testing.assert_close(decode_residual(residual, previous), previous, rtol=0, atol=0) + assert residual.num_tokens == 6 + assert residual.storage_bytes == 0 + + +@pytest.mark.parametrize( + "values", + [ + torch.ones(2, 3), + torch.ones(2, 1, 0), + torch.ones(2, 1, 3, dtype=torch.int32), + ], +) +def test_invalid_values_shape_and_dtype(values): + with pytest.raises((ValueError, TypeError)): + pack_kv(values, kind="value") + + +@pytest.mark.parametrize( + "options", + [ + {"group_size": 0}, + {"group_size": -2}, + {"kind": "other"}, + ], +) +def test_invalid_codec_options(options): + with pytest.raises((ValueError, TypeError)): + pack_kv(torch.ones(2, 1, 4), **{"kind": "value", **options}) + + +@pytest.mark.parametrize("group_size", [True, 2.5]) +def test_group_size_conversion_matches_flashloop(group_size): + values = torch.tensor([[[0.0, 0.5, 1.0, 15.0]]]) + packed = pack_kv(values, kind="value", group_size=group_size) + expected_payload, expected = scalar_reference(values, "value", int(group_size)) + assert packed.group_size == int(group_size) + torch.testing.assert_close(packed.payload, expected_payload, rtol=0, atol=0) + torch.testing.assert_close(unpack_kv(packed), expected, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "positions", + [ + torch.tensor([[1, 2]]), + torch.tensor([-1, 2]), + torch.tensor([1, 6]), + torch.tensor([2, 1]), + torch.tensor([2, 2]), + torch.tensor([1]), + ], +) +def test_invalid_sparse_positions(positions): + with pytest.raises((ValueError, TypeError)): + encode_residual(torch.ones(2, 1, 4), torch.ones(6, 1, 4), kind="key", positions=positions) + + +def test_residual_shape_and_dtype_mismatches(): + previous = torch.ones(3, 1, 4) + for current in (torch.ones(2, 1, 4), torch.ones(3, 2, 4), previous.half()): + with pytest.raises((ValueError, TypeError)): + encode_residual(current, previous, kind="value") + residual = encode_residual(previous + 1, previous, kind="value") + for wrong in (previous[:2], previous.half(), torch.tensor(1.0)): + with pytest.raises((ValueError, TypeError)): + encode_residual(previous, wrong, kind="value") + with pytest.raises((ValueError, TypeError)): + encode_residual(previous, wrong, kind="value", positions=torch.arange(3)) + with pytest.raises((ValueError, TypeError)): + decode_residual(residual, wrong) + + +@pytest.mark.gpu +@pytest.mark.parametrize("kind", ["key", "value"]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) +def test_cuda_codec_and_sparse_residual_match_cpu_equations(kind, dtype): + previous = torch.randn(7, 2, 5, generator=torch.Generator().manual_seed(12)).to(dtype) + positions = torch.tensor([0, 3, 6]) + current = previous[positions] + 0.2 + expected_payload, decoded_delta = scalar_reference( + (current - previous[positions]).cuda(), kind, 2 + ) + expected = previous.clone() + expected[positions] += decoded_delta + residual = encode_residual( + current.cuda(), previous.cuda(), kind=kind, positions=positions.cuda(), group_size=2 + ) + assert residual.data.payload.is_cuda and residual.positions.is_cuda + torch.testing.assert_close(residual.data.payload.cpu(), expected_payload, rtol=0, atol=0) + torch.testing.assert_close( + decode_residual(residual, previous.cuda()).cpu(), expected, rtol=0, atol=0 + ) + transferred = encode_residual( + current.cuda(), previous.cuda(), kind=kind, positions=positions, group_size=2 + ) + assert transferred.positions.is_cuda and transferred.positions.dtype == torch.int32 + torch.testing.assert_close(transferred.data.payload, residual.data.payload, rtol=0, atol=0) + torch.testing.assert_close( + decode_residual(transferred, previous.cuda()).cpu(), expected, rtol=0, atol=0 + ) diff --git a/vllm_rlt/kernels/kv_quantization.py b/vllm_rlt/kernels/kv_quantization.py new file mode 100644 index 00000000..4ec8e1c1 --- /dev/null +++ b/vllm_rlt/kernels/kv_quantization.py @@ -0,0 +1,208 @@ +"""Affine INT4 KV packing and cross-loop residual reconstruction. + +Inputs are logical ``[token, KV head, channel]`` tensors for one request/layer. +K is grouped across stored token rows per channel; V across channels per token. + +Adapted from FlashLoop packing.py and cache.py at aeaadee10d75b7aa8c4a42a809f15c75d6d3bdc6. +Copyright (c) 2026 FlashLoop contributors. MIT License. +""" + +from dataclasses import dataclass +from typing import Literal + +import torch + +KVKind = Literal["key", "value"] +_DTYPES = {torch.float16, torch.bfloat16, torch.float32, torch.float64} + + +def _validate_inputs(values: torch.Tensor, *, kind: KVKind, group_size: int) -> None: + if group_size <= 0: + raise ValueError("group_size must be positive") + if kind not in {"key", "value"}: + raise ValueError("kind must be 'key' or 'value'") + if values.ndim != 3 or values.shape[2] <= 0: + raise ValueError("KV values must have shape [tokens, heads, positive channels]") + if values.dtype not in _DTYPES: + raise ValueError("KV values must use a floating-point dtype") + + +def _codes_for_group( + values: torch.Tensor, *, reduce_dim: int +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + work = values.float() + minimum = work.amin(dim=reduce_dim, keepdim=True) + span = work.amax(dim=reduce_dim, keepdim=True) - minimum + scale = torch.where(span > 0, span / 15.0, torch.ones_like(span)) + codes = (work - minimum).div_(scale).round_().clamp_(0, 15).to(torch.uint8) + return codes, minimum.to(torch.float16), scale.to(torch.float16) + + +def _pack_channel_nibbles(codes: torch.Tensor) -> torch.Tensor: + payload = codes[..., 0::2].clone(memory_format=torch.contiguous_format) + payload[..., : codes.shape[-1] // 2] |= codes[..., 1::2] << 4 + return payload + + +def _unpack_channel_nibbles(payload: torch.Tensor, channels: int) -> torch.Tensor: + codes = torch.empty((*payload.shape[:-1], channels), dtype=torch.float32, device=payload.device) + codes[..., 0::2] = payload & 0x0F + codes[..., 1::2] = (payload >> 4)[..., : channels // 2] + return codes + + +@dataclass(frozen=True) +class PackedInt4KV: + """Encoded tensor and FP16 affine parameters; treat its tensors as immutable. + + ``payload`` is uint8 ``[T, H, ceil(D/2)]``. For K, parameters are + ``[ceil(T/G), H, D]``; for V, ``[T, H, ceil(D/G)]``. An odd last channel + leaves the high nibble zero. Constant groups use code zero and scale one. + """ + + payload: torch.Tensor + minimum: torch.Tensor + scale: torch.Tensor + original_shape: tuple[int, int, int] + original_dtype: torch.dtype + kind: KVKind + group_size: int + + @property + def storage_bytes(self) -> int: + """Payload and parameter bytes, excluding allocator overhead and decode scratch.""" + return sum(t.numel() * t.element_size() for t in (self.payload, self.minimum, self.scale)) + + +@torch.no_grad() +def pack_kv(values: torch.Tensor, *, kind: KVKind, group_size: int = 64) -> PackedInt4KV: + """Quantize logical KV in FP32, storing uint8 payload and FP16 parameters.""" + group_size = int(group_size) + _validate_inputs(values, kind=kind, group_size=group_size) + shape = tuple(values.shape) + axis = 0 if kind == "key" else 2 + codes = torch.empty_like(values, dtype=torch.uint8) + minima, scales = [], [] + for start in range(0, shape[axis], group_size): + length = min(group_size, shape[axis] - start) + group_codes, minimum, scale = _codes_for_group( + values.narrow(axis, start, length), reduce_dim=axis + ) + codes.narrow(axis, start, length).copy_(group_codes) + minima.append(minimum) + scales.append(scale) + metadata_shape = list(shape) + metadata_shape[axis] = (shape[axis] + group_size - 1) // group_size + minimum = ( + torch.cat(minima, dim=axis) + if minima + else torch.empty(metadata_shape, device=values.device, dtype=torch.float16) + ) + scale = torch.cat(scales, dim=axis) if scales else torch.empty_like(minimum) + payload = _pack_channel_nibbles(codes) + return PackedInt4KV(payload, minimum, scale, shape, values.dtype, kind, group_size) + + +@torch.no_grad() +def unpack_kv(packed: PackedInt4KV) -> torch.Tensor: + """Reconstruct with stored parameters in FP32, then cast to compute dtype.""" + codes = _unpack_channel_nibbles(packed.payload, packed.original_shape[2]) + axis = 0 if packed.kind == "key" else 2 + length = packed.original_shape[axis] + group_size = min(packed.group_size, length) + minimum = packed.minimum.repeat_interleave(group_size, axis).narrow(axis, 0, length) + scale = packed.scale.repeat_interleave(group_size, axis).narrow(axis, 0, length) + return codes.mul_(scale).add_(minimum).to(packed.original_dtype) + + +def _validate_positions(positions: torch.Tensor, rows: int, tokens: int) -> None: + if positions.ndim != 1: + raise ValueError("positions must be one-dimensional") + if positions.numel() != rows: + raise ValueError("positions must identify every compact residual row") + if ( + rows + and ( + (positions[0] < 0) | (positions[-1] >= tokens) | (positions[1:] <= positions[:-1]).any() + ).item() + ): + raise ValueError("positions must be strictly increasing and within the logical token range") + + +@dataclass(frozen=True) +class Int4Residual: + """One depth's compact residual and its logical scatter map. + + ``positions=None`` denotes a dense update; an empty map denotes inheritance + at every position. Positions affect scatter only: K groups follow compact + row order, as in the pinned upstream cache. + """ + + data: PackedInt4KV + positions: torch.Tensor | None + num_tokens: int + + @property + def storage_bytes(self) -> int: + mapping_bytes = ( + 0 if self.positions is None else self.positions.numel() * self.positions.element_size() + ) + return self.data.storage_bytes + mapping_bytes + + +@torch.no_grad() +def encode_residual( + current: torch.Tensor, + reconstructed_previous: torch.Tensor, + *, + kind: KVKind, + positions: torch.Tensor | None = None, + group_size: int = 64, +) -> Int4Residual: + """Encode ``current - reconstructed_previous`` in the input dtype. + + Optional positions map compact rows into the full predecessor and are copied + as contiguous int32 on the input device, matching FlashLoop's index format. + """ + if ( + current.dtype != reconstructed_previous.dtype + or current.device != reconstructed_previous.device + ): + raise ValueError("current and predecessor must have matching dtype and device") + if positions is None: + if current.shape != reconstructed_previous.shape: + raise ValueError("dense residual requires matching current and predecessor shapes") + predecessor = reconstructed_previous + else: + if current.shape[1:] != reconstructed_previous.shape[1:]: + raise ValueError("current and predecessor must have matching heads and channels") + positions = positions.to( + device=current.device, + dtype=torch.int32, + copy=True, + memory_format=torch.contiguous_format, + ) + _validate_positions(positions, current.shape[0], reconstructed_previous.shape[0]) + predecessor = reconstructed_previous.index_select(0, positions) + data = pack_kv(current - predecessor, kind=kind, group_size=group_size) + return Int4Residual(data, positions, reconstructed_previous.shape[0]) + + +@torch.no_grad() +def decode_residual(residual: Int4Residual, previous: torch.Tensor) -> torch.Tensor: + """Return a new full tensor; absent rows retain their previous values.""" + expected_shape = (residual.num_tokens, *residual.data.original_shape[1:]) + if previous.shape != expected_shape: + raise ValueError("predecessor shape does not match residual logical extent") + if ( + previous.dtype != residual.data.original_dtype + or previous.device != residual.data.payload.device + ): + raise ValueError("predecessor dtype and device must match residual") + delta = unpack_kv(residual.data) + if residual.positions is None: + return previous + delta + result = previous.clone() + indices = residual.positions.long() + result.index_copy_(0, indices, previous.index_select(0, indices).add_(delta)) + return result From 0dacc65c37dca496497f1faf662da72802259ec2 Mon Sep 17 00:00:00 2001 From: Howard Yang <284181151+HowardYangAI@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:07:53 +0800 Subject: [PATCH 02/15] feat: add selectable NVFP4 KV reference codec --- NOTICE | 2 +- ...zation.py => test_int4_kv_quantization.py} | 0 tests/test_nvfp4_kv_quantization.py | 403 ++++++++++++++++++ vllm_rlt/kernels/kv_quantization.py | 208 --------- vllm_rlt/kernels/kv_quantization/__init__.py | 138 ++++++ vllm_rlt/kernels/kv_quantization/_common.py | 38 ++ vllm_rlt/kernels/kv_quantization/int4.py | 93 ++++ vllm_rlt/kernels/kv_quantization/nvfp4.py | 122 ++++++ 8 files changed, 795 insertions(+), 209 deletions(-) rename tests/{test_kv_quantization.py => test_int4_kv_quantization.py} (100%) create mode 100644 tests/test_nvfp4_kv_quantization.py delete mode 100644 vllm_rlt/kernels/kv_quantization.py create mode 100644 vllm_rlt/kernels/kv_quantization/__init__.py create mode 100644 vllm_rlt/kernels/kv_quantization/_common.py create mode 100644 vllm_rlt/kernels/kv_quantization/int4.py create mode 100644 vllm_rlt/kernels/kv_quantization/nvfp4.py diff --git a/NOTICE b/NOTICE index 8b7cefe2..21e5eab1 100644 --- a/NOTICE +++ b/NOTICE @@ -13,7 +13,7 @@ vllm-rlt modifies the execution structure, configuration handling, model loading and attention integration for standalone loop-level scheduling and depth-aware paged KV caching. It does not include an upstream CDB runtime implementation. -The affine INT4 reference codec in vllm_rlt/kernels/kv_quantization.py adapts +The affine INT4 reference codec in vllm_rlt/kernels/kv_quantization/ adapts FlashLoop's packing.py and cross-loop residual arithmetic in cache.py, revision aeaadee10d75b7aa8c4a42a809f15c75d6d3bdc6, to vllm-rlt's logical KV tensor layout. https://github.com/Superone77/FlashLoop diff --git a/tests/test_kv_quantization.py b/tests/test_int4_kv_quantization.py similarity index 100% rename from tests/test_kv_quantization.py rename to tests/test_int4_kv_quantization.py diff --git a/tests/test_nvfp4_kv_quantization.py b/tests/test_nvfp4_kv_quantization.py new file mode 100644 index 00000000..5058017b --- /dev/null +++ b/tests/test_nvfp4_kv_quantization.py @@ -0,0 +1,403 @@ +import numpy as np +import pytest +import torch + +from vllm_rlt.kernels.kv_quantization import ( + KVResidual, + PackedInt4KV, + PackedNVFP4KV, + decode_residual, + encode_residual, + pack_kv, + unpack_kv, +) +from vllm_rlt.kernels.kv_quantization.int4 import PackedInt4KV as BackendPackedInt4KV + +_FP4_VALUES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0) +_FP8_VALUES = tuple( + fraction / 512 if exponent == 0 else (1 + fraction / 8) * 2 ** (exponent - 7) + for exponent in range(16) + for fraction in range(8 if exponent < 15 else 7) +) + + +def _nearest_code(value, table): + return min(range(len(table)), key=lambda code: (abs(value - table[code]), code % 2)) + + +def scalar_nvfp4(values, kind): + """Enumerate finite E2M1/E4M3 values independently of Torch's casts/kernels.""" + source = values.float().cpu().numpy() + tokens, heads, channels = source.shape + maximum = np.max(np.abs(source)) if source.size else np.float32(0) + with np.errstate(over="ignore"): + global_scale = ( + np.float32(np.float32(np.float32(1) / maximum) * np.float32(2688)) + if maximum + else np.float32(1) + ) + global_scale = min(global_scale, np.float32(np.finfo(np.float32).max)) + reciprocal_global_scale = np.float32(np.float32(1) / global_scale) + payload = np.zeros((tokens, heads, (channels + 1) // 2), dtype=np.uint8) + scale_shape = ( + ((tokens + 15) // 16, heads, channels) + if kind == "key" + else (tokens, heads, (channels + 15) // 16) + ) + scales = np.zeros(scale_shape, dtype=np.uint8) + decoded = np.empty(source.shape, dtype=np.float32) + for head in range(heads): + for fixed in range(channels if kind == "key" else tokens): + for start in range(0, tokens if kind == "key" else channels, 16): + if kind == "key": + indices = [(row, head, fixed) for row in range(start, min(start + 16, tokens))] + scale_index = (start // 16, head, fixed) + else: + indices = [ + (fixed, head, col) for col in range(start, min(start + 16, channels)) + ] + scale_index = (fixed, head, start // 16) + group = np.array([source[index] for index in indices]) + block_maximum = np.max(np.abs(group)) + scale = np.float32(global_scale * np.float32(block_maximum * np.float32(1 / 6))) + scale_code = _nearest_code(min(float(scale), 448), _FP8_VALUES) + scales[scale_index] = scale_code + stored_scale = np.float32(_FP8_VALUES[scale_code]) + effective_scale = np.float32(stored_scale * reciprocal_global_scale) + decode_scale = np.float32(stored_scale / global_scale) + with np.errstate(over="ignore"): + multiplier = ( + np.float32(np.float32(1) / effective_scale) + if effective_scale + else np.float32(0) + ) + for index, value in zip(indices, group): + # Robustness extension: a positive scale can have an + # unrepresentable reciprocal while value / scale is finite. + normalized = np.float32( + value / effective_scale + if not np.isfinite(multiplier) + else value * multiplier + ) + code = _nearest_code(abs(float(normalized)), _FP4_VALUES) + negative = np.signbit(normalized) + signed_code = code | (8 if negative else 0) + token, _, channel = index + payload[token, head, channel // 2] |= signed_code << (4 * (channel % 2)) + quantized = np.float32(-_FP4_VALUES[code] if negative else _FP4_VALUES[code]) + decoded[token, head, channel] = np.float32(quantized * decode_scale) + return payload, scales, global_scale, torch.from_numpy(decoded).to(values.dtype) + + +def assert_matches_scalar(values, kind): + expected_payload, expected_scale, expected_global_scale, expected = scalar_nvfp4(values, kind) + packed = pack_kv(values, kind=kind, format="nvfp4") + assert isinstance(packed, PackedNVFP4KV) + assert packed.format == "nvfp4" and packed.axis == (0 if kind == "key" else 2) + assert packed.group_size == 16 + assert packed.original_shape == tuple(values.shape) and packed.original_dtype == values.dtype + assert packed.payload.dtype == torch.uint8 + assert packed.scale.dtype == torch.float8_e4m3fn + assert packed.global_scale.dtype == torch.float32 and packed.global_scale.ndim == 0 + torch.testing.assert_close( + packed.payload.cpu(), torch.from_numpy(expected_payload), rtol=0, atol=0 + ) + torch.testing.assert_close( + packed.scale.cpu().view(torch.uint8), torch.from_numpy(expected_scale), rtol=0, atol=0 + ) + assert packed.global_scale.item() == expected_global_scale + torch.testing.assert_close(unpack_kv(packed).cpu(), expected, rtol=0, atol=0) + tokens, heads, channels = values.shape + scale_bytes = ( + ((tokens + 15) // 16) * heads * channels + if kind == "key" + else tokens * heads * ((channels + 15) // 16) + ) + assert packed.storage_bytes == tokens * heads * ((channels + 1) // 2) + scale_bytes + 4 + return packed + + +@pytest.mark.parametrize("kind", ["key", "value"]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32, torch.float64]) +@pytest.mark.parametrize( + "shape", + [ + (5, 2, 17), + (2, 1, 16), + (1, 2, 31), + (2, 1, 1), + (0, 2, 17), + (3, 0, 17), + (16, 1, 3), + (17, 2, 3), + (33, 2, 17), + ], +) +def test_nvfp4_layout_partial_empty_and_noncontiguous(shape, dtype, kind): + values = torch.randn( + (*shape[:2], shape[2] * 2), generator=torch.Generator().manual_seed(831) + ).to(dtype)[..., ::2] + original = values.clone() + packed = assert_matches_scalar(values, kind) + torch.testing.assert_close(values, original, rtol=0, atol=0) + if shape[2] % 2: + assert torch.count_nonzero(packed.payload[..., -1] >> 4) == 0 + + +@pytest.mark.parametrize("kind", ["key", "value"]) +def test_nvfp4_group_axis_and_partial_tail(kind): + values = torch.zeros(33, 1, 2) + values[:16, 0, 0] = 6 + values[16:32, 0, 0] = 12 + values[32, 0, 0] = 24 + values[:, 0, 1] = 2688 + if kind == "value": + values = values.transpose(0, 2) + packed = assert_matches_scalar(values, kind) + scales = packed.scale[:, 0, 0] if kind == "key" else packed.scale[0, 0, :] + assert packed.global_scale.item() == 1 + torch.testing.assert_close(scales.float(), torch.tensor([1.0, 2.0, 4.0]), rtol=0, atol=0) + torch.testing.assert_close(unpack_kv(packed), values, rtol=0, atol=0) + + +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +def test_nvfp4_zero_groups_and_negative_zero(dtype): + values = torch.zeros(2, 1, 17, dtype=dtype) + values[..., 1::2] = -0.0 + values[..., -1] = -0.0 + packed = assert_matches_scalar(values, "key") + assert packed.global_scale.item() == 1 + assert torch.count_nonzero(packed.scale.float()) == 0 + assert packed.payload[0, 0, 0].item() == 0x80 + assert packed.payload[0, 0, -1].item() == 0x08 + assert torch.equal(torch.signbit(unpack_kv(packed)), torch.signbit(values)) + + +def test_nvfp4_tiny_inputs_keep_scales_and_reconstruction_finite(): + values = torch.tensor([1e-42, -1e-42, 0.0, -0.0]).repeat(4).reshape(1, 1, 16) + packed = pack_kv(values, kind="key", format="nvfp4") + decoded = unpack_kv(packed) + assert torch.isfinite(packed.global_scale) and torch.isfinite(packed.scale.float()).all() + assert torch.isfinite(decoded).all() and torch.count_nonzero(decoded) == 0 + assert torch.equal(torch.signbit(decoded), torch.signbit(values)) + + +@pytest.mark.parametrize("device", ["cpu", pytest.param("cuda", marks=pytest.mark.gpu)]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32, torch.float64]) +@pytest.mark.parametrize("anchor", [False, True]) +def test_nvfp4_tiny_values_survive_reciprocal_overflow(dtype, anchor, device): + tiny = 2**-24 if dtype == torch.float16 else 1e-38 + values = torch.tensor([tiny, -tiny, 0.0, -0.0], dtype=dtype).repeat(4).reshape(1, 1, 16) + if anchor: + anchored = torch.zeros_like(values) + anchored[..., 0] = tiny * (16 if dtype == torch.float16 else 800) + values = torch.cat((values, anchored), dim=2) + packed = pack_kv(values.to(device), kind="key", format="nvfp4") + decoded = unpack_kv(packed).cpu() + if device == "cpu": + expected_payload, expected_scale, _, expected = scalar_nvfp4(values, "key") + torch.testing.assert_close(decoded, expected, rtol=0, atol=0) + torch.testing.assert_close( + packed.payload.cpu(), torch.from_numpy(expected_payload), rtol=0, atol=0 + ) + torch.testing.assert_close( + packed.scale.cpu().view(torch.uint8), torch.from_numpy(expected_scale), rtol=0, atol=0 + ) + assert torch.isfinite(decoded).all() + nonzero = values != 0 + assert (decoded[nonzero] != 0).all() + assert ((decoded[nonzero].float() / values[nonzero].float() - 1).abs() < 0.1).all() + assert torch.equal(torch.signbit(decoded), torch.signbit(values)) + + +@pytest.mark.parametrize("kind", ["key", "value"]) +def test_nvfp4_fp4_midpoints_nextafter_and_even_codes(kind): + midpoints = np.array([0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5], dtype=np.float32) + probes = np.stack( + (np.nextafter(midpoints, -np.inf), midpoints, np.nextafter(midpoints, np.inf)), axis=1 + ).reshape(-1) + values = torch.zeros(len(probes) + 1, 1, 16) + values[:-1, 0, 0] = torch.from_numpy(probes) + values[:-1, 0, 1] = -torch.from_numpy(probes) + values[:-1, 0, 2] = 6 + values[-1, 0, 0] = 2688 # Fix g=1 and each probe's effective scale=1. + if kind == "key": + values = values.transpose(0, 2) + packed = assert_matches_scalar(values, kind) + assert packed.global_scale.item() == 1 + if kind == "key": + ties = torch.stack( + [ + (packed.payload[0, 0, channel // 2] >> (4 * (channel % 2))) & 0x0F + for channel in range(1, len(probes), 3) + ] + ) + else: + ties = packed.payload[1:-1:3, 0, 0] & 0x0F + assert ties.tolist() == [0, 2, 2, 4, 4, 6, 6] + + +@pytest.mark.parametrize("source_dtype", [torch.float16, torch.bfloat16]) +def test_nvfp4_all_low_precision_magnitudes_choose_nearest_even_code(source_dtype): + bits = torch.arange(32768, dtype=torch.int32).to(torch.int16) + probes = bits.view(source_dtype).float() + probes = probes[torch.isfinite(probes) & (probes <= 6)] + values = torch.zeros(len(probes) + 1, 1, 16) + values[:-1, 0, 0] = probes + values[:-1, 0, 1] = -probes + values[:-1, 0, 2] = 6 + values[-1, 0, 0] = 2688 # Keep global and probe block scales exactly one. + expected = torch.tensor( + [_nearest_code(value, _FP4_VALUES) for value in probes.tolist()], dtype=torch.uint8 + ) + packed = pack_kv(values, kind="value", format="nvfp4") + assert packed.global_scale.item() == 1 + assert torch.equal(packed.payload[:-1, 0, 0], expected | ((expected | 8) << 4)) + + +@pytest.mark.parametrize("kind", ["key", "value"]) +def test_nvfp4_fp8_scale_rounding_and_finite_maximum(kind): + targets = np.array([0.0001, 0.0009765625, 1.0625, 1.1875, 432, 448], dtype=np.float32) + probes = np.stack( + (np.nextafter(targets, -np.inf), targets, np.nextafter(targets, np.inf)), axis=1 + ).reshape(-1) + values = torch.zeros(len(probes) + 1, 1, 16) + values[:-1, 0, 0] = torch.from_numpy(probes) * 6 + values[-1, 0, 0] = 2688 + if kind == "key": + values = values.transpose(0, 2) + packed = assert_matches_scalar(values, kind) + anchor = (0, 0, -1) if kind == "key" else (-1, 0, 0) + assert packed.scale[anchor].float().item() == 448 + assert torch.isfinite(packed.scale.float()).all() + assert 0 in packed.scale.view(torch.uint8) + + +@pytest.mark.parametrize("group_size", [1, 8, 32, 64]) +@pytest.mark.parametrize("kind", ["key", "value"]) +def test_nvfp4_requires_sixteen_element_groups(group_size, kind): + with pytest.raises(ValueError): + pack_kv(torch.ones(1, 1, 16), kind=kind, format="nvfp4", group_size=group_size) + + +@pytest.mark.parametrize("kind", ["key", "value"]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +def test_nvfp4_dense_and_nested_sparse_residuals(kind, dtype): + generator = torch.Generator().manual_seed(712) + raw_previous = torch.randn(65, 2, 17, generator=generator).to(dtype) + reconstructed = unpack_kv(pack_kv(raw_previous, kind=kind, format="nvfp4")) + _, _, _, expected = scalar_nvfp4(raw_previous, kind) + torch.testing.assert_close(reconstructed, expected, rtol=0, atol=0) + active = torch.arange(0, 65, 3) + for positions in (None, active, active[::2], active[:0]): + raw_rows = raw_previous if positions is None else raw_previous[positions] + current = raw_rows + torch.randn(raw_rows.shape, generator=generator).to(dtype) * 0.3 + previous_rows = expected if positions is None else expected[positions] + _, _, _, expected_delta = scalar_nvfp4(current - previous_rows, kind) + saved_current, saved_previous = current.clone(), reconstructed.clone() + residual = encode_residual( + current, reconstructed, kind=kind, format="nvfp4", positions=positions + ) + assert isinstance(residual, KVResidual) and isinstance(residual.data, PackedNVFP4KV) + previous_input = reconstructed + reconstructed = decode_residual(residual, previous_input) + torch.testing.assert_close(current, saved_current, rtol=0, atol=0) + torch.testing.assert_close(previous_input, saved_previous, rtol=0, atol=0) + mapping_bytes = 0 if positions is None else 4 * positions.numel() + assert residual.storage_bytes == residual.data.storage_bytes + mapping_bytes + if positions is None: + expected = expected + expected_delta + raw_previous = current + else: + inactive = torch.ones(raw_previous.shape[0], dtype=torch.bool) + inactive[positions] = False + torch.testing.assert_close( + reconstructed[inactive], saved_previous[inactive], rtol=0, atol=0 + ) + expected[positions] = expected[positions] + expected_delta + raw_previous[positions] = current + torch.testing.assert_close(reconstructed, expected, rtol=0, atol=0) + + +@pytest.mark.gpu +@pytest.mark.parametrize("kind", ["key", "value"]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) +@pytest.mark.parametrize("tokens", [7, 65]) +def test_nvfp4_cuda_matches_scalar_and_sparse_reconstruction(dtype, kind, tokens): + previous = torch.randn(tokens, 2, 17, generator=torch.Generator().manual_seed(68)).to(dtype) + packed = assert_matches_scalar(previous.cuda(), kind) + previous = unpack_kv(packed).cpu() + positions = torch.arange(0, tokens, 3) + current = previous[positions] + 0.2 + _, _, _, delta = scalar_nvfp4(current - previous[positions], kind) + expected = previous.clone() + expected[positions] += delta + residual = encode_residual( + current.cuda(), previous.cuda(), kind=kind, format="nvfp4", positions=positions.cuda() + ) + torch.testing.assert_close( + decode_residual(residual, previous.cuda()).cpu(), expected, rtol=0, atol=0 + ) + + +@pytest.mark.parametrize("kind,axis", [("key", 0), ("value", 2)]) +def test_default_int4_selection_and_exports(kind, axis): + assert PackedInt4KV is BackendPackedInt4KV + values = torch.randn(65, 1, 17, generator=torch.Generator().manual_seed(412)) + legacy = pack_kv(values, kind=kind) + explicit = pack_kv(values, kind=kind, format="int4") + assert isinstance(legacy, PackedInt4KV) + assert legacy.format == explicit.format == "int4" + assert legacy.axis == explicit.axis == axis + assert legacy.group_size == explicit.group_size == 64 + for name in ("payload", "minimum", "scale"): + torch.testing.assert_close(getattr(legacy, name), getattr(explicit, name), rtol=0, atol=0) + torch.testing.assert_close(unpack_kv(legacy), unpack_kv(explicit), rtol=0, atol=0) + residual = encode_residual(values, unpack_kv(legacy), kind=kind) + assert isinstance(residual, KVResidual) and isinstance(residual.data, PackedInt4KV) + + +def test_bfloat16_mixed_format_residual_chain(): + generator = torch.Generator().manual_seed(614) + raw_previous = torch.randn(9, 2, 17, generator=generator).bfloat16() + base = pack_kv(raw_previous, kind="key", group_size=None) + reconstructed = unpack_kv(base) + expected = reconstructed.clone() + torch.testing.assert_close(reconstructed, raw_previous, rtol=0, atol=0.25) + formats = [base.format] + for format, positions in (("nvfp4", None), ("int4", torch.tensor([0, 3, 7]))): + raw_rows = raw_previous if positions is None else raw_previous[positions] + current = raw_rows + torch.randn(raw_rows.shape, generator=generator).bfloat16() * 0.3 + previous_rows = expected if positions is None else expected[positions] + delta = current - previous_rows + expected_delta = unpack_kv(pack_kv(delta, kind="key", format=format, group_size=None)) + residual = encode_residual( + current, reconstructed, kind="key", format=format, group_size=None, positions=positions + ) + formats.append(residual.data.format) + assert residual.data.group_size == (16 if format == "nvfp4" else 64) + previous_input = reconstructed + saved_previous = previous_input.clone() + reconstructed = decode_residual(residual, previous_input) + torch.testing.assert_close(previous_input, saved_previous, rtol=0, atol=0) + if positions is None: + expected = expected + expected_delta + raw_previous = current + else: + expected[positions] = expected[positions] + expected_delta + inactive = torch.ones(9, dtype=torch.bool) + inactive[positions] = False + torch.testing.assert_close( + reconstructed[inactive], saved_previous[inactive], rtol=0, atol=0 + ) + assert reconstructed.shape == raw_previous.shape + assert reconstructed.dtype == torch.bfloat16 + torch.testing.assert_close(reconstructed, expected, rtol=0, atol=0) + active_rows = reconstructed if positions is None else reconstructed[positions] + torch.testing.assert_close(active_rows, current, rtol=0, atol=0.25) + assert formats == ["int4", "nvfp4", "int4"] + + +def test_unknown_format_is_rejected(): + with pytest.raises(ValueError, match="format"): + pack_kv(torch.ones(1, 1, 16), kind="key", format="unknown") diff --git a/vllm_rlt/kernels/kv_quantization.py b/vllm_rlt/kernels/kv_quantization.py deleted file mode 100644 index 4ec8e1c1..00000000 --- a/vllm_rlt/kernels/kv_quantization.py +++ /dev/null @@ -1,208 +0,0 @@ -"""Affine INT4 KV packing and cross-loop residual reconstruction. - -Inputs are logical ``[token, KV head, channel]`` tensors for one request/layer. -K is grouped across stored token rows per channel; V across channels per token. - -Adapted from FlashLoop packing.py and cache.py at aeaadee10d75b7aa8c4a42a809f15c75d6d3bdc6. -Copyright (c) 2026 FlashLoop contributors. MIT License. -""" - -from dataclasses import dataclass -from typing import Literal - -import torch - -KVKind = Literal["key", "value"] -_DTYPES = {torch.float16, torch.bfloat16, torch.float32, torch.float64} - - -def _validate_inputs(values: torch.Tensor, *, kind: KVKind, group_size: int) -> None: - if group_size <= 0: - raise ValueError("group_size must be positive") - if kind not in {"key", "value"}: - raise ValueError("kind must be 'key' or 'value'") - if values.ndim != 3 or values.shape[2] <= 0: - raise ValueError("KV values must have shape [tokens, heads, positive channels]") - if values.dtype not in _DTYPES: - raise ValueError("KV values must use a floating-point dtype") - - -def _codes_for_group( - values: torch.Tensor, *, reduce_dim: int -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - work = values.float() - minimum = work.amin(dim=reduce_dim, keepdim=True) - span = work.amax(dim=reduce_dim, keepdim=True) - minimum - scale = torch.where(span > 0, span / 15.0, torch.ones_like(span)) - codes = (work - minimum).div_(scale).round_().clamp_(0, 15).to(torch.uint8) - return codes, minimum.to(torch.float16), scale.to(torch.float16) - - -def _pack_channel_nibbles(codes: torch.Tensor) -> torch.Tensor: - payload = codes[..., 0::2].clone(memory_format=torch.contiguous_format) - payload[..., : codes.shape[-1] // 2] |= codes[..., 1::2] << 4 - return payload - - -def _unpack_channel_nibbles(payload: torch.Tensor, channels: int) -> torch.Tensor: - codes = torch.empty((*payload.shape[:-1], channels), dtype=torch.float32, device=payload.device) - codes[..., 0::2] = payload & 0x0F - codes[..., 1::2] = (payload >> 4)[..., : channels // 2] - return codes - - -@dataclass(frozen=True) -class PackedInt4KV: - """Encoded tensor and FP16 affine parameters; treat its tensors as immutable. - - ``payload`` is uint8 ``[T, H, ceil(D/2)]``. For K, parameters are - ``[ceil(T/G), H, D]``; for V, ``[T, H, ceil(D/G)]``. An odd last channel - leaves the high nibble zero. Constant groups use code zero and scale one. - """ - - payload: torch.Tensor - minimum: torch.Tensor - scale: torch.Tensor - original_shape: tuple[int, int, int] - original_dtype: torch.dtype - kind: KVKind - group_size: int - - @property - def storage_bytes(self) -> int: - """Payload and parameter bytes, excluding allocator overhead and decode scratch.""" - return sum(t.numel() * t.element_size() for t in (self.payload, self.minimum, self.scale)) - - -@torch.no_grad() -def pack_kv(values: torch.Tensor, *, kind: KVKind, group_size: int = 64) -> PackedInt4KV: - """Quantize logical KV in FP32, storing uint8 payload and FP16 parameters.""" - group_size = int(group_size) - _validate_inputs(values, kind=kind, group_size=group_size) - shape = tuple(values.shape) - axis = 0 if kind == "key" else 2 - codes = torch.empty_like(values, dtype=torch.uint8) - minima, scales = [], [] - for start in range(0, shape[axis], group_size): - length = min(group_size, shape[axis] - start) - group_codes, minimum, scale = _codes_for_group( - values.narrow(axis, start, length), reduce_dim=axis - ) - codes.narrow(axis, start, length).copy_(group_codes) - minima.append(minimum) - scales.append(scale) - metadata_shape = list(shape) - metadata_shape[axis] = (shape[axis] + group_size - 1) // group_size - minimum = ( - torch.cat(minima, dim=axis) - if minima - else torch.empty(metadata_shape, device=values.device, dtype=torch.float16) - ) - scale = torch.cat(scales, dim=axis) if scales else torch.empty_like(minimum) - payload = _pack_channel_nibbles(codes) - return PackedInt4KV(payload, minimum, scale, shape, values.dtype, kind, group_size) - - -@torch.no_grad() -def unpack_kv(packed: PackedInt4KV) -> torch.Tensor: - """Reconstruct with stored parameters in FP32, then cast to compute dtype.""" - codes = _unpack_channel_nibbles(packed.payload, packed.original_shape[2]) - axis = 0 if packed.kind == "key" else 2 - length = packed.original_shape[axis] - group_size = min(packed.group_size, length) - minimum = packed.minimum.repeat_interleave(group_size, axis).narrow(axis, 0, length) - scale = packed.scale.repeat_interleave(group_size, axis).narrow(axis, 0, length) - return codes.mul_(scale).add_(minimum).to(packed.original_dtype) - - -def _validate_positions(positions: torch.Tensor, rows: int, tokens: int) -> None: - if positions.ndim != 1: - raise ValueError("positions must be one-dimensional") - if positions.numel() != rows: - raise ValueError("positions must identify every compact residual row") - if ( - rows - and ( - (positions[0] < 0) | (positions[-1] >= tokens) | (positions[1:] <= positions[:-1]).any() - ).item() - ): - raise ValueError("positions must be strictly increasing and within the logical token range") - - -@dataclass(frozen=True) -class Int4Residual: - """One depth's compact residual and its logical scatter map. - - ``positions=None`` denotes a dense update; an empty map denotes inheritance - at every position. Positions affect scatter only: K groups follow compact - row order, as in the pinned upstream cache. - """ - - data: PackedInt4KV - positions: torch.Tensor | None - num_tokens: int - - @property - def storage_bytes(self) -> int: - mapping_bytes = ( - 0 if self.positions is None else self.positions.numel() * self.positions.element_size() - ) - return self.data.storage_bytes + mapping_bytes - - -@torch.no_grad() -def encode_residual( - current: torch.Tensor, - reconstructed_previous: torch.Tensor, - *, - kind: KVKind, - positions: torch.Tensor | None = None, - group_size: int = 64, -) -> Int4Residual: - """Encode ``current - reconstructed_previous`` in the input dtype. - - Optional positions map compact rows into the full predecessor and are copied - as contiguous int32 on the input device, matching FlashLoop's index format. - """ - if ( - current.dtype != reconstructed_previous.dtype - or current.device != reconstructed_previous.device - ): - raise ValueError("current and predecessor must have matching dtype and device") - if positions is None: - if current.shape != reconstructed_previous.shape: - raise ValueError("dense residual requires matching current and predecessor shapes") - predecessor = reconstructed_previous - else: - if current.shape[1:] != reconstructed_previous.shape[1:]: - raise ValueError("current and predecessor must have matching heads and channels") - positions = positions.to( - device=current.device, - dtype=torch.int32, - copy=True, - memory_format=torch.contiguous_format, - ) - _validate_positions(positions, current.shape[0], reconstructed_previous.shape[0]) - predecessor = reconstructed_previous.index_select(0, positions) - data = pack_kv(current - predecessor, kind=kind, group_size=group_size) - return Int4Residual(data, positions, reconstructed_previous.shape[0]) - - -@torch.no_grad() -def decode_residual(residual: Int4Residual, previous: torch.Tensor) -> torch.Tensor: - """Return a new full tensor; absent rows retain their previous values.""" - expected_shape = (residual.num_tokens, *residual.data.original_shape[1:]) - if previous.shape != expected_shape: - raise ValueError("predecessor shape does not match residual logical extent") - if ( - previous.dtype != residual.data.original_dtype - or previous.device != residual.data.payload.device - ): - raise ValueError("predecessor dtype and device must match residual") - delta = unpack_kv(residual.data) - if residual.positions is None: - return previous + delta - result = previous.clone() - indices = residual.positions.long() - result.index_copy_(0, indices, previous.index_select(0, indices).add_(delta)) - return result diff --git a/vllm_rlt/kernels/kv_quantization/__init__.py b/vllm_rlt/kernels/kv_quantization/__init__.py new file mode 100644 index 00000000..d3226611 --- /dev/null +++ b/vllm_rlt/kernels/kv_quantization/__init__.py @@ -0,0 +1,138 @@ +"""Logical INT4/NVFP4 KV codecs and cross-loop residual reconstruction. + +Inputs are [token, KV head, channel] tensors for one request/layer. +Residual arithmetic adapts FlashLoop cache.py at +aeaadee10d75b7aa8c4a42a809f15c75d6d3bdc6. +Copyright (c) 2026 FlashLoop contributors. MIT License; see NOTICE. +""" + +from dataclasses import dataclass +from typing import Literal + +import torch + +from . import int4, nvfp4 +from ._common import KVKind +from .int4 import PackedInt4KV +from .nvfp4 import PackedNVFP4KV + +KVFormat = Literal["int4", "nvfp4"] +PackedKV = PackedInt4KV | PackedNVFP4KV + + +def pack_kv( + values: torch.Tensor, + *, + kind: KVKind, + format: KVFormat = "int4", + group_size: int | None = None, +) -> PackedKV: + """Encode logical KV; defaults are INT4 G=64 or NVFP4 G=16.""" + if format == "int4": + return int4.pack(values, kind=kind, group_size=64 if group_size is None else group_size) + if format == "nvfp4": + return nvfp4.pack(values, kind=kind, group_size=16 if group_size is None else group_size) + raise ValueError("format must be 'int4' or 'nvfp4'") + + +def unpack_kv(packed: PackedKV) -> torch.Tensor: + """Decode using the format carried by the packed tensor.""" + if isinstance(packed, PackedInt4KV): + return int4.unpack(packed) + if isinstance(packed, PackedNVFP4KV): + return nvfp4.unpack(packed) + raise TypeError("expected PackedInt4KV or PackedNVFP4KV") + + +def _validate_positions(positions: torch.Tensor, rows: int, tokens: int) -> None: + if positions.ndim != 1: + raise ValueError("positions must be one-dimensional") + if positions.numel() != rows: + raise ValueError("positions must identify every compact residual row") + if ( + rows + and ( + (positions[0] < 0) | (positions[-1] >= tokens) | (positions[1:] <= positions[:-1]).any() + ).item() + ): + raise ValueError("positions must be strictly increasing and within the logical token range") + + +@dataclass(frozen=True) +class KVResidual: + """One depth's compact residual and its logical scatter map. + + ``positions=None`` denotes a dense update; an empty map denotes inheritance + at every position. K groups follow compact row order for both formats; + V groups remain within each compact row, along channels. + """ + + data: PackedKV + positions: torch.Tensor | None + num_tokens: int + + @property + def storage_bytes(self) -> int: + mapping_bytes = ( + 0 if self.positions is None else self.positions.numel() * self.positions.element_size() + ) + return self.data.storage_bytes + mapping_bytes + + +@torch.no_grad() +def encode_residual( + current: torch.Tensor, + reconstructed_previous: torch.Tensor, + *, + kind: KVKind, + positions: torch.Tensor | None = None, + format: KVFormat = "int4", + group_size: int | None = None, +) -> KVResidual: + """Encode ``current - reconstructed_previous`` in the input dtype. + + Optional positions map compact rows into the full predecessor and are copied + as contiguous int32 on the input device, matching FlashLoop's index format. + """ + if ( + current.dtype != reconstructed_previous.dtype + or current.device != reconstructed_previous.device + ): + raise ValueError("current and predecessor must have matching dtype and device") + if positions is None: + if current.shape != reconstructed_previous.shape: + raise ValueError("dense residual requires matching current and predecessor shapes") + predecessor = reconstructed_previous + else: + if current.shape[1:] != reconstructed_previous.shape[1:]: + raise ValueError("current and predecessor must have matching heads and channels") + positions = positions.to( + device=current.device, + dtype=torch.int32, + copy=True, + memory_format=torch.contiguous_format, + ) + _validate_positions(positions, current.shape[0], reconstructed_previous.shape[0]) + predecessor = reconstructed_previous.index_select(0, positions) + data = pack_kv(current - predecessor, kind=kind, format=format, group_size=group_size) + return KVResidual(data, positions, reconstructed_previous.shape[0]) + + +@torch.no_grad() +def decode_residual(residual: KVResidual, previous: torch.Tensor) -> torch.Tensor: + """Return a new full tensor; absent rows retain their previous values.""" + expected_shape = (residual.num_tokens, *residual.data.original_shape[1:]) + if previous.shape != expected_shape: + raise ValueError("predecessor shape does not match residual logical extent") + if ( + previous.dtype != residual.data.original_dtype + or previous.device != residual.data.payload.device + ): + raise ValueError("predecessor dtype and device must match residual") + delta = unpack_kv(residual.data) + if residual.positions is None: + return previous + delta + result = previous.clone() + indices = residual.positions.long() + result.index_copy_(0, indices, previous.index_select(0, indices).add_(delta)) + return result diff --git a/vllm_rlt/kernels/kv_quantization/_common.py b/vllm_rlt/kernels/kv_quantization/_common.py new file mode 100644 index 00000000..177e3a5b --- /dev/null +++ b/vllm_rlt/kernels/kv_quantization/_common.py @@ -0,0 +1,38 @@ +"""Shared logical KV validation and nibble packing. + +Adapted from FlashLoop packing.py at aeaadee10d75b7aa8c4a42a809f15c75d6d3bdc6. +Copyright (c) 2026 FlashLoop contributors. MIT License; see NOTICE. +""" + +from typing import Literal + +import torch + +KVKind = Literal["key", "value"] +_DTYPES = {torch.float16, torch.bfloat16, torch.float32, torch.float64} + + +def _validate_inputs(values: torch.Tensor, *, kind: KVKind, group_size: int) -> None: + if group_size <= 0: + raise ValueError("group_size must be positive") + if kind not in {"key", "value"}: + raise ValueError("kind must be 'key' or 'value'") + if values.ndim != 3 or values.shape[2] <= 0: + raise ValueError("KV values must have shape [tokens, heads, positive channels]") + if values.dtype not in _DTYPES: + raise ValueError("KV values must use a floating-point dtype") + + +def _pack_channel_nibbles(codes: torch.Tensor) -> torch.Tensor: + payload = codes[..., 0::2].clone(memory_format=torch.contiguous_format) + payload[..., : codes.shape[-1] // 2] |= codes[..., 1::2] << 4 + return payload + + +def _unpack_channel_nibbles( + payload: torch.Tensor, channels: int, *, dtype: torch.dtype = torch.float32 +) -> torch.Tensor: + codes = torch.empty((*payload.shape[:-1], channels), dtype=dtype, device=payload.device) + codes[..., 0::2] = payload & 0x0F + codes[..., 1::2] = (payload >> 4)[..., : channels // 2] + return codes diff --git a/vllm_rlt/kernels/kv_quantization/int4.py b/vllm_rlt/kernels/kv_quantization/int4.py new file mode 100644 index 00000000..b9b7a7e0 --- /dev/null +++ b/vllm_rlt/kernels/kv_quantization/int4.py @@ -0,0 +1,93 @@ +"""Affine INT4 reference codec for logical [token, KV head, channel] tensors. + +Adapted from FlashLoop packing.py at aeaadee10d75b7aa8c4a42a809f15c75d6d3bdc6. +Copyright (c) 2026 FlashLoop contributors. MIT License; see NOTICE. +""" + +from dataclasses import dataclass +from typing import ClassVar, Literal + +import torch + +from ._common import KVKind, _pack_channel_nibbles, _unpack_channel_nibbles, _validate_inputs + + +def _codes_for_group( + values: torch.Tensor, *, reduce_dim: int +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + work = values.float() + minimum = work.amin(dim=reduce_dim, keepdim=True) + span = work.amax(dim=reduce_dim, keepdim=True) - minimum + scale = torch.where(span > 0, span / 15.0, torch.ones_like(span)) + codes = (work - minimum).div_(scale).round_().clamp_(0, 15).to(torch.uint8) + return codes, minimum.to(torch.float16), scale.to(torch.float16) + + +@dataclass(frozen=True) +class PackedInt4KV: + """Encoded tensor and FP16 affine parameters; treat its tensors as immutable. + + ``payload`` is uint8 ``[T, H, ceil(D/2)]``. For K, parameters are + ``[ceil(T/G), H, D]``; for V, ``[T, H, ceil(D/G)]``. An odd last channel + leaves the high nibble zero. Constant groups use code zero and scale one. + """ + + format: ClassVar[Literal["int4"]] = "int4" + + payload: torch.Tensor + minimum: torch.Tensor + scale: torch.Tensor + original_shape: tuple[int, int, int] + original_dtype: torch.dtype + kind: KVKind + group_size: int + + @property + def axis(self) -> int: + return 0 if self.kind == "key" else 2 + + @property + def storage_bytes(self) -> int: + """Payload and parameter bytes, excluding allocator overhead and decode scratch.""" + return sum(t.numel() * t.element_size() for t in (self.payload, self.minimum, self.scale)) + + +@torch.no_grad() +def pack(values: torch.Tensor, *, kind: KVKind, group_size: int = 64) -> PackedInt4KV: + """Quantize logical KV in FP32, storing uint8 payload and FP16 parameters.""" + group_size = int(group_size) + _validate_inputs(values, kind=kind, group_size=group_size) + shape = tuple(values.shape) + axis = 0 if kind == "key" else 2 + codes = torch.empty_like(values, dtype=torch.uint8) + minima, scales = [], [] + for start in range(0, shape[axis], group_size): + length = min(group_size, shape[axis] - start) + group_codes, minimum, scale = _codes_for_group( + values.narrow(axis, start, length), reduce_dim=axis + ) + codes.narrow(axis, start, length).copy_(group_codes) + minima.append(minimum) + scales.append(scale) + metadata_shape = list(shape) + metadata_shape[axis] = (shape[axis] + group_size - 1) // group_size + minimum = ( + torch.cat(minima, dim=axis) + if minima + else torch.empty(metadata_shape, device=values.device, dtype=torch.float16) + ) + scale = torch.cat(scales, dim=axis) if scales else torch.empty_like(minimum) + payload = _pack_channel_nibbles(codes) + return PackedInt4KV(payload, minimum, scale, shape, values.dtype, kind, group_size) + + +@torch.no_grad() +def unpack(packed: PackedInt4KV) -> torch.Tensor: + """Reconstruct with stored parameters in FP32, then cast to compute dtype.""" + codes = _unpack_channel_nibbles(packed.payload, packed.original_shape[2]) + axis = packed.axis + length = packed.original_shape[axis] + group_size = min(packed.group_size, length) + minimum = packed.minimum.repeat_interleave(group_size, axis).narrow(axis, 0, length) + scale = packed.scale.repeat_interleave(group_size, axis).narrow(axis, 0, length) + return codes.mul_(scale).add_(minimum).to(packed.original_dtype) diff --git a/vllm_rlt/kernels/kv_quantization/nvfp4.py b/vllm_rlt/kernels/kv_quantization/nvfp4.py new file mode 100644 index 00000000..64e2c8e3 --- /dev/null +++ b/vllm_rlt/kernels/kv_quantization/nvfp4.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""NVFP4 reference for logical [token, KV head, channel] tensors. + +Adapted from vLLM ref_nvfp4_quant / ref_nvfp4_quant_dequant at +c7a760751977e4e3396d586956d8767a9bc44a82 in +vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py. +Unlike the hardware layout, this logical format also supports partial groups. +""" + +from dataclasses import dataclass +from typing import ClassVar, Literal + +import torch + +from ._common import KVKind, _pack_channel_nibbles, _unpack_channel_nibbles, _validate_inputs + + +@dataclass(frozen=True) +class PackedNVFP4KV: + """Uint8 payload, E4M3 block scales and an FP32 global encoding multiplier. + + K groups 16 tokens per channel; V groups 16 channels per token. + ``scale`` is [ceil(T/16), H, D] for K or [T, H, ceil(D/16)] for V. + ``global_scale`` is shared by this encoded tensor. Decode multiplies + E2M1 values by ``scale / global_scale``. Treat tensors as immutable. + """ + + format: ClassVar[Literal["nvfp4"]] = "nvfp4" + + payload: torch.Tensor + scale: torch.Tensor + global_scale: torch.Tensor + original_shape: tuple[int, int, int] + original_dtype: torch.dtype + kind: KVKind + group_size: int = 16 + + @property + def axis(self) -> int: + return 0 if self.kind == "key" else 2 + + @property + def storage_bytes(self) -> int: + return sum( + t.numel() * t.element_size() for t in (self.payload, self.scale, self.global_scale) + ) + + +@torch.no_grad() +def pack(values: torch.Tensor, *, kind: KVKind, group_size: int = 16) -> PackedNVFP4KV: + """Encode E2M1 with nearest-even rounding and FP8-rounded block scales.""" + group_size = int(group_size) + _validate_inputs(values, kind=kind, group_size=group_size) + if group_size != 16: + raise ValueError("NVFP4 requires group_size=16") + shape = tuple(values.shape) + axis = 0 if kind == "key" else 2 + global_scale = torch.ones((), device=values.device, dtype=torch.float32) + if values.numel(): + amax = values.abs().amax().float() + global_scale = torch.where(amax > 0, 2688.0 / amax, global_scale).clamp_max_( + torch.finfo(torch.float32).max + ) + inverse_global = global_scale.reciprocal() + codes = torch.empty_like(values, dtype=torch.uint8) + scale_shape = list(shape) + scale_shape[axis] = (shape[axis] + 15) // 16 + scales = torch.empty( + scale_shape, + dtype=torch.float8_e4m3fn, + device=values.device, + ) + # bucketize keeps equality in the lower bin. Move odd lower-code boundaries + # down one FP32 ULP so midpoint ties choose the even upper code instead. + boundaries = torch.tensor( + [0.25, 0.75 - 2**-24, 1.25, 1.75 - 2**-23, 2.5, 3.5 - 2**-22, 5.0], + device=values.device, + dtype=torch.float32, + ) + for start in range(0, shape[axis], 16): + length = min(16, shape[axis] - start) + work = values.narrow(axis, start, length).float() + scale = (global_scale * (work.abs().amax(dim=axis, keepdim=True) * (1.0 / 6))).clamp_( + 0, 448 + ) + scale = scale.to(torch.float8_e4m3fn) + effective_scale = scale.float() * inverse_global + multiplier = effective_scale.reciprocal() + finite_multiplier = torch.isfinite(multiplier) + normalized = work * torch.where(finite_multiplier, multiplier, 0.0) + # Preserve tiny values when only the FP32 reciprocal overflows. + normalized = torch.where( + (effective_scale > 0) & ~finite_multiplier, work / effective_scale, normalized + ).clamp_(-6, 6) + magnitude = normalized.abs().contiguous() + group_codes = torch.bucketize(magnitude, boundaries, out_int32=True).to(torch.uint8) + group_codes |= torch.signbit(normalized).to(torch.uint8) << 3 + codes.narrow(axis, start, length).copy_(group_codes) + scales.select(axis, start // 16).copy_(scale.squeeze(axis)) + return PackedNVFP4KV( + _pack_channel_nibbles(codes), scales, global_scale, shape, values.dtype, kind + ) + + +@torch.no_grad() +def unpack(packed: PackedNVFP4KV) -> torch.Tensor: + """Decode with stored block/global scales in FP32, then cast to compute dtype.""" + channels = packed.original_shape[2] + codes = _unpack_channel_nibbles(packed.payload, channels, dtype=torch.uint8) + magnitudes = torch.tensor( + [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], + device=packed.payload.device, + dtype=torch.float32, + ) + values = torch.index_select(magnitudes, 0, (codes & 7).flatten().to(torch.int32)).reshape( + codes.shape + ) + values.copysign_((codes << 4).view(torch.int8)) + axis = packed.axis + scale = (packed.scale.float() / packed.global_scale).repeat_interleave(16, dim=axis) + return values.mul_(scale.narrow(axis, 0, packed.original_shape[axis])).to(packed.original_dtype) From c8a9770409f6af02772299d90d19feb45bfbd4b7 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Fri, 6 Nov 2026 12:29:37 +0100 Subject: [PATCH 03/15] perf(kv): vectorize INT4/NVFP4 packing and add paged helpers Replace per-group Python loops with padded reshape reductions: INT4 key packing 0.091s -> 0.003s and NVFP4 key packing 0.680s -> 0.002s for 32k x 8 x 128 on GH200, with byte-identical payloads and 313 tests passing. Adds a codec registry and pack_paged/unpack_paged. (cherry picked from commit 606d15d8dce0c6d1de45017eece6139164978bab) --- tests/test_int4_kv_quantization.py | 13 ++++ vllm_rlt/kernels/kv_quantization/__init__.py | 32 +++++++-- vllm_rlt/kernels/kv_quantization/_common.py | 14 ++++ vllm_rlt/kernels/kv_quantization/int4.py | 58 +++++++++------ vllm_rlt/kernels/kv_quantization/nvfp4.py | 76 ++++++++++++-------- 5 files changed, 137 insertions(+), 56 deletions(-) diff --git a/tests/test_int4_kv_quantization.py b/tests/test_int4_kv_quantization.py index c35d85bc..ba271d92 100644 --- a/tests/test_int4_kv_quantization.py +++ b/tests/test_int4_kv_quantization.py @@ -49,6 +49,19 @@ def scalar_reference(values, kind, group_size): return torch.from_numpy(payload), torch.from_numpy(decoded).to(values.dtype) +def test_paged_round_trip_and_registry_dispatch(): + from vllm_rlt.kernels.kv_quantization import pack_paged, unpack_paged + + torch.manual_seed(3) + pool = torch.randn(4, 16, 2, 8, dtype=torch.bfloat16) + for kind in ("key", "value"): + packed, paged_shape = pack_paged(pool, kind=kind, format="int4", group_size=4) + restored = unpack_paged(packed, paged_shape) + logical = pack_kv(pool.reshape(64, 2, 8), kind=kind, format="int4", group_size=4) + torch.testing.assert_close(restored, unpack_kv(logical).reshape(pool.shape)) + assert restored.shape == pool.shape + + def test_known_nibble_order_and_ties_to_even(): values = torch.tensor([0, 0.5, 1.5, 2.5, 14.5, 15]).reshape(1, 1, 6) packed = pack_kv(values, kind="value", group_size=6) diff --git a/vllm_rlt/kernels/kv_quantization/__init__.py b/vllm_rlt/kernels/kv_quantization/__init__.py index d3226611..67f5121b 100644 --- a/vllm_rlt/kernels/kv_quantization/__init__.py +++ b/vllm_rlt/kernels/kv_quantization/__init__.py @@ -19,6 +19,8 @@ KVFormat = Literal["int4", "nvfp4"] PackedKV = PackedInt4KV | PackedNVFP4KV +CODECS = {"int4": int4, "nvfp4": nvfp4} + def pack_kv( values: torch.Tensor, @@ -28,11 +30,31 @@ def pack_kv( group_size: int | None = None, ) -> PackedKV: """Encode logical KV; defaults are INT4 G=64 or NVFP4 G=16.""" - if format == "int4": - return int4.pack(values, kind=kind, group_size=64 if group_size is None else group_size) - if format == "nvfp4": - return nvfp4.pack(values, kind=kind, group_size=16 if group_size is None else group_size) - raise ValueError("format must be 'int4' or 'nvfp4'") + codec = CODECS.get(format) + if codec is None: + raise ValueError("format must be 'int4' or 'nvfp4'") + if group_size is None: + group_size = 64 if format == "int4" else 16 + return codec.pack(values, kind=kind, group_size=group_size) + + +def pack_paged(pool: torch.Tensor, *, kind: KVKind, format: KVFormat = "int4", group_size=None): + """Pack a paged pool ``[pages, page_size, heads, dim]`` as one logical KV.""" + pages, page = pool.shape[:2] + packed = pack_kv( + pool.reshape(pages * page, *pool.shape[2:]), + kind=kind, + format=format, + group_size=group_size, + ) + return packed, (pages, page) + + +def unpack_paged(packed: PackedKV, paged_shape: tuple[int, int]) -> torch.Tensor: + """Restore a pool packed by :func:`pack_paged` to its original page layout.""" + pages, page = paged_shape + logical = unpack_kv(packed) + return logical.reshape(pages, page, *logical.shape[1:]) def unpack_kv(packed: PackedKV) -> torch.Tensor: diff --git a/vllm_rlt/kernels/kv_quantization/_common.py b/vllm_rlt/kernels/kv_quantization/_common.py index 177e3a5b..1e82e027 100644 --- a/vllm_rlt/kernels/kv_quantization/_common.py +++ b/vllm_rlt/kernels/kv_quantization/_common.py @@ -23,6 +23,20 @@ def _validate_inputs(values: torch.Tensor, *, kind: KVKind, group_size: int) -> raise ValueError("KV values must use a floating-point dtype") +def _pad_repeat(values: torch.Tensor, axis: int, multiple: int) -> torch.Tensor: + """Pad ``axis`` to a multiple by repeating the last slice (keeps extremes).""" + length = values.shape[axis] + pad = (-length) % multiple + if pad == 0: + return values + index = [slice(None)] * values.ndim + index[axis] = slice(length - 1, length) + tail = values[tuple(index)] + shape = list(values.shape) + shape[axis] = pad + return torch.cat([values, tail.expand(shape)], dim=axis) + + def _pack_channel_nibbles(codes: torch.Tensor) -> torch.Tensor: payload = codes[..., 0::2].clone(memory_format=torch.contiguous_format) payload[..., : codes.shape[-1] // 2] |= codes[..., 1::2] << 4 diff --git a/vllm_rlt/kernels/kv_quantization/int4.py b/vllm_rlt/kernels/kv_quantization/int4.py index b9b7a7e0..c9a959b9 100644 --- a/vllm_rlt/kernels/kv_quantization/int4.py +++ b/vllm_rlt/kernels/kv_quantization/int4.py @@ -9,7 +9,13 @@ import torch -from ._common import KVKind, _pack_channel_nibbles, _unpack_channel_nibbles, _validate_inputs +from ._common import ( + KVKind, + _pack_channel_nibbles, + _pad_repeat, + _unpack_channel_nibbles, + _validate_inputs, +) def _codes_for_group( @@ -59,26 +65,38 @@ def pack(values: torch.Tensor, *, kind: KVKind, group_size: int = 64) -> PackedI _validate_inputs(values, kind=kind, group_size=group_size) shape = tuple(values.shape) axis = 0 if kind == "key" else 2 - codes = torch.empty_like(values, dtype=torch.uint8) - minima, scales = [], [] - for start in range(0, shape[axis], group_size): - length = min(group_size, shape[axis] - start) - group_codes, minimum, scale = _codes_for_group( - values.narrow(axis, start, length), reduce_dim=axis - ) - codes.narrow(axis, start, length).copy_(group_codes) - minima.append(minimum) - scales.append(scale) - metadata_shape = list(shape) - metadata_shape[axis] = (shape[axis] + group_size - 1) // group_size - minimum = ( - torch.cat(minima, dim=axis) - if minima - else torch.empty(metadata_shape, device=values.device, dtype=torch.float16) - ) - scale = torch.cat(scales, dim=axis) if scales else torch.empty_like(minimum) + work = values.float() + padded = _pad_repeat(work, axis, group_size) + if kind == "key": + groups = padded.shape[0] // group_size + grouped = padded.view(groups, group_size, *shape[1:]) + minimum = grouped.amin(dim=1, keepdim=True) + span = grouped.amax(dim=1, keepdim=True) - minimum + scale = torch.where(span > 0, span / 15.0, torch.ones_like(span)) + codes = ((grouped - minimum) / scale).round_().clamp_(0, 15).to(torch.uint8) + codes = codes.view(padded.shape)[: shape[0]] + minimum = minimum.squeeze(1) + scale = scale.squeeze(1) + else: + groups = padded.shape[2] // group_size + grouped = padded.view(shape[0], shape[1], groups, group_size) + minimum = grouped.amin(dim=-1, keepdim=True) + span = grouped.amax(dim=-1, keepdim=True) - minimum + scale = torch.where(span > 0, span / 15.0, torch.ones_like(span)) + codes = ((grouped - minimum) / scale).round_().clamp_(0, 15).to(torch.uint8) + codes = codes.view(padded.shape)[..., : shape[2]] + minimum = minimum.squeeze(-1) + scale = scale.squeeze(-1) payload = _pack_channel_nibbles(codes) - return PackedInt4KV(payload, minimum, scale, shape, values.dtype, kind, group_size) + return PackedInt4KV( + payload, + minimum.to(torch.float16), + scale.to(torch.float16), + shape, + values.dtype, + kind, + group_size, + ) @torch.no_grad() diff --git a/vllm_rlt/kernels/kv_quantization/nvfp4.py b/vllm_rlt/kernels/kv_quantization/nvfp4.py index 64e2c8e3..a964ab8a 100644 --- a/vllm_rlt/kernels/kv_quantization/nvfp4.py +++ b/vllm_rlt/kernels/kv_quantization/nvfp4.py @@ -13,7 +13,13 @@ import torch -from ._common import KVKind, _pack_channel_nibbles, _unpack_channel_nibbles, _validate_inputs +from ._common import ( + KVKind, + _pack_channel_nibbles, + _pad_repeat, + _unpack_channel_nibbles, + _validate_inputs, +) @dataclass(frozen=True) @@ -56,48 +62,56 @@ def pack(values: torch.Tensor, *, kind: KVKind, group_size: int = 16) -> PackedN raise ValueError("NVFP4 requires group_size=16") shape = tuple(values.shape) axis = 0 if kind == "key" else 2 + work = values.float() global_scale = torch.ones((), device=values.device, dtype=torch.float32) if values.numel(): - amax = values.abs().amax().float() + amax = work.abs().amax() global_scale = torch.where(amax > 0, 2688.0 / amax, global_scale).clamp_max_( torch.finfo(torch.float32).max ) inverse_global = global_scale.reciprocal() - codes = torch.empty_like(values, dtype=torch.uint8) - scale_shape = list(shape) - scale_shape[axis] = (shape[axis] + 15) // 16 - scales = torch.empty( - scale_shape, - dtype=torch.float8_e4m3fn, - device=values.device, - ) - # bucketize keeps equality in the lower bin. Move odd lower-code boundaries - # down one FP32 ULP so midpoint ties choose the even upper code instead. boundaries = torch.tensor( [0.25, 0.75 - 2**-24, 1.25, 1.75 - 2**-23, 2.5, 3.5 - 2**-22, 5.0], device=values.device, dtype=torch.float32, ) - for start in range(0, shape[axis], 16): - length = min(16, shape[axis] - start) - work = values.narrow(axis, start, length).float() - scale = (global_scale * (work.abs().amax(dim=axis, keepdim=True) * (1.0 / 6))).clamp_( - 0, 448 + padded = _pad_repeat(work, axis, 16) + if kind == "key": + groups = padded.shape[0] // 16 + grouped = padded.view(groups, 16, *shape[1:]) + amax = grouped.abs().amax(dim=1, keepdim=True) + scale = (global_scale * (amax * (1.0 / 6))).clamp_(0, 448).to(torch.float8_e4m3fn) + effective = scale.float() * inverse_global + multiplier = effective.reciprocal() + finite = torch.isfinite(multiplier) + normalized = grouped * torch.where(finite, multiplier, 0.0) + normalized = torch.where((effective > 0) & ~finite, grouped / effective, normalized).clamp_( + -6, 6 + ) + codes = torch.bucketize(normalized.abs().contiguous(), boundaries, out_int32=True).to( + torch.uint8 + ) + codes |= torch.signbit(normalized).to(torch.uint8) << 3 + codes = codes.view(padded.shape)[: shape[0]] + scales = scale.squeeze(1) + else: + groups = padded.shape[2] // 16 + grouped = padded.view(shape[0], shape[1], groups, 16) + amax = grouped.abs().amax(dim=-1, keepdim=True) + scale = (global_scale * (amax * (1.0 / 6))).clamp_(0, 448).to(torch.float8_e4m3fn) + effective = scale.float() * inverse_global + multiplier = effective.reciprocal() + finite = torch.isfinite(multiplier) + normalized = grouped * torch.where(finite, multiplier, 0.0) + normalized = torch.where((effective > 0) & ~finite, grouped / effective, normalized).clamp_( + -6, 6 + ) + codes = torch.bucketize(normalized.abs().contiguous(), boundaries, out_int32=True).to( + torch.uint8 ) - scale = scale.to(torch.float8_e4m3fn) - effective_scale = scale.float() * inverse_global - multiplier = effective_scale.reciprocal() - finite_multiplier = torch.isfinite(multiplier) - normalized = work * torch.where(finite_multiplier, multiplier, 0.0) - # Preserve tiny values when only the FP32 reciprocal overflows. - normalized = torch.where( - (effective_scale > 0) & ~finite_multiplier, work / effective_scale, normalized - ).clamp_(-6, 6) - magnitude = normalized.abs().contiguous() - group_codes = torch.bucketize(magnitude, boundaries, out_int32=True).to(torch.uint8) - group_codes |= torch.signbit(normalized).to(torch.uint8) << 3 - codes.narrow(axis, start, length).copy_(group_codes) - scales.select(axis, start // 16).copy_(scale.squeeze(axis)) + codes |= torch.signbit(normalized).to(torch.uint8) << 3 + codes = codes.view(padded.shape)[..., : shape[2]] + scales = scale.squeeze(-1) return PackedNVFP4KV( _pack_channel_nibbles(codes), scales, global_scale, shape, values.dtype, kind ) From e552a8a4458aa188979556e929beda613a29265d Mon Sep 17 00:00:00 2001 From: Howard Yang <284181151+HowardYangAI@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:39:14 +0800 Subject: [PATCH 04/15] refactor(kv): add codec protocol and registry - Dispatch pack_kv/unpack_kv through KV_CODECS - Expose group_axis/scale_shape on codecs and packed tensors - Share one grouped path for K and V - Mark decode and position checks as reference/debug paths --- tests/test_kv_codec.py | 82 +++++++++++++++ vllm_rlt/kernels/kv_quantization/__init__.py | 102 +++++++++++++------ vllm_rlt/kernels/kv_quantization/_common.py | 46 ++++++++- vllm_rlt/kernels/kv_quantization/int4.py | 76 ++++++++------ vllm_rlt/kernels/kv_quantization/nvfp4.py | 97 +++++++++++------- 5 files changed, 301 insertions(+), 102 deletions(-) create mode 100644 tests/test_kv_codec.py diff --git a/tests/test_kv_codec.py b/tests/test_kv_codec.py new file mode 100644 index 00000000..0ea2f39b --- /dev/null +++ b/tests/test_kv_codec.py @@ -0,0 +1,82 @@ +from dataclasses import dataclass, fields + +import pytest +import torch + +from vllm_rlt.kernels.kv_quantization import ( + KV_CODECS, + decode_residual, + encode_residual, + pack_kv, + unpack_kv, +) + +_FORMATS = [("int4", 64), ("int4", 16), ("nvfp4", 16)] + + +@pytest.mark.parametrize("format,group_size", _FORMATS) +@pytest.mark.parametrize("kind,axis", [("key", 0), ("value", 2)]) +@pytest.mark.parametrize("shape", [(64, 2, 64), (5, 2, 7), (0, 2, 7)]) +def test_codec_metadata_and_actual_storage(format, group_size, kind, axis, shape, record_property): + codec = KV_CODECS[format] + values = torch.ones(shape, dtype=torch.bfloat16) + packed = pack_kv(values, kind=kind, format=format, group_size=group_size) + assert packed.format == codec.format == format + assert codec.storage_dtype == packed.payload.dtype + assert packed.group_axis == codec.group_axis(kind) == axis + expected_shape = list(shape) + expected_shape[axis] = (shape[axis] + group_size - 1) // group_size + assert ( + packed.scale_shape + == codec.scale_shape(shape, group_size, kind=kind) + == tuple(expected_shape) + ) + tensor_bytes = sum( + tensor.numel() * tensor.element_size() + for field in fields(packed) + if isinstance(tensor := getattr(packed, field.name), torch.Tensor) + ) + assert packed.storage_bytes == codec.storage_bytes(packed) == tensor_bytes + if values.numel(): + bits_per_element = tensor_bytes * 8 / values.numel() + record_property("bits_per_element", bits_per_element) + if shape == (64, 2, 64): + expected = 4 + 32 / group_size if format == "int4" else 4.5 + 32 / values.numel() + assert bits_per_element == expected + + +def test_registered_codec_works_without_format_dispatch_changes(monkeypatch): + @dataclass(frozen=True) + class Packed: + payload: torch.Tensor + original_dtype: torch.dtype + format = "test_bf16" + + @property + def original_shape(self): + return tuple(self.payload.shape) + + class Codec: + default_group_size = 1 + + @staticmethod + def pack(values, *, kind, group_size): + assert kind == "key" and group_size == 1 + return Packed(values.clone(), values.dtype) + + @staticmethod + def unpack(packed): + return packed.payload.clone() + + monkeypatch.setitem(KV_CODECS, "test_bf16", Codec) + values = torch.arange(12, dtype=torch.bfloat16).reshape(3, 1, 4) + previous = unpack_kv(pack_kv(values, kind="key", format="test_bf16")) + torch.testing.assert_close(previous, values, atol=0, rtol=0) + positions = torch.tensor([0, 2]) + current = previous[positions] + 0.5 + residual = encode_residual( + current, previous, kind="key", format="test_bf16", positions=positions + ) + expected = previous.clone() + expected[positions] = current + torch.testing.assert_close(decode_residual(residual, previous), expected, atol=0, rtol=0) diff --git a/vllm_rlt/kernels/kv_quantization/__init__.py b/vllm_rlt/kernels/kv_quantization/__init__.py index 67f5121b..9313467c 100644 --- a/vllm_rlt/kernels/kv_quantization/__init__.py +++ b/vllm_rlt/kernels/kv_quantization/__init__.py @@ -7,39 +7,82 @@ """ from dataclasses import dataclass -from typing import Literal +from typing import Any, Protocol, TypeVar import torch from . import int4, nvfp4 -from ._common import KVKind -from .int4 import PackedInt4KV -from .nvfp4 import PackedNVFP4KV +from ._common import KVKind, PackedGroupKV +from .int4 import PackedInt4KV as PackedInt4KV +from .nvfp4 import PackedNVFP4KV as PackedNVFP4KV -KVFormat = Literal["int4", "nvfp4"] -PackedKV = PackedInt4KV | PackedNVFP4KV +_Packed = TypeVar("_Packed", bound=PackedGroupKV) -CODECS = {"int4": int4, "nvfp4": nvfp4} + +class KVCodec(Protocol[_Packed]): + """Logical codec contract; pack/unpack here are reference operations. + + Each codec accepts only the packed type its own ``pack`` returns. + + Group axes are indices in [token, head, channel], resolved by kind. + Serving attention stays format-specific and need not materialize decoded KV. + """ + + format: str + storage_dtype: torch.dtype + default_group_size: int + + def group_axis(self, kind: KVKind) -> int: ... + + def scale_shape( + self, shape: tuple[int, int, int], group_size: int, *, kind: KVKind + ) -> tuple[int, ...]: ... + + def pack(self, values: torch.Tensor, *, kind: KVKind, group_size: int) -> _Packed: ... + + def unpack(self, packed: _Packed) -> torch.Tensor: ... + + def storage_bytes(self, packed: _Packed) -> int: ... + + +KV_CODECS: dict[str, KVCodec[Any]] = {"int4": int4, "nvfp4": nvfp4} def pack_kv( values: torch.Tensor, *, kind: KVKind, - format: KVFormat = "int4", + format: str = "int4", group_size: int | None = None, -) -> PackedKV: - """Encode logical KV; defaults are INT4 G=64 or NVFP4 G=16.""" - codec = CODECS.get(format) - if codec is None: - raise ValueError("format must be 'int4' or 'nvfp4'") - if group_size is None: - group_size = 64 if format == "int4" else 16 - return codec.pack(values, kind=kind, group_size=group_size) - - -def pack_paged(pool: torch.Tensor, *, kind: KVKind, format: KVFormat = "int4", group_size=None): - """Pack a paged pool ``[pages, page_size, heads, dim]`` as one logical KV.""" +) -> PackedGroupKV: + """Reference encode; defaults are INT4 G=64 or NVFP4 G=16.""" + if format not in KV_CODECS: + raise ValueError(f"unknown KV format: {format!r}") + codec = KV_CODECS[format] + return codec.pack( + values, kind=kind, group_size=codec.default_group_size if group_size is None else group_size + ) + + +def unpack_kv(packed: PackedGroupKV) -> torch.Tensor: + """Reference/debug full decode using the packed tensor's registered format.""" + if packed.format not in KV_CODECS: + raise ValueError(f"unknown KV format: {packed.format!r}") + return KV_CODECS[packed.format].unpack(packed) + + +def pack_paged( + pool: torch.Tensor, + *, + kind: KVKind, + format: str = "int4", + group_size: int | None = None, +) -> tuple[PackedGroupKV, tuple[int, int]]: + """Encode a pool ``[pages, page, heads, dim]`` as one logical KV tensor. + + Groups follow logical token order and may span physical pages. The returned + paged shape is what :func:`unpack_paged` needs to rebuild the layout. + """ pages, page = pool.shape[:2] packed = pack_kv( pool.reshape(pages * page, *pool.shape[2:]), @@ -50,22 +93,13 @@ def pack_paged(pool: torch.Tensor, *, kind: KVKind, format: KVFormat = "int4", g return packed, (pages, page) -def unpack_paged(packed: PackedKV, paged_shape: tuple[int, int]) -> torch.Tensor: - """Restore a pool packed by :func:`pack_paged` to its original page layout.""" +def unpack_paged(packed: PackedGroupKV, paged_shape: tuple[int, int]) -> torch.Tensor: + """Restore a pool encoded by :func:`pack_paged` to its page layout.""" pages, page = paged_shape logical = unpack_kv(packed) return logical.reshape(pages, page, *logical.shape[1:]) -def unpack_kv(packed: PackedKV) -> torch.Tensor: - """Decode using the format carried by the packed tensor.""" - if isinstance(packed, PackedInt4KV): - return int4.unpack(packed) - if isinstance(packed, PackedNVFP4KV): - return nvfp4.unpack(packed) - raise TypeError("expected PackedInt4KV or PackedNVFP4KV") - - def _validate_positions(positions: torch.Tensor, rows: int, tokens: int) -> None: if positions.ndim != 1: raise ValueError("positions must be one-dimensional") @@ -89,7 +123,7 @@ class KVResidual: V groups remain within each compact row, along channels. """ - data: PackedKV + data: PackedGroupKV positions: torch.Tensor | None num_tokens: int @@ -108,13 +142,15 @@ def encode_residual( *, kind: KVKind, positions: torch.Tensor | None = None, - format: KVFormat = "int4", + format: str = "int4", group_size: int | None = None, ) -> KVResidual: """Encode ``current - reconstructed_previous`` in the input dtype. Optional positions map compact rows into the full predecessor and are copied as contiguous int32 on the input device, matching FlashLoop's index format. + Position checks read device scalars and synchronize; this is a reference/debug + path, not a serving hot path. """ if ( current.dtype != reconstructed_previous.dtype diff --git a/vllm_rlt/kernels/kv_quantization/_common.py b/vllm_rlt/kernels/kv_quantization/_common.py index 1e82e027..63c92a6c 100644 --- a/vllm_rlt/kernels/kv_quantization/_common.py +++ b/vllm_rlt/kernels/kv_quantization/_common.py @@ -4,7 +4,7 @@ Copyright (c) 2026 FlashLoop contributors. MIT License; see NOTICE. """ -from typing import Literal +from typing import Literal, Protocol import torch @@ -12,6 +12,50 @@ _DTYPES = {torch.float16, torch.bfloat16, torch.float32, torch.float64} +class PackedGroupKV(Protocol): + """Logical tensor encoded in groups along one axis. + + ``group_axis`` and ``scale_shape`` let callers size and place parameters + without branching on format. Concrete formats carry their own quantization + parameters, and physical page ownership is managed separately. + """ + + @property + def format(self) -> str: ... + + @property + def payload(self) -> torch.Tensor: ... + + @property + def original_shape(self) -> tuple[int, int, int]: ... + + @property + def original_dtype(self) -> torch.dtype: ... + + @property + def group_size(self) -> int: ... + + @property + def group_axis(self) -> int: ... + + @property + def scale_shape(self) -> tuple[int, ...]: ... + + @property + def storage_bytes(self) -> int: ... + + +def group_axis(kind: KVKind) -> int: + return 0 if kind == "key" else 2 + + +def scale_shape(shape: tuple[int, int, int], group_size: int, *, kind: KVKind) -> tuple[int, ...]: + dimensions = list(shape) + axis = group_axis(kind) + dimensions[axis] = (dimensions[axis] + group_size - 1) // group_size + return tuple(dimensions) + + def _validate_inputs(values: torch.Tensor, *, kind: KVKind, group_size: int) -> None: if group_size <= 0: raise ValueError("group_size must be positive") diff --git a/vllm_rlt/kernels/kv_quantization/int4.py b/vllm_rlt/kernels/kv_quantization/int4.py index c9a959b9..5bfc1407 100644 --- a/vllm_rlt/kernels/kv_quantization/int4.py +++ b/vllm_rlt/kernels/kv_quantization/int4.py @@ -9,14 +9,25 @@ import torch +from . import _common from ._common import ( KVKind, _pack_channel_nibbles, _pad_repeat, _unpack_channel_nibbles, _validate_inputs, + group_axis, ) +format = "int4" +storage_dtype = torch.uint8 +default_group_size = 64 + + +def scale_shape(shape: tuple[int, int, int], group_size: int, *, kind: KVKind) -> tuple[int, ...]: + """Protocol hook: metadata shape for ``shape`` at ``group_size``.""" + return _common.scale_shape(shape, group_size, kind=kind) + def _codes_for_group( values: torch.Tensor, *, reduce_dim: int @@ -50,7 +61,15 @@ class PackedInt4KV: @property def axis(self) -> int: - return 0 if self.kind == "key" else 2 + return group_axis(self.kind) + + @property + def group_axis(self) -> int: + return self.axis + + @property + def scale_shape(self) -> tuple[int, ...]: + return tuple(self.scale.shape) @property def storage_bytes(self) -> int: @@ -58,50 +77,47 @@ def storage_bytes(self) -> int: return sum(t.numel() * t.element_size() for t in (self.payload, self.minimum, self.scale)) +def storage_bytes(packed: PackedInt4KV) -> int: + return packed.storage_bytes + + @torch.no_grad() -def pack(values: torch.Tensor, *, kind: KVKind, group_size: int = 64) -> PackedInt4KV: - """Quantize logical KV in FP32, storing uint8 payload and FP16 parameters.""" +def pack( + values: torch.Tensor, *, kind: KVKind, group_size: int = default_group_size +) -> PackedInt4KV: + """Quantize logical KV in FP32, storing uint8 payload and FP16 parameters. + + Groups are formed by one padded reshape along the group axis; the padded + tail repeats the last slice so padding never moves an extreme. + Parameters are stored as FP16: a group whose minimum magnitude or + ``span / 15`` exceeds 65504 overflows to infinity, so float32/float64 + inputs must keep both within FP16 range. + """ group_size = int(group_size) _validate_inputs(values, kind=kind, group_size=group_size) - shape = tuple(values.shape) - axis = 0 if kind == "key" else 2 + tokens, heads, channels = values.shape + shape = (tokens, heads, channels) + axis = group_axis(kind) work = values.float() padded = _pad_repeat(work, axis, group_size) if kind == "key": - groups = padded.shape[0] // group_size - grouped = padded.view(groups, group_size, *shape[1:]) - minimum = grouped.amin(dim=1, keepdim=True) - span = grouped.amax(dim=1, keepdim=True) - minimum - scale = torch.where(span > 0, span / 15.0, torch.ones_like(span)) - codes = ((grouped - minimum) / scale).round_().clamp_(0, 15).to(torch.uint8) + grouped = padded.view(padded.shape[0] // group_size, group_size, *shape[1:]) + codes, minimum, scale = _codes_for_group(grouped, reduce_dim=1) codes = codes.view(padded.shape)[: shape[0]] - minimum = minimum.squeeze(1) - scale = scale.squeeze(1) + minimum, scale = minimum.squeeze(1), scale.squeeze(1) else: - groups = padded.shape[2] // group_size - grouped = padded.view(shape[0], shape[1], groups, group_size) - minimum = grouped.amin(dim=-1, keepdim=True) - span = grouped.amax(dim=-1, keepdim=True) - minimum - scale = torch.where(span > 0, span / 15.0, torch.ones_like(span)) - codes = ((grouped - minimum) / scale).round_().clamp_(0, 15).to(torch.uint8) + grouped = padded.view(shape[0], shape[1], padded.shape[2] // group_size, group_size) + codes, minimum, scale = _codes_for_group(grouped, reduce_dim=3) codes = codes.view(padded.shape)[..., : shape[2]] - minimum = minimum.squeeze(-1) - scale = scale.squeeze(-1) - payload = _pack_channel_nibbles(codes) + minimum, scale = minimum.squeeze(3), scale.squeeze(3) return PackedInt4KV( - payload, - minimum.to(torch.float16), - scale.to(torch.float16), - shape, - values.dtype, - kind, - group_size, + _pack_channel_nibbles(codes), minimum, scale, shape, values.dtype, kind, group_size ) @torch.no_grad() def unpack(packed: PackedInt4KV) -> torch.Tensor: - """Reconstruct with stored parameters in FP32, then cast to compute dtype.""" + """Reference/debug full decode; serving kernels should decode tiles in-register.""" codes = _unpack_channel_nibbles(packed.payload, packed.original_shape[2]) axis = packed.axis length = packed.original_shape[axis] diff --git a/vllm_rlt/kernels/kv_quantization/nvfp4.py b/vllm_rlt/kernels/kv_quantization/nvfp4.py index a964ab8a..9ac29755 100644 --- a/vllm_rlt/kernels/kv_quantization/nvfp4.py +++ b/vllm_rlt/kernels/kv_quantization/nvfp4.py @@ -13,14 +13,25 @@ import torch +from . import _common from ._common import ( KVKind, _pack_channel_nibbles, _pad_repeat, _unpack_channel_nibbles, _validate_inputs, + group_axis, ) +format = "nvfp4" +storage_dtype = torch.uint8 +default_group_size = 16 + + +def scale_shape(shape: tuple[int, int, int], group_size: int, *, kind: KVKind) -> tuple[int, ...]: + """Protocol hook: metadata shape for ``shape`` at ``group_size``.""" + return _common.scale_shape(shape, group_size, kind=kind) + @dataclass(frozen=True) class PackedNVFP4KV: @@ -44,7 +55,15 @@ class PackedNVFP4KV: @property def axis(self) -> int: - return 0 if self.kind == "key" else 2 + return group_axis(self.kind) + + @property + def group_axis(self) -> int: + return self.axis + + @property + def scale_shape(self) -> tuple[int, ...]: + return tuple(self.scale.shape) @property def storage_bytes(self) -> int: @@ -53,15 +72,26 @@ def storage_bytes(self) -> int: ) +def storage_bytes(packed: PackedNVFP4KV) -> int: + return packed.storage_bytes + + @torch.no_grad() -def pack(values: torch.Tensor, *, kind: KVKind, group_size: int = 16) -> PackedNVFP4KV: - """Encode E2M1 with nearest-even rounding and FP8-rounded block scales.""" +def pack( + values: torch.Tensor, *, kind: KVKind, group_size: int = default_group_size +) -> PackedNVFP4KV: + """Encode E2M1 with nearest-even rounding and FP8-rounded block scales. + + Groups are formed by one padded reshape along the group axis. ``global_scale`` + is derived from the whole tensor, so it stays a single FP32 scalar. + """ group_size = int(group_size) _validate_inputs(values, kind=kind, group_size=group_size) if group_size != 16: raise ValueError("NVFP4 requires group_size=16") - shape = tuple(values.shape) - axis = 0 if kind == "key" else 2 + tokens, heads, channels = values.shape + shape = (tokens, heads, channels) + axis = group_axis(kind) work = values.float() global_scale = torch.ones((), device=values.device, dtype=torch.float32) if values.numel(): @@ -70,6 +100,8 @@ def pack(values: torch.Tensor, *, kind: KVKind, group_size: int = 16) -> PackedN torch.finfo(torch.float32).max ) inverse_global = global_scale.reciprocal() + # bucketize keeps equality in the lower bin. Move odd lower-code boundaries + # down one FP32 ULP so midpoint ties choose the even upper code instead. boundaries = torch.tensor( [0.25, 0.75 - 2**-24, 1.25, 1.75 - 2**-23, 2.5, 3.5 - 2**-22, 5.0], device=values.device, @@ -77,41 +109,30 @@ def pack(values: torch.Tensor, *, kind: KVKind, group_size: int = 16) -> PackedN ) padded = _pad_repeat(work, axis, 16) if kind == "key": - groups = padded.shape[0] // 16 - grouped = padded.view(groups, 16, *shape[1:]) - amax = grouped.abs().amax(dim=1, keepdim=True) - scale = (global_scale * (amax * (1.0 / 6))).clamp_(0, 448).to(torch.float8_e4m3fn) - effective = scale.float() * inverse_global - multiplier = effective.reciprocal() - finite = torch.isfinite(multiplier) - normalized = grouped * torch.where(finite, multiplier, 0.0) - normalized = torch.where((effective > 0) & ~finite, grouped / effective, normalized).clamp_( - -6, 6 - ) - codes = torch.bucketize(normalized.abs().contiguous(), boundaries, out_int32=True).to( - torch.uint8 - ) - codes |= torch.signbit(normalized).to(torch.uint8) << 3 + grouped = padded.view(padded.shape[0] // 16, 16, *shape[1:]) + reduce_dim = 1 + else: + grouped = padded.view(shape[0], shape[1], padded.shape[2] // 16, 16) + reduce_dim = 3 + amax = grouped.abs().amax(dim=reduce_dim, keepdim=True) + scale = (global_scale * (amax * (1.0 / 6))).clamp_(0, 448).to(torch.float8_e4m3fn) + effective = scale.float() * inverse_global + multiplier = effective.reciprocal() + finite = torch.isfinite(multiplier) + normalized = grouped * torch.where(finite, multiplier, 0.0) + # Preserve tiny values when only the FP32 reciprocal overflows. + normalized = torch.where((effective > 0) & ~finite, grouped / effective, normalized).clamp_( + -6, 6 + ) + codes = torch.bucketize(normalized.abs().contiguous(), boundaries, out_int32=True).to( + torch.uint8 + ) + codes |= torch.signbit(normalized).to(torch.uint8) << 3 + scales = scale.squeeze(reduce_dim) + if kind == "key": codes = codes.view(padded.shape)[: shape[0]] - scales = scale.squeeze(1) else: - groups = padded.shape[2] // 16 - grouped = padded.view(shape[0], shape[1], groups, 16) - amax = grouped.abs().amax(dim=-1, keepdim=True) - scale = (global_scale * (amax * (1.0 / 6))).clamp_(0, 448).to(torch.float8_e4m3fn) - effective = scale.float() * inverse_global - multiplier = effective.reciprocal() - finite = torch.isfinite(multiplier) - normalized = grouped * torch.where(finite, multiplier, 0.0) - normalized = torch.where((effective > 0) & ~finite, grouped / effective, normalized).clamp_( - -6, 6 - ) - codes = torch.bucketize(normalized.abs().contiguous(), boundaries, out_int32=True).to( - torch.uint8 - ) - codes |= torch.signbit(normalized).to(torch.uint8) << 3 codes = codes.view(padded.shape)[..., : shape[2]] - scales = scale.squeeze(-1) return PackedNVFP4KV( _pack_channel_nibbles(codes), scales, global_scale, shape, values.dtype, kind ) @@ -119,7 +140,7 @@ def pack(values: torch.Tensor, *, kind: KVKind, group_size: int = 16) -> PackedN @torch.no_grad() def unpack(packed: PackedNVFP4KV) -> torch.Tensor: - """Decode with stored block/global scales in FP32, then cast to compute dtype.""" + """Reference/debug full decode; serving kernels should decode tiles in-register.""" channels = packed.original_shape[2] codes = _unpack_channel_nibbles(packed.payload, channels, dtype=torch.uint8) magnitudes = torch.tensor( From 307f6b5712c918dcff759671ce01495ef0bbd783 Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Wed, 11 Nov 2026 13:13:00 +0800 Subject: [PATCH 05/15] feat(kv): store INT4/NVFP4 KV pages in the cache manager Pages that leave the latest kv_quantization_window tokens are packed into per-block slots; recent KV stays in a bounded BF16 staging pool. Attention decodes the batch's pages to BF16 and runs the existing backend. The memory plan charges packed pages, staging, the gather and padded codec scratch. Off by default. Async scheduling, CUDA graphs, speculative decoding, preemption, PD, prefix caching and the shared layout are rejected. --- tests/test_kv_quantization_integration.py | 659 ++++++++++++++++++++++ vllm_rlt/config.py | 12 + vllm_rlt/core/kv_cache_manager.py | 447 ++++++++++++++- vllm_rlt/core/memory.py | 85 ++- vllm_rlt/engine/stage_engine.py | 35 +- vllm_rlt/entrypoints/runtime_args.py | 12 + 6 files changed, 1239 insertions(+), 11 deletions(-) create mode 100644 tests/test_kv_quantization_integration.py diff --git a/tests/test_kv_quantization_integration.py b/tests/test_kv_quantization_integration.py new file mode 100644 index 00000000..802f01b2 --- /dev/null +++ b/tests/test_kv_quantization_integration.py @@ -0,0 +1,659 @@ +"""Quantized KV storage: packed pages plus a BF16 staging window (FlashLoop policy).""" + +import weakref + +import pytest +import torch +from torch.utils._python_dispatch import TorchDispatchMode + +from tests.helpers import tiny_ouro_config +from vllm_rlt import CacheConfig, ExecutionConfig, SamplingParams, SchedulerConfig +from vllm_rlt.core.kv_cache_manager import ( + KVCacheManager, + codec_page_scratch_bytes, + codec_scratch_bytes, + packed_page_bytes, + staging_pages_bound, +) +from vllm_rlt.core.memory import plan_cache +from vllm_rlt.engine.llm_engine import LLMEngine +from vllm_rlt.engine.stage_engine import StageEngine +from vllm_rlt.kernels.kv_quantization import pack_kv, unpack_kv +from vllm_rlt.kernels.paged_attention import torch_paged_attention +from vllm_rlt.models import OuroForCausalLM + + +def make_cache(**overrides): + values = dict( + num_layers=2, + num_kv_heads=2, + head_dim=8, + num_blocks=32, + block_size=4, + max_loops=2, + dtype=torch.bfloat16, + kv_quantization="int4", + kv_quantization_group_size=2, + kv_quantization_window=4, + ) + values.update(overrides) + return KVCacheManager(**values) + + +def write_tokens(cache, request_id, depth, positions, generator, layers=None): + shape = (len(positions), cache.num_kv_heads, cache.head_dim) + written = {} + for layer in range(cache.num_layers) if layers is None else layers: + k = torch.randn(shape, generator=generator).to(cache.dtype) + v = torch.randn(shape, generator=generator).to(cache.dtype) + cache.write(layer, [request_id] * len(positions), [depth] * len(positions), positions, k, v) + written[layer] = (k, v) + return written + + +def fake_quantize(values, kind, format, group_size): + return unpack_kv(pack_kv(values, kind=kind, format=format, group_size=group_size)) + + +_PAGE_CASES = [("int4", 2, 4), ("nvfp4", 16, 16)] + + +@pytest.mark.parametrize("format,group_size,block_size", _PAGE_CASES) +def test_pages_pack_once_after_leaving_the_window(format, group_size, block_size): + cache = make_cache( + block_size=block_size, + kv_quantization=format, + kv_quantization_group_size=group_size, + kv_quantization_window=block_size, + kv_quantization_staging_blocks=8, + ) + generator = torch.Generator().manual_seed(0) + assert cache.allocate("r", 8 * block_size) + tokens = 3 * block_size + 1 + exact = write_tokens(cache, "r", 0, list(range(tokens)), generator) + assert cache.num_free_staging_blocks == 8 - 4 + # (tokens - window) // block_size = 2 pages have left the window. + assert cache.ready_pages() == 2 + assert cache.pack_ready() == 2 and cache.ready_pages() == 0 + assert cache.num_free_staging_blocks == 8 - 2 + for layer, (k, v) in exact.items(): + read_k, read_v = cache.read(layer, "r", 0) + for actual, values, kind in ((read_k, k, "key"), (read_v, v, "value")): + for page in range(2): + rows = slice(page * block_size, (page + 1) * block_size) + expected = fake_quantize(values[rows], kind, format, group_size) + assert torch.equal(actual[rows], expected) + assert torch.equal(actual[2 * block_size :], values[2 * block_size :]) + before = cache.read(0, "r", 0) + with pytest.raises(RuntimeError, match="packed KV page"): + write_tokens(cache, "r", 0, [0], generator) + # The next page packs only once it leaves the window; packed pages never change. + write_tokens(cache, "r", 0, list(range(tokens, 4 * block_size)), generator) + assert cache.pack_ready() == 1 + after = cache.read(0, "r", 0) + packed = slice(0, 2 * block_size) + assert torch.equal(after[0][packed], before[0][packed]) + assert torch.equal(after[1][packed], before[1][packed]) + + +# Realistic sizes: tiny groups or heads make per-group metadata outweigh the payload. +# Bytes per group of parameters, and per (page, layer, kind) for a global scale. +@pytest.mark.parametrize( + "format,group_size,block_size,group_bytes,global_bytes", + [("int4", 64, 64, 4, 0), ("nvfp4", 16, 16, 1, 4)], +) +def test_storage_is_packed_and_staging_is_bounded( + format, group_size, block_size, group_bytes, global_bytes +): + cache = make_cache( + head_dim=64, + block_size=block_size, + kv_quantization=format, + kv_quantization_group_size=group_size, + kv_quantization_staging_blocks=5, + ) + # BF16 pages exist only for staging; every block id owns a packed slot. + assert cache.key_cache.shape[0] == cache.value_cache.shape[0] == 5 + packed_bytes = sum( + storage.numel() * storage.element_size() + for pool in cache._packed_pools.values() + for storage in pool.values() + ) + expected = packed_page_bytes( + format=format, + group_size=group_size, + block_size=block_size, + num_kv_heads=2, + head_dim=64, + num_layers=2, + dtype=torch.bfloat16, + ) + assert packed_bytes == cache.num_blocks * expected == cache.num_blocks * cache.bytes_per_block + # Per block id: 4-bit codes, group parameters, and any per-page global scale. + elements = 2 * 2 * block_size * 2 * 64 # K/V x layers x tokens x heads x dim + assert cache.bytes_per_block == ( + elements // 2 + elements // group_size * group_bytes + 2 * 2 * global_bytes + ) + unquantized = make_cache(head_dim=64, block_size=block_size, kv_quantization=None) + assert unquantized.bytes_per_block == elements * 2 + assert cache.bytes_per_block * 3 < unquantized.bytes_per_block + # Staging never exceeds the logical page count. + assert make_cache(kv_quantization_staging_blocks=1000).num_staging_blocks == 32 + + +def test_staging_bound_covers_window_and_one_step(): + # 64-token pages and window: each request keeps 2 pages plus 1 partial page + # for its share of the step; the 128-token step adds 2 pages per plane. + bound = staging_pages_bound(block_size=64, window=64, tokens_per_step=128, requests=8, planes=4) + assert bound == 4 * (8 * 3 + 2) + # The step's tokens are shared by all requests, not counted once per request. + large = staging_pages_bound( + block_size=64, window=64, tokens_per_step=2048, requests=64, planes=4 + ) + assert large == 4 * (64 * 3 + 32) + + +def test_staging_bound_holds_for_every_split_of_a_step(): + block_size, window, step, requests = 4, 4, 9, 3 + bound = staging_pages_bound( + block_size=block_size, window=window, tokens_per_step=step, requests=requests, planes=1 + ) + worst = 0 + # Worst case per request: the most unpacked tokens left after packing, then its share. + retained = window + block_size - 1 + for first in range(step + 1): + for second in range(step + 1 - first): + shares = (first, second, step - first - second) + worst = max(worst, sum(-(-(retained + t) // block_size) for t in shares)) + assert worst <= bound + + +def test_a_page_waits_for_every_layer_of_its_plane(): + cache = make_cache() + generator = torch.Generator().manual_seed(1) + assert cache.allocate("r", 32) + write_tokens(cache, "r", 0, list(range(12)), generator, layers=[0]) + assert cache.ready_pages() == 0 + write_tokens(cache, "r", 0, list(range(8)), generator, layers=[1]) + # Layer 1 reaches 8 tokens: (8 - 4) // 4 = 1 page. + assert cache.ready_pages() == 1 + write_tokens(cache, "r", 0, list(range(8, 12)), generator, layers=[1]) + assert cache.ready_pages() == 2 + + +def test_each_request_and_plane_packs_independently(): + cache = make_cache() + generator = torch.Generator().manual_seed(2) + assert cache.allocate("a", 32) and cache.allocate("b", 32) + write_tokens(cache, "a", 0, list(range(12)), generator) + write_tokens(cache, "a", 1, list(range(8)), generator) + write_tokens(cache, "b", 0, list(range(4)), generator) + assert cache.pack_ready() == 2 + 1 + assert cache._allocations["a"].packed_pages == [2, 1] + assert cache._allocations["b"].packed_pages == [0, 0] + + +def dense_pool(cache, request_ids, depths, layer): + """Dense BF16 pool and tables rebuilt from read(), as unquantized pages would hold.""" + pages, tables = [], [] + for request_id, depth in zip(request_ids, depths): + k, v = cache.read(layer, request_id, depth) + count = -(-k.shape[0] // cache.block_size) + pad = count * cache.block_size - k.shape[0] + shape = (count, cache.block_size, cache.num_kv_heads, cache.head_dim) + k = torch.cat([k, k.new_zeros(pad, *k.shape[1:])]).view(shape) + v = torch.cat([v, v.new_zeros(pad, *v.shape[1:])]).view(shape) + tables.append(list(range(len(pages), len(pages) + count))) + pages.extend(zip(k, v)) + keys = torch.stack([k for k, _ in pages]) + values = torch.stack([v for _, v in pages]) + width = max(len(table) for table in tables) + tables = torch.tensor([table + [0] * (width - len(table)) for table in tables]) + return keys, values, tables.to(torch.int32) + + +@pytest.mark.parametrize("format,group_size,block_size", _PAGE_CASES) +def test_attention_reads_packed_and_staged_pages(format, group_size, block_size): + cache = make_cache( + block_size=block_size, + kv_quantization=format, + kv_quantization_group_size=group_size, + kv_quantization_window=block_size, + ) + generator = torch.Generator().manual_seed(3) + lengths = {"a": 3 * block_size + 2, "b": 2 * block_size - 1} + for request_id, length in lengths.items(): + assert cache.allocate(request_id, 4 * block_size) + for depth in range(2): + write_tokens(cache, request_id, depth, list(range(length)), generator) + assert cache.pack_ready() > 0 + rows = [("a", 0, 0), ("a", 1, block_size + 1), ("a", 0, lengths["a"] - 1), ("b", 1, 3)] + ids, depths, positions = (list(column) for column in zip(*rows)) + q = torch.randn(len(rows), 4, 8, generator=generator).to(torch.bfloat16) + for layer in range(cache.num_layers): + actual = cache.attend(layer, ids, depths, positions, q) + keys, values, tables = dense_pool(cache, ids, depths, layer) + lengths_tensor = torch.tensor([p + 1 for p in positions], dtype=torch.int32) + assert torch.equal(actual, torch_paged_attention(q, keys, values, tables, lengths_tensor)) + + +def test_finalize_copies_between_staging_pages(): + cache = make_cache(kv_quantization_staging_blocks=8) + generator = torch.Generator().manual_seed(4) + assert cache.allocate("r", 32) + for position in range(10): + written = write_tokens(cache, "r", 0, [position], generator) + cache.finalize_token("r", position, 0) + for layer, (k, v) in written.items(): + deeper_k, deeper_v = cache.read(layer, "r", 1, position + 1) + assert torch.equal(deeper_k[position], k[0]) + assert torch.equal(deeper_v[position], v[0]) + cache.pack_ready() + assert cache._allocations["r"].packed_pages == [1, 1] + # A finalized deeper plane packs to the same values as the exit plane. + for layer in range(cache.num_layers): + assert all( + torch.equal(a, b) for a, b in zip(cache.read(layer, "r", 0), cache.read(layer, "r", 1)) + ) + + +def test_staging_pages_are_returned_and_exhaustion_is_reported(): + cache = make_cache(kv_quantization_staging_blocks=2) + generator = torch.Generator().manual_seed(5) + assert cache.allocate("r", 32) + write_tokens(cache, "r", 0, list(range(8)), generator) + assert cache.num_free_staging_blocks == 0 + with pytest.raises(RuntimeError, match="staging pool is exhausted"): + write_tokens(cache, "r", 0, [8], generator) + cache.free("r") + assert cache.num_free_staging_blocks == 2 + assert not any(cache._packed_page) + + +def test_truncation_and_snapshots_respect_packed_pages(): + cache = make_cache() + assert cache.allocate("r", 32) + write_tokens(cache, "r", 0, list(range(12)), torch.Generator().manual_seed(6)) + assert cache.pack_ready() == 2 + cache.truncate_suffix("r", 8) + with pytest.raises(ValueError, match="packed KV"): + cache.truncate_suffix("r", 7) + with pytest.raises(ValueError, match="snapshots"): + cache.snapshot("r") + + +@pytest.mark.parametrize( + "overrides,message", + [ + (dict(kv_quantization="fp8"), "unknown KV quantization"), + (dict(kv_quantization_group_size=3), "multiple of kv_quantization_group_size"), + (dict(kv_quantization="nvfp4", block_size=16, kv_quantization_group_size=8), "NVFP4"), + (dict(kv_quantization_window=0), "window must be a positive"), + (dict(kv_quantization_staging_blocks=0), "staging_blocks must be a positive"), + (dict(layout="shared"), "requires last_exited"), + (dict(enable_prefix_caching=True), "prefix caching"), + (dict(recurrent_layers=[1]), "every cache layer"), + ], +) +def test_rejects_unsupported_configuration(overrides, message): + with pytest.raises(ValueError, match=message): + make_cache(**overrides) + + +def tiny_model(device="cpu", dtype=torch.float32): + torch.manual_seed(123) + return OuroForCausalLM(tiny_ouro_config()).to(device=device, dtype=dtype) + + +def make_engine(cache_config, *, device="cpu", dtype=torch.bfloat16, backend="torch", **options): + options.setdefault( + "scheduler_config", SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=8) + ) + return LLMEngine( + tiny_model(device, dtype), cache_config=cache_config, attention_backend=backend, **options + ) + + +def run(engine, prompts, *, max_tokens=24): + """Generate, keeping each request's KV as read just before it is freed.""" + captured = {} + cache = engine.cache_manager + original_free = cache.free + + def free(request_id): + if cache.has_allocation(request_id): + for depth in range(cache.max_loops): + for layer in range(cache.num_layers): + k, v = cache.read(layer, request_id, depth) + captured[(request_id, depth, layer)] = (k.cpu(), v.cpu()) + return original_free(request_id) + + cache.free = free + params = SamplingParams(max_tokens=max_tokens, exit_threshold=0.7, ignore_eos=True) + for index, prompt in enumerate(prompts): + engine.add_request(str(index), prompt, params) + outputs = {} + for _ in range(2000): + if not engine.has_unfinished_requests(): + break + for output in engine.step(): + if output.finished: + outputs[output.request_id] = output + assert cache.num_used_blocks == 0 + return [outputs[str(index)] for index in range(len(prompts))], captured + + +def same_run(left, right): + (left_outputs, left_kv), (right_outputs, right_kv) = left, right + return ( + [o.token_ids for o in left_outputs] == [o.token_ids for o in right_outputs] + and [o.exit_depths for o in left_outputs] == [o.exit_depths for o in right_outputs] + and left_kv.keys() == right_kv.keys() + and all( + torch.equal(a, b) + for key in left_kv + for a, b in zip(left_kv[key], right_kv[key], strict=True) + ) + ) + + +def fake_quantized_engine(format, group_size, block_size, window, **options): + """Unquantized pages, with each step's ready pages fake-quantized in place. + + This re-implements the packing policy on dense BF16 pages, independently of + the manager's packed storage, staging pool and attention gather. + """ + engine = make_engine(CacheConfig(num_blocks=256, block_size=block_size), **options) + cache = engine.cache_manager + + def pack_kv_step(): + engine.model_runner.synchronize() + for allocation in cache._allocations.values(): + for plane, layers in enumerate(allocation.written): + frontier = min(written.prefix for written in layers) + end = max(0, frontier - window) // block_size + for page in range(allocation.packed_pages[plane], end): + block = allocation.block_tables[plane][page] + for layer in range(cache.num_layers): + for pool, kind in ((cache.key_cache, "key"), (cache.value_cache, "value")): + pool[block, layer] = fake_quantize( + pool[block, layer], kind, format, group_size + ) + allocation.packed_pages[plane] = max(allocation.packed_pages[plane], end) + if cache.device.type == "cuda": + torch.cuda.synchronize(cache.device) + + engine._pack_kv = pack_kv_step + return engine + + +def quantized_config(format, group_size, block_size, window, **overrides): + return CacheConfig( + num_blocks=256, + block_size=block_size, + kv_quantization=format, + kv_quantization_group_size=group_size, + kv_quantization_window=window, + **overrides, + ) + + +_ENGINE_CASES = [ + pytest.param("cpu", "torch", id="cpu-bf16-torch"), + pytest.param("cuda", "torch", id="cuda-bf16-torch", marks=pytest.mark.gpu), + pytest.param("cuda", "triton", id="cuda-bf16-triton", marks=pytest.mark.gpu), +] + + +@pytest.mark.parametrize("format,group_size,block_size", _PAGE_CASES) +@pytest.mark.parametrize("device,backend", _ENGINE_CASES) +def test_packed_engine_matches_fake_quantized_reference( + device, backend, format, group_size, block_size +): + # The production run never reads the pools between steps, so on CUDA only + # _pack_kv's own synchronization orders packing against later reads. + prompts = [list(range(2, 2 + 3 * block_size)), [5, 6, 7]] + window = block_size + options = dict(device=device, backend=backend) + engine = make_engine(quantized_config(format, group_size, block_size, window), **options) + cache = engine.cache_manager + packed_pages = [] + original = cache.pack_ready + + def pack_ready(): + count = original() + packed_pages.append(max(max(a.packed_pages) for a in cache._allocations.values())) + return count + + cache.pack_ready = pack_ready + production = run(engine, prompts) + reference_engine = fake_quantized_engine(format, group_size, block_size, window, **options) + reference = run(reference_engine, prompts) + assert all(len(output.token_ids) == 24 for output in production[0]) + # The prompt alone packs (3 * block_size - window) // block_size = 2 pages, + # so a third page was packed from decode tokens and attended afterwards. + assert max(packed_pages) > 2 + # Without packing, two requests need more pages than the staging pool holds. + assert cache.num_staging_blocks < cache.required_blocks(3 * block_size + 23) * 2 + assert same_run(production, reference) + + +def test_reference_comparison_detects_wrong_packed_pages(): + # Negative control: corrupting packed slots must break the comparison. + format, group_size, block_size = "int4", 2, 4 + prompts = [list(range(2, 14)), [5, 6, 7]] + reference = run(fake_quantized_engine(format, group_size, block_size, block_size), prompts) + engine = make_engine(quantized_config(format, group_size, block_size, block_size)) + cache = engine.cache_manager + original = cache.pack_ready + + def corrupt(): + count = original() + for pool in cache._packed_pools.values(): + for storage in pool.values(): + storage.zero_() + return count + + cache.pack_ready = corrupt + assert not same_run(run(engine, prompts), reference) + engine = make_engine(quantized_config(format, group_size, block_size, block_size)) + assert same_run(run(engine, prompts), reference) + + +@pytest.mark.parametrize("device,backend", _ENGINE_CASES) +def test_window_past_every_sequence_matches_unquantized(device, backend): + # Nothing packs, so attention reads staged BF16 pages only. + prompts = [list(range(2, 14)), [5, 6, 7]] + options = dict(device=device, backend=backend) + exact = run(make_engine(CacheConfig(num_blocks=256, block_size=4), **options), prompts) + staged = run(make_engine(quantized_config("int4", 2, 4, 1000), **options), prompts) + assert same_run(staged, exact) + + +def test_memory_plan_budgets_packed_blocks_and_staging(): + model = tiny_model(dtype=torch.bfloat16) + config = model.config + scheduler = SchedulerConfig(max_num_seqs=2, max_num_batched_tokens=8) + budget = 1 << 20 + plain = CacheConfig(block_size=64, kv_cache_memory_bytes=budget) + quantized = CacheConfig( + block_size=64, + kv_cache_memory_bytes=budget, + kv_quantization="int4", + kv_quantization_window=64, + ) + plain_blocks, plain_plan = plan_cache(model, plain, scheduler, ExecutionConfig(), "torch") + blocks, plan = plan_cache(model, quantized, scheduler, ExecutionConfig(), "torch") + packed = packed_page_bytes( + format="int4", + group_size=64, + block_size=64, + num_kv_heads=config.num_key_value_heads, + head_dim=config.head_dim, + num_layers=config.num_hidden_layers, + dtype=torch.bfloat16, + ) + staging = staging_pages_bound( + block_size=64, window=64, tokens_per_step=8, requests=2, planes=config.total_ut_steps + ) + assert plan["bytes_per_block"] == packed + assert plan["kv_staging_blocks"] == staging + assert plan["kv_staging_bytes"] == staging * plain_plan["bytes_per_block"] + assert blocks == (budget - plan["kv_staging_bytes"]) // packed + assert packed * 2 < plain_plan["bytes_per_block"] + assert blocks > plain_blocks + # A budget below the staging bound: the manager stages at most one BF16 + # page per block id, so only those pages are charged. + page = plain_plan["bytes_per_block"] + tight = (staging // 2) * (packed + page) + packed + small = CacheConfig( + block_size=64, + kv_cache_memory_bytes=tight, + kv_quantization="int4", + kv_quantization_window=64, + ) + blocks, plan = plan_cache(model, small, scheduler, ExecutionConfig(), "torch") + assert blocks == staging // 2 + assert plan["kv_staging_blocks"] == blocks + assert plan["kv_staging_bytes"] == blocks * page + assert blocks * packed + plan["kv_staging_bytes"] <= tight + + +class _PeakNewStorage(TorchDispatchMode): + """Peak bytes of tensor storage allocated inside the block and still alive.""" + + def __init__(self, existing): + super().__init__() + self.existing = {tensor.untyped_storage().data_ptr() for tensor in existing} + self.live: dict[int, int] = {} + self.current = self.peak = 0 + + def _track(self, tensor): + storage = tensor.untyped_storage() + key, size = storage.data_ptr(), storage.nbytes() + if not size or key in self.existing or key in self.live: + return + self.live[key] = size + self.current += size + self.peak = max(self.peak, self.current) + + def release(): + if self.live.pop(key, None) is not None: + self.current -= size + + weakref.finalize(storage, release) + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + output = func(*args, **(kwargs or {})) + for tensor in output if isinstance(output, (tuple, list)) else (output,): + if isinstance(tensor, torch.Tensor): + self._track(tensor) + return output + + +@pytest.mark.parametrize( + "format,group_size,heads,head_dim,layers,block_size", + [ + # head_dim < G pads V channels 8x; a fixed bytes-per-element rate misses it. + ("int4", 64, 2, 8, 24, 64), + ("int4", 64, 1, 65, 2, 64), + ("int4", 2, 2, 8, 2, 4), + ("int4", 64, 4, 128, 2, 64), + ("nvfp4", 16, 2, 8, 2, 16), + ("nvfp4", 16, 4, 128, 2, 16), + ], +) +def test_codec_scratch_covers_padded_groups( + format, group_size, heads, head_dim, layers, block_size +): + pages = 8 # One pack chunk. + cache = make_cache( + num_layers=layers, + num_kv_heads=heads, + head_dim=head_dim, + block_size=block_size, + max_loops=1, + kv_quantization=format, + kv_quantization_group_size=group_size, + kv_quantization_window=block_size, + kv_quantization_staging_blocks=pages + 2, + ) + assert cache.allocate("r", (pages + 1) * block_size) + write_tokens( + cache, "r", 0, list(range((pages + 1) * block_size)), torch.Generator().manual_seed(0) + ) + pools = [cache.key_cache, cache.value_cache] + pools += [storage for pool in cache._packed_pools.values() for storage in pool.values()] + per_page = codec_page_scratch_bytes( + group_size=group_size, block_size=block_size, num_kv_heads=heads, head_dim=head_dim + ) + with _PeakNewStorage(pools) as packing: + assert cache.pack_ready() == pages + assert packing.peak <= pages * layers * per_page + blocks = torch.tensor(cache._allocations["r"].block_tables[0][:pages]) + for kind in ("key", "value"): + with _PeakNewStorage(pools) as decoding: + cache._decode_pages(kind, 0, blocks) + assert decoding.peak <= pages * per_page + chunk = codec_scratch_bytes( + group_size=group_size, + block_size=block_size, + num_kv_heads=heads, + head_dim=head_dim, + num_layers=layers, + ) + assert chunk >= max(64, pages * layers) * per_page + + +@pytest.mark.parametrize( + "options,message", + [ + (dict(execution_config=ExecutionConfig(async_scheduling=True)), "synchronous"), + (dict(scheduler_config=SchedulerConfig(enable_preemption=True)), "preemption"), + ], +) +def test_engine_rejects_unsupported_execution(options, message): + with pytest.raises(ValueError, match=message): + make_engine(quantized_config("int4", 2, 4, 4), **options) + + +def test_pd_is_rejected_when_the_worker_attaches_its_connector(): + from unittest.mock import Mock + + from tests.test_stage_engine import cpu_connector + from vllm_rlt.engine.stage_config import StageConfig + from vllm_rlt.engine.worker import StageWorker + + channel = Mock() + options = dict( + seed=0, + dtype="float32", + revision=None, + model_config=None, + engine=dict(cache_config=quantized_config("int4", 2, 2, 2)), + ) + with pytest.raises(ValueError, match="PD transfer"): + StageWorker( + StageConfig(stage_id="p", role="prefill"), + "cpu", + channel, + tiny_model(), + options, + connector_factory=cpu_connector, + ) + assert not any(call.args[0].get("kind") == "ready" for call in channel.send.call_args_list) + + +def test_pd_is_rejected_when_the_connector_is_passed_to_the_engine(): + from tests.test_stage_engine import cpu_connector + from vllm_rlt.engine.stage_config import StageConfig + + engine = StageEngine(tiny_model(), stage_config=StageConfig(role="prefill")) + connector = cpu_connector(engine, notify=lambda *args, **kwargs: None) + with pytest.raises(ValueError, match="PD transfer"): + StageEngine( + tiny_model(), + stage_config=StageConfig(role="prefill"), + cache_config=quantized_config("int4", 2, 2, 2), + connector=connector, + ) diff --git a/vllm_rlt/config.py b/vllm_rlt/config.py index 648781d7..93f46bad 100644 --- a/vllm_rlt/config.py +++ b/vllm_rlt/config.py @@ -20,6 +20,14 @@ class CacheConfig: incremental_allocation: bool = False # Admission headroom for incremental allocation; otherwise effective watermark is zero. watermark_ratio: float = 0.0 + # "int4" or "nvfp4": pages leaving the latest kv_quantization_window tokens + # move from a small BF16 staging pool to packed storage. + kv_quantization: str | None = None + # None selects the codec default (INT4 G=64, NVFP4 G=16); must divide block_size. + kv_quantization_group_size: int | None = None + kv_quantization_window: int = 64 + # None sizes the staging pool for max_num_seqs requests at the batch token limit. + kv_quantization_staging_blocks: int | None = None def __post_init__(self): for name in ("enable_prefix_caching", "incremental_allocation"): @@ -45,6 +53,10 @@ def __post_init__(self): raise ValueError("specify num_blocks or kv_cache_memory_bytes, not both") if type(self.memory_reserve_bytes) is not int or self.memory_reserve_bytes < 0: raise ValueError("memory_reserve_bytes must be a nonnegative integer") + if self.kv_quantization not in (None, "int4", "nvfp4"): + raise ValueError("kv_quantization must be None, 'int4', or 'nvfp4'") + if self.kv_quantization_staging_blocks is not None: + _positive("kv_quantization_staging_blocks", self.kv_quantization_staging_blocks) @dataclass(frozen=True) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index dfcdf0df..6ea5b241 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -4,15 +4,23 @@ SHARED overwrites a single plane and needs no exit copies. Optional incremental allocation separates logical capacity from physical pages. Immutable full-depth prompt blocks can be shared through the prefix cache. + +Optional INT4/NVFP4 quantization follows FlashLoop's group-plus-window policy. +Block ids then name logical pages, each owning one slot in a packed pool. A +page is staged in a small BF16 pool while it is written and moves to its packed +slot once it leaves the latest ``window`` tokens. Attention reads both through +a Torch reference path; fused kernels are future work. """ import hashlib import struct +import weakref from collections import OrderedDict from collections.abc import Callable, Sequence from copy import deepcopy -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields, replace from numbers import Integral +from typing import Any import torch @@ -23,10 +31,77 @@ create_backend, validate_backend_name, ) +from vllm_rlt.kernels.kv_quantization import KV_CODECS, pack_kv, unpack_kv +from vllm_rlt.kernels.kv_quantization._common import KVKind from vllm_rlt.request import ExecutionBatch from vllm_rlt.worker.buffers import stage_metadata from vllm_rlt.worker.metadata import DeviceKVMetadata, KVMetadata, MetadataBuilder +# Pages per codec call when attention decodes or the engine packs, which +# bounds FP32 codec temporaries independently of context and batch size. +_DECODE_CHUNK_PAGES = 64 +_PACK_CHUNK_PAGES = 8 + + +def staging_pages_bound( + *, block_size: int, window: int, tokens_per_step: int, requests: int, planes: int +) -> int: + """BF16 staging pages that quantized KV can need at once. + + After packing, each request's plane keeps at most ``window + block_size - 1`` + unpacked tokens, starting on a page boundary. One step then writes at most + ``tokens_per_step`` tokens per plane, shared by all requests; request ``r``'s + ``t_r`` tokens add at most ``t_r // block_size + 1`` pages. + """ + retained = -(-(window + block_size - 1) // block_size) + return planes * (requests * (retained + 1) + tokens_per_step // block_size) + + +def codec_page_scratch_bytes( + *, group_size: int, block_size: int, num_kv_heads: int, head_dim: int +) -> int: + """Codec temporaries per page of one layer and kind, in a pack or decode call. + + V groups pad channels to a multiple of the group size, so temporaries follow + the padded shape. Measured CPU peaks stay below 12 bytes per padded element + plus 16 per group; the element term is doubled for allocator rounding. + """ + channels = -(-head_dim // group_size) * group_size + elements = block_size * num_kv_heads * channels + return 24 * elements + 16 * -(-elements // group_size) + + +def codec_scratch_bytes( + *, group_size: int, block_size: int, num_kv_heads: int, head_dim: int, num_layers: int +) -> int: + """Codec temporaries of the largest pack or decode chunk.""" + pages = max(_DECODE_CHUNK_PAGES, _PACK_CHUNK_PAGES * num_layers) + return pages * codec_page_scratch_bytes( + group_size=group_size, + block_size=block_size, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + ) + + +def packed_page_bytes( + *, + format: str, + group_size: int, + block_size: int, + num_kv_heads: int, + head_dim: int, + num_layers: int, + dtype: torch.dtype, +) -> int: + """Bytes of one block id's packed K and V slots across every layer.""" + page = torch.zeros((block_size, num_kv_heads, head_dim), dtype=dtype) + kinds: tuple[KVKind, ...] = ("key", "value") + return num_layers * sum( + pack_kv(page, kind=kind, format=format, group_size=group_size).storage_bytes + for kind in kinds + ) + @dataclass class _WrittenPositions: @@ -57,6 +132,8 @@ class _Allocation: written: list[list[_WrittenPositions]] transfer_leases: set[str] = field(default_factory=set) release_requested: bool = False + # Quantized KV: leading pages per plane that live in the packed pool. + packed_pages: list[int] = field(default_factory=list) @dataclass(frozen=True) @@ -85,6 +162,14 @@ class KVCacheManager: are zero-based throughout the interface. """ + # Set by the constructor's positive-integer validation loop. + num_layers: int + num_kv_heads: int + head_dim: int + num_blocks: int + block_size: int + max_loops: int + def __init__( self, num_layers: int, @@ -103,6 +188,10 @@ def __init__( recurrent_layers: Sequence[int] | None = None, *, attention: AttentionBackend | None = None, + kv_quantization: str | None = None, + kv_quantization_group_size: int | None = None, + kv_quantization_window: int = 64, + kv_quantization_staging_blocks: int | None = None, ): for name, value in ( ("num_layers", num_layers), @@ -140,11 +229,20 @@ def __init__( ) self.attention_info = self.attention.info self.backend = self.attention_info["selected_backend"] - shape = (num_blocks, num_layers, block_size, num_kv_heads, head_dim) + pool_blocks = self._init_quantization( + kv_quantization, + kv_quantization_group_size, + kv_quantization_window, + kv_quantization_staging_blocks, + enable_prefix_caching=enable_prefix_caching, + ) + # With quantization these BF16 pages only stage recent KV. + shape = (pool_blocks, num_layers, block_size, num_kv_heads, head_dim) self.key_cache = torch.empty(shape, device=self.device, dtype=dtype) self.value_cache = torch.empty_like(self.key_cache) # Resolve an implicit CUDA index to the actual storage device once. self.device = self.key_cache.device + self._init_packed_pool() self._free_blocks = list(reversed(range(num_blocks))) self._allocations: dict[str, _Allocation] = {} self.enable_prefix_caching = enable_prefix_caching @@ -158,6 +256,91 @@ def __init__( if enable_prefix_caching and layout != "last_exited": raise ValueError("prefix caching requires last_exited KV") + def _init_quantization( + self, format, group_size, window, staging_blocks, *, enable_prefix_caching + ) -> int: + """Validate the quantized layout; return how many BF16 pages to allocate.""" + self.kv_quantization = format + self.kv_quantization_group_size = None + self.kv_quantization_window = window + if format is None: + return self.num_blocks + if format not in KV_CODECS: + raise ValueError(f"unknown KV quantization format: {format!r}") + if group_size is None: + group_size = KV_CODECS[format].default_group_size + if type(group_size) is not int or group_size <= 0: + raise ValueError("kv_quantization_group_size must be a positive integer") + if format == "nvfp4" and group_size != 16: + raise ValueError("NVFP4 requires kv_quantization_group_size=16") + # The newest token must stay staged until finalize copies it to deeper planes. + if type(window) is not int or window <= 0: + raise ValueError("kv_quantization_window must be a positive integer") + # K groups then never span pages, so each page packs on its own. + if self.block_size % group_size: + raise ValueError("block_size must be a multiple of kv_quantization_group_size") + # SHARED rewrites a token's KV at every depth, after its page may be packed. + if self.layout != "last_exited": + raise ValueError("KV quantization requires last_exited KV") + # Shared prompt pages would need cross-request pack ownership. + if enable_prefix_caching: + raise ValueError("KV quantization does not support prefix caching") + # A page packs once every layer of its plane has left the window. + if len(self.recurrent_layers) != self.num_layers: + raise ValueError("KV quantization requires every cache layer to be recurrent") + if staging_blocks is None: + staging_blocks = self.num_blocks + if type(staging_blocks) is not int or staging_blocks <= 0: + raise ValueError("kv_quantization_staging_blocks must be a positive integer") + self.kv_quantization_group_size = group_size + # More staging pages than logical pages can never be used. + return min(staging_blocks, self.num_blocks) + + def _init_packed_pool(self) -> None: + # Per block id: its BF16 staging page or -1, and whether it is packed. + self._staging_of = [-1] * self.num_blocks + self._packed_page = [False] * self.num_blocks + self._free_staging = list(reversed(range(self.key_cache.shape[0]))) + self._packed_templates: dict[str, Any] = {} + self._packed_pools: dict[str, dict[str, torch.Tensor]] = {} + # Packing and page release bump this; cached per-batch index plans then rebuild. + self._epoch = 0 + self._write_slots: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary() + self._attention_plans: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary() + if self.kv_quantization is None: + return + page = torch.zeros( + (self.block_size, self.num_kv_heads, self.head_dim), + dtype=self.dtype, + device=self.device, + ) + kinds: tuple[KVKind, ...] = ("key", "value") + for kind in kinds: + # The codec's own output for one page fixes every pool's shape and dtype. + # Registered codecs return dataclasses; the protocol type does not say so. + template: Any = pack_kv( + page, + kind=kind, + format=self.kv_quantization, + group_size=self.kv_quantization_group_size, + ) + pool = {} + for item in fields(template): + value = getattr(template, item.name) + if isinstance(value, torch.Tensor): + pool[item.name] = torch.empty( + (self.num_blocks, self.num_layers, *value.shape), + dtype=value.dtype, + device=self.device, + ) + self._packed_templates[kind] = template + self._packed_pools[kind] = pool + # Pages join along tokens into one codec call unless a parameter is + # per-tensor (NVFP4's global scale), which must stay per page and layer. + self._joinable = all( + storage.ndim > 2 for pool in self._packed_pools.values() for storage in pool.values() + ) + @property def attention_capabilities(self) -> BackendCapabilities: return backend_capabilities(self.attention) @@ -172,8 +355,23 @@ def num_free_blocks(self) -> int: def num_used_blocks(self) -> int: return self.num_blocks - self.num_free_blocks + @property + def num_staging_blocks(self) -> int: + return self.key_cache.shape[0] + + @property + def num_free_staging_blocks(self) -> int: + return len(self._free_staging) + @property def bytes_per_block(self) -> int: + """Bytes one block id owns: its BF16 page, or its packed slot when quantized.""" + if self.kv_quantization is not None: + return sum( + storage[0].numel() * storage.element_size() + for pool in self._packed_pools.values() + for storage in pool.values() + ) return ( 2 * self.num_layers @@ -193,6 +391,7 @@ def _drop_refs(self, blocks): self._refs[b] -= 1 assert self._refs[b] >= 0 if not self._refs[b]: + self._release_page(b) self._free_blocks.append(b) def _claim(self, count): @@ -313,6 +512,7 @@ def allocate(self, request_id: str, max_tokens: int, *, initial_tokens=None, pre [_WrittenPositions(prefix=length) for _ in range(self.num_layers)] for _ in range(self.storage_depths) ], + packed_pages=[0] * self.storage_depths, ) self.prefix_queries += 1 self.prefix_hits += length @@ -347,6 +547,8 @@ def truncate_suffix(self, request_id: str, frontier: int) -> None: raise ValueError("invalid KV truncation frontier") if allocation.transfer_leases: raise ValueError("cannot truncate KV during a transfer") + if any(pages * self.block_size > frontier for pages in allocation.packed_pages): + raise ValueError("cannot truncate packed KV") for table in allocation.block_tables: if any(self._refs[b] > 1 for b in table[frontier // self.block_size :]): raise ValueError("cannot truncate shared prefix pages") @@ -473,6 +675,8 @@ def snapshot(self, request_id: str) -> KVSnapshot: Copying page by page avoids a request-sized temporary on a device that is already full, which is exactly when preemption happens. """ + if self.kv_quantization is not None: + raise ValueError("snapshots of quantized KV are not supported") allocation = self._get_allocation(request_id) if allocation.transfer_leases: raise RuntimeError("cannot snapshot KV during a transfer") @@ -497,6 +701,8 @@ def restore(self, request_id: str, snapshot: KVSnapshot) -> None: decision belongs to the scheduler. The copies go on the current stream; the caller decides when other streams may read them. """ + if self.kv_quantization is not None: + raise ValueError("snapshots of quantized KV are not supported") allocation = self._get_allocation(request_id) if allocation.max_tokens != snapshot.max_tokens: raise ValueError("snapshot capacity does not match the allocation") @@ -510,6 +716,221 @@ def restore(self, request_id: str, snapshot: KVSnapshot) -> None: self.value_cache[block].copy_(snapshot.values[row]) allocation.written = deepcopy(snapshot.written) + # Quantized storage. Block ids are logical pages: each owns a packed slot in + # every layer and borrows a BF16 staging page while it is being written. + + def _release_page(self, block: int) -> None: + if self.kv_quantization is None: + return + slot = self._staging_of[block] + if slot >= 0: + self._free_staging.append(slot) + self._staging_of[block] = -1 + self._packed_page[block] = False + self._epoch += 1 + + def _stage_page(self, block: int) -> int: + if self._packed_page[block]: + raise RuntimeError(f"cannot write packed KV page {block}") + slot = self._staging_of[block] + if slot < 0: + if not self._free_staging: + raise RuntimeError( + "BF16 KV staging pool is exhausted; raise kv_quantization_staging_blocks" + ) + slot = self._staging_of[block] = self._free_staging.pop() + return slot + + def _staging_writes(self, batch: KVMetadata) -> torch.Tensor: + """Staging page of every written row, assigned on the batch's first write.""" + cached = self._write_slots.get(batch) + if cached is not None and cached[0] == self._epoch: + return cached[1] + slots = [ + self._stage_page( + allocation.block_tables[self._plane(depth)][position // self.block_size] + ) + for allocation, depth, position in batch.rows + ] + tensor = torch.tensor(slots, dtype=torch.long, device=self.device) + self._write_slots[batch] = (self._epoch, tensor) + return tensor + + def _page_plan(self, pages: Sequence[int]): + """Index tensors that gather ``pages`` from the staging and packed pools.""" + staged, slots, packed, blocks = [], [], [], [] + for index, block in enumerate(pages): + if self._packed_page[block]: + packed.append(index) + blocks.append(block) + elif self._staging_of[block] >= 0: + staged.append(index) + slots.append(self._staging_of[block]) + else: + raise RuntimeError(f"KV page {block} was read before it was written") + + def tensor(values): + return torch.tensor(values, dtype=torch.long, device=self.device) + + return len(pages), tensor(staged), tensor(slots), tensor(packed), tensor(blocks) + + def _gather_pages(self, layer: int, plan) -> tuple[torch.Tensor, torch.Tensor]: + """Return BF16 ``[pages, block_size, heads, dim]`` K/V for a page plan.""" + count, staged, slots, packed, blocks = plan + # Keep at least one page so empty batches still pass a valid pool. + shape = (max(count, 1), self.block_size, self.num_kv_heads, self.head_dim) + keys = torch.empty(shape, dtype=self.dtype, device=self.device) + values = torch.empty_like(keys) + if staged.numel(): + keys[staged] = self.key_cache[slots, layer] + values[staged] = self.value_cache[slots, layer] + # Decode in chunks so FP32 temporaries stay bounded for long contexts. + for start in range(0, packed.numel(), _DECODE_CHUNK_PAGES): + rows = packed[start : start + _DECODE_CHUNK_PAGES] + chunk = blocks[start : start + _DECODE_CHUNK_PAGES] + keys[rows] = self._decode_pages("key", layer, chunk) + values[rows] = self._decode_pages("value", layer, chunk) + return keys, values + + def _decode_pages(self, kind: str, layer: int, blocks: torch.Tensor) -> torch.Tensor: + template = self._packed_templates[kind] + tensors = { + name: storage[blocks, layer] for name, storage in self._packed_pools[kind].items() + } + count, page = blocks.numel(), template.original_shape + if self._joinable: + joined = replace( + template, + original_shape=(count * page[0], *page[1:]), + **{name: tensor.flatten(0, 1) for name, tensor in tensors.items()}, + ) + return unpack_kv(joined).view(count, *page) + return torch.stack( + [ + unpack_kv(replace(template, **{name: t[index] for name, t in tensors.items()})) + for index in range(count) + ] + ) + + def _encode_pages(self, kind: KVKind, pages: torch.Tensor, blocks: list[int]) -> None: + """Pack ``[pages, layers, block_size, heads, dim]`` into each block's slot.""" + items = pages.flatten(0, 1) + page_ids = [block for block in blocks for _ in range(self.num_layers)] + layer_ids = list(range(self.num_layers)) * len(blocks) + pool = self._packed_pools[kind] + group = self.kv_quantization_group_size + if self._joinable: + packed = pack_kv( + items.flatten(0, 1), kind=kind, format=self.kv_quantization, group_size=group + ) + index = ( + torch.tensor(page_ids, dtype=torch.long, device=self.device), + torch.tensor(layer_ids, dtype=torch.long, device=self.device), + ) + for name, storage in pool.items(): + storage[index] = getattr(packed, name).view(len(page_ids), *storage.shape[2:]) + return + for item, page, layer in zip(items, page_ids, layer_ids): + packed = pack_kv(item, kind=kind, format=self.kv_quantization, group_size=group) + for name, storage in pool.items(): + storage[page, layer] = getattr(packed, name) + + def _ready_pages(self, allocation: _Allocation, plane: int) -> range: + # A page packs once every layer has written past its end plus the window. + frontier = min(written.prefix for written in allocation.written[plane]) + end = max(0, frontier - self.kv_quantization_window) // self.block_size + return range(allocation.packed_pages[plane], end) + + def ready_pages(self) -> int: + """Pages that have left the BF16 window but are not packed yet.""" + if self.kv_quantization is None: + return 0 + return sum( + len(self._ready_pages(allocation, plane)) + for allocation in self._allocations.values() + for plane in range(self.storage_depths) + ) + + @torch.no_grad() + def pack_ready(self) -> int: + """Pack every ready page and free its staging page; return the page count. + + The caller must have finished all device work that writes or reads these + staging pages, and later work must observe the packed slots. Each page is + packed once, per layer, so NVFP4 keeps one global scale per page and layer. + """ + if self.kv_quantization is None: + return 0 + items = [ + (allocation, plane, page, allocation.block_tables[plane][page]) + for allocation in self._allocations.values() + for plane in range(self.storage_depths) + for page in self._ready_pages(allocation, plane) + ] + if not items: + return 0 + blocks = [block for *_, block in items] + if any(self._staging_of[block] < 0 for block in blocks): + raise RuntimeError("a ready KV page has no staged storage") + kinds: tuple[tuple[torch.Tensor, KVKind], ...] = ( + (self.key_cache, "key"), + (self.value_cache, "value"), + ) + for start in range(0, len(blocks), _PACK_CHUNK_PAGES): + chunk = blocks[start : start + _PACK_CHUNK_PAGES] + slots = torch.tensor( + [self._staging_of[block] for block in chunk], dtype=torch.long, device=self.device + ) + for cache, kind in kinds: + self._encode_pages(kind, cache[slots], chunk) + for allocation, plane, page, block in items: + self._free_staging.append(self._staging_of[block]) + self._staging_of[block] = -1 + self._packed_page[block] = True + allocation.packed_pages[plane] = page + 1 + self._epoch += 1 + return len(items) + + def _attention_plan(self, batch: KVMetadata): + """Gather plan and remapped tables for every page a row actually reads.""" + cached = self._attention_plans.get(batch) + if cached is not None and cached[0] == self._epoch: + return cached[1] + tables = batch.block_tables.tolist() + lengths = batch.context_lengths.tolist() + index: dict[int, int] = {} + remapped = [] + for table, length in zip(tables, lengths): + needed = (length + self.block_size - 1) // self.block_size + # Entries past a row's context are never read; point them at page 0. + remapped.append( + [index.setdefault(b, len(index)) if j < needed else 0 for j, b in enumerate(table)] + ) + plan = ( + self._page_plan(list(index)), + torch.tensor(remapped, dtype=torch.int32, device=self.device).reshape( + batch.block_tables.shape + ), + ) + self._attention_plans[batch] = (self._epoch, plan) + return plan + + def _attend_quantized(self, layer: int, batch: KVMetadata, q: torch.Tensor) -> torch.Tensor: + """Reference path: decode the batch's pages to BF16, then run the backend.""" + plan, tables = self._attention_plan(batch) + keys, values = self._gather_pages(layer, plan) + if batch.cu_seqlens_q is not None: + return self.attention.prefill( + q, + keys, + values, + tables, + batch.context_lengths, + batch.cu_seqlens_q, + batch.max_seqlen_q, + ) + return self.attention(q, keys, values, tables, batch.context_lengths) + def _plane(self, depth: int) -> int: return depth if self.layout == "last_exited" else 0 @@ -667,6 +1088,8 @@ def _write_prepared( self._validate_layer(layer) self._require_live_batch(batch) if isinstance(batch, DeviceKVMetadata): + if self.kv_quantization is not None: + raise RuntimeError("quantized KV requires host-prepared metadata") self._validate_tensor(k, len(batch.position_ids), "k") self._validate_tensor(v, len(batch.position_ids), "v") if batch.write_mask is not None: @@ -683,8 +1106,9 @@ def _write_prepared( self._validate_tensor(v, len(batch.position_ids), "v") if not batch.rows: return - self.key_cache[batch.write_blocks, layer, batch.write_offsets] = k[: len(batch.rows)] - self.value_cache[batch.write_blocks, layer, batch.write_offsets] = v[: len(batch.rows)] + blocks = batch.write_blocks if self.kv_quantization is None else self._staging_writes(batch) + self.key_cache[blocks, layer, batch.write_offsets] = k[: len(batch.rows)] + self.value_cache[blocks, layer, batch.write_offsets] = v[: len(batch.rows)] for allocation, depth, position in batch.rows: allocation.written[self._plane(depth)][layer].add(position) @@ -719,6 +1143,8 @@ def _attend_prepared( self._require_live_batch(batch) self._validate_tensor(q, len(batch.position_ids), "q", query=True) if isinstance(batch, DeviceKVMetadata): + if self.kv_quantization is not None: + raise RuntimeError("quantized KV requires host-prepared metadata") return self.attention( q, self.key_cache[:, layer], @@ -730,6 +1156,8 @@ def _attend_prepared( return torch.empty_like(q) for allocation, depth, position in batch.rows: self._require_prefix(allocation, layer, depth, position + 1) + if self.kv_quantization is not None: + return self._attend_quantized(layer, batch, q) if batch.cu_seqlens_q is not None: return self.attention.prefill( q, @@ -760,6 +1188,12 @@ def finalize_token(self, request_id: str, position: int, exit_depth: int) -> Non allocation.block_tables[self._plane(depth)][logical_page] for depth in range(exit_depth + 1, self.max_loops) ] + if self.kv_quantization is not None: + # The window keeps the newest token's page staged in every plane. + source = self._staging_of[source] + if source < 0: + raise RuntimeError("finalized KV page is not staged") + destinations = [self._stage_page(destination) for destination in destinations] # Basic-index copies avoid a blocking host->device index tensor on the # boundary stream. That transfer would serialize final core and routing. for destination in destinations: @@ -791,6 +1225,11 @@ def read(self, layer: int, request_id: str, depth: int, length: int | None = Non ): raise ValueError("read length exceeds the request's reserved token budget") self._require_prefix(allocation, layer, depth, length) + if self.kv_quantization is not None: + table = allocation.block_tables[self._plane(depth)] + pages = table[: (length + self.block_size - 1) // self.block_size] + keys, values = self._gather_pages(layer, self._page_plan(pages)) + return keys.flatten(0, 1)[:length], values.flatten(0, 1)[:length] positions = torch.arange(length, device=self.device) table = torch.tensor( allocation.block_tables[self._plane(depth)], device=self.device, dtype=torch.long diff --git a/vllm_rlt/core/memory.py b/vllm_rlt/core/memory.py index e9d2d4cf..2a443195 100644 --- a/vllm_rlt/core/memory.py +++ b/vllm_rlt/core/memory.py @@ -5,7 +5,13 @@ import torch from vllm_rlt.config import cuda_graph_rows -from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.core.kv_cache_manager import ( + KVCacheManager, + codec_scratch_bytes, + packed_page_bytes, + staging_pages_bound, +) +from vllm_rlt.kernels.kv_quantization import KV_CODECS from vllm_rlt.worker.buffers import execution_buffer_bytes from vllm_rlt.worker.cuda_graph import speculative_capture_sizes @@ -50,6 +56,44 @@ def graph_memory_reserve(model, scheduler, execution, speculative=None): return pools * execution.cuda_graph_memory_reserve_bytes + retained_logits +def _quantized_plan(model, cache, scheduler): + """Packed bytes per block id, staging pages and reference-attention costs.""" + if cache.kv_quantization is None: + return None + parameter = next(model.parameters()) + config = model.config + group = cache.kv_quantization_group_size or KV_CODECS[cache.kv_quantization].default_group_size + packed = packed_page_bytes( + format=cache.kv_quantization, + group_size=group, + block_size=cache.block_size, + num_kv_heads=config.num_key_value_heads, + head_dim=config.head_dim, + num_layers=config.num_hidden_layers, + dtype=parameter.dtype, + ) + planes = config.total_ut_steps if cache.layout == "last_exited" else 1 + staging = cache.kv_quantization_staging_blocks or staging_pages_bound( + block_size=cache.block_size, + window=cache.kv_quantization_window, + tokens_per_step=scheduler.max_num_batched_tokens, + requests=scheduler.max_num_seqs, + planes=planes, + ) + page_elements = cache.block_size * config.num_key_value_heads * config.head_dim + # Reference attention decodes each page a batch reads into BF16 K/V, one + # layer at a time, so it can hold one layer of every block id at once. + gather = 2 * page_elements * parameter.element_size() + scratch = codec_scratch_bytes( + group_size=group, + block_size=cache.block_size, + num_kv_heads=config.num_key_value_heads, + head_dim=config.head_dim, + num_layers=config.num_hidden_layers, + ) + return packed, staging, gather, scratch + + @torch.inference_mode() def plan_cache( model, cache, scheduler, execution, backend, *, speculative=None, staged_prefill=None @@ -64,13 +108,36 @@ def plan_cache( * config.head_dim * parameter.element_size() ) + quantized = _quantized_plan(model, cache, scheduler) + staging_page, staging, quantized_scratch, gather = per_block, 0, 0, 0 + if quantized is not None: + # A block id then owns a packed slot; BF16 pages only stage recent KV. + per_block, staging, gather, quantized_scratch = quantized + + def kv_blocks(available, cost_per_block): + if quantized is None: + return budget_blocks(available, cost_per_block) + # The manager stages at most min(staging, blocks) BF16 pages. + blocks = budget_blocks(available, cost_per_block + staging_page) + if blocks >= staging: + blocks = budget_blocks(available - staging * staging_page, cost_per_block) + return blocks + + def quantized_info(blocks): + if quantized is None: + return {} + pages = min(staging, blocks) + return dict(kv_staging_blocks=pages, kv_staging_bytes=pages * staging_page) + + def fixed(blocks, source): + return blocks, {"source": source, "bytes_per_block": per_block, **quantized_info(blocks)} + if cache.num_blocks is not None: - return cache.num_blocks, {"source": "blocks", "bytes_per_block": per_block} + return fixed(cache.num_blocks, "blocks") if cache.kv_cache_memory_bytes is not None: - blocks = budget_blocks(cache.kv_cache_memory_bytes, per_block) - return blocks, {"source": "bytes", "bytes_per_block": per_block} + return fixed(kv_blocks(cache.kv_cache_memory_bytes, per_block), "bytes") if device.type != "cuda": - return 256, {"source": "cpu_default", "bytes_per_block": per_block} + return fixed(256, "cpu_default") torch.cuda.synchronize(device) # Return unused allocator segments before measuring driver-free memory. # In particular, a previous engine may have left its KV pool cached here. @@ -161,6 +228,7 @@ def plan_cache( + metadata + cache.memory_reserve_bytes + graph_reserve + + quantized_scratch ) # More blocks than all concurrent full-context requests can use only waste # memory, particularly for tiny-model GPU tests with very small pages. @@ -169,7 +237,11 @@ def plan_cache( * (config.total_ut_steps if cache.layout == "last_exited" else 1) * scheduler.max_num_seqs ) - blocks = min(budget_blocks(budget, per_block), useful_blocks) + blocks = min(kv_blocks(budget, per_block + gather), useful_blocks) + extra = quantized_info(blocks) + if quantized is not None: + extra["kv_gather_bytes_per_block"] = gather + extra["kv_quantized_scratch_bytes"] = quantized_scratch return blocks, dict( source="cuda_profile", bytes_per_block=per_block, @@ -180,4 +252,5 @@ def plan_cache( static_buffer_bytes=buffers, context_scratch_bytes=context_scratch, kv_budget_bytes=int(budget), + **extra, ) diff --git a/vllm_rlt/engine/stage_engine.py b/vllm_rlt/engine/stage_engine.py index a83d5101..17dd5e5b 100644 --- a/vllm_rlt/engine/stage_engine.py +++ b/vllm_rlt/engine/stage_engine.py @@ -2,6 +2,8 @@ from dataclasses import dataclass, replace from typing import TYPE_CHECKING, cast +import torch + from vllm_rlt.attention import BackendCapabilities, create_backend from vllm_rlt.config import ( CacheConfig, @@ -70,6 +72,17 @@ def __init__( self.execution_config = resolve_cuda_graph_rows( execution_config or ExecutionConfig(), scheduler_config, speculative_config ) + if cache_config.kv_quantization is not None: + # Pages pack between synchronous steps, and only the manager's + # host-prepared write/attend path knows where a page lives. + if self.execution_config.async_scheduling: + raise ValueError("KV quantization requires synchronous scheduling") + if self.execution_config.cuda_graphs: + raise ValueError("KV quantization does not support CUDA graphs") + if speculative_config is not None: + raise ValueError("KV quantization does not support speculative decoding") + if scheduler_config.enable_preemption: + raise ValueError("KV quantization does not support preemption") staged_prefill = requires_staged_prefill( scheduler_config, produces_tokens=self.stage_config.produces_tokens ) @@ -194,6 +207,10 @@ def __init__( incremental_allocation=cache_config.incremental_allocation, watermark_ratio=cache_config.watermark_ratio, recurrent_layers=getattr(model, "recurrent_kv_layers", None), + kv_quantization=cache_config.kv_quantization, + kv_quantization_group_size=cache_config.kv_quantization_group_size, + kv_quantization_window=cache_config.kv_quantization_window, + kv_quantization_staging_blocks=self.memory_plan.get("kv_staging_blocks"), ) self.scheduler = Scheduler( scheduler_config, @@ -269,6 +286,9 @@ def connector(self): @connector.setter def connector(self, connector): + # StageWorker attaches its connector after construction, so check here. + if connector is not None and self.cache_manager.kv_quantization is not None: + raise ValueError("KV quantization does not support PD transfer") self._connector = connector self._connector_blocked = set() self.scheduler.execution_ready = ( @@ -483,7 +503,9 @@ def _step(self) -> list[RequestOutput]: # Only host-dependent decisions collect now. Prompt/PRELUDE submissions # advance logically under their event without a readback wait. result = ticket.collect() if ticket.has_result else None - return self._update(batch, result) + outputs = self._update(batch, result) + self._pack_kv() + return outputs except Exception: if self.connector is not None: raise # Transfer leases remain quarantined until peer shutdown. @@ -493,6 +515,17 @@ def _step(self) -> list[RequestOutput]: self.abort_request(scheduled_request.request.request_id) raise + def _pack_kv(self): + cache = self.cache_manager + if cache.kv_quantization is None or not cache.ready_pages(): + return + # Drain every stream so packing reads complete staging pages, and finish + # before a later step reuses a freed staging page or reads a packed slot. + self.model_runner.synchronize() + cache.pack_ready() + if cache.device.type == "cuda": + torch.cuda.synchronize(cache.device) + def _schedule(self, *, prefer_recurrent=False): self._connector_blocked.clear() return self.scheduler.schedule(prefer_recurrent=prefer_recurrent) diff --git a/vllm_rlt/entrypoints/runtime_args.py b/vllm_rlt/entrypoints/runtime_args.py index 883d884b..5c91f9be 100644 --- a/vllm_rlt/entrypoints/runtime_args.py +++ b/vllm_rlt/entrypoints/runtime_args.py @@ -36,6 +36,14 @@ def add_runtime_args(parser): help="Copy metadata to the GPU instead of reading mapped host memory", ) parser.add_argument("--kv-layout", choices=["last_exited", "shared"], default="last_exited") + parser.add_argument( + "--kv-quantization", + choices=["int4", "nvfp4"], + help="Pack KV pages outside the latest window (Torch reference attention path)", + ) + parser.add_argument("--kv-quantization-group-size", type=int) + parser.add_argument("--kv-quantization-window", type=int, default=64) + parser.add_argument("--kv-quantization-staging-blocks", type=int) parser.add_argument("--gpu-memory-utilization", type=float, default=0.9) parser.add_argument("--kv-cache-memory-bytes", type=int) parser.add_argument("--memory-reserve-bytes", type=int, default=256 * 1024 * 1024) @@ -109,6 +117,10 @@ def runtime_configs(args): gpu_memory_utilization=args.gpu_memory_utilization, kv_cache_memory_bytes=args.kv_cache_memory_bytes, memory_reserve_bytes=args.memory_reserve_bytes, + kv_quantization=args.kv_quantization, + kv_quantization_group_size=args.kv_quantization_group_size, + kv_quantization_window=args.kv_quantization_window, + kv_quantization_staging_blocks=args.kv_quantization_staging_blocks, ), exit_config=ExitConfig( mode=args.exit_mode, From 460638b6d7843804c32b68963dde6970abe2da5a Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Sat, 14 Nov 2026 07:31:04 +0800 Subject: [PATCH 06/15] perf(kv): decode packed KV pages with a Triton kernel On CUDA, _gather_pages decodes INT4/NVFP4 pages straight into the BF16 attention input instead of calling the Torch reference codec per page. The kernel matches the reference bit for bit, including signed zeros, odd head_dim and head_dim smaller than the group size. CPU keeps the Torch path. Measured on L20 (8 seqs x 1024 prompt, 128 decode tokens, single run): INT4 49.3 -> 82.2 tok/s, NVFP4 2.0 -> 75.1 tok/s, BF16 baseline 105.2 tok/s. --- tests/test_triton_kv_decode.py | 62 ++++++++ vllm_rlt/core/kv_cache_manager.py | 16 +++ .../kernels/kv_quantization/triton_decode.py | 134 ++++++++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 tests/test_triton_kv_decode.py create mode 100644 vllm_rlt/kernels/kv_quantization/triton_decode.py diff --git a/tests/test_triton_kv_decode.py b/tests/test_triton_kv_decode.py new file mode 100644 index 00000000..9abd72b9 --- /dev/null +++ b/tests/test_triton_kv_decode.py @@ -0,0 +1,62 @@ +import pytest +import torch + +from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.kernels.kv_quantization.triton_decode import decode_pages_into + +# (format, group_size, block_size, heads, head_dim): odd and sub-group head_dim included. +CASES = [ + ("int4", 64, 64, 16, 128), + ("int4", 2, 4, 2, 8), + ("int4", 64, 64, 2, 8), + ("int4", 16, 32, 3, 65), + ("nvfp4", 16, 16, 2, 8), + ("nvfp4", 16, 64, 16, 128), + ("nvfp4", 16, 32, 3, 65), +] + + +@pytest.mark.gpu +@pytest.mark.parametrize("format,group,block,heads,dim", CASES) +@pytest.mark.parametrize("kind", ["key", "value"]) +def test_triton_decode_matches_torch_reference_bit_for_bit(format, group, block, heads, dim, kind): + torch.manual_seed(0) + layers, num_blocks = 3, 40 + cache = KVCacheManager( + num_layers=layers, + num_kv_heads=heads, + head_dim=dim, + num_blocks=num_blocks, + block_size=block, + max_loops=1, + dtype=torch.bfloat16, + device="cuda", + kv_quantization=format, + kv_quantization_group_size=group, + kv_quantization_window=block, + ) + blocks = torch.randperm(num_blocks, device="cuda")[:17] + pages = (torch.randn(len(blocks), layers, block, heads, dim, device="cuda") * 3).to( + torch.bfloat16 + ) + # A constant page exercises zero spans and signed zeros. + pages[0] = 0 + cache._encode_pages(kind, pages, blocks.tolist()) + for layer in range(layers): + reference = cache._decode_pages(kind, layer, blocks) + # NaN-filled output and shuffled rows catch unwritten or misplaced pages. + out = torch.full( + (len(blocks) + 3, block, heads, dim), float("nan"), dtype=torch.bfloat16, device="cuda" + ) + rows = torch.randperm(len(blocks) + 3, device="cuda")[: len(blocks)] + decode_pages_into( + out, + rows, + blocks, + cache._packed_pools[kind], + layer, + format=format, + kind=kind, + group_size=group, + ) + assert torch.equal(out[rows].view(torch.int16), reference.view(torch.int16)) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 6ea5b241..e6959f36 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -784,6 +784,22 @@ def _gather_pages(self, layer: int, plan) -> tuple[torch.Tensor, torch.Tensor]: if staged.numel(): keys[staged] = self.key_cache[slots, layer] values[staged] = self.value_cache[slots, layer] + if self.device.type == "cuda": + # Decode straight into the destination rows; no FP32 temporaries. + from vllm_rlt.kernels.kv_quantization.triton_decode import decode_pages_into + + for kind, out in (("key", keys), ("value", values)): + decode_pages_into( + out, + packed, + blocks, + self._packed_pools[kind], + layer, + format=self.kv_quantization, + kind=kind, + group_size=self.kv_quantization_group_size, + ) + return keys, values # Decode in chunks so FP32 temporaries stay bounded for long contexts. for start in range(0, packed.numel(), _DECODE_CHUNK_PAGES): rows = packed[start : start + _DECODE_CHUNK_PAGES] diff --git a/vllm_rlt/kernels/kv_quantization/triton_decode.py b/vllm_rlt/kernels/kv_quantization/triton_decode.py new file mode 100644 index 00000000..3b4dea3b --- /dev/null +++ b/vllm_rlt/kernels/kv_quantization/triton_decode.py @@ -0,0 +1,134 @@ +"""Gather packed KV pages of one layer and decode them straight into BF16 pages. + +The Torch reference (``unpack_kv``) materializes FP32 codes and expanded +parameters, then the caller copies the result into the attention input. This +kernel reads each page's payload and group parameters once and writes the +decoded value into its destination row, matching the reference bit for bit: +arithmetic stays in FP32 with the same operation order and no FMA contraction. +""" + +import torch +import triton +import triton.language as tl + +_TILE = 1024 + + +@triton.jit +def _decode_pages( + Out, + Rows, + Blocks, + Payload, + First, + Second, + out_page, + pool_block, + pool_layer, + param_block, + param_layer, + global_block, + global_layer, + layer, + TOKENS: tl.constexpr, + HEADS: tl.constexpr, + DIM: tl.constexpr, + GROUP: tl.constexpr, + IS_KEY: tl.constexpr, + IS_NVFP4: tl.constexpr, + TILE: tl.constexpr, +): + page = tl.program_id(0) + token = tl.program_id(1) + index = tl.program_id(2) * TILE + tl.arange(0, TILE) + valid = index < HEADS * DIM + head, dim = index // DIM, index % DIM + # Promote before multiplying: pool offsets exceed int32 for large caches. + block = tl.load(Blocks + page).to(tl.int64) + row = tl.load(Rows + page).to(tl.int64) + packed_dim = (DIM + 1) // 2 + payload = Payload + block * pool_block + layer * pool_layer + byte = tl.load(payload + (token * HEADS + head) * packed_dim + dim // 2, mask=valid, other=0) + code = ((byte >> ((dim % 2) * 4).to(tl.uint8)) & 0x0F).to(tl.int32) + if IS_KEY: + param = ((token // GROUP) * HEADS + head) * DIM + dim + else: + groups = (DIM + GROUP - 1) // GROUP + param = (token * HEADS + head) * groups + dim // GROUP + base = block * param_block + layer * param_layer + if IS_NVFP4: + # E2M1: magnitude code m -> 0, .5, 1, 1.5, 2, 3, 4, 6; bit 3 is the sign. + m = code & 7 + exponent = m >> 1 + mantissa = (m & 1).to(tl.float32) + normal = (1.0 + 0.5 * mantissa) * tl.exp2((exponent - 1).to(tl.float32)) + magnitude = tl.where(exponent == 0, 0.5 * mantissa, normal) + # Multiply rather than negate so code 0b1000 decodes to -0 like copysign. + value = magnitude * tl.where((code & 8) != 0, -1.0, 1.0) + scale = tl.load(First + base + param, mask=valid, other=0.0).to(tl.float32) + global_scale = tl.load(Second + block * global_block + layer * global_layer) + value = value * tl.div_rn(scale, global_scale) + else: + scale = tl.load(Second + base + param, mask=valid, other=0.0).to(tl.float32) + minimum = tl.load(First + base + param, mask=valid, other=0.0).to(tl.float32) + value = code.to(tl.float32) * scale + minimum + out = Out + row * out_page + token * HEADS * DIM + index + tl.store(out, value.to(Out.dtype.element_ty), mask=valid) + + +def decode_pages_into( + out: torch.Tensor, + rows: torch.Tensor, + blocks: torch.Tensor, + pool: dict[str, torch.Tensor], + layer: int, + *, + format: str, + kind: str, + group_size: int, +) -> None: + """Write decoded ``pool[blocks, layer]`` pages into ``out[rows]``. + + ``out`` is contiguous ``[pages, tokens, heads, dim]``; ``pool`` holds the + manager's ``[blocks, layers, ...]`` storages, each contiguous. + """ + count = blocks.numel() + if not count: + return + _, tokens, heads, dim = out.shape + payload = pool["payload"] + if format == "nvfp4": + first, second, global_scale = pool["scale"], pool["scale"], pool["global_scale"] + global_strides = global_scale.stride()[:2] + elif format == "int4": + first, second, global_scale = pool["minimum"], pool["scale"], None + global_strides = (0, 0) + else: + raise ValueError(f"unknown KV format: {format!r}") + for tensor in (out, payload, first, second): + if not tensor.is_contiguous(): + raise ValueError("decode_pages_into requires contiguous tensors") + grid = (count, tokens, triton.cdiv(heads * dim, _TILE)) + _decode_pages[grid]( + out, + rows, + blocks, + payload, + first, + second if global_scale is None else global_scale, + out.stride(0), + payload.stride(0), + payload.stride(1), + first.stride(0), + first.stride(1), + *global_strides, + layer, + TOKENS=tokens, + HEADS=heads, + DIM=dim, + GROUP=group_size, + IS_KEY=kind == "key", + IS_NVFP4=format == "nvfp4", + TILE=min(_TILE, triton.next_power_of_2(heads * dim)), + enable_fp_fusion=False, + ) From c8c79429bbdfd8721408fc4b30cb8222d75ba855 Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Sat, 14 Nov 2026 09:07:00 +0800 Subject: [PATCH 07/15] perf(kv): encode NVFP4 pages in one batched call Pack every page of a flush together, keeping one FP32 global scale per page. The result matches packing each page with nvfp4.pack bit for bit. --- tests/test_nvfp4_batch_pack.py | 23 ++++++++ vllm_rlt/core/kv_cache_manager.py | 11 ++++ .../kernels/kv_quantization/nvfp4_batch.py | 56 +++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 tests/test_nvfp4_batch_pack.py create mode 100644 vllm_rlt/kernels/kv_quantization/nvfp4_batch.py diff --git a/tests/test_nvfp4_batch_pack.py b/tests/test_nvfp4_batch_pack.py new file mode 100644 index 00000000..530358b2 --- /dev/null +++ b/tests/test_nvfp4_batch_pack.py @@ -0,0 +1,23 @@ +import pytest +import torch + +from vllm_rlt.kernels.kv_quantization import nvfp4 +from vllm_rlt.kernels.kv_quantization.nvfp4_batch import pack_batch + + +@pytest.mark.gpu +@pytest.mark.parametrize("block,heads,dim", [(16, 2, 8), (64, 16, 128), (32, 3, 65), (16, 1, 5)]) +@pytest.mark.parametrize("kind", ["key", "value"]) +def test_batched_nvfp4_pack_matches_per_page_bit_for_bit(block, heads, dim, kind): + torch.manual_seed(0) + x = (torch.randn(11, block, heads, dim, device="cuda") * 3).to(torch.bfloat16) + # All-zero, tiny and huge pages exercise the per-entry global scale. + x[0] = 0 + x[1] *= 1e-4 + x[2] *= 300 + payload, scale, global_scale = pack_batch(x, kind=kind) + for i in range(len(x)): + reference = nvfp4.pack(x[i], kind=kind) + assert torch.equal(payload[i], reference.payload) + assert torch.equal(scale[i].view(torch.uint8), reference.scale.view(torch.uint8)) + assert torch.equal(global_scale[i], reference.global_scale) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index e6959f36..acf91bb2 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -846,6 +846,17 @@ def _encode_pages(self, kind: KVKind, pages: torch.Tensor, blocks: list[int]) -> for name, storage in pool.items(): storage[index] = getattr(packed, name).view(len(page_ids), *storage.shape[2:]) return + if self.kv_quantization == "nvfp4" and self.device.type == "cuda": + # One global scale per page and layer, computed for the whole batch at once. + from vllm_rlt.kernels.kv_quantization.nvfp4_batch import pack_batch + + payload, scale, global_scale = pack_batch(items, kind=kind) + page_index = torch.tensor(page_ids, dtype=torch.long, device=self.device) + layer_index = torch.tensor(layer_ids, dtype=torch.long, device=self.device) + pool["payload"][page_index, layer_index] = payload + pool["scale"][page_index, layer_index] = scale + pool["global_scale"][page_index, layer_index] = global_scale + return for item, page, layer in zip(items, page_ids, layer_ids): packed = pack_kv(item, kind=kind, format=self.kv_quantization, group_size=group) for name, storage in pool.items(): diff --git a/vllm_rlt/kernels/kv_quantization/nvfp4_batch.py b/vllm_rlt/kernels/kv_quantization/nvfp4_batch.py new file mode 100644 index 00000000..f3b095f9 --- /dev/null +++ b/vllm_rlt/kernels/kv_quantization/nvfp4_batch.py @@ -0,0 +1,56 @@ +"""Batched NVFP4 encode for many [tokens, heads, dim] pages in one call. + +Each batch entry keeps its own FP32 global scale. Every operation mirrors +nvfp4.pack with a leading batch axis, so the result matches packing each +entry on its own bit for bit. +""" + +import torch + +from ._common import KVKind, _pack_channel_nibbles, _pad_repeat, group_axis + +_BOUNDARIES = [0.25, 0.75 - 2**-24, 1.25, 1.75 - 2**-23, 2.5, 3.5 - 2**-22, 5.0] + + +@torch.no_grad() +def pack_batch( + values: torch.Tensor, *, kind: KVKind +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Encode ``[N, tokens, heads, dim]`` into (payload, scale, global_scale[N]).""" + count, tokens, heads, channels = values.shape + axis = group_axis(kind) + 1 + work = values.float() + ones = torch.ones((count,), device=values.device, dtype=torch.float32) + global_scale = ones + if values.numel(): + amax = work.abs().flatten(1).amax(dim=1) + global_scale = torch.where(amax > 0, 2688.0 / amax, ones).clamp_max_( + torch.finfo(torch.float32).max + ) + inverse_global = global_scale.reciprocal().view(count, 1, 1, 1, 1) + scale_global = global_scale.view(count, 1, 1, 1, 1) + boundaries = torch.tensor(_BOUNDARIES, device=values.device, dtype=torch.float32) + padded = _pad_repeat(work, axis, 16) + if kind == "key": + grouped = padded.view(count, padded.shape[1] // 16, 16, heads, channels) + reduce_dim = 2 + else: + grouped = padded.view(count, tokens, heads, padded.shape[3] // 16, 16) + reduce_dim = 4 + amax = grouped.abs().amax(dim=reduce_dim, keepdim=True) + scale = (scale_global * (amax * (1.0 / 6))).clamp_(0, 448).to(torch.float8_e4m3fn) + effective = scale.float() * inverse_global + multiplier = effective.reciprocal() + finite = torch.isfinite(multiplier) + normalized = grouped * torch.where(finite, multiplier, 0.0) + normalized = torch.where((effective > 0) & ~finite, grouped / effective, normalized).clamp_( + -6, 6 + ) + codes = torch.bucketize(normalized.abs().contiguous(), boundaries, out_int32=True).to( + torch.uint8 + ) + codes |= torch.signbit(normalized).to(torch.uint8) << 3 + scales = scale.squeeze(reduce_dim) + codes = codes.view(padded.shape) + codes = codes[:, :tokens] if kind == "key" else codes[..., :channels] + return _pack_channel_nibbles(codes), scales, global_scale From 454ca9853f781ecc7f427c59380030f160e41f34 Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Sun, 15 Nov 2026 00:34:02 +0800 Subject: [PATCH 08/15] perf(kv): attend over packed KV pages in place during decode A fused Triton decode kernel decodes INT4/NVFP4 tiles in registers right before the dot product, instead of rewriting every history page to BF16 each step. Staged and packed pages are resolved per token through a per-page sources tensor. Prefill keeps the decode-to-BF16 path. Output matches decode-then-attend within one BF16 ulp. This step decodes NVFP4 with the same arithmetic as triton_decode. One attention layer, 16 heads, head_dim 128, 1152 tokens, RTX PRO 5000: INT4 fused 112 us vs decode+attend 164 us at 8 rows; 386 vs 770 us at 32 rows. NVFP4 fused 208 us vs decode+attend 161 us at 8 rows; 781 vs 788 us at 32 rows. INT4 is close to pure BF16 attention (108 / 377 us). NVFP4 is not yet a win; the next two commits fix that. --- tests/test_triton_packed_attention.py | 82 +++++ vllm_rlt/core/kv_cache_manager.py | 31 +- .../triton_attention_packed.py | 335 ++++++++++++++++++ 3 files changed, 446 insertions(+), 2 deletions(-) create mode 100644 tests/test_triton_packed_attention.py create mode 100644 vllm_rlt/kernels/kv_quantization/triton_attention_packed.py diff --git a/tests/test_triton_packed_attention.py b/tests/test_triton_packed_attention.py new file mode 100644 index 00000000..6075b005 --- /dev/null +++ b/tests/test_triton_packed_attention.py @@ -0,0 +1,82 @@ +import pytest +import torch + +from vllm_rlt.core.kv_cache_manager import KVCacheManager +from vllm_rlt.kernels.kv_quantization.triton_attention_packed import packed_paged_attention +from vllm_rlt.kernels.triton_attention import paged_attention + +# (format, group_size, block_size, q_heads, kv_heads, head_dim) +CASES = [ + ("int4", 64, 64, 16, 2, 128), + ("int4", 2, 4, 4, 2, 8), + ("int4", 16, 32, 6, 3, 65), + ("nvfp4", 16, 16, 4, 2, 8), + ("nvfp4", 16, 64, 16, 2, 128), + ("nvfp4", 16, 32, 6, 3, 65), +] + + +@pytest.mark.gpu +@pytest.mark.parametrize("format,group,block,heads,kv_heads,dim", CASES) +def test_packed_attention_matches_decode_then_attend(format, group, block, heads, kv_heads, dim): + torch.manual_seed(0) + layers, num_blocks, slots = 2, 48, 12 + cache = KVCacheManager( + num_layers=layers, + num_kv_heads=kv_heads, + head_dim=dim, + num_blocks=num_blocks, + block_size=block, + max_loops=1, + dtype=torch.bfloat16, + device="cuda", + kv_quantization=format, + kv_quantization_group_size=group, + kv_quantization_window=block, + ) + packed = torch.randperm(num_blocks, device="cuda")[:20] + for kind in ("key", "value"): + pages = (torch.randn(len(packed), layers, block, kv_heads, dim, device="cuda") * 2).to( + torch.bfloat16 + ) + cache._encode_pages(kind, pages, packed.tolist()) + staged_slots = torch.randperm(cache.key_cache.shape[0], device="cuda")[:slots] + cache.key_cache.normal_() + cache.value_cache.normal_() + # Remapped pages: packed first, then staged; sources follow the module contract. + sources = torch.cat([-(packed + 1), staged_slots]).to(torch.int32) + pages_total = len(sources) + rows = 5 + pages_per_row = 7 + tables = torch.stack( + [torch.randperm(pages_total, device="cuda")[:pages_per_row] for _ in range(rows)] + ).to(torch.int32) + # Lengths: empty, partial page, one token, full table, mid-page. + lengths = torch.tensor( + [0, block // 2 + 1, 1, pages_per_row * block, 3 * block + 1], device="cuda" + ).to(torch.int32) + q = torch.randn(rows, heads, dim, device="cuda", dtype=torch.bfloat16) + for layer in range(layers): + keys = torch.empty(pages_total, block, kv_heads, dim, dtype=torch.bfloat16, device="cuda") + values = torch.empty_like(keys) + keys[: len(packed)] = cache._decode_pages("key", layer, packed) + values[: len(packed)] = cache._decode_pages("value", layer, packed) + keys[len(packed) :] = cache.key_cache[staged_slots, layer] + values[len(packed) :] = cache.value_cache[staged_slots, layer] + expected = paged_attention(q, keys, values, tables, lengths) + actual = packed_paged_attention( + q, + cache.key_cache[:, layer], + cache.value_cache[:, layer], + cache._packed_pools["key"], + cache._packed_pools["value"], + layer, + tables, + sources, + lengths, + format=format, + group_size=group, + ) + assert torch.equal(actual[0], torch.zeros_like(actual[0])) + # Tile reductions may be ordered differently: allow one BF16 ulp. + torch.testing.assert_close(actual, expected, atol=0, rtol=2**-7) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index acf91bb2..08f964c2 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -33,6 +33,7 @@ ) from vllm_rlt.kernels.kv_quantization import KV_CODECS, pack_kv, unpack_kv from vllm_rlt.kernels.kv_quantization._common import KVKind +from vllm_rlt.kernels.paged_attention import TritonPagedAttention from vllm_rlt.request import ExecutionBatch from vllm_rlt.worker.buffers import stage_metadata from vllm_rlt.worker.metadata import DeviceKVMetadata, KVMetadata, MetadataBuilder @@ -933,18 +934,44 @@ def _attention_plan(self, batch: KVMetadata): remapped.append( [index.setdefault(b, len(index)) if j < needed else 0 for j, b in enumerate(table)] ) + # Per remapped page: its staging slot, or -(block + 1) once packed. + sources = [ + -(block + 1) if self._packed_page[block] else self._staging_of[block] + for block in index + ] plan = ( self._page_plan(list(index)), torch.tensor(remapped, dtype=torch.int32, device=self.device).reshape( batch.block_tables.shape ), + torch.tensor(sources or [0], dtype=torch.int32, device=self.device), ) self._attention_plans[batch] = (self._epoch, plan) return plan def _attend_quantized(self, layer: int, batch: KVMetadata, q: torch.Tensor) -> torch.Tensor: - """Reference path: decode the batch's pages to BF16, then run the backend.""" - plan, tables = self._attention_plan(batch) + """Decode reads packed pages in place; prefill decodes pages to BF16 first.""" + plan, tables, sources = self._attention_plan(batch) + kernel = getattr(self.attention, "kernel", self.attention) + if batch.cu_seqlens_q is None and isinstance(kernel, TritonPagedAttention): + # Same result as decode-then-attend, without writing BF16 pages. + from vllm_rlt.kernels.kv_quantization.triton_attention_packed import ( + packed_paged_attention, + ) + + return packed_paged_attention( + q, + self.key_cache[:, layer], + self.value_cache[:, layer], + self._packed_pools["key"], + self._packed_pools["value"], + layer, + tables, + sources, + batch.context_lengths, + format=self.kv_quantization, + group_size=self.kv_quantization_group_size, + ) keys, values = self._gather_pages(layer, plan) if batch.cu_seqlens_q is not None: return self.attention.prefill( diff --git a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py new file mode 100644 index 00000000..d566a6d5 --- /dev/null +++ b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py @@ -0,0 +1,335 @@ +"""Single-query paged attention that reads INT4/NVFP4 KV pages in place. + +The reference path decodes every page a batch reads into BF16 pages and then +runs ``paged_attention``, so each decode step rewrites the whole history. This +kernel decodes each [BLOCK_T, head_dim] tile in registers right before its dot +product, as vLLM's TurboQuant decode does. A page is either still staged in +the BF16 cache or packed; ``SOURCES`` says which, per page of the remapped +table: a staging slot (>= 0) or ``-(block + 1)`` for a packed block. Tiles may +span pages, so each token selects its source with masked loads. + +Decode uses the same FP32 operations as ``triton_decode`` (no FMA contraction) +and rounds to the cache dtype; the softmax reuses the reference tiling, split +layout and merge. Results usually equal decode-then-``paged_attention`` bit for +bit, but the compiler may order a tile reduction differently, so rare outputs +differ by one BF16 ulp. +""" + +import torch +import triton +import triton.language as tl + +from vllm_rlt.kernels.triton_attention import NUM_SPLITS, _paged_attention_merge_kernel + + +@triton.jit +def _mul_rn(a, b, EXACT: tl.constexpr): + # PTX never contracts mul.rn into an FMA, so a later add rounds separately, + # as in triton_decode (compiled with enable_fp_fusion=False). + if EXACT: + return tl.inline_asm_elementwise( + "mul.rn.f32 $0, $1, $2;", "=r,r,r", [a, b], dtype=tl.float32, is_pure=True, pack=1 + ) + return a * b + + +@triton.jit +def _e2m1(code): + # Same arithmetic as triton_decode: m -> 0, .5, 1, 1.5, 2, 3, 4, 6; bit 3 is + # the sign, multiplied in so code 0b1000 decodes to -0. + m = code & 7 + exponent = m >> 1 + mantissa = (m & 1).to(tl.float32) + normal = (1.0 + 0.5 * mantissa) * tl.exp2((exponent - 1).to(tl.float32)) + magnitude = tl.where(exponent == 0, 0.5 * mantissa, normal) + return magnitude * tl.where((code & 8) != 0, -1.0, 1.0) + + +@triton.jit +def _packed_tile( + Payload, + First, + Second, + Global, + block, + offsets, + dims, + mask, + kv_head, + layer, + pool_block, + pool_layer, + param_block, + param_layer, + global_block, + global_layer, + HEADS: tl.constexpr, + DIM: tl.constexpr, + GROUP: tl.constexpr, + IS_KEY: tl.constexpr, + IS_NVFP4: tl.constexpr, + EXACT: tl.constexpr, +): + """Decode one [tokens, dims] tile of a packed page into FP32.""" + packed_dim = (DIM + 1) // 2 + payload = Payload + block[:, None] * pool_block + layer * pool_layer + byte = tl.load( + payload + (offsets[:, None] * HEADS + kv_head) * packed_dim + dims[None, :] // 2, + mask=mask, + other=0, + ) + code = ((byte >> ((dims[None, :] % 2) * 4).to(tl.uint8)) & 0x0F).to(tl.int32) + if IS_KEY: + param = ((offsets[:, None] // GROUP) * HEADS + kv_head) * DIM + dims[None, :] + else: + groups = (DIM + GROUP - 1) // GROUP + param = (offsets[:, None] * HEADS + kv_head) * groups + dims[None, :] // GROUP + base = block[:, None] * param_block + layer * param_layer + if IS_NVFP4: + scale = tl.load(First + base + param, mask=mask, other=0.0).to(tl.float32) + global_scale = tl.load( + Global + block[:, None] * global_block + layer * global_layer, mask=mask, other=1.0 + ) + value = _e2m1(code) * tl.div_rn(scale, global_scale) + else: + minimum = tl.load(First + base + param, mask=mask, other=0.0).to(tl.float32) + scale = tl.load(Second + base + param, mask=mask, other=0.0).to(tl.float32) + value = _mul_rn(code.to(tl.float32), scale, EXACT) + minimum + return value + + +@triton.jit +def _packed_attention_kernel( + Q, + K, + V, + KPayload, + KFirst, + KSecond, + KGlobal, + VPayload, + VFirst, + VSecond, + VGlobal, + TABLES, + SOURCES, + LENGTHS, + PART_M, + PART_L, + PART_O, + layer, + q_batch_stride: tl.constexpr, + q_head_stride: tl.constexpr, + q_dim_stride: tl.constexpr, + k_block_stride: tl.constexpr, + k_token_stride: tl.constexpr, + k_head_stride: tl.constexpr, + v_block_stride: tl.constexpr, + v_token_stride: tl.constexpr, + v_head_stride: tl.constexpr, + table_stride: tl.constexpr, + kp_block: tl.constexpr, + kp_layer: tl.constexpr, + kq_block: tl.constexpr, + kq_layer: tl.constexpr, + kg_block: tl.constexpr, + kg_layer: tl.constexpr, + vp_block: tl.constexpr, + vp_layer: tl.constexpr, + vq_block: tl.constexpr, + vq_layer: tl.constexpr, + vg_block: tl.constexpr, + vg_layer: tl.constexpr, + HEAD_DIM: tl.constexpr, + HEAD_GROUPS: tl.constexpr, + NUM_HEADS: tl.constexpr, + KV_HEADS: tl.constexpr, + PAGE_SIZE: tl.constexpr, + GROUP: tl.constexpr, + IS_NVFP4: tl.constexpr, + EXACT: tl.constexpr, + SCALE: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_T: tl.constexpr, + SPLITS: tl.constexpr, +): + row = tl.program_id(0) + head = tl.program_id(1) + split = tl.program_id(2) + kv_head = head // HEAD_GROUPS + length = tl.load(LENGTHS + row) + split_tokens = ((length + SPLITS - 1) // SPLITS + BLOCK_T - 1) // BLOCK_T * BLOCK_T + begin = split * split_tokens + end = tl.minimum(length, begin + split_tokens) + dims = tl.arange(0, BLOCK_D) + valid_dims = dims < HEAD_DIM + query = tl.load( + Q + row * q_batch_stride + head * q_head_stride + dims * q_dim_stride, + mask=valid_dims, + other=0, + ).to(tl.float32) + maximum = -float("inf") + normalizer = 0.0 + accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32) + for start in range(begin, end, BLOCK_T): + positions = start + tl.arange(0, BLOCK_T) + valid_tokens = positions < end + mask = valid_tokens[:, None] & valid_dims[None, :] + offsets = positions % PAGE_SIZE + pages = tl.load( + TABLES + row * table_stride + positions // PAGE_SIZE, mask=valid_tokens, other=0 + ) + sources = tl.load(SOURCES + pages, mask=valid_tokens, other=0).to(tl.int64) + staged = mask & (sources[:, None] >= 0) + packed = mask & (sources[:, None] < 0) + slot = tl.maximum(sources, 0) + block = tl.maximum(-sources - 1, 0) + keys = tl.load( + K + + slot[:, None] * k_block_stride + + offsets[:, None] * k_token_stride + + kv_head * k_head_stride + + dims[None, :], + mask=staged, + other=0, + ) + values = tl.load( + V + + slot[:, None] * v_block_stride + + offsets[:, None] * v_token_stride + + kv_head * v_head_stride + + dims[None, :], + mask=staged, + other=0, + ) + if tl.max(tl.max(packed.to(tl.int32), axis=1), axis=0) > 0: + decoded_keys = _packed_tile( + KPayload, KFirst, KSecond, KGlobal, block, offsets, dims, packed, kv_head, + layer, kp_block, kp_layer, kq_block, kq_layer, kg_block, kg_layer, + KV_HEADS, HEAD_DIM, GROUP, True, IS_NVFP4, EXACT, + ) + decoded_values = _packed_tile( + VPayload, VFirst, VSecond, VGlobal, block, offsets, dims, packed, kv_head, + layer, vp_block, vp_layer, vq_block, vq_layer, vg_block, vg_layer, + KV_HEADS, HEAD_DIM, GROUP, False, IS_NVFP4, EXACT, + ) + # Round like the decoded BF16 pages the reference attends over. + keys = tl.where(packed, decoded_keys.to(keys.dtype), keys) + values = tl.where(packed, decoded_values.to(values.dtype), values) + keys = keys.to(tl.float32) + values = values.to(tl.float32) + scores = tl.sum(keys * query[None, :], axis=1) * SCALE + scores = tl.where(valid_tokens, scores, -float("inf")) + next_maximum = tl.maximum(maximum, tl.max(scores, axis=0)) + correction = tl.exp(maximum - next_maximum) + weights = tl.exp(scores - next_maximum) + accumulator = accumulator * correction + tl.sum(weights[:, None] * values, axis=0) + normalizer = normalizer * correction + tl.sum(weights, axis=0) + maximum = next_maximum + part = (row * NUM_HEADS + head) * SPLITS + split + tl.store(PART_M + part, maximum) + tl.store(PART_L + part, normalizer) + tl.store(PART_O + part * BLOCK_D + dims, accumulator) + + +def _pool_args(pool, format): + """(payload, first, second, global) tensors plus their block/layer strides.""" + payload = pool["payload"] + if format == "nvfp4": + first = second = pool["scale"] + global_scale = pool["global_scale"] + global_strides = global_scale.stride()[:2] + elif format == "int4": + first, second = pool["minimum"], pool["scale"] + global_scale, global_strides = second, (0, 0) + else: + raise ValueError(f"unknown KV format: {format!r}") + for tensor in (payload, first, second): + if not tensor.is_contiguous(): + raise ValueError("packed KV pools must be contiguous") + if first.stride() != second.stride(): + raise ValueError("INT4 minimum and scale pools must share a layout") + tensors = (payload, first, second, global_scale) + strides = (*payload.stride()[:2], *first.stride()[:2], *global_strides) + return tensors, strides + + +def packed_paged_attention( + q: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + key_pool: dict[str, torch.Tensor], + value_pool: dict[str, torch.Tensor], + layer: int, + block_tables: torch.Tensor, + sources: torch.Tensor, + context_lengths: torch.Tensor, + *, + format: str, + group_size: int, +) -> torch.Tensor: + """Decode attention over staged BF16 and packed pages of one layer. + + ``key_cache``/``value_cache`` are the staging pages of this layer, + ``[slots, block_size, kv_heads, dim]`` with unit dim stride. ``block_tables`` + index ``sources`` (see the module docstring). + """ + output = torch.empty_like(q) + if q.shape[0] == 0: + return output + rows, heads = q.shape[0], q.shape[1] + _, page_size, kv_heads, dim = key_cache.shape + if key_cache.stride(-1) != 1 or value_cache.stride(-1) != 1: + raise ValueError("staging caches need a unit dim stride") + block_d = triton.next_power_of_2(dim) + part_m = torch.empty((rows, heads, NUM_SPLITS), device=q.device, dtype=torch.float32) + part_l = torch.empty_like(part_m) + part_o = torch.empty((rows, heads, NUM_SPLITS, block_d), device=q.device, dtype=torch.float32) + k_tensors, k_strides = _pool_args(key_pool, format) + v_tensors, v_strides = _pool_args(value_pool, format) + _packed_attention_kernel[(rows, heads, NUM_SPLITS)]( + q, + key_cache, + value_cache, + *k_tensors, + *v_tensors, + block_tables, + sources, + context_lengths, + part_m, + part_l, + part_o, + layer, + *q.stride(), + *key_cache.stride()[:3], + *value_cache.stride()[:3], + block_tables.stride(0), + *k_strides, + *v_strides, + HEAD_DIM=dim, + HEAD_GROUPS=heads // kv_heads, + NUM_HEADS=heads, + KV_HEADS=kv_heads, + PAGE_SIZE=page_size, + GROUP=group_size, + IS_NVFP4=format == "nvfp4", + EXACT=torch.version.hip is None, + SCALE=dim**-0.5, + BLOCK_D=block_d, + BLOCK_T=32, + SPLITS=NUM_SPLITS, + num_warps=4, + ) + _paged_attention_merge_kernel[(rows, heads)]( + part_m, + part_l, + part_o, + output, + *output.stride(), + HEAD_DIM=dim, + NUM_HEADS=heads, + BLOCK_D=block_d, + SPLITS=NUM_SPLITS, + num_warps=4, + ) + return output From f3b3ada4b7cc2766968d4c00efb13e344238faab Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Sun, 15 Nov 2026 00:34:02 +0800 Subject: [PATCH 09/15] perf(kv): build NVFP4 E2M1 values from bits in the fused kernel The reference decode computes (1 + 0.5 m) * exp2(e - 1), selects the subnormal case and multiplies in the sign: an exp2, several FP32 multiplies and selects per element of every tile. E2M1 has only eight magnitudes, and each is an exact FP32 whose bits follow directly from the 3-bit code: (m + 252) << 22 for m >= 2, 252 << 22 for m = 1, zero for m = 0, with code bit 3 moved to the FP32 sign bit. The values are identical bit for bit (including -0 for 0b1000), so the output does not change; tests pass unchanged. NVFP4 fused attention, one layer, 1152 tokens: 208 -> 155 us at 8 rows, 781 -> 570 us at 32 rows (two runs, within 0.2 us). INT4 is unaffected. --- .../kv_quantization/triton_attention_packed.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py index d566a6d5..814c09ce 100644 --- a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py +++ b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py @@ -35,14 +35,13 @@ def _mul_rn(a, b, EXACT: tl.constexpr): @triton.jit def _e2m1(code): - # Same arithmetic as triton_decode: m -> 0, .5, 1, 1.5, 2, 3, 4, 6; bit 3 is - # the sign, multiplied in so code 0b1000 decodes to -0. + # Build the FP32 bits of E2M1 directly: magnitude m = 2..7 is exponent + # (m >> 1) - 1 with mantissa bit m & 1, i.e. (m + 252) << 22; m = 1 is 0.5 + # (252 << 22); m = 0 is zero. Bit 3 becomes the FP32 sign, so 0b1000 is -0. m = code & 7 - exponent = m >> 1 - mantissa = (m & 1).to(tl.float32) - normal = (1.0 + 0.5 * mantissa) * tl.exp2((exponent - 1).to(tl.float32)) - magnitude = tl.where(exponent == 0, 0.5 * mantissa, normal) - return magnitude * tl.where((code & 8) != 0, -1.0, 1.0) + magnitude = tl.where(m >= 2, m + 252, tl.where(m == 1, 252, 0)) << 22 + bits = magnitude | ((code & 8) << 28) + return bits.to(tl.float32, bitcast=True) @triton.jit From a81515fbe04d36bda02f65aef5dfa86620f99d0b Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Sun, 15 Nov 2026 00:34:02 +0800 Subject: [PATCH 10/15] perf(kv): load the NVFP4 global scale once per token The global scale is one FP32 per page and layer, but the tile loaded it with a [tokens, dims] pointer, i.e. 128 identical loads per token, and divided per element. Loading it with a per-token pointer (masked by whether the token has any valid channel) and broadcasting removes those redundant loads. The division still happens per element with div_rn, so values are unchanged. NVFP4 fused attention, one layer, 1152 tokens: 155 -> 143 us at 8 rows, 570 -> 528 us at 32 rows. End to end on Ouro-1.4B, 8 seqs x 1024 prompt, 128 decode tokens, the full fused path reaches INT4 148.3 and NVFP4 151.9 tok/s (decode-then-attend: 137.7 and 134.6; BF16 160.6). --- .../kernels/kv_quantization/triton_attention_packed.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py index 814c09ce..c7967513 100644 --- a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py +++ b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py @@ -86,10 +86,13 @@ def _packed_tile( base = block[:, None] * param_block + layer * param_layer if IS_NVFP4: scale = tl.load(First + base + param, mask=mask, other=0.0).to(tl.float32) + # One global scale per page and layer: load it once per token. + tokens = tl.max(mask.to(tl.int32), axis=1) > 0 global_scale = tl.load( - Global + block[:, None] * global_block + layer * global_layer, mask=mask, other=1.0 + Global + block * global_block + layer * global_layer, mask=tokens, other=1.0 ) - value = _e2m1(code) * tl.div_rn(scale, global_scale) + ratio = tl.div_rn(scale, global_scale[:, None]) + value = _e2m1(code) * ratio else: minimum = tl.load(First + base + param, mask=mask, other=0.0).to(tl.float32) scale = tl.load(Second + base + param, mask=mask, other=0.0).to(tl.float32) From a59304f59a521d9582c667ecadc87311dfa6d38b Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Sat, 14 Nov 2026 16:28:11 +0800 Subject: [PATCH 11/15] docs(kv): record why NVFP4 decode differs by one ulp The fused kernel matched decode-then-attend exactly for INT4 but one output in 20480 differed for NVFP4 at head_dim 128. Traced it with three probes: - Decoding a tile in isolation and comparing against triton_decode gave 0 of 65536 differences for both K and V, so the arithmetic is not the cause. - Dumping the tile the kernel attends over showed keys and values identical to the decoded pages, but 16 of 32 scores off by one ulp. - TTGIR for both kernels showed the same reduction structure and the same number of splits, but different tile layouts: the reference and the INT4 path use sizePerThread [1, 8], NVFP4 uses [1, 16]. The cause is the scale width. A BF16 key element is two bytes and an INT4 scale is two bytes, so eight elements fill a 16-byte vector. An NVFP4 E4M3 scale is one byte, so Triton fills sixteen and assigns the tile [1, 16]; the FP32 score dot product then associates as a different tree. Tried and rejected: tl.max_contiguous on the scale index (compiles, does not change the choice), an explicit two-level sum reshaped into the reference's 8-then-16 tree (compiles to a different binary, still differs), and mul.rn through inline asm on the NVFP4 product (no effect). Document the caveat in the module docstring instead of forcing the layout. --- .../kv_quantization/triton_attention_packed.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py index c7967513..8855e2a1 100644 --- a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py +++ b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py @@ -10,9 +10,16 @@ Decode uses the same FP32 operations as ``triton_decode`` (no FMA contraction) and rounds to the cache dtype; the softmax reuses the reference tiling, split -layout and merge. Results usually equal decode-then-``paged_attention`` bit for -bit, but the compiler may order a tile reduction differently, so rare outputs -differ by one BF16 ulp. +layout and merge. INT4 results equal decode-then-``paged_attention`` bit for +bit. NVFP4 does too except for rare single outputs (1 in 20480 at head_dim 128, +none at 8 or 65), because its E4M3 scale is one byte where the BF16 key tile is +two: Triton fills a 16-byte vector from the scale load and gives the tile +``sizePerThread`` [1, 16] instead of the reference kernel's [1, 8], so the FP32 +score dot product associates as a different tree. The decoded values themselves +are identical; only that reduction order differs. ``tl.max_contiguous`` on the +scale index does not change the choice, and an explicit two-level sum in the +reference's tree did not either, so this is left as a documented caveat rather +than paid for with a layout-forcing workaround. """ import torch From ca06bb7ce20cb1c8e976d31fc6d24e0ab9b18ea7 Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Sat, 14 Nov 2026 16:29:03 +0800 Subject: [PATCH 12/15] test(kv): require bit-exact INT4 output from the fused kernel The test relaxed every case to one BF16 ulp. Only NVFP4 needs that: INT4 reaches decode-then-attend exactly at all three shapes (dim 8, 65, 128), so assert equality there and keep the tolerance for NVFP4 alone, with the reason next to it. --- tests/test_triton_packed_attention.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_triton_packed_attention.py b/tests/test_triton_packed_attention.py index 6075b005..bc9a7803 100644 --- a/tests/test_triton_packed_attention.py +++ b/tests/test_triton_packed_attention.py @@ -78,5 +78,12 @@ def test_packed_attention_matches_decode_then_attend(format, group, block, heads group_size=group, ) assert torch.equal(actual[0], torch.zeros_like(actual[0])) - # Tile reductions may be ordered differently: allow one BF16 ulp. - torch.testing.assert_close(actual, expected, atol=0, rtol=2**-7) + if format == "int4": + # Same arithmetic and same tile layout as the reference: exact. + assert torch.equal(actual, expected) + else: + # The one-byte E4M3 scale makes Triton lay the tile out as + # [1, 16] where the reference gets [1, 8], so the FP32 score dot + # product associates differently. Decoded values are identical; + # allow the single ulp that reduction order can move. + torch.testing.assert_close(actual, expected, atol=0, rtol=2**-7) From 4cad536676e42b55fa24776f5ee7bf4d5ef13dd7 Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Mon, 16 Nov 2026 02:17:10 +0800 Subject: [PATCH 13/15] perf(kv): decode NVFP4 pages with the shared bit-built E2M1 triton_decode (the prefill path) still built E2M1 values with exp2 and a sign multiply, while the fused decode kernel had switched to assembling the FP32 bits directly. Both files also carried their own copy of the pool argument dispatch. Move _e2m1 and _pool_args into triton_decode and import them from the fused kernel. The decode kernel now takes the global scale as its own argument instead of overloading Second, and drops the unused TOKENS constexpr. The bit-built values are identical to the exp2 form, including -0 for code 0b1000; the bit-for-bit decode tests still pass (272 KV tests, --run-gpu). decode_pages_into, 512 pages x 64 tokens x 8 heads x 128, K and V, RTX PRO 5000 Blackwell, median of 5 do_bench runs, two alternating rounds: NVFP4 166.0 -> 157.6 us INT4 164.9 -> 164.9 us (unchanged code, control) The fused kernel is unchanged: 144 / 531 us at 8 / 32 rows before and after. --- .../triton_attention_packed.py | 34 +-------- .../kernels/kv_quantization/triton_decode.py | 74 +++++++++++-------- 2 files changed, 43 insertions(+), 65 deletions(-) diff --git a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py index 8855e2a1..267addaa 100644 --- a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py +++ b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py @@ -26,6 +26,7 @@ import triton import triton.language as tl +from vllm_rlt.kernels.kv_quantization.triton_decode import _e2m1, _pool_args from vllm_rlt.kernels.triton_attention import NUM_SPLITS, _paged_attention_merge_kernel @@ -40,17 +41,6 @@ def _mul_rn(a, b, EXACT: tl.constexpr): return a * b -@triton.jit -def _e2m1(code): - # Build the FP32 bits of E2M1 directly: magnitude m = 2..7 is exponent - # (m >> 1) - 1 with mantissa bit m & 1, i.e. (m + 252) << 22; m = 1 is 0.5 - # (252 << 22); m = 0 is zero. Bit 3 becomes the FP32 sign, so 0b1000 is -0. - m = code & 7 - magnitude = tl.where(m >= 2, m + 252, tl.where(m == 1, 252, 0)) << 22 - bits = magnitude | ((code & 8) << 28) - return bits.to(tl.float32, bitcast=True) - - @triton.jit def _packed_tile( Payload, @@ -241,28 +231,6 @@ def _packed_attention_kernel( tl.store(PART_O + part * BLOCK_D + dims, accumulator) -def _pool_args(pool, format): - """(payload, first, second, global) tensors plus their block/layer strides.""" - payload = pool["payload"] - if format == "nvfp4": - first = second = pool["scale"] - global_scale = pool["global_scale"] - global_strides = global_scale.stride()[:2] - elif format == "int4": - first, second = pool["minimum"], pool["scale"] - global_scale, global_strides = second, (0, 0) - else: - raise ValueError(f"unknown KV format: {format!r}") - for tensor in (payload, first, second): - if not tensor.is_contiguous(): - raise ValueError("packed KV pools must be contiguous") - if first.stride() != second.stride(): - raise ValueError("INT4 minimum and scale pools must share a layout") - tensors = (payload, first, second, global_scale) - strides = (*payload.stride()[:2], *first.stride()[:2], *global_strides) - return tensors, strides - - def packed_paged_attention( q: torch.Tensor, key_cache: torch.Tensor, diff --git a/vllm_rlt/kernels/kv_quantization/triton_decode.py b/vllm_rlt/kernels/kv_quantization/triton_decode.py index 3b4dea3b..bbd05591 100644 --- a/vllm_rlt/kernels/kv_quantization/triton_decode.py +++ b/vllm_rlt/kernels/kv_quantization/triton_decode.py @@ -14,6 +14,40 @@ _TILE = 1024 +@triton.jit +def _e2m1(code): + # E2M1 magnitudes 0, .5, 1, 1.5, 2, 3, 4, 6 as FP32 bits: m = 2..7 is + # (m + 252) << 22, m = 1 is 252 << 22. Bit 3 is the sign, so 0b1000 is -0. + m = code & 7 + magnitude = tl.where(m >= 2, m + 252, tl.where(m == 1, 252, 0)) << 22 + return (magnitude | ((code & 8) << 28)).to(tl.float32, bitcast=True) + + +def _pool_args(pool, format): + """(payload, first, second, global) tensors plus their [block, layer] strides. + + INT4 reads minimum and scale as first and second; NVFP4 reads its E4M3 + scale as first and its per-page FP32 global scale as global. + """ + payload = pool["payload"] + if format == "nvfp4": + first = second = pool["scale"] + global_scale = pool["global_scale"] + global_strides = global_scale.stride()[:2] + elif format == "int4": + first, second = pool["minimum"], pool["scale"] + global_scale, global_strides = second, (0, 0) + else: + raise ValueError(f"unknown KV format: {format!r}") + for tensor in (payload, first, second): + if not tensor.is_contiguous(): + raise ValueError("packed KV pools must be contiguous") + if first.stride() != second.stride(): + raise ValueError("INT4 minimum and scale pools must share a layout") + tensors = (payload, first, second, global_scale) + return tensors, (*payload.stride()[:2], *first.stride()[:2], *global_strides) + + @triton.jit def _decode_pages( Out, @@ -22,6 +56,7 @@ def _decode_pages( Payload, First, Second, + Global, out_page, pool_block, pool_layer, @@ -30,7 +65,6 @@ def _decode_pages( global_block, global_layer, layer, - TOKENS: tl.constexpr, HEADS: tl.constexpr, DIM: tl.constexpr, GROUP: tl.constexpr, @@ -57,17 +91,9 @@ def _decode_pages( param = (token * HEADS + head) * groups + dim // GROUP base = block * param_block + layer * param_layer if IS_NVFP4: - # E2M1: magnitude code m -> 0, .5, 1, 1.5, 2, 3, 4, 6; bit 3 is the sign. - m = code & 7 - exponent = m >> 1 - mantissa = (m & 1).to(tl.float32) - normal = (1.0 + 0.5 * mantissa) * tl.exp2((exponent - 1).to(tl.float32)) - magnitude = tl.where(exponent == 0, 0.5 * mantissa, normal) - # Multiply rather than negate so code 0b1000 decodes to -0 like copysign. - value = magnitude * tl.where((code & 8) != 0, -1.0, 1.0) scale = tl.load(First + base + param, mask=valid, other=0.0).to(tl.float32) - global_scale = tl.load(Second + block * global_block + layer * global_layer) - value = value * tl.div_rn(scale, global_scale) + global_scale = tl.load(Global + block * global_block + layer * global_layer) + value = _e2m1(code) * tl.div_rn(scale, global_scale) else: scale = tl.load(Second + base + param, mask=valid, other=0.0).to(tl.float32) minimum = tl.load(First + base + param, mask=valid, other=0.0).to(tl.float32) @@ -96,34 +122,18 @@ def decode_pages_into( if not count: return _, tokens, heads, dim = out.shape - payload = pool["payload"] - if format == "nvfp4": - first, second, global_scale = pool["scale"], pool["scale"], pool["global_scale"] - global_strides = global_scale.stride()[:2] - elif format == "int4": - first, second, global_scale = pool["minimum"], pool["scale"], None - global_strides = (0, 0) - else: - raise ValueError(f"unknown KV format: {format!r}") - for tensor in (out, payload, first, second): - if not tensor.is_contiguous(): - raise ValueError("decode_pages_into requires contiguous tensors") + if not out.is_contiguous(): + raise ValueError("decode_pages_into requires a contiguous output") + tensors, strides = _pool_args(pool, format) grid = (count, tokens, triton.cdiv(heads * dim, _TILE)) _decode_pages[grid]( out, rows, blocks, - payload, - first, - second if global_scale is None else global_scale, + *tensors, out.stride(0), - payload.stride(0), - payload.stride(1), - first.stride(0), - first.stride(1), - *global_strides, + *strides, layer, - TOKENS=tokens, HEADS=heads, DIM=dim, GROUP=group_size, From 9743cf24294f2dc27bc128b43a25a07587786d72 Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Mon, 16 Nov 2026 02:17:15 +0800 Subject: [PATCH 14/15] style(kv): pass ruff format and trim the NVFP4 caveat CI runs `ruff format --check .`, which failed on two lines this branch added: the hand-packed _packed_tile calls and the sources comprehension in _attention_plan. Apply the formatter. Shorten the module docstring's NVFP4 note to the cause and its effect; the probes and rejected workarounds are in a59304f's message. The test comment now points at the docstring instead of repeating it. --- tests/test_triton_packed_attention.py | 6 +- vllm_rlt/core/kv_cache_manager.py | 3 +- .../triton_attention_packed.py | 68 ++++++++++++++----- 3 files changed, 52 insertions(+), 25 deletions(-) diff --git a/tests/test_triton_packed_attention.py b/tests/test_triton_packed_attention.py index bc9a7803..38a38761 100644 --- a/tests/test_triton_packed_attention.py +++ b/tests/test_triton_packed_attention.py @@ -79,11 +79,7 @@ def test_packed_attention_matches_decode_then_attend(format, group, block, heads ) assert torch.equal(actual[0], torch.zeros_like(actual[0])) if format == "int4": - # Same arithmetic and same tile layout as the reference: exact. assert torch.equal(actual, expected) else: - # The one-byte E4M3 scale makes Triton lay the tile out as - # [1, 16] where the reference gets [1, 8], so the FP32 score dot - # product associates differently. Decoded values are identical; - # allow the single ulp that reduction order can move. + # One ulp from a different score reduction order (module docstring). torch.testing.assert_close(actual, expected, atol=0, rtol=2**-7) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 08f964c2..27800dd9 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -936,8 +936,7 @@ def _attention_plan(self, batch: KVMetadata): ) # Per remapped page: its staging slot, or -(block + 1) once packed. sources = [ - -(block + 1) if self._packed_page[block] else self._staging_of[block] - for block in index + -(block + 1) if self._packed_page[block] else self._staging_of[block] for block in index ] plan = ( self._page_plan(list(index)), diff --git a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py index 267addaa..87ec5105 100644 --- a/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py +++ b/vllm_rlt/kernels/kv_quantization/triton_attention_packed.py @@ -8,18 +8,12 @@ table: a staging slot (>= 0) or ``-(block + 1)`` for a packed block. Tiles may span pages, so each token selects its source with masked loads. -Decode uses the same FP32 operations as ``triton_decode`` (no FMA contraction) -and rounds to the cache dtype; the softmax reuses the reference tiling, split -layout and merge. INT4 results equal decode-then-``paged_attention`` bit for -bit. NVFP4 does too except for rare single outputs (1 in 20480 at head_dim 128, -none at 8 or 65), because its E4M3 scale is one byte where the BF16 key tile is -two: Triton fills a 16-byte vector from the scale load and gives the tile -``sizePerThread`` [1, 16] instead of the reference kernel's [1, 8], so the FP32 -score dot product associates as a different tree. The decoded values themselves -are identical; only that reduction order differs. ``tl.max_contiguous`` on the -scale index does not change the choice, and an explicit two-level sum in the -reference's tree did not either, so this is left as a documented caveat rather -than paid for with a layout-forcing workaround. +Decode uses the same FP32 operations as ``triton_decode`` and rounds to the +cache dtype; the softmax reuses the reference tiling, split layout and merge. +INT4 matches decode-then-``paged_attention`` bit for bit. NVFP4 decodes the +same values, but its one-byte E4M3 scale makes Triton lay the tile out as +[1, 16] instead of [1, 8], so the FP32 score sum can differ by one ulp in rare +outputs (head_dim 128). """ import torch @@ -203,14 +197,52 @@ def _packed_attention_kernel( ) if tl.max(tl.max(packed.to(tl.int32), axis=1), axis=0) > 0: decoded_keys = _packed_tile( - KPayload, KFirst, KSecond, KGlobal, block, offsets, dims, packed, kv_head, - layer, kp_block, kp_layer, kq_block, kq_layer, kg_block, kg_layer, - KV_HEADS, HEAD_DIM, GROUP, True, IS_NVFP4, EXACT, + KPayload, + KFirst, + KSecond, + KGlobal, + block, + offsets, + dims, + packed, + kv_head, + layer, + kp_block, + kp_layer, + kq_block, + kq_layer, + kg_block, + kg_layer, + KV_HEADS, + HEAD_DIM, + GROUP, + True, + IS_NVFP4, + EXACT, ) decoded_values = _packed_tile( - VPayload, VFirst, VSecond, VGlobal, block, offsets, dims, packed, kv_head, - layer, vp_block, vp_layer, vq_block, vq_layer, vg_block, vg_layer, - KV_HEADS, HEAD_DIM, GROUP, False, IS_NVFP4, EXACT, + VPayload, + VFirst, + VSecond, + VGlobal, + block, + offsets, + dims, + packed, + kv_head, + layer, + vp_block, + vp_layer, + vq_block, + vq_layer, + vg_block, + vg_layer, + KV_HEADS, + HEAD_DIM, + GROUP, + False, + IS_NVFP4, + EXACT, ) # Round like the decoded BF16 pages the reference attends over. keys = tl.where(packed, decoded_keys.to(keys.dtype), keys) From 8e35940c8e6985f772d370abf733569fc13136e1 Mon Sep 17 00:00:00 2001 From: HowardYangAI <13777842855@163.com> Date: Mon, 16 Nov 2026 03:05:12 +0800 Subject: [PATCH 15/15] perf(kv): attend over shared pages through one decode, not the fused kernel The fused kernel decodes each page once per row that reads it. In decode every row is a different sequence or loop plane, so that is once per page. But without packed prefill (the default), a prompt chunk runs as single-query rows that all read the same prefix, and every one of them re-decoded it: with 8 x 1024-token prompts each fused call had 1024 rows and _gather_pages was never called. _attention_plan already walks every row's pages, so it now counts page reads against unique pages and keeps the fused kernel only while each page is read fewer than twice on average. Other batches decode their pages once into BF16 and run the backend as before. Ouro-1.4B, 8 seqs x 1024 prompt, 128 decode tokens, RTX PRO 5000 Blackwell, warm Triton cache, two runs each (307f6b5 is the PyTorch-decode baseline): prefill s (307f6b5 / before / after) decode tok/s (before / after) INT4 3.00 / 4.89 / 2.73 156.3 / 158.0 NVFP4 18.2 / 6.62 / 2.90 153.1 / 154.4 Decode is unchanged, as expected: those batches still take the fused path. --- vllm_rlt/core/kv_cache_manager.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/vllm_rlt/core/kv_cache_manager.py b/vllm_rlt/core/kv_cache_manager.py index 27800dd9..6cefb44a 100644 --- a/vllm_rlt/core/kv_cache_manager.py +++ b/vllm_rlt/core/kv_cache_manager.py @@ -928,8 +928,10 @@ def _attention_plan(self, batch: KVMetadata): lengths = batch.context_lengths.tolist() index: dict[int, int] = {} remapped = [] + reads = 0 for table, length in zip(tables, lengths): needed = (length + self.block_size - 1) // self.block_size + reads += needed # Entries past a row's context are never read; point them at page 0. remapped.append( [index.setdefault(b, len(index)) if j < needed else 0 for j, b in enumerate(table)] @@ -944,15 +946,19 @@ def _attention_plan(self, batch: KVMetadata): batch.block_tables.shape ), torch.tensor(sources or [0], dtype=torch.int32, device=self.device), + # The fused kernel decodes a page once per row that reads it, so it + # only pays off when rows rarely share pages (decode, not the rows + # of a prompt chunk, which all read the same prefix). + reads < 2 * len(index), ) self._attention_plans[batch] = (self._epoch, plan) return plan def _attend_quantized(self, layer: int, batch: KVMetadata, q: torch.Tensor) -> torch.Tensor: - """Decode reads packed pages in place; prefill decodes pages to BF16 first.""" - plan, tables, sources = self._attention_plan(batch) + """Decode rows read packed pages in place; rows sharing pages decode them to BF16 first.""" + plan, tables, sources, fused = self._attention_plan(batch) kernel = getattr(self.attention, "kernel", self.attention) - if batch.cu_seqlens_q is None and isinstance(kernel, TritonPagedAttention): + if fused and batch.cu_seqlens_q is None and isinstance(kernel, TritonPagedAttention): # Same result as decode-then-attend, without writing BF16 pages. from vllm_rlt.kernels.kv_quantization.triton_attention_packed import ( packed_paged_attention,