From 36aaa7f316a787e7674c6e83e8ac3ec51cf6b760 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 17 Jul 2026 12:43:32 +0200 Subject: [PATCH 01/23] feat: native pure-Rust u64 FastPFOR codec Adds a 64-bit (`u64`) FastPFOR codec to the pure-Rust side, which was previously 32-bit only. `FastPForWide128`/`FastPForWide256` compress `u64` values via the (now un-gated) `BlockCodec64` trait: FastPFOR-packed aligned blocks plus a variable-byte tail for the sub-block remainder. The wire format is byte-identical to the C++ `CppFastPFor128`/`CppFastPFor256` `encode64`/`decode64` paths, verified by parity unit tests and a 30-minute differential fuzz run (13.8M executions, no crashes). - bitpacking_wide: generic scalar packer for `u64` at widths 0..=64, whose bit-ordering is cross-checked against the proven u32 kernels. - fastpfor64: `FastPForWide` with 65-entry exception tables, a 2-word exception bitmap, and u64 exception values. - Un-gate `BlockCodec64` so pure-Rust builds get 64-bit support. - Add the `fastpfor_u64` differential fuzz target and a README example. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 21 + fuzz/Cargo.toml | 6 + fuzz/fuzz_targets/fastpfor_u64.rs | 61 ++ src/codec.rs | 8 +- src/lib.rs | 3 +- .../integer_compression/bitpacking_wide.rs | 106 +++ src/rust/integer_compression/fastpfor64.rs | 673 ++++++++++++++++++ src/rust/integer_compression/mod.rs | 2 + src/rust/mod.rs | 2 + src/test_utils.rs | 7 +- 10 files changed, 877 insertions(+), 12 deletions(-) create mode 100644 fuzz/fuzz_targets/fastpfor_u64.rs create mode 100644 src/rust/integer_compression/bitpacking_wide.rs create mode 100644 src/rust/integer_compression/fastpfor64.rs diff --git a/README.md b/README.md index 2f3cc20..ba83415 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,27 @@ codec.decode_blocks(&encoded, Some(u32::try_from(blocks.len() * 256).expect("blo assert_eq!(decoded, input); ``` +### 64-bit integers (`u64`) + +`FastPForWide128` / `FastPForWide256` compress `u64` values via the `BlockCodec64` +trait. The wire format is byte-compatible with the C++ `CppFastPFor128` / +`CppFastPFor256` `encode64` / `decode64` paths. + +```rust +use fastpfor::{BlockCodec64, FastPForWide256}; + +let mut codec = FastPForWide256::default(); +let input: Vec = (0..600).map(|i| i * 1_000_000_000).collect(); + +let mut encoded = Vec::new(); +codec.encode64(&input, &mut encoded).unwrap(); + +let mut decoded = Vec::new(); +codec.decode64(&encoded, &mut decoded).unwrap(); + +assert_eq!(decoded, input); +``` + ### C++ Wrapper (`cpp` feature) Enable the `cpp` feature in `Cargo.toml`: diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 69e9628..09bf44d 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -62,3 +62,9 @@ name = "compare_fastpfor_128" path = "fuzz_targets/compare_fastpfor_128.rs" test = false doc = false + +[[bin]] +name = "fastpfor_u64" +path = "fuzz_targets/fastpfor_u64.rs" +test = false +doc = false diff --git a/fuzz/fuzz_targets/fastpfor_u64.rs b/fuzz/fuzz_targets/fastpfor_u64.rs new file mode 100644 index 0000000..5ccef1a --- /dev/null +++ b/fuzz/fuzz_targets/fastpfor_u64.rs @@ -0,0 +1,61 @@ +#![no_main] + +//! Differential fuzz of the 64-bit FastPFOR codec. +//! +//! For arbitrary `u64` input, the pure-Rust `FastPForWide` and the C++ +//! `CppFastPFor128` / `CppFastPFor256` must produce bit-identical compressed +//! output, and every decoder must reproduce the original input — including +//! decoding the other implementation's bytes. + +use fastpfor::cpp::{CppFastPFor128, CppFastPFor256}; +use fastpfor::{BlockCodec64, FastPForWide128, FastPForWide256}; +use libfuzzer_sys::fuzz_target; + +#[derive(arbitrary::Arbitrary, Debug)] +struct Input { + data: Vec, + use_256: bool, +} + +fn check(rust: &mut impl BlockCodec64, cpp: &mut impl BlockCodec64, data: &[u64], name: &str) { + let mut rust_enc = Vec::new(); + rust.encode64(data, &mut rust_enc).expect("Rust encode64 failed"); + + let mut cpp_enc = Vec::new(); + cpp.encode64(data, &mut cpp_enc).expect("C++ encode64 failed"); + + assert_eq!(rust_enc, cpp_enc, "{name}: Rust and C++ encode64 bytes differ"); + + let mut rust_dec = Vec::new(); + rust.decode64(&rust_enc, &mut rust_dec) + .expect("Rust decode64 of own output failed"); + assert_eq!(rust_dec, data, "{name}: Rust roundtrip mismatch"); + + let mut cross = Vec::new(); + rust.decode64(&cpp_enc, &mut cross) + .expect("Rust decode64 of C++ output failed"); + assert_eq!(cross, data, "{name}: Rust could not decode C++ output"); + + let mut cpp_dec = Vec::new(); + cpp.decode64(&rust_enc, &mut cpp_dec) + .expect("C++ decode64 of Rust output failed"); + assert_eq!(cpp_dec, data, "{name}: C++ could not decode Rust output"); +} + +fuzz_target!(|input: Input| { + if input.use_256 { + check( + &mut FastPForWide256::default(), + &mut CppFastPFor256::default(), + &input.data, + "FastPForWide256", + ); + } else { + check( + &mut FastPForWide128::default(), + &mut CppFastPFor128::default(), + &input.data, + "FastPForWide128", + ); + } +}); diff --git a/src/codec.rs b/src/codec.rs index 9f55563..d72d740 100644 --- a/src/codec.rs +++ b/src/codec.rs @@ -93,13 +93,13 @@ pub trait BlockCodec: Default { /// Codec that supports compressing 64-bit integers into a 32-bit word stream. /// -/// Only three C++ codecs implement this trait: `CppFastPFor128`, -/// `CppFastPFor256`, and `CppVarInt`. For simple use, call -/// `encode64` / `decode64` directly on the struct — no trait import required. +/// Implemented by the pure-Rust [`FastPForWide`](crate::FastPForWide) codecs and, +/// with the `cpp` feature, by `CppFastPFor128`, `CppFastPFor256`, and `CppVarInt`. +/// For simple use, call `encode64` / `decode64` directly on the struct — no trait +/// import required. /// /// Import `BlockCodec64` only when writing generic code over multiple codecs /// that support 64-bit compression. -#[cfg(feature = "cpp")] pub trait BlockCodec64 { /// Compress 64-bit integers into a 32-bit word stream. fn encode64(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()>; diff --git a/src/lib.rs b/src/lib.rs index a62c545..cc74a6a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,6 @@ pub mod cpp; pub(crate) mod rust; mod codec; -#[cfg(feature = "cpp")] pub use codec::BlockCodec64; pub use codec::{AnyLenCodec, BlockCodec, slice_to_blocks}; @@ -31,7 +30,7 @@ pub use bytemuck::Pod; #[cfg(feature = "rust")] pub use rust::{ CompositeCodec, FastPFor, FastPFor128, FastPFor256, FastPForBlock128, FastPForBlock256, - JustCopy, VariableByte, + FastPForWide, FastPForWide128, FastPForWide256, JustCopy, VariableByte, }; // `src/test_utils.rs` uses `fastpfor::...`; alias this crate for unit tests only. diff --git a/src/rust/integer_compression/bitpacking_wide.rs b/src/rust/integer_compression/bitpacking_wide.rs new file mode 100644 index 0000000..f5df4b3 --- /dev/null +++ b/src/rust/integer_compression/bitpacking_wide.rs @@ -0,0 +1,106 @@ +//! Generic scalar bit-packing for 64-bit values. +//! +//! Packs and unpacks groups of 32 values at any bit width `0..=64` using the same +//! little-endian bitstream layout as the hand-unrolled 32-bit kernels in +//! [`bitpacking`](super::bitpacking): value `j` occupies bits `[j*bit, (j+1)*bit)` +//! of the concatenated stream. Each call moves exactly `bit` `u32` words. + +const fn low_mask(bit: u8) -> u64 { + if bit >= 64 { u64::MAX } else { (1u64 << bit) - 1 } +} + +/// Packs 32 values from `input[inpos..]` into `output[outpos..]` at `bit` bits each. +pub fn pack_wide(input: &[u64], inpos: usize, output: &mut [u32], outpos: usize, bit: u8) { + if bit == 0 { + return; + } + let mask = u128::from(low_mask(bit)); + let mut acc: u128 = 0; + let mut filled: u32 = 0; + let mut out = outpos; + for j in 0..32 { + acc |= (u128::from(input[inpos + j]) & mask) << filled; + filled += u32::from(bit); + while filled >= 32 { + output[out] = acc as u32; + out += 1; + acc >>= 32; + filled -= 32; + } + } +} + +/// Unpacks 32 values from `input[inpos..]` into `output[outpos..]` at `bit` bits each. +pub fn unpack_wide(input: &[u32], inpos: usize, output: &mut [u64], outpos: usize, bit: u8) { + if bit == 0 { + output[outpos..outpos + 32].fill(0); + return; + } + let mask = u128::from(low_mask(bit)); + let mut acc: u128 = 0; + let mut avail: u32 = 0; + let mut inp = inpos; + for j in 0..32 { + while avail < u32::from(bit) { + acc |= u128::from(input[inp]) << avail; + inp += 1; + avail += 32; + } + output[outpos + j] = (acc & mask) as u64; + acc >>= u32::from(bit); + avail -= u32::from(bit); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rust::integer_compression::{bitpacking, bitunpacking}; + + /// The wide packer must produce byte-identical output to the proven u32 kernels + /// for every width `1..=32`, validating its bit ordering without needing C++. + #[test] + fn wide_matches_u32_kernels() { + let values32: [u32; 32] = std::array::from_fn(|i| (i as u32).wrapping_mul(2_654_435_761)); + + for bit in 1..=32u8 { + let mask = if bit == 32 { u32::MAX } else { (1u32 << bit) - 1 }; + let masked32: [u32; 32] = std::array::from_fn(|i| values32[i] & mask); + let masked64: [u64; 32] = std::array::from_fn(|i| u64::from(masked32[i])); + + let mut out_ref = vec![0u32; bit as usize]; + bitpacking::fast_pack(&masked32, 0, &mut out_ref, 0, bit); + + let mut out_wide = vec![0u32; bit as usize]; + pack_wide(&masked64, 0, &mut out_wide, 0, bit); + + assert_eq!(out_ref, out_wide, "pack mismatch at bit={bit}"); + + let mut back_ref = vec![0u32; 32]; + bitunpacking::fast_unpack(&out_ref, 0, &mut back_ref, 0, bit); + let mut back_wide = vec![0u64; 32]; + unpack_wide(&out_wide, 0, &mut back_wide, 0, bit); + + for i in 0..32 { + assert_eq!(u64::from(back_ref[i]), back_wide[i], "unpack mismatch at bit={bit}"); + } + } + } + + #[test] + fn wide_roundtrip_all_widths() { + for bit in 0..=64u8 { + let mask = low_mask(bit); + let values: [u64; 32] = + std::array::from_fn(|i| (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15) & mask); + + let mut packed = vec![0u32; bit as usize]; + pack_wide(&values, 0, &mut packed, 0, bit); + + let mut back = vec![0u64; 32]; + unpack_wide(&packed, 0, &mut back, 0, bit); + + assert_eq!(values.to_vec(), back, "roundtrip mismatch at bit={bit}"); + } + } +} diff --git a/src/rust/integer_compression/fastpfor64.rs b/src/rust/integer_compression/fastpfor64.rs new file mode 100644 index 0000000..bf51544 --- /dev/null +++ b/src/rust/integer_compression/fastpfor64.rs @@ -0,0 +1,673 @@ +//! 64-bit ([`u64`]) `FastPFOR` codec. +//! +//! This is the widened counterpart of the 32-bit [`FastPFor`](super::fastpfor::FastPFor) +//! codec: the algorithm and wire format are identical except that values, exceptions, +//! and the exception bitmap are 64 bits wide. It is byte-compatible with the C++ +//! `CppFastPFor128` / `CppFastPFor256` `encode64` / `decode64` paths. +//! +//! Like the 32-bit codec, `FastPFOR` only handles complete blocks, so a +//! [`VariableByte`] tail encodes the sub-block remainder. [`FastPForWide`] +//! bundles both and implements [`BlockCodec64`]. + +use std::cmp::min; +use std::io::Cursor; + +use bytemuck::{cast_slice, cast_slice_mut}; +use bytes::{Buf as _, BufMut as _, BytesMut}; + +use crate::codec::default_max_decoded_len; +use crate::helpers::{AsUsize, GetWithErr, greatest_multiple}; +use crate::rust::cursor::IncrementCursor; +use crate::rust::integer_compression::bitpacking_wide::{pack_wide, unpack_wide}; +use crate::{BlockCodec64, FastPForError, FastPForResult}; + +/// Overhead cost (in bits) for storing each exception's position in the block. +const OVERHEAD_OF_EACH_EXCEPT: u32 = 8; + +/// Default page size in number of integers. +const DEFAULT_PAGE_SIZE: u32 = 65536; + +/// Number of frequency/exception buckets: one per possible bit width `0..=64`. +const WIDTHS: usize = 65; + +/// [`FastPForWide`] with 128-value blocks. +pub type FastPForWide128 = FastPForWide<128>; + +/// [`FastPForWide`] with 256-value blocks. +pub type FastPForWide256 = FastPForWide<256>; + +fn bits64(value: u64) -> usize { + 64 - value.leading_zeros().as_usize() +} + +/// 64-bit `FastPFOR` codec: `FastPFOR`-packed blocks plus a variable-byte tail. +/// +/// `N` is the block size (128 or 256 values). Use [`FastPForWide128`] or +/// [`FastPForWide256`], or call [`BlockCodec64::encode64`] / [`decode64`](BlockCodec64::decode64). +#[derive(Debug)] +pub struct FastPForWide { + exception_buffers: [Vec; WIDTHS], + bytes_container: BytesMut, + page_size: u32, + data_pointers: [usize; WIDTHS], + freqs: [u32; WIDTHS], + optimal_bits: u8, + exception_count: u8, + max_bits: u8, +} + +impl Default for FastPForWide { + fn default() -> Self { + Self::new(DEFAULT_PAGE_SIZE) + } +} + +impl FastPForWide { + fn new(page_size: u32) -> Self { + Self { + bytes_container: BytesMut::with_capacity( + (3 * page_size / N as u32 + page_size) as usize, + ), + page_size, + exception_buffers: std::array::from_fn(|_| Vec::new()), + data_pointers: [0; WIDTHS], + freqs: [0; WIDTHS], + optimal_bits: 0, + exception_count: 0, + max_bits: 0, + } + } + + fn compress_blocks( + &mut self, + input: &[u64], + input_length: u32, + input_offset: &mut Cursor, + output: &mut [u32], + output_offset: &mut Cursor, + ) { + let inlength = greatest_multiple(input_length, N as u32); + let final_inpos = input_offset.position() as u32 + inlength; + while input_offset.position() as u32 != final_inpos { + let this_size = min(self.page_size, final_inpos - input_offset.position() as u32); + self.encode_page(input, this_size, input_offset, output, output_offset); + } + } + + fn decode_headless_blocks( + &mut self, + input: &[u32], + inlength: u32, + input_offset: &mut Cursor, + output: &mut [u64], + output_offset: &mut Cursor, + ) -> FastPForResult<()> { + let mynvalue = greatest_multiple(inlength, N as u32); + let final_out = output_offset.position() as u32 + mynvalue; + while output_offset.position() as u32 != final_out { + let this_size = min(self.page_size, final_out - output_offset.position() as u32); + self.decode_page(input, input_offset, output, output_offset, this_size)?; + } + Ok(()) + } + + fn encode_page( + &mut self, + input: &[u64], + this_size: u32, + input_offset: &mut Cursor, + output: &mut [u32], + output_offset: &mut Cursor, + ) { + let header_pos = output_offset.position() as usize; + output_offset.increment(); + let mut tmp_output_offset = output_offset.position() as u32; + + self.data_pointers.fill(0); + self.bytes_container.clear(); + + let mut tmp_input_offset = input_offset.position() as u32; + let final_input_offset = tmp_input_offset + this_size - N as u32; + while tmp_input_offset <= final_input_offset { + self.best_bit_from_data(input, tmp_input_offset); + self.bytes_container.put_u8(self.optimal_bits); + self.bytes_container.put_u8(self.exception_count); + if self.exception_count > 0 { + self.bytes_container.put_u8(self.max_bits); + let index = usize::from(self.max_bits - self.optimal_bits); + let needed = self.data_pointers[index] + usize::from(self.exception_count); + if needed > self.exception_buffers[index].len() { + let new_cap = needed.saturating_mul(2).next_multiple_of(32); + self.exception_buffers[index].resize(new_cap, 0); + } + for k in 0..N as u32 { + if (input[(k + tmp_input_offset) as usize] >> self.optimal_bits) != 0 { + self.bytes_container.put_u8(k as u8); + self.exception_buffers[index][self.data_pointers[index]] = + input[(k + tmp_input_offset) as usize] >> self.optimal_bits; + self.data_pointers[index] += 1; + } + } + } + for k in (0..N as u32).step_by(32) { + pack_wide( + input, + (tmp_input_offset + k) as usize, + output, + tmp_output_offset as usize, + self.optimal_bits, + ); + tmp_output_offset += u32::from(self.optimal_bits); + } + tmp_input_offset += N as u32; + } + input_offset.set_position(u64::from(tmp_input_offset)); + output[header_pos] = tmp_output_offset - header_pos as u32; + let byte_size = self.bytes_container.len(); + while (self.bytes_container.len() & 3) != 0 { + self.bytes_container.put_u8(0); + } + output[tmp_output_offset as usize] = byte_size as u32; + tmp_output_offset += 1; + let how_many_ints = self.bytes_container.len() / 4; + let meta_u32s: &[u32] = cast_slice(self.bytes_container.chunk()); + output[tmp_output_offset as usize..][..how_many_ints] + .copy_from_slice(&meta_u32s[..how_many_ints]); + tmp_output_offset += how_many_ints as u32; + + let mut bitmap: u64 = 0; + for k in 2..=64 { + if self.data_pointers[k] != 0 { + bitmap |= 1u64 << (k - 1); + } + } + output[tmp_output_offset as usize] = bitmap as u32; + output[tmp_output_offset as usize + 1] = (bitmap >> 32) as u32; + tmp_output_offset += 2; + + for k in 2..=64 { + if self.data_pointers[k] != 0 { + output[tmp_output_offset as usize] = self.data_pointers[k] as u32; + tmp_output_offset += 1; + let mut j = 0; + while j < self.data_pointers[k] { + pack_wide( + &self.exception_buffers[k], + j, + output, + tmp_output_offset as usize, + k as u8, + ); + tmp_output_offset += k as u32; + j += 32; + } + let overflow = j as u32 - self.data_pointers[k] as u32; + tmp_output_offset -= (overflow * k as u32) / 32; + } + } + output_offset.set_position(u64::from(tmp_output_offset)); + } + + fn best_bit_from_data(&mut self, input: &[u64], pos: u32) { + self.freqs.fill(0); + let k_end = min(pos + N as u32, input.len() as u32); + for k in pos..k_end { + self.freqs[bits64(input[k as usize])] += 1; + } + + self.optimal_bits = 64; + while self.freqs[self.optimal_bits as usize] == 0 { + self.optimal_bits -= 1; + } + self.max_bits = self.optimal_bits; + + let mut best_cost = u32::from(self.optimal_bits) * N as u32; + let mut num_exceptions: u32 = 0; + self.exception_count = 0; + + for bits in (0..self.optimal_bits).rev() { + num_exceptions += self.freqs[bits as usize + 1]; + if num_exceptions == N as u32 { + break; + } + let diff = u32::from(self.max_bits - bits); + let mut cost = num_exceptions * OVERHEAD_OF_EACH_EXCEPT + + num_exceptions * diff + + u32::from(bits) * N as u32 + + 8; + if diff == 1 { + cost -= num_exceptions; + } + if cost < best_cost { + best_cost = cost; + self.optimal_bits = bits; + self.exception_count = num_exceptions as u8; + } + } + } + + #[expect(clippy::too_many_lines)] + fn decode_page( + &mut self, + input: &[u32], + input_offset: &mut Cursor, + output: &mut [u64], + output_offset: &mut Cursor, + this_size: u32, + ) -> FastPForResult<()> { + let n = u32::try_from(input.len()) + .map_err(|_| FastPForError::InvalidInputLength(input.len()))?; + + let init_pos = + u32::try_from(input_offset.position()).map_err(|_| FastPForError::NotEnoughData)?; + let where_meta = input.get_val(init_pos)?; + input_offset.increment(); + let mut inexcept = init_pos + .checked_add(where_meta) + .ok_or(FastPForError::NotEnoughData)?; + let bytesize = input.get_val(inexcept)?; + inexcept = inexcept + .checked_add(1) + .ok_or(FastPForError::NotEnoughData)?; + let input_bytes: &[u8] = cast_slice(input); + let mut byte_pos = (inexcept as usize) + .checked_mul(4) + .filter(|&bp| bp <= input_bytes.len()) + .ok_or(FastPForError::NotEnoughData)?; + let length = bytesize.div_ceil(4); + inexcept = inexcept + .checked_add(length) + .ok_or(FastPForError::NotEnoughData)?; + + let bitmap_lo = input.get_val(inexcept)?; + let bitmap_hi = input.get_val( + inexcept + .checked_add(1) + .ok_or(FastPForError::NotEnoughData)?, + )?; + let bitmap = u64::from(bitmap_lo) | (u64::from(bitmap_hi) << 32); + inexcept = inexcept + .checked_add(2) + .ok_or(FastPForError::NotEnoughData)?; + + for k in 2..=64u32 { + if (bitmap & (1u64 << (k - 1))) != 0 { + let size = input.get_val(inexcept)?; + inexcept = inexcept + .checked_add(1) + .ok_or(FastPForError::NotEnoughData)?; + if size > self.page_size { + return Err(FastPForError::NotEnoughData); + } + let rounded_up = size.next_multiple_of(32) as usize; + if self.exception_buffers[k as usize].len() < rounded_up { + self.exception_buffers[k as usize].resize(rounded_up, 0); + } + let mut j: u32 = 0; + while j.checked_add(32).is_some_and(|j32| j32 <= size) + && inexcept.checked_add(k).is_some_and(|ie| ie <= n) + { + unpack_wide( + input, + inexcept as usize, + &mut self.exception_buffers[k as usize], + j as usize, + k as u8, + ); + inexcept += k; + j += 32; + } + if j < size { + let words_needed = (size - j).saturating_mul(k).div_ceil(32); + let avail = n - inexcept.min(n); + if avail < words_needed { + return Err(FastPForError::NotEnoughData); + } + let copy_len = words_needed as usize; + let mut tail_buf = [0u32; 128]; + if copy_len == 0 { + return Err(FastPForError::NotEnoughData); + } + let start = inexcept as usize; + let src = input + .get(start..start + copy_len) + .ok_or(FastPForError::NotEnoughData)?; + tail_buf[..copy_len].copy_from_slice(src); + unpack_wide( + &tail_buf, + 0, + &mut self.exception_buffers[k as usize], + j as usize, + k as u8, + ); + inexcept += k; + j += 32; + } + let overflow = j - size; + inexcept -= (overflow * k) / 32; + } + } + + self.data_pointers.fill(0); + let mut tmp_output_offset = output_offset.position() as u32; + let mut tmp_input_offset = input_offset.position() as u32; + + let run_end = this_size / N as u32; + for _ in 0..run_end { + let bits = input_bytes.get_val(byte_pos)?; + if bits > 64 { + return Err(FastPForError::NotEnoughData); + } + byte_pos += 1; + let num_exceptions = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + for k in (0..N as u32).step_by(32) { + let in_start = tmp_input_offset as usize; + let out_start = (tmp_output_offset + k) as usize; + let in_end = in_start + .checked_add(usize::from(bits)) + .ok_or(FastPForError::NotEnoughData)?; + if in_end > input.len() { + return Err(FastPForError::NotEnoughData); + } + let out_end = out_start + .checked_add(32) + .ok_or(FastPForError::OutputBufferTooSmall)?; + if out_end > output.len() { + return Err(FastPForError::OutputBufferTooSmall); + } + unpack_wide(input, in_start, output, out_start, bits); + tmp_input_offset += u32::from(bits); + } + if num_exceptions > 0 { + let maxbits = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + let index = maxbits + .checked_sub(bits) + .ok_or(FastPForError::NotEnoughData)?; + if maxbits > 64 || index == 0 || index > 64 { + return Err(FastPForError::NotEnoughData); + } + let index = usize::from(index); + if index == 1 { + for _ in 0..num_exceptions { + let pos = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + if u32::from(pos) >= N as u32 { + return Err(FastPForError::NotEnoughData); + } + let out_idx = tmp_output_offset as usize + pos as usize; + if out_idx >= output.len() { + return Err(FastPForError::OutputBufferTooSmall); + } + output[out_idx] |= 1u64 << bits; + } + } else { + for _ in 0..num_exceptions { + let pos = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + if u32::from(pos) >= N as u32 { + return Err(FastPForError::NotEnoughData); + } + let out_idx = tmp_output_offset as usize + pos as usize; + if out_idx >= output.len() { + return Err(FastPForError::OutputBufferTooSmall); + } + let ptr = self.data_pointers[index]; + let except_value = self.exception_buffers[index].get_val(ptr)?; + output[out_idx] |= except_value << bits; + self.data_pointers[index] += 1; + } + } + } + tmp_output_offset += N as u32; + } + output_offset.set_position(u64::from(tmp_output_offset)); + input_offset.set_position(u64::from(inexcept)); + Ok(()) + } +} + +/// Variable-byte (LEB128 with high-bit terminator) encoding of the `u64` tail. +/// +/// Matches the C++ `VariableByte::encodeToByteArray` layout: every byte but the +/// last carries 7 payload bits with the high bit clear; the final byte has its +/// high bit set. Output is padded with zero bytes to a whole number of `u32` words. +fn vbyte_encode64(input: &[u64], out: &mut Vec) { + if input.is_empty() { + return; + } + let start = out.len(); + let capacity = input.len() * 3 + 4; + out.resize(start + capacity, 0); + let bytes: &mut [u8] = cast_slice_mut(&mut out[start..]); + let mut byte_pos = 0; + for &value in input { + let mut v = value; + while v >= 0x80 { + bytes[byte_pos] = (v as u8) & 0x7F; + byte_pos += 1; + v >>= 7; + } + bytes[byte_pos] = (v as u8) | 0x80; + byte_pos += 1; + } + while byte_pos % 4 != 0 { + bytes[byte_pos] = 0; + byte_pos += 1; + } + out.truncate(start + byte_pos / 4); +} + +/// Inverse of [`vbyte_encode64`]. Trailing zero padding decodes to no value +/// because a padding byte never has the high-bit terminator. +fn vbyte_decode64(input: &[u32], out: &mut Vec) -> FastPForResult<()> { + if input.is_empty() { + return Ok(()); + } + let bytes: &[u8] = cast_slice(input); + let byte_len = bytes.len(); + let mut byte_pos = 0; + while byte_pos < byte_len { + let mut v: u64 = 0; + let mut shift = 0u32; + loop { + if byte_pos >= byte_len { + return Ok(()); + } + let c = bytes[byte_pos]; + byte_pos += 1; + if shift >= 64 { + return Err(FastPForError::NotEnoughData); + } + if c >= 0x80 { + v |= u64::from(c & 0x7F) << shift; + out.push(v); + break; + } + v |= u64::from(c) << shift; + shift += 7; + } + } + Ok(()) +} + +impl BlockCodec64 for FastPForWide { + fn encode64(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()> { + let rounded = (input.len() / N) * N; + let n_values = rounded as u32; + + let start = out.len(); + if rounded == 0 { + out.push(0); + } else { + let capacity = rounded * 3 + 1024; + out.resize(start + 1 + capacity, 0); + out[start] = n_values; + + let mut in_off = Cursor::new(0u32); + let mut out_off = Cursor::new(0u32); + self.compress_blocks( + &input[..rounded], + n_values, + &mut in_off, + &mut out[start + 1..], + &mut out_off, + ); + let written = 1 + out_off.position() as usize; + out.truncate(start + written); + } + + vbyte_encode64(&input[rounded..], out); + Ok(()) + } + + fn decode64(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { + let Some((&block_n_values, rest)) = input.split_first() else { + return Ok(()); + }; + if block_n_values % N as u32 != 0 { + return Err(FastPForError::NotEnoughData); + } + if block_n_values.as_usize() > default_max_decoded_len(input.len()) { + return Err(FastPForError::NotEnoughData); + } + let n_blocks = block_n_values.as_usize() / N; + + let consumed = if n_blocks == 0 { + 1 + } else { + let start = out.len(); + out.resize(start + n_blocks * N, 0); + let mut in_off = Cursor::new(0u32); + let mut out_off = Cursor::new(0u32); + self.decode_headless_blocks( + rest, + block_n_values, + &mut in_off, + &mut out[start..], + &mut out_off, + )?; + 1 + in_off.position() as usize + }; + + let tail_input = input.get(consumed..).ok_or(FastPForError::NotEnoughData)?; + vbyte_decode64(tail_input, out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn roundtrip(input: &[u64]) { + let mut codec = FastPForWide::::default(); + let mut encoded = Vec::new(); + codec.encode64(input, &mut encoded).unwrap(); + let mut decoded = Vec::new(); + codec.decode64(&encoded, &mut decoded).unwrap(); + assert_eq!(decoded, input, "roundtrip mismatch (N={N})"); + } + + #[test] + fn empty() { + roundtrip::<128>(&[]); + roundtrip::<256>(&[]); + } + + #[test] + fn single_value() { + roundtrip::<128>(&[42]); + roundtrip::<256>(&[u64::MAX]); + } + + #[test] + fn sub_block_tail_only() { + let data: Vec = (0..10).collect(); + roundtrip::<256>(&data); + } + + #[test] + fn exact_block() { + let data: Vec = (0..128).collect(); + roundtrip::<128>(&data); + } + + #[test] + fn blocks_with_remainder() { + let data: Vec = (0..600).collect(); + roundtrip::<256>(&data); + } + + #[test] + fn large_values_and_exceptions() { + let data: Vec = (0..1024u32) + .map(|i| if i % 7 == 0 { 1u64 << 60 } else { u64::from(i) }) + .collect(); + roundtrip::<128>(&data); + } + + #[test] + fn full_width_values() { + let data: Vec = (0..256u32).map(|i| u64::MAX - u64::from(i)).collect(); + roundtrip::<256>(&data); + } + + #[test] + fn spans_multiple_pages() { + let data: Vec = (0..70_000u64).map(|i| i.wrapping_mul(0x1_0001)).collect(); + roundtrip::<128>(&data); + } + + #[cfg(feature = "cpp")] + mod cpp_parity { + use super::*; + use crate::BlockCodec64; + use crate::cpp::{CppFastPFor128, CppFastPFor256}; + + fn cases() -> Vec> { + vec![ + vec![], + vec![42], + vec![u64::MAX], + (0..10).collect(), + (0..128).collect(), + (0..256).collect(), + (0..600).collect(), + (0..1024u32) + .map(|i| if i % 7 == 0 { 1u64 << 60 } else { u64::from(i) }) + .collect(), + (0..256u32).map(|i| u64::MAX - u64::from(i)).collect(), + (0..5000u64).map(|i| i.wrapping_mul(0x1_0001)).collect(), + ] + } + + fn assert_parity(rust: &mut FastPForWide, cpp: &mut impl BlockCodec64) { + for data in cases() { + let mut rust_enc = Vec::new(); + rust.encode64(&data, &mut rust_enc).unwrap(); + let mut cpp_enc = Vec::new(); + cpp.encode64(&data, &mut cpp_enc).unwrap(); + assert_eq!(rust_enc, cpp_enc, "encode64 bytes differ for {data:?}"); + + let mut rust_dec = Vec::new(); + rust.decode64(&cpp_enc, &mut rust_dec).unwrap(); + assert_eq!(rust_dec, data, "Rust failed to decode C++ output"); + + let mut cpp_dec = Vec::new(); + cpp.decode64(&rust_enc, &mut cpp_dec).unwrap(); + assert_eq!(cpp_dec, data, "C++ failed to decode Rust output"); + } + } + + #[test] + fn parity_128() { + assert_parity(&mut FastPForWide::<128>::default(), &mut CppFastPFor128::default()); + } + + #[test] + fn parity_256() { + assert_parity(&mut FastPForWide::<256>::default(), &mut CppFastPFor256::default()); + } + } +} diff --git a/src/rust/integer_compression/mod.rs b/src/rust/integer_compression/mod.rs index 7aed001..76b6ffe 100644 --- a/src/rust/integer_compression/mod.rs +++ b/src/rust/integer_compression/mod.rs @@ -1,5 +1,7 @@ pub mod bitpacking; +pub mod bitpacking_wide; pub mod bitunpacking; pub mod fastpfor; +pub mod fastpfor64; pub mod just_copy; pub mod variable_byte; diff --git a/src/rust/mod.rs b/src/rust/mod.rs index 92a2b05..b69e782 100644 --- a/src/rust/mod.rs +++ b/src/rust/mod.rs @@ -5,6 +5,8 @@ mod integer_compression; pub use composite::CompositeCodec; /// Type-safe block codec with block size encoded in the type. pub use integer_compression::fastpfor::{FastPFor, FastPForBlock128, FastPForBlock256}; +/// 64-bit ([`u64`]) `FastPFOR` codec implementing [`BlockCodec64`](crate::BlockCodec64). +pub use integer_compression::fastpfor64::{FastPForWide, FastPForWide128, FastPForWide256}; /// Pass-through codec — implements [`AnyLenCodec`](crate::codec::AnyLenCodec). pub use integer_compression::just_copy::JustCopy; /// Variable-byte codec — implements [`AnyLenCodec`](crate::codec::AnyLenCodec). diff --git a/src/test_utils.rs b/src/test_utils.rs index 2041a84..21c60e2 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -10,10 +10,8 @@ // noise without benefit. #![allow(dead_code, missing_docs)] -#[cfg(feature = "cpp")] -use fastpfor::BlockCodec64; #[allow(unused_imports)] -use fastpfor::{AnyLenCodec, BlockCodec, FastPForResult, slice_to_blocks}; +use fastpfor::{AnyLenCodec, BlockCodec, BlockCodec64, FastPForResult, slice_to_blocks}; #[cfg(feature = "rust")] use fastpfor::{ FastPFor128, FastPFor256, FastPForBlock128, FastPForBlock256, JustCopy, VariableByte, @@ -48,7 +46,6 @@ pub fn roundtrip_full(data: &[u32], expected_len assert_eq!(decompressed, data); } -#[cfg(feature = "cpp")] pub fn roundtrip64(data: &[u64]) { let mut codec = C::default(); let mut compressed = Vec::new(); @@ -100,14 +97,12 @@ pub fn block_decompress( Ok(out) } -#[cfg(feature = "cpp")] pub fn compress64(data: &[u64]) -> FastPForResult> { let mut compressed = Vec::new(); C::default().encode64(data, &mut compressed)?; Ok(compressed) } -#[cfg(feature = "cpp")] pub fn decompress64(compressed: &[u32]) -> FastPForResult> { let mut out = Vec::new(); C::default().decode64(compressed, &mut out)?; From 9d7673714209b4d07b173a5dd0afc4c33ed999df Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 10:44:25 +0000 Subject: [PATCH 02/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../integer_compression/bitpacking_wide.rs | 18 +++++++++++++++--- src/rust/integer_compression/fastpfor64.rs | 10 ++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/rust/integer_compression/bitpacking_wide.rs b/src/rust/integer_compression/bitpacking_wide.rs index f5df4b3..139ec05 100644 --- a/src/rust/integer_compression/bitpacking_wide.rs +++ b/src/rust/integer_compression/bitpacking_wide.rs @@ -6,7 +6,11 @@ //! of the concatenated stream. Each call moves exactly `bit` `u32` words. const fn low_mask(bit: u8) -> u64 { - if bit >= 64 { u64::MAX } else { (1u64 << bit) - 1 } + if bit >= 64 { + u64::MAX + } else { + (1u64 << bit) - 1 + } } /// Packs 32 values from `input[inpos..]` into `output[outpos..]` at `bit` bits each. @@ -64,7 +68,11 @@ mod tests { let values32: [u32; 32] = std::array::from_fn(|i| (i as u32).wrapping_mul(2_654_435_761)); for bit in 1..=32u8 { - let mask = if bit == 32 { u32::MAX } else { (1u32 << bit) - 1 }; + let mask = if bit == 32 { + u32::MAX + } else { + (1u32 << bit) - 1 + }; let masked32: [u32; 32] = std::array::from_fn(|i| values32[i] & mask); let masked64: [u64; 32] = std::array::from_fn(|i| u64::from(masked32[i])); @@ -82,7 +90,11 @@ mod tests { unpack_wide(&out_wide, 0, &mut back_wide, 0, bit); for i in 0..32 { - assert_eq!(u64::from(back_ref[i]), back_wide[i], "unpack mismatch at bit={bit}"); + assert_eq!( + u64::from(back_ref[i]), + back_wide[i], + "unpack mismatch at bit={bit}" + ); } } } diff --git a/src/rust/integer_compression/fastpfor64.rs b/src/rust/integer_compression/fastpfor64.rs index bf51544..2c5ba2d 100644 --- a/src/rust/integer_compression/fastpfor64.rs +++ b/src/rust/integer_compression/fastpfor64.rs @@ -662,12 +662,18 @@ mod tests { #[test] fn parity_128() { - assert_parity(&mut FastPForWide::<128>::default(), &mut CppFastPFor128::default()); + assert_parity( + &mut FastPForWide::<128>::default(), + &mut CppFastPFor128::default(), + ); } #[test] fn parity_256() { - assert_parity(&mut FastPForWide::<256>::default(), &mut CppFastPFor256::default()); + assert_parity( + &mut FastPForWide::<256>::default(), + &mut CppFastPFor256::default(), + ); } } } From 53e4c1929a2bfee6214eb910f9977010d12b532b Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 17 Jul 2026 12:48:39 +0200 Subject: [PATCH 03/23] docs: de-vibe comments and use one sentence per line Explain the Rust code on its own terms rather than by reference to the C++ implementation, drop a redundant test comment, and write comments as short single-sentence lines. Co-Authored-By: Claude Opus 4.8 (1M context) --- fuzz/fuzz_targets/fastpfor_u64.rs | 7 ++--- src/codec.rs | 10 +++---- .../integer_compression/bitpacking_wide.rs | 10 +++---- src/rust/integer_compression/fastpfor64.rs | 30 +++++++++---------- 4 files changed, 26 insertions(+), 31 deletions(-) diff --git a/fuzz/fuzz_targets/fastpfor_u64.rs b/fuzz/fuzz_targets/fastpfor_u64.rs index 5ccef1a..48a8b7f 100644 --- a/fuzz/fuzz_targets/fastpfor_u64.rs +++ b/fuzz/fuzz_targets/fastpfor_u64.rs @@ -2,10 +2,9 @@ //! Differential fuzz of the 64-bit FastPFOR codec. //! -//! For arbitrary `u64` input, the pure-Rust `FastPForWide` and the C++ -//! `CppFastPFor128` / `CppFastPFor256` must produce bit-identical compressed -//! output, and every decoder must reproduce the original input — including -//! decoding the other implementation's bytes. +//! For any `u64` input, `FastPForWide` and the C++ codec must produce identical compressed bytes. +//! Every decoder must reproduce the original input. +//! Each side must also decode the other's output. use fastpfor::cpp::{CppFastPFor128, CppFastPFor256}; use fastpfor::{BlockCodec64, FastPForWide128, FastPForWide256}; diff --git a/src/codec.rs b/src/codec.rs index d72d740..6bf9cf1 100644 --- a/src/codec.rs +++ b/src/codec.rs @@ -93,13 +93,11 @@ pub trait BlockCodec: Default { /// Codec that supports compressing 64-bit integers into a 32-bit word stream. /// -/// Implemented by the pure-Rust [`FastPForWide`](crate::FastPForWide) codecs and, -/// with the `cpp` feature, by `CppFastPFor128`, `CppFastPFor256`, and `CppVarInt`. -/// For simple use, call `encode64` / `decode64` directly on the struct — no trait -/// import required. +/// Implemented by the pure-Rust [`FastPForWide`](crate::FastPForWide) codecs. +/// With the `cpp` feature, `CppFastPFor128`, `CppFastPFor256`, and `CppVarInt` also implement it. +/// For simple use, call `encode64` / `decode64` directly on the struct. /// -/// Import `BlockCodec64` only when writing generic code over multiple codecs -/// that support 64-bit compression. +/// Import `BlockCodec64` only when writing generic code over several 64-bit codecs. pub trait BlockCodec64 { /// Compress 64-bit integers into a 32-bit word stream. fn encode64(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()>; diff --git a/src/rust/integer_compression/bitpacking_wide.rs b/src/rust/integer_compression/bitpacking_wide.rs index 139ec05..d3ae71f 100644 --- a/src/rust/integer_compression/bitpacking_wide.rs +++ b/src/rust/integer_compression/bitpacking_wide.rs @@ -1,9 +1,9 @@ //! Generic scalar bit-packing for 64-bit values. //! -//! Packs and unpacks groups of 32 values at any bit width `0..=64` using the same -//! little-endian bitstream layout as the hand-unrolled 32-bit kernels in -//! [`bitpacking`](super::bitpacking): value `j` occupies bits `[j*bit, (j+1)*bit)` -//! of the concatenated stream. Each call moves exactly `bit` `u32` words. +//! Packs and unpacks groups of 32 values at any bit width `0..=64`. +//! The layout is a little-endian bitstream, matching the hand-unrolled 32-bit kernels in [`bitpacking`](super::bitpacking). +//! Value `j` occupies bits `[j*bit, (j+1)*bit)` of the concatenated stream. +//! Each call moves exactly `bit` `u32` words. const fn low_mask(bit: u8) -> u64 { if bit >= 64 { @@ -61,8 +61,6 @@ mod tests { use super::*; use crate::rust::integer_compression::{bitpacking, bitunpacking}; - /// The wide packer must produce byte-identical output to the proven u32 kernels - /// for every width `1..=32`, validating its bit ordering without needing C++. #[test] fn wide_matches_u32_kernels() { let values32: [u32; 32] = std::array::from_fn(|i| (i as u32).wrapping_mul(2_654_435_761)); diff --git a/src/rust/integer_compression/fastpfor64.rs b/src/rust/integer_compression/fastpfor64.rs index 2c5ba2d..973a349 100644 --- a/src/rust/integer_compression/fastpfor64.rs +++ b/src/rust/integer_compression/fastpfor64.rs @@ -1,13 +1,12 @@ //! 64-bit ([`u64`]) `FastPFOR` codec. //! -//! This is the widened counterpart of the 32-bit [`FastPFor`](super::fastpfor::FastPFor) -//! codec: the algorithm and wire format are identical except that values, exceptions, -//! and the exception bitmap are 64 bits wide. It is byte-compatible with the C++ -//! `CppFastPFor128` / `CppFastPFor256` `encode64` / `decode64` paths. +//! This is the widened counterpart of the 32-bit [`FastPFor`](super::fastpfor::FastPFor). +//! Values, exceptions, and the exception bitmap are 64 bits wide instead of 32. +//! The output is byte-compatible with the C++ `CppFastPFor128` / `CppFastPFor256` 64-bit paths. //! -//! Like the 32-bit codec, `FastPFOR` only handles complete blocks, so a -//! [`VariableByte`] tail encodes the sub-block remainder. [`FastPForWide`] -//! bundles both and implements [`BlockCodec64`]. +//! `FastPFOR` only handles complete blocks. +//! A [`VariableByte`] tail encodes the sub-block remainder. +//! [`FastPForWide`] bundles both and implements [`BlockCodec64`]. use std::cmp::min; use std::io::Cursor; @@ -42,8 +41,8 @@ fn bits64(value: u64) -> usize { /// 64-bit `FastPFOR` codec: `FastPFOR`-packed blocks plus a variable-byte tail. /// -/// `N` is the block size (128 or 256 values). Use [`FastPForWide128`] or -/// [`FastPForWide256`], or call [`BlockCodec64::encode64`] / [`decode64`](BlockCodec64::decode64). +/// `N` is the block size (128 or 256 values). +/// Use [`FastPForWide128`] or [`FastPForWide256`], or call [`encode64`](BlockCodec64::encode64) / [`decode64`](BlockCodec64::decode64). #[derive(Debug)] pub struct FastPForWide { exception_buffers: [Vec; WIDTHS], @@ -428,11 +427,12 @@ impl FastPForWide { } } -/// Variable-byte (LEB128 with high-bit terminator) encoding of the `u64` tail. +/// Variable-byte encoding of the `u64` tail. /// -/// Matches the C++ `VariableByte::encodeToByteArray` layout: every byte but the -/// last carries 7 payload bits with the high bit clear; the final byte has its -/// high bit set. Output is padded with zero bytes to a whole number of `u32` words. +/// Each value is emitted little-endian in 7-bit groups. +/// Every byte but the last has its high bit clear. +/// The final byte sets its high bit as a terminator. +/// The stream is zero-padded to a whole number of `u32` words. fn vbyte_encode64(input: &[u64], out: &mut Vec) { if input.is_empty() { return; @@ -459,8 +459,8 @@ fn vbyte_encode64(input: &[u64], out: &mut Vec) { out.truncate(start + byte_pos / 4); } -/// Inverse of [`vbyte_encode64`]. Trailing zero padding decodes to no value -/// because a padding byte never has the high-bit terminator. +/// Inverse of [`vbyte_encode64`]. +/// Trailing zero padding decodes to no value, since a padding byte never sets the terminator bit. fn vbyte_decode64(input: &[u32], out: &mut Vec) -> FastPForResult<()> { if input.is_empty() { return Ok(()); From 10ced0037eba7ed767693340c186bbe9a2ed4ea6 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 17 Jul 2026 13:48:45 +0200 Subject: [PATCH 04/23] refactor: one codec type per block size for u32 and u64 Consolidate the separate FastPForWide128/256 types into the existing FastPFor128/FastPFor256: each now implements AnyLenCodec (u32) and BlockCodec64 (u64), mirroring the C++ codecs that expose both widths from one type. The u64 engine is retained as a private field, so buffer reuse is preserved and the public API drops the extra FastPForWide* names. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 10 +-- fuzz/fuzz_targets/fastpfor_u64.rs | 12 +-- src/codec.rs | 2 +- src/lib.rs | 2 +- src/rust/composite.rs | 1 + src/rust/fastpfor_codec.rs | 86 ++++++++++++++++++++++ src/rust/integer_compression/fastpfor64.rs | 18 ++--- src/rust/mod.rs | 11 +-- 8 files changed, 109 insertions(+), 33 deletions(-) create mode 100644 src/rust/fastpfor_codec.rs diff --git a/README.md b/README.md index ba83415..237261b 100644 --- a/README.md +++ b/README.md @@ -60,14 +60,14 @@ assert_eq!(decoded, input); ### 64-bit integers (`u64`) -`FastPForWide128` / `FastPForWide256` compress `u64` values via the `BlockCodec64` -trait. The wire format is byte-compatible with the C++ `CppFastPFor128` / -`CppFastPFor256` `encode64` / `decode64` paths. +The same `FastPFor128` / `FastPFor256` codecs also compress `u64` values via the +`BlockCodec64` trait (`encode64` / `decode64`). The wire format is byte-compatible +with the C++ `CppFastPFor128` / `CppFastPFor256` 64-bit paths. ```rust -use fastpfor::{BlockCodec64, FastPForWide256}; +use fastpfor::{BlockCodec64, FastPFor256}; -let mut codec = FastPForWide256::default(); +let mut codec = FastPFor256::default(); let input: Vec = (0..600).map(|i| i * 1_000_000_000).collect(); let mut encoded = Vec::new(); diff --git a/fuzz/fuzz_targets/fastpfor_u64.rs b/fuzz/fuzz_targets/fastpfor_u64.rs index 48a8b7f..ef070df 100644 --- a/fuzz/fuzz_targets/fastpfor_u64.rs +++ b/fuzz/fuzz_targets/fastpfor_u64.rs @@ -2,12 +2,12 @@ //! Differential fuzz of the 64-bit FastPFOR codec. //! -//! For any `u64` input, `FastPForWide` and the C++ codec must produce identical compressed bytes. +//! For any `u64` input, the Rust and C++ codecs must produce identical compressed bytes. //! Every decoder must reproduce the original input. //! Each side must also decode the other's output. use fastpfor::cpp::{CppFastPFor128, CppFastPFor256}; -use fastpfor::{BlockCodec64, FastPForWide128, FastPForWide256}; +use fastpfor::{BlockCodec64, FastPFor128, FastPFor256}; use libfuzzer_sys::fuzz_target; #[derive(arbitrary::Arbitrary, Debug)] @@ -44,17 +44,17 @@ fn check(rust: &mut impl BlockCodec64, cpp: &mut impl BlockCodec64, data: &[u64] fuzz_target!(|input: Input| { if input.use_256 { check( - &mut FastPForWide256::default(), + &mut FastPFor256::default(), &mut CppFastPFor256::default(), &input.data, - "FastPForWide256", + "FastPFor256", ); } else { check( - &mut FastPForWide128::default(), + &mut FastPFor128::default(), &mut CppFastPFor128::default(), &input.data, - "FastPForWide128", + "FastPFor128", ); } }); diff --git a/src/codec.rs b/src/codec.rs index 6bf9cf1..52e1ea7 100644 --- a/src/codec.rs +++ b/src/codec.rs @@ -93,7 +93,7 @@ pub trait BlockCodec: Default { /// Codec that supports compressing 64-bit integers into a 32-bit word stream. /// -/// Implemented by the pure-Rust [`FastPForWide`](crate::FastPForWide) codecs. +/// Implemented by the pure-Rust [`FastPFor128`](crate::FastPFor128) and [`FastPFor256`](crate::FastPFor256) codecs. /// With the `cpp` feature, `CppFastPFor128`, `CppFastPFor256`, and `CppVarInt` also implement it. /// For simple use, call `encode64` / `decode64` directly on the struct. /// diff --git a/src/lib.rs b/src/lib.rs index cc74a6a..3a17e00 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,7 +30,7 @@ pub use bytemuck::Pod; #[cfg(feature = "rust")] pub use rust::{ CompositeCodec, FastPFor, FastPFor128, FastPFor256, FastPForBlock128, FastPForBlock256, - FastPForWide, FastPForWide128, FastPForWide256, JustCopy, VariableByte, + JustCopy, VariableByte, }; // `src/test_utils.rs` uses `fastpfor::...`; alias this crate for unit tests only. diff --git a/src/rust/composite.rs b/src/rust/composite.rs index adea3c1..9bcd2fd 100644 --- a/src/rust/composite.rs +++ b/src/rust/composite.rs @@ -40,6 +40,7 @@ use crate::helpers::AsUsize; /// codec.decode(&encoded, &mut decoded, None).unwrap(); /// assert_eq!(decoded, data); /// ``` +#[derive(Debug)] pub struct CompositeCodec { block: Blocks, tail: Tail, diff --git a/src/rust/fastpfor_codec.rs b/src/rust/fastpfor_codec.rs new file mode 100644 index 0000000..7fd9719 --- /dev/null +++ b/src/rust/fastpfor_codec.rs @@ -0,0 +1,86 @@ +//! Public any-length `FastPFOR` codecs supporting both 32- and 64-bit integers. +//! +//! [`FastPFor128`] and [`FastPFor256`] are the primary entry points. +//! Each implements [`AnyLenCodec`] for `u32` and [`BlockCodec64`] for `u64`. +//! Aligned blocks are coded with `FastPFOR` and the sub-block remainder with variable-byte coding. + +use crate::codec::{AnyLenCodec, BlockCodec64}; +use crate::rust::VariableByte; +use crate::rust::composite::CompositeCodec; +use crate::rust::integer_compression::fastpfor::{FastPForBlock128, FastPForBlock256}; +use crate::rust::integer_compression::fastpfor64::FastPForWide; +use crate::FastPForResult; + +macro_rules! define_fastpfor { + ($(#[$meta:meta])* $name:ident, $block:ty, $n:literal) => { + $(#[$meta])* + #[derive(Debug, Default)] + pub struct $name { + narrow: CompositeCodec<$block, VariableByte>, + wide: FastPForWide<$n>, + } + + impl AnyLenCodec for $name { + fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { + self.narrow.encode(input, out) + } + + fn decode( + &mut self, + input: &[u32], + out: &mut Vec, + expected_len: Option, + ) -> FastPForResult<()> { + self.narrow.decode(input, out, expected_len) + } + } + + impl BlockCodec64 for $name { + fn encode64(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()> { + self.wide.encode64(input, out) + } + + fn decode64(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { + self.wide.decode64(input, out) + } + } + }; +} + +define_fastpfor! { + /// Any-length `FastPFOR` codec with 128-value blocks. + /// + /// Compresses `u32` via [`AnyLenCodec`] and `u64` via [`BlockCodec64`]. + FastPFor128, FastPForBlock128, 128 +} + +define_fastpfor! { + /// Any-length `FastPFOR` codec with 256-value blocks. + /// + /// Compresses `u32` via [`AnyLenCodec`] and `u64` via [`BlockCodec64`]. + FastPFor256, FastPForBlock256, 256 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn one_codec_handles_both_widths() { + let mut codec = FastPFor256::default(); + + let data32: Vec = (0..600).collect(); + let mut enc32 = Vec::new(); + codec.encode(&data32, &mut enc32).unwrap(); + let mut dec32 = Vec::new(); + codec.decode(&enc32, &mut dec32, None).unwrap(); + assert_eq!(dec32, data32); + + let data64: Vec = (0..600).map(|i| i * 1_000_000_000).collect(); + let mut enc64 = Vec::new(); + codec.encode64(&data64, &mut enc64).unwrap(); + let mut dec64 = Vec::new(); + codec.decode64(&enc64, &mut dec64).unwrap(); + assert_eq!(dec64, data64); + } +} diff --git a/src/rust/integer_compression/fastpfor64.rs b/src/rust/integer_compression/fastpfor64.rs index 973a349..68afd82 100644 --- a/src/rust/integer_compression/fastpfor64.rs +++ b/src/rust/integer_compression/fastpfor64.rs @@ -1,12 +1,12 @@ -//! 64-bit ([`u64`]) `FastPFOR` codec. +//! 64-bit ([`u64`]) `FastPFOR` engine. //! //! This is the widened counterpart of the 32-bit [`FastPFor`](super::fastpfor::FastPFor). //! Values, exceptions, and the exception bitmap are 64 bits wide instead of 32. //! The output is byte-compatible with the C++ `CppFastPFor128` / `CppFastPFor256` 64-bit paths. //! -//! `FastPFOR` only handles complete blocks. -//! A [`VariableByte`] tail encodes the sub-block remainder. -//! [`FastPForWide`] bundles both and implements [`BlockCodec64`]. +//! [`FastPForWide`] is the 64-bit half of the public [`FastPFor128`](crate::FastPFor128) / +//! [`FastPFor256`](crate::FastPFor256) codecs and is not exported on its own. +//! It handles complete blocks, then a [`VariableByte`] tail encodes the sub-block remainder. use std::cmp::min; use std::io::Cursor; @@ -29,20 +29,14 @@ const DEFAULT_PAGE_SIZE: u32 = 65536; /// Number of frequency/exception buckets: one per possible bit width `0..=64`. const WIDTHS: usize = 65; -/// [`FastPForWide`] with 128-value blocks. -pub type FastPForWide128 = FastPForWide<128>; - -/// [`FastPForWide`] with 256-value blocks. -pub type FastPForWide256 = FastPForWide<256>; - fn bits64(value: u64) -> usize { 64 - value.leading_zeros().as_usize() } -/// 64-bit `FastPFOR` codec: `FastPFOR`-packed blocks plus a variable-byte tail. +/// 64-bit `FastPFOR` engine: `FastPFOR`-packed blocks plus a variable-byte tail. /// /// `N` is the block size (128 or 256 values). -/// Use [`FastPForWide128`] or [`FastPForWide256`], or call [`encode64`](BlockCodec64::encode64) / [`decode64`](BlockCodec64::decode64). +/// This is the internal `u64` engine behind [`FastPFor128`](crate::FastPFor128) and [`FastPFor256`](crate::FastPFor256). #[derive(Debug)] pub struct FastPForWide { exception_buffers: [Vec; WIDTHS], diff --git a/src/rust/mod.rs b/src/rust/mod.rs index b69e782..b889a19 100644 --- a/src/rust/mod.rs +++ b/src/rust/mod.rs @@ -1,19 +1,14 @@ mod composite; mod cursor; +mod fastpfor_codec; mod integer_compression; pub use composite::CompositeCodec; +/// Any-length `FastPFOR` codecs supporting both `u32` and `u64`. +pub use fastpfor_codec::{FastPFor128, FastPFor256}; /// Type-safe block codec with block size encoded in the type. pub use integer_compression::fastpfor::{FastPFor, FastPForBlock128, FastPForBlock256}; -/// 64-bit ([`u64`]) `FastPFOR` codec implementing [`BlockCodec64`](crate::BlockCodec64). -pub use integer_compression::fastpfor64::{FastPForWide, FastPForWide128, FastPForWide256}; /// Pass-through codec — implements [`AnyLenCodec`](crate::codec::AnyLenCodec). pub use integer_compression::just_copy::JustCopy; /// Variable-byte codec — implements [`AnyLenCodec`](crate::codec::AnyLenCodec). pub use integer_compression::variable_byte::VariableByte; - -/// `FastPForBlock256` blocks + `VariableByte` remainder — the most common composite. -pub type FastPFor256 = CompositeCodec; - -/// `FastPForBlock128` blocks + `VariableByte` remainder. -pub type FastPFor128 = CompositeCodec; From 989b9a3d2691295846d72ffadc79debae2fe0bc8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:49:08 +0000 Subject: [PATCH 05/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/rust/fastpfor_codec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rust/fastpfor_codec.rs b/src/rust/fastpfor_codec.rs index 7fd9719..85e6c44 100644 --- a/src/rust/fastpfor_codec.rs +++ b/src/rust/fastpfor_codec.rs @@ -4,12 +4,12 @@ //! Each implements [`AnyLenCodec`] for `u32` and [`BlockCodec64`] for `u64`. //! Aligned blocks are coded with `FastPFOR` and the sub-block remainder with variable-byte coding. +use crate::FastPForResult; use crate::codec::{AnyLenCodec, BlockCodec64}; use crate::rust::VariableByte; use crate::rust::composite::CompositeCodec; use crate::rust::integer_compression::fastpfor::{FastPForBlock128, FastPForBlock256}; use crate::rust::integer_compression::fastpfor64::FastPForWide; -use crate::FastPForResult; macro_rules! define_fastpfor { ($(#[$meta:meta])* $name:ident, $block:ty, $n:literal) => { From b3d15e2137c8ecc70050b2d7b63082ddb9e9444e Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 17 Jul 2026 13:50:45 +0200 Subject: [PATCH 06/23] Update fuzz/fuzz_targets/fastpfor_u64.rs --- fuzz/fuzz_targets/fastpfor_u64.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/fuzz/fuzz_targets/fastpfor_u64.rs b/fuzz/fuzz_targets/fastpfor_u64.rs index ef070df..deaaeff 100644 --- a/fuzz/fuzz_targets/fastpfor_u64.rs +++ b/fuzz/fuzz_targets/fastpfor_u64.rs @@ -1,11 +1,5 @@ #![no_main] -//! Differential fuzz of the 64-bit FastPFOR codec. -//! -//! For any `u64` input, the Rust and C++ codecs must produce identical compressed bytes. -//! Every decoder must reproduce the original input. -//! Each side must also decode the other's output. - use fastpfor::cpp::{CppFastPFor128, CppFastPFor256}; use fastpfor::{BlockCodec64, FastPFor128, FastPFor256}; use libfuzzer_sys::fuzz_target; From 276df9fa0f8639d4594f77a7cfa217dd11c4c62a Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 17 Jul 2026 14:09:00 +0200 Subject: [PATCH 07/23] refactor: share one width-generic page engine between u32 and u64 The u32 and u64 codecs had near-identical encode_page/decode_page/best_bit implementations differing only in element width. Extract a single generic FastPForEngine parameterized by a FastPForInt trait that abstracts the width-specific pieces (bit width, exception bitmap word count, pack/unpack, shifts). u32 keeps its hand-unrolled bit-packing kernels via trait dispatch, so there is no performance regression; u64 uses the generic wide packer. fastpfor.rs and fastpfor64.rs become thin wrappers over the engine. Net -296 lines with the page algorithm now written once. Verified byte-identical: u32 encode_compare fuzz (Rust==C++, 808K runs) and u64 differential fuzz (619K runs) both clean, plus the full test suite. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/helpers.rs | 7 - src/rust/integer_compression/fastpfor.rs | 445 +-------------- src/rust/integer_compression/fastpfor64.rs | 400 +------------- .../integer_compression/fastpfor_engine.rs | 515 ++++++++++++++++++ src/rust/integer_compression/mod.rs | 1 + 5 files changed, 536 insertions(+), 832 deletions(-) create mode 100644 src/rust/integer_compression/fastpfor_engine.rs diff --git a/src/helpers.rs b/src/helpers.rs index cf0d438..d61f276 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -6,13 +6,6 @@ pub fn greatest_multiple(value: u32, factor: u32) -> u32 { value - value % factor } -/// Returns the number of bits needed to represent `i`. -/// Returns 0 for input 0. -#[cfg_attr(feature = "cpp", allow(dead_code))] -pub fn bits(i: u32) -> usize { - 32 - i.leading_zeros().as_usize() -} - pub trait AsUsize: Eq + Copy { fn as_usize(self) -> usize; diff --git a/src/rust/integer_compression/fastpfor.rs b/src/rust/integer_compression/fastpfor.rs index fa6f6bc..fabf276 100644 --- a/src/rust/integer_compression/fastpfor.rs +++ b/src/rust/integer_compression/fastpfor.rs @@ -1,13 +1,9 @@ -use std::array; -use std::cmp::min; use std::io::Cursor; use bytemuck::cast_slice; -use bytes::{Buf as _, BufMut as _, BytesMut}; -use crate::helpers::{AsUsize, GetWithErr, bits, greatest_multiple}; -use crate::rust::cursor::IncrementCursor; -use crate::rust::integer_compression::{bitpacking, bitunpacking}; +use crate::helpers::AsUsize; +use crate::rust::integer_compression::fastpfor_engine::FastPForEngine; use crate::{BlockCodec, FastPForError, FastPForResult}; mod sealed { @@ -20,9 +16,6 @@ mod sealed { impl BlockSize for [u32; 256] {} } -/// Overhead cost (in bits) for storing each exception's position in the block -const OVERHEAD_OF_EACH_EXCEPT: u32 = 8; - /// Default page size in number of integers. const DEFAULT_PAGE_SIZE: u32 = 65536; @@ -52,23 +45,7 @@ pub type FastPForBlock256 = FastPFor<256>; /// ``` #[derive(Debug)] pub struct FastPFor { - /// Exception values indexed by bit width difference - exception_buffers: [Vec; 33], - /// Metadata buffer for encoding/decoding - bytes_container: BytesMut, - /// Maximum integers per page - page_size: u32, - /// Position trackers for exception arrays - data_pointers: [usize; 33], - /// Frequency count for each bit width: - /// `freqs[i]` = count of values needing exactly i bits - freqs: [u32; 33], - /// Optimal number of bits chosen for the current block - optimal_bits: u8, - /// Number of exceptions that don't fit in the optimal bit width - exception_count: u8, - /// Maximum bit width required for any value in the block - max_bits: u8, + engine: FastPForEngine, } impl Default for FastPFor @@ -82,9 +59,9 @@ where } impl FastPFor { - /// Creates a new `FastPForBlock` with a codec with the given page size. + /// Creates a new codec with the given page size. /// - /// Returns an error if `page_size` is not a multiple of 128. + /// Returns an error if `page_size` is not a multiple of the block size. /// Use [`Default`] for the default page size. pub fn new(page_size: u32) -> FastPForResult { if page_size % N as u32 != 0 { @@ -94,401 +71,9 @@ impl FastPFor { }); } Ok(Self { - bytes_container: BytesMut::with_capacity( - (3 * page_size / N as u32 + page_size) as usize, - ), - page_size, - exception_buffers: array::from_fn(|_| Vec::new()), - data_pointers: [0; 33], - freqs: [0; 33], - optimal_bits: 0, - exception_count: 0, - max_bits: 0, + engine: FastPForEngine::new(page_size), }) } - - fn compress_blocks( - &mut self, - input: &[u32], - input_length: u32, - input_offset: &mut Cursor, - output: &mut [u32], - output_offset: &mut Cursor, - ) { - let inlength = greatest_multiple(input_length, N as u32); - let final_inpos = input_offset.position() as u32 + inlength; - while input_offset.position() as u32 != final_inpos { - let this_size = min(self.page_size, final_inpos - input_offset.position() as u32); - self.encode_page(input, this_size, input_offset, output, output_offset); - } - } - - fn decode_headless_blocks( - &mut self, - input: &[u32], - inlength: u32, - input_offset: &mut Cursor, - output: &mut [u32], - output_offset: &mut Cursor, - ) -> FastPForResult<()> { - let mynvalue = greatest_multiple(inlength, N as u32); - let final_out = output_offset.position() as u32 + mynvalue; - while output_offset.position() as u32 != final_out { - let this_size = min(self.page_size, final_out - output_offset.position() as u32); - self.decode_page(input, input_offset, output, output_offset, this_size)?; - } - Ok(()) - } - - /// Encodes a page using optimal bit width per block. - /// - /// For each block: - /// - Determines best bit width, bitpacks regular values, - /// - Stores exceptions with positions. - /// - Writes header, packed data, metadata bytes, and exception values. - /// - /// # Arguments - /// * `this_size` - Must be multiple of `block_size` - /// * `input_offset` - Advanced by `this_size` - /// * `output_offset` - Advanced by compressed size - fn encode_page( - &mut self, - input: &[u32], - this_size: u32, - input_offset: &mut Cursor, - output: &mut [u32], - output_offset: &mut Cursor, - ) { - let header_pos = output_offset.position() as usize; - output_offset.increment(); - let mut tmp_output_offset = output_offset.position() as u32; - - // Data pointers to 0 - self.data_pointers.fill(0); - self.bytes_container.clear(); - - let mut tmp_input_offset = input_offset.position() as u32; - let final_input_offset = tmp_input_offset + this_size - N as u32; - while tmp_input_offset <= final_input_offset { - self.best_bit_from_data(input, tmp_input_offset); - self.bytes_container.put_u8(self.optimal_bits); - self.bytes_container.put_u8(self.exception_count); - if self.exception_count > 0 { - self.bytes_container.put_u8(self.max_bits); - let index = usize::from(self.max_bits - self.optimal_bits); - let needed = self.data_pointers[index] + usize::from(self.exception_count); - if needed > self.exception_buffers[index].len() { - // Grow to the next multiple of 32 above 2×needed, to amortize resizes. - let new_cap = needed.saturating_mul(2).next_multiple_of(32); - self.exception_buffers[index].resize(new_cap, 0); - } - for k in 0..N as u32 { - if (input[(k + tmp_input_offset) as usize] >> self.optimal_bits) != 0 { - self.bytes_container.put_u8(k as u8); - self.exception_buffers[index][self.data_pointers[index]] = - input[(k + tmp_input_offset) as usize] >> self.optimal_bits; - self.data_pointers[index] += 1; - } - } - } - for k in (0..N as u32).step_by(32) { - bitpacking::fast_pack( - input, - (tmp_input_offset + k) as usize, - output, - tmp_output_offset as usize, - self.optimal_bits, - ); - tmp_output_offset += u32::from(self.optimal_bits); - } - tmp_input_offset += N as u32; - } - input_offset.set_position(u64::from(tmp_input_offset)); - output[header_pos] = tmp_output_offset - header_pos as u32; - let byte_size = self.bytes_container.len(); - while (self.bytes_container.len() & 3) != 0 { - self.bytes_container.put_u8(0); - } - // Output should have 3 position as 4 - output[tmp_output_offset as usize] = byte_size as u32; - tmp_output_offset += 1; - let how_many_ints = self.bytes_container.len() / 4; - // Match C++ memcpy: copy metadata bytes as u32s in one shot (native byte order). - let meta_u32s: &[u32] = cast_slice(self.bytes_container.chunk()); - output[tmp_output_offset as usize..][..how_many_ints] - .copy_from_slice(&meta_u32s[..how_many_ints]); - tmp_output_offset += how_many_ints as u32; - let mut bitmap = 0; - for k in 2..=32 { - if self.data_pointers[k] != 0 { - bitmap |= 1 << (k - 1); - } - } - output[tmp_output_offset as usize] = bitmap; - tmp_output_offset += 1; - - for k in 2..=32 { - if self.data_pointers[k] != 0 { - output[tmp_output_offset as usize] = self.data_pointers[k] as u32; - tmp_output_offset += 1; - let mut j = 0; - while j < self.data_pointers[k] { - bitpacking::fast_pack( - &self.exception_buffers[k], - j, - output, - tmp_output_offset as usize, - k as u8, - ); - tmp_output_offset += k as u32; - j += 32; - } - - // Overflow adjustment - let overflow = j as u32 - self.data_pointers[k] as u32; - tmp_output_offset -= (overflow * k as u32) / 32; - } - } - output_offset.set_position(u64::from(tmp_output_offset)); - } - - /// Computes optimal bit width minimizing total storage cost. - /// - /// Analyzes frequency distribution to balance regular value bits against exception overhead. - fn best_bit_from_data(&mut self, input: &[u32], pos: u32) { - self.freqs.fill(0); - let k_end = min(pos + N as u32, input.len() as u32); - for k in pos..k_end { - self.freqs[bits(input[k as usize])] += 1; - } - - self.optimal_bits = 32; - while self.freqs[self.optimal_bits as usize] == 0 { - self.optimal_bits -= 1; - } - self.max_bits = self.optimal_bits; - - let mut best_cost = u32::from(self.optimal_bits) * N as u32; - let mut num_exceptions: u32 = 0; - self.exception_count = 0; - - for bits in (0..self.optimal_bits).rev() { - num_exceptions += self.freqs[bits as usize + 1]; - if num_exceptions == N as u32 { - break; - } - let diff = u32::from(self.max_bits - bits); - let mut cost = num_exceptions * OVERHEAD_OF_EACH_EXCEPT - + num_exceptions * diff - + u32::from(bits) * N as u32 - + 8; - if diff == 1 { - cost -= num_exceptions; - } - if cost < best_cost { - best_cost = cost; - self.optimal_bits = bits; - self.exception_count = num_exceptions as u8; - } - } - } - - /// Decodes a compressed page. - /// - /// Reads header to locate exception data, loads exceptions by bit width, - /// unpacks regular values per block, patches in exceptions by position. - /// - /// # Arguments - /// * `this_size` - Expected decompressed integer count - /// * `input_offset` - Advanced by bytes read - /// * `output_offset` - Advanced by `this_size` - #[expect(clippy::too_many_lines)] - fn decode_page( - &mut self, - input: &[u32], - input_offset: &mut Cursor, - output: &mut [u32], - output_offset: &mut Cursor, - this_size: u32, - ) -> FastPForResult<()> { - let n = u32::try_from(input.len()) - .map_err(|_| FastPForError::InvalidInputLength(input.len()))?; - - let init_pos = - u32::try_from(input_offset.position()).map_err(|_| FastPForError::NotEnoughData)?; - let where_meta = input.get_val(init_pos)?; - input_offset.increment(); - let mut inexcept = init_pos - .checked_add(where_meta) - .ok_or(FastPForError::NotEnoughData)?; - let bytesize = input.get_val(inexcept)?; - inexcept = inexcept - .checked_add(1) - .ok_or(FastPForError::NotEnoughData)?; - // Point a byte cursor directly at the metadata region in `input`, - // mirrors C++ `const uint8_t *bytep = reinterpret_cast(inexcept)`. - // The C++ encoder uses a raw `memcpy` of bytes into the u32 output (no endian - // conversion), and the decoder does a raw reinterpret_cast back -- both native byte - // order. `cast_slice` is the exact Rust equivalent: a safe, zero-copy native view. - let input_bytes: &[u8] = cast_slice(input); - let mut byte_pos = (inexcept as usize) - .checked_mul(4) - .filter(|&bp| bp <= input_bytes.len()) - .ok_or(FastPForError::NotEnoughData)?; - let length = bytesize.div_ceil(4); - inexcept = inexcept - .checked_add(length) - .ok_or(FastPForError::NotEnoughData)?; - - let bitmap = input.get_val(inexcept)?; - inexcept = inexcept - .checked_add(1) - .ok_or(FastPForError::NotEnoughData)?; - - for k in 2..=32 { - if (bitmap & (1 << (k - 1))) != 0 { - let size = input.get_val(inexcept)?; - inexcept = inexcept - .checked_add(1) - .ok_or(FastPForError::NotEnoughData)?; - // Reject adversarial inputs: exceptions can't exceed the page size. - if size > self.page_size { - return Err(FastPForError::NotEnoughData); - } - // Ensure the buffer is large enough for `size` values, rounded up - // to the next group of 32 for the bitunpacking calls. - let rounded_up = size.next_multiple_of(32) as usize; - if self.exception_buffers[k as usize].len() < rounded_up { - self.exception_buffers[k as usize].resize(rounded_up, 0); - } - let mut j: u32 = 0; - // Process full groups directly from input - while j.checked_add(32).is_some_and(|j32| j32 <= size) - && inexcept.checked_add(k).is_some_and(|ie| ie <= n) - { - bitunpacking::fast_unpack( - input, - inexcept as usize, - &mut self.exception_buffers[k as usize], - j as usize, - k as u8, - ); - inexcept += k; // safe: loop guard checked inexcept + k <= n <= u32::MAX - j += 32; // safe: loop guard checked j + 32 <= size - } - // Handle the final partial group using a stack buffer (mirrors C++ buffer[PACKSIZE*2]) - if j < size { - let words_needed = (size - j) // safe: j < size - .saturating_mul(k) - .div_ceil(32); - let avail = n - inexcept.min(n); - if avail < words_needed { - return Err(FastPForError::NotEnoughData); - } - let copy_len = words_needed as usize; - let mut tail_buf = [0u32; 64]; - if copy_len == 0 { - return Err(FastPForError::NotEnoughData); - } - let start = inexcept as usize; - let src = input - .get(start..start + copy_len) - .ok_or(FastPForError::NotEnoughData)?; - tail_buf[..copy_len].copy_from_slice(src); - let tail_inpos = 0; - bitunpacking::fast_unpack( - &tail_buf, - tail_inpos, - &mut self.exception_buffers[k as usize], - j as usize, - k as u8, - ); - inexcept += k; - j += 32; - } - let overflow = j - size; - inexcept -= (overflow * k) / 32; - } - } - - self.data_pointers.fill(0); - let mut tmp_output_offset = output_offset.position() as u32; - let mut tmp_input_offset = input_offset.position() as u32; - - let run_end = this_size / N as u32; - for _ in 0..run_end { - let bits = input_bytes.get_val(byte_pos)?; - if bits > 32 { - return Err(FastPForError::NotEnoughData); - } - byte_pos += 1; - let num_exceptions = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - for k in (0..N as u32).step_by(32) { - let in_start = tmp_input_offset as usize; - let out_start = (tmp_output_offset + k) as usize; - let in_end = in_start - .checked_add(usize::from(bits)) - .ok_or(FastPForError::NotEnoughData)?; - if in_end > input.len() { - return Err(FastPForError::NotEnoughData); - } - let out_end = out_start - .checked_add(32) - .ok_or(FastPForError::OutputBufferTooSmall)?; - if out_end > output.len() { - return Err(FastPForError::OutputBufferTooSmall); - } - bitunpacking::fast_unpack(input, in_start, output, out_start, bits); - tmp_input_offset += u32::from(bits); - } - if num_exceptions > 0 { - let maxbits = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - let index = maxbits - .checked_sub(bits) - .ok_or(FastPForError::NotEnoughData)?; - if maxbits > 32 || index == 0 || index > 32 { - return Err(FastPForError::NotEnoughData); - } - let index = usize::from(index); - if index == 1 { - for _ in 0..num_exceptions { - let pos = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - if u32::from(pos) >= N as u32 { - return Err(FastPForError::NotEnoughData); - } - let out_idx = tmp_output_offset as usize + pos as usize; - if out_idx >= output.len() { - return Err(FastPForError::OutputBufferTooSmall); - } - output[out_idx] |= 1 << bits; - } - } else { - for _ in 0..num_exceptions { - let pos = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - if u32::from(pos) >= N as u32 { - return Err(FastPForError::NotEnoughData); - } - let out_idx = tmp_output_offset as usize + pos as usize; - if out_idx >= output.len() { - return Err(FastPForError::OutputBufferTooSmall); - } - let ptr = self.data_pointers[index]; - let except_value = self.exception_buffers[index].get_val(ptr)?; - output[out_idx] |= except_value << bits; - self.data_pointers[index] += 1; - } - } - } - tmp_output_offset += N as u32; - } - output_offset.set_position(u64::from(tmp_output_offset)); - input_offset.set_position(u64::from(inexcept)); - Ok(()) - } } impl BlockCodec for FastPFor @@ -515,7 +100,7 @@ where // Write length header then compress. out[start] = n_values; - self.compress_blocks( + self.engine.compress_blocks( flat, n_values, &mut in_off, @@ -563,7 +148,7 @@ where let mut in_off = Cursor::new(0u32); let mut out_off = Cursor::new(0u32); - self.decode_headless_blocks( + self.engine.decode_headless_blocks( rest, block_n_values, &mut in_off, @@ -601,14 +186,13 @@ mod tests { #[test] fn test_empty_blocks_ok() { - // Empty input encodes to length header [0] (matches C++ FastPFor) and decodes cleanly. + // Empty input encodes to length header [0] and decodes cleanly. let enc = block_compress::(&[]).unwrap(); assert_eq!(enc, [0]); let dec = block_decompress::(&enc, Some(0)).unwrap(); assert!(dec.is_empty()); } - // Tests ported from C++ #[test] fn test_constant_sequence() { block_roundtrip::(&vec![42u32; 65536]); @@ -647,20 +231,16 @@ mod tests { block_roundtrip::(&input); } - // ── Error / edge tests not covered by `tests/decode_validation.rs` ───── - // - // `AnyLenCodec::decode` treats an empty slice as tail-only and succeeds; an empty - // `decode_blocks` input is still invalid. Headless decode is internal-only. - #[test] fn uncompress_zero_input_length_err() { - // Truly empty input (no header word at all) is invalid — C++ would crash reading *in. + // Truly empty input (no header word at all) is invalid. block_decompress::(&[], None).unwrap_err(); } #[test] fn headless_uncompress_zero_inlength_128_ok() { FastPForBlock128::default() + .engine .decode_headless_blocks( &[], 0, @@ -673,7 +253,6 @@ mod tests { #[test] fn decode_where_meta_overflow() { - // `decode_headless_blocks` only: no `AnyLenCodec` entry point passes this layout. let data: Vec = (0..256u32) .map(|i| if i % 2 == 0 { 1u32 << 30 } else { 3 }) .collect(); @@ -685,6 +264,7 @@ mod tests { let out_length = padded[1]; assert!( FastPForBlock256::default() + .engine .decode_headless_blocks( &padded, out_length, @@ -706,7 +286,6 @@ mod tests { /// `decode_blocks` with `expected_len: None` and header=0 returns `Ok` with empty output. #[test] fn decode_blocks_header_only_input() { - // Input with just the length header [0]: no blocks to decode. let input = vec![0u32]; let out = block_decompress::(&input, None).unwrap(); assert!(out.is_empty()); diff --git a/src/rust/integer_compression/fastpfor64.rs b/src/rust/integer_compression/fastpfor64.rs index 68afd82..1ec4e12 100644 --- a/src/rust/integer_compression/fastpfor64.rs +++ b/src/rust/integer_compression/fastpfor64.rs @@ -1,4 +1,4 @@ -//! 64-bit ([`u64`]) `FastPFOR` engine. +//! 64-bit ([`u64`]) `FastPFOR` codec. //! //! This is the widened counterpart of the 32-bit [`FastPFor`](super::fastpfor::FastPFor). //! Values, exceptions, and the exception bitmap are 64 bits wide instead of 32. @@ -8,416 +8,32 @@ //! [`FastPFor256`](crate::FastPFor256) codecs and is not exported on its own. //! It handles complete blocks, then a [`VariableByte`] tail encodes the sub-block remainder. -use std::cmp::min; use std::io::Cursor; use bytemuck::{cast_slice, cast_slice_mut}; -use bytes::{Buf as _, BufMut as _, BytesMut}; use crate::codec::default_max_decoded_len; -use crate::helpers::{AsUsize, GetWithErr, greatest_multiple}; -use crate::rust::cursor::IncrementCursor; -use crate::rust::integer_compression::bitpacking_wide::{pack_wide, unpack_wide}; +use crate::helpers::AsUsize; +use crate::rust::integer_compression::fastpfor_engine::FastPForEngine; use crate::{BlockCodec64, FastPForError, FastPForResult}; -/// Overhead cost (in bits) for storing each exception's position in the block. -const OVERHEAD_OF_EACH_EXCEPT: u32 = 8; - /// Default page size in number of integers. const DEFAULT_PAGE_SIZE: u32 = 65536; -/// Number of frequency/exception buckets: one per possible bit width `0..=64`. -const WIDTHS: usize = 65; - -fn bits64(value: u64) -> usize { - 64 - value.leading_zeros().as_usize() -} - -/// 64-bit `FastPFOR` engine: `FastPFOR`-packed blocks plus a variable-byte tail. +/// 64-bit `FastPFOR` codec: `FastPFOR`-packed blocks plus a variable-byte tail. /// /// `N` is the block size (128 or 256 values). /// This is the internal `u64` engine behind [`FastPFor128`](crate::FastPFor128) and [`FastPFor256`](crate::FastPFor256). #[derive(Debug)] pub struct FastPForWide { - exception_buffers: [Vec; WIDTHS], - bytes_container: BytesMut, - page_size: u32, - data_pointers: [usize; WIDTHS], - freqs: [u32; WIDTHS], - optimal_bits: u8, - exception_count: u8, - max_bits: u8, + engine: FastPForEngine, } impl Default for FastPForWide { fn default() -> Self { - Self::new(DEFAULT_PAGE_SIZE) - } -} - -impl FastPForWide { - fn new(page_size: u32) -> Self { Self { - bytes_container: BytesMut::with_capacity( - (3 * page_size / N as u32 + page_size) as usize, - ), - page_size, - exception_buffers: std::array::from_fn(|_| Vec::new()), - data_pointers: [0; WIDTHS], - freqs: [0; WIDTHS], - optimal_bits: 0, - exception_count: 0, - max_bits: 0, - } - } - - fn compress_blocks( - &mut self, - input: &[u64], - input_length: u32, - input_offset: &mut Cursor, - output: &mut [u32], - output_offset: &mut Cursor, - ) { - let inlength = greatest_multiple(input_length, N as u32); - let final_inpos = input_offset.position() as u32 + inlength; - while input_offset.position() as u32 != final_inpos { - let this_size = min(self.page_size, final_inpos - input_offset.position() as u32); - self.encode_page(input, this_size, input_offset, output, output_offset); - } - } - - fn decode_headless_blocks( - &mut self, - input: &[u32], - inlength: u32, - input_offset: &mut Cursor, - output: &mut [u64], - output_offset: &mut Cursor, - ) -> FastPForResult<()> { - let mynvalue = greatest_multiple(inlength, N as u32); - let final_out = output_offset.position() as u32 + mynvalue; - while output_offset.position() as u32 != final_out { - let this_size = min(self.page_size, final_out - output_offset.position() as u32); - self.decode_page(input, input_offset, output, output_offset, this_size)?; - } - Ok(()) - } - - fn encode_page( - &mut self, - input: &[u64], - this_size: u32, - input_offset: &mut Cursor, - output: &mut [u32], - output_offset: &mut Cursor, - ) { - let header_pos = output_offset.position() as usize; - output_offset.increment(); - let mut tmp_output_offset = output_offset.position() as u32; - - self.data_pointers.fill(0); - self.bytes_container.clear(); - - let mut tmp_input_offset = input_offset.position() as u32; - let final_input_offset = tmp_input_offset + this_size - N as u32; - while tmp_input_offset <= final_input_offset { - self.best_bit_from_data(input, tmp_input_offset); - self.bytes_container.put_u8(self.optimal_bits); - self.bytes_container.put_u8(self.exception_count); - if self.exception_count > 0 { - self.bytes_container.put_u8(self.max_bits); - let index = usize::from(self.max_bits - self.optimal_bits); - let needed = self.data_pointers[index] + usize::from(self.exception_count); - if needed > self.exception_buffers[index].len() { - let new_cap = needed.saturating_mul(2).next_multiple_of(32); - self.exception_buffers[index].resize(new_cap, 0); - } - for k in 0..N as u32 { - if (input[(k + tmp_input_offset) as usize] >> self.optimal_bits) != 0 { - self.bytes_container.put_u8(k as u8); - self.exception_buffers[index][self.data_pointers[index]] = - input[(k + tmp_input_offset) as usize] >> self.optimal_bits; - self.data_pointers[index] += 1; - } - } - } - for k in (0..N as u32).step_by(32) { - pack_wide( - input, - (tmp_input_offset + k) as usize, - output, - tmp_output_offset as usize, - self.optimal_bits, - ); - tmp_output_offset += u32::from(self.optimal_bits); - } - tmp_input_offset += N as u32; - } - input_offset.set_position(u64::from(tmp_input_offset)); - output[header_pos] = tmp_output_offset - header_pos as u32; - let byte_size = self.bytes_container.len(); - while (self.bytes_container.len() & 3) != 0 { - self.bytes_container.put_u8(0); - } - output[tmp_output_offset as usize] = byte_size as u32; - tmp_output_offset += 1; - let how_many_ints = self.bytes_container.len() / 4; - let meta_u32s: &[u32] = cast_slice(self.bytes_container.chunk()); - output[tmp_output_offset as usize..][..how_many_ints] - .copy_from_slice(&meta_u32s[..how_many_ints]); - tmp_output_offset += how_many_ints as u32; - - let mut bitmap: u64 = 0; - for k in 2..=64 { - if self.data_pointers[k] != 0 { - bitmap |= 1u64 << (k - 1); - } - } - output[tmp_output_offset as usize] = bitmap as u32; - output[tmp_output_offset as usize + 1] = (bitmap >> 32) as u32; - tmp_output_offset += 2; - - for k in 2..=64 { - if self.data_pointers[k] != 0 { - output[tmp_output_offset as usize] = self.data_pointers[k] as u32; - tmp_output_offset += 1; - let mut j = 0; - while j < self.data_pointers[k] { - pack_wide( - &self.exception_buffers[k], - j, - output, - tmp_output_offset as usize, - k as u8, - ); - tmp_output_offset += k as u32; - j += 32; - } - let overflow = j as u32 - self.data_pointers[k] as u32; - tmp_output_offset -= (overflow * k as u32) / 32; - } - } - output_offset.set_position(u64::from(tmp_output_offset)); - } - - fn best_bit_from_data(&mut self, input: &[u64], pos: u32) { - self.freqs.fill(0); - let k_end = min(pos + N as u32, input.len() as u32); - for k in pos..k_end { - self.freqs[bits64(input[k as usize])] += 1; - } - - self.optimal_bits = 64; - while self.freqs[self.optimal_bits as usize] == 0 { - self.optimal_bits -= 1; + engine: FastPForEngine::new(DEFAULT_PAGE_SIZE), } - self.max_bits = self.optimal_bits; - - let mut best_cost = u32::from(self.optimal_bits) * N as u32; - let mut num_exceptions: u32 = 0; - self.exception_count = 0; - - for bits in (0..self.optimal_bits).rev() { - num_exceptions += self.freqs[bits as usize + 1]; - if num_exceptions == N as u32 { - break; - } - let diff = u32::from(self.max_bits - bits); - let mut cost = num_exceptions * OVERHEAD_OF_EACH_EXCEPT - + num_exceptions * diff - + u32::from(bits) * N as u32 - + 8; - if diff == 1 { - cost -= num_exceptions; - } - if cost < best_cost { - best_cost = cost; - self.optimal_bits = bits; - self.exception_count = num_exceptions as u8; - } - } - } - - #[expect(clippy::too_many_lines)] - fn decode_page( - &mut self, - input: &[u32], - input_offset: &mut Cursor, - output: &mut [u64], - output_offset: &mut Cursor, - this_size: u32, - ) -> FastPForResult<()> { - let n = u32::try_from(input.len()) - .map_err(|_| FastPForError::InvalidInputLength(input.len()))?; - - let init_pos = - u32::try_from(input_offset.position()).map_err(|_| FastPForError::NotEnoughData)?; - let where_meta = input.get_val(init_pos)?; - input_offset.increment(); - let mut inexcept = init_pos - .checked_add(where_meta) - .ok_or(FastPForError::NotEnoughData)?; - let bytesize = input.get_val(inexcept)?; - inexcept = inexcept - .checked_add(1) - .ok_or(FastPForError::NotEnoughData)?; - let input_bytes: &[u8] = cast_slice(input); - let mut byte_pos = (inexcept as usize) - .checked_mul(4) - .filter(|&bp| bp <= input_bytes.len()) - .ok_or(FastPForError::NotEnoughData)?; - let length = bytesize.div_ceil(4); - inexcept = inexcept - .checked_add(length) - .ok_or(FastPForError::NotEnoughData)?; - - let bitmap_lo = input.get_val(inexcept)?; - let bitmap_hi = input.get_val( - inexcept - .checked_add(1) - .ok_or(FastPForError::NotEnoughData)?, - )?; - let bitmap = u64::from(bitmap_lo) | (u64::from(bitmap_hi) << 32); - inexcept = inexcept - .checked_add(2) - .ok_or(FastPForError::NotEnoughData)?; - - for k in 2..=64u32 { - if (bitmap & (1u64 << (k - 1))) != 0 { - let size = input.get_val(inexcept)?; - inexcept = inexcept - .checked_add(1) - .ok_or(FastPForError::NotEnoughData)?; - if size > self.page_size { - return Err(FastPForError::NotEnoughData); - } - let rounded_up = size.next_multiple_of(32) as usize; - if self.exception_buffers[k as usize].len() < rounded_up { - self.exception_buffers[k as usize].resize(rounded_up, 0); - } - let mut j: u32 = 0; - while j.checked_add(32).is_some_and(|j32| j32 <= size) - && inexcept.checked_add(k).is_some_and(|ie| ie <= n) - { - unpack_wide( - input, - inexcept as usize, - &mut self.exception_buffers[k as usize], - j as usize, - k as u8, - ); - inexcept += k; - j += 32; - } - if j < size { - let words_needed = (size - j).saturating_mul(k).div_ceil(32); - let avail = n - inexcept.min(n); - if avail < words_needed { - return Err(FastPForError::NotEnoughData); - } - let copy_len = words_needed as usize; - let mut tail_buf = [0u32; 128]; - if copy_len == 0 { - return Err(FastPForError::NotEnoughData); - } - let start = inexcept as usize; - let src = input - .get(start..start + copy_len) - .ok_or(FastPForError::NotEnoughData)?; - tail_buf[..copy_len].copy_from_slice(src); - unpack_wide( - &tail_buf, - 0, - &mut self.exception_buffers[k as usize], - j as usize, - k as u8, - ); - inexcept += k; - j += 32; - } - let overflow = j - size; - inexcept -= (overflow * k) / 32; - } - } - - self.data_pointers.fill(0); - let mut tmp_output_offset = output_offset.position() as u32; - let mut tmp_input_offset = input_offset.position() as u32; - - let run_end = this_size / N as u32; - for _ in 0..run_end { - let bits = input_bytes.get_val(byte_pos)?; - if bits > 64 { - return Err(FastPForError::NotEnoughData); - } - byte_pos += 1; - let num_exceptions = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - for k in (0..N as u32).step_by(32) { - let in_start = tmp_input_offset as usize; - let out_start = (tmp_output_offset + k) as usize; - let in_end = in_start - .checked_add(usize::from(bits)) - .ok_or(FastPForError::NotEnoughData)?; - if in_end > input.len() { - return Err(FastPForError::NotEnoughData); - } - let out_end = out_start - .checked_add(32) - .ok_or(FastPForError::OutputBufferTooSmall)?; - if out_end > output.len() { - return Err(FastPForError::OutputBufferTooSmall); - } - unpack_wide(input, in_start, output, out_start, bits); - tmp_input_offset += u32::from(bits); - } - if num_exceptions > 0 { - let maxbits = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - let index = maxbits - .checked_sub(bits) - .ok_or(FastPForError::NotEnoughData)?; - if maxbits > 64 || index == 0 || index > 64 { - return Err(FastPForError::NotEnoughData); - } - let index = usize::from(index); - if index == 1 { - for _ in 0..num_exceptions { - let pos = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - if u32::from(pos) >= N as u32 { - return Err(FastPForError::NotEnoughData); - } - let out_idx = tmp_output_offset as usize + pos as usize; - if out_idx >= output.len() { - return Err(FastPForError::OutputBufferTooSmall); - } - output[out_idx] |= 1u64 << bits; - } - } else { - for _ in 0..num_exceptions { - let pos = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - if u32::from(pos) >= N as u32 { - return Err(FastPForError::NotEnoughData); - } - let out_idx = tmp_output_offset as usize + pos as usize; - if out_idx >= output.len() { - return Err(FastPForError::OutputBufferTooSmall); - } - let ptr = self.data_pointers[index]; - let except_value = self.exception_buffers[index].get_val(ptr)?; - output[out_idx] |= except_value << bits; - self.data_pointers[index] += 1; - } - } - } - tmp_output_offset += N as u32; - } - output_offset.set_position(u64::from(tmp_output_offset)); - input_offset.set_position(u64::from(inexcept)); - Ok(()) } } @@ -501,7 +117,7 @@ impl BlockCodec64 for FastPForWide { let mut in_off = Cursor::new(0u32); let mut out_off = Cursor::new(0u32); - self.compress_blocks( + self.engine.compress_blocks( &input[..rounded], n_values, &mut in_off, @@ -535,7 +151,7 @@ impl BlockCodec64 for FastPForWide { out.resize(start + n_blocks * N, 0); let mut in_off = Cursor::new(0u32); let mut out_off = Cursor::new(0u32); - self.decode_headless_blocks( + self.engine.decode_headless_blocks( rest, block_n_values, &mut in_off, diff --git a/src/rust/integer_compression/fastpfor_engine.rs b/src/rust/integer_compression/fastpfor_engine.rs new file mode 100644 index 0000000..b7f4407 --- /dev/null +++ b/src/rust/integer_compression/fastpfor_engine.rs @@ -0,0 +1,515 @@ +//! Width-generic `FastPFOR` page engine shared by the 32- and 64-bit codecs. +//! +//! The block-splitting, best-bit search, exception handling, and metadata layout are +//! identical for `u32` and `u64`; only the element width differs. +//! [`FastPForInt`] abstracts the width-specific pieces so a single [`FastPForEngine`] +//! implements the algorithm once. +//! `u32` keeps its hand-unrolled bit-packing kernels; `u64` uses the generic wide packer. + +use std::array; +use std::cmp::min; +use std::io::Cursor; + +use bytemuck::cast_slice; +use bytes::{Buf as _, BufMut as _, BytesMut}; + +use crate::helpers::{GetWithErr, greatest_multiple}; +use crate::rust::cursor::IncrementCursor; +use crate::rust::integer_compression::{bitpacking, bitpacking_wide, bitunpacking}; +use crate::{FastPForError, FastPForResult}; + +/// Overhead cost (in bits) for storing each exception's position in the block. +const OVERHEAD_OF_EACH_EXCEPT: u32 = 8; + +/// One frequency/exception bucket per possible bit width, up to the widest supported (`u64`). +const WIDTHS: usize = 65; + +/// Element type of a `FastPFOR` stream: [`u32`] or [`u64`]. +/// +/// Implementors supply the width-specific operations the engine needs. +/// The exception bitmap spans [`BITMAP_WORDS`](Self::BITMAP_WORDS) output words. +pub trait FastPForInt: Copy + 'static { + /// Bit width of the element: 32 or 64. + const WIDTH: u8; + /// Output words occupied by the exception bitmap: 1 for `u32`, 2 for `u64`. + const BITMAP_WORDS: u32; + /// The zero value. + const ZERO: Self; + + /// Number of significant bits, i.e. `WIDTH - leading_zeros` (0 for a zero value). + fn significant_bits(self) -> u8; + /// Logical right shift by `n`, where `n < WIDTH`. + fn shr(self, n: u8) -> Self; + /// Whether the value is zero. + fn is_zero(self) -> bool; + /// `*dst |= val << shift`, where `shift < WIDTH`. + fn or_shl_assign(dst: &mut Self, val: Self, shift: u8); + /// `1 << shift`, where `shift < WIDTH`. + fn one_shl(shift: u8) -> Self; + + /// Pack 32 values at `bit` bits each into `out`. + fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8); + /// Unpack 32 values at `bit` bits each from `src`. + fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8); + + /// Write the exception bitmap as [`BITMAP_WORDS`](Self::BITMAP_WORDS) words at `out`. + fn write_bitmap(bitmap: u64, out: &mut [u32]); + /// Read the exception bitmap from [`BITMAP_WORDS`](Self::BITMAP_WORDS) words at `pos`. + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult; +} + +#[allow(clippy::use_self, reason = "u32 literals here are stream words, not the Self element type")] +impl FastPForInt for u32 { + const WIDTH: u8 = 32; + const BITMAP_WORDS: u32 = 1; + const ZERO: Self = 0; + + fn significant_bits(self) -> u8 { + (32 - self.leading_zeros()) as u8 + } + fn shr(self, n: u8) -> Self { + self >> n + } + fn is_zero(self) -> bool { + self == 0 + } + fn or_shl_assign(dst: &mut Self, val: Self, shift: u8) { + *dst |= val << shift; + } + fn one_shl(shift: u8) -> Self { + 1 << shift + } + fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { + bitpacking::fast_pack(src, inpos, out, outpos, bit); + } + fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { + bitunpacking::fast_unpack(src, inpos, out, outpos, bit); + } + fn write_bitmap(bitmap: u64, out: &mut [u32]) { + out[0] = bitmap as u32; + } + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { + let word: u32 = input.get_val(pos)?; + Ok(u64::from(word)) + } +} + +#[allow(clippy::use_self, reason = "u32 literals here are stream words, not the Self element type")] +impl FastPForInt for u64 { + const WIDTH: u8 = 64; + const BITMAP_WORDS: u32 = 2; + const ZERO: Self = 0; + + fn significant_bits(self) -> u8 { + (64 - self.leading_zeros()) as u8 + } + fn shr(self, n: u8) -> Self { + self >> n + } + fn is_zero(self) -> bool { + self == 0 + } + fn or_shl_assign(dst: &mut Self, val: Self, shift: u8) { + *dst |= val << shift; + } + fn one_shl(shift: u8) -> Self { + 1 << shift + } + fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { + bitpacking_wide::pack_wide(src, inpos, out, outpos, bit); + } + fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { + bitpacking_wide::unpack_wide(src, inpos, out, outpos, bit); + } + fn write_bitmap(bitmap: u64, out: &mut [u32]) { + out[0] = bitmap as u32; + out[1] = (bitmap >> 32) as u32; + } + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { + let lo: u32 = input.get_val(pos)?; + let hi_pos = pos.checked_add(1).ok_or(FastPForError::NotEnoughData)?; + let hi: u32 = input.get_val(hi_pos)?; + Ok(u64::from(lo) | (u64::from(hi) << 32)) + } +} + +/// Shared `FastPFOR` scratch state and page codec for a block size `N` and element type `T`. +#[derive(Debug)] +pub struct FastPForEngine { + /// Exception values grouped by `max_bits - optimal_bits`. + exception_buffers: [Vec; WIDTHS], + /// Per-block metadata (bit widths, exception counts, positions). + bytes_container: BytesMut, + /// Maximum integers per page. + page_size: u32, + /// Write positions into `exception_buffers`. + data_pointers: [usize; WIDTHS], + /// Count of values needing exactly `i` bits. + freqs: [u32; WIDTHS], + /// Chosen bit width for the current block. + optimal_bits: u8, + /// Exceptions that exceed `optimal_bits`. + exception_count: u8, + /// Widest value in the current block. + max_bits: u8, +} + +impl FastPForEngine { + pub fn new(page_size: u32) -> Self { + Self { + bytes_container: BytesMut::with_capacity( + (3 * page_size / N as u32 + page_size) as usize, + ), + page_size, + exception_buffers: array::from_fn(|_| Vec::new()), + data_pointers: [0; WIDTHS], + freqs: [0; WIDTHS], + optimal_bits: 0, + exception_count: 0, + max_bits: 0, + } + } + + pub fn compress_blocks( + &mut self, + input: &[T], + input_length: u32, + input_offset: &mut Cursor, + output: &mut [u32], + output_offset: &mut Cursor, + ) { + let inlength = greatest_multiple(input_length, N as u32); + let final_inpos = input_offset.position() as u32 + inlength; + while input_offset.position() as u32 != final_inpos { + let this_size = min(self.page_size, final_inpos - input_offset.position() as u32); + self.encode_page(input, this_size, input_offset, output, output_offset); + } + } + + pub fn decode_headless_blocks( + &mut self, + input: &[u32], + inlength: u32, + input_offset: &mut Cursor, + output: &mut [T], + output_offset: &mut Cursor, + ) -> FastPForResult<()> { + let mynvalue = greatest_multiple(inlength, N as u32); + let final_out = output_offset.position() as u32 + mynvalue; + while output_offset.position() as u32 != final_out { + let this_size = min(self.page_size, final_out - output_offset.position() as u32); + self.decode_page(input, input_offset, output, output_offset, this_size)?; + } + Ok(()) + } + + fn encode_page( + &mut self, + input: &[T], + this_size: u32, + input_offset: &mut Cursor, + output: &mut [u32], + output_offset: &mut Cursor, + ) { + let header_pos = output_offset.position() as usize; + output_offset.increment(); + let mut tmp_output_offset = output_offset.position() as u32; + + self.data_pointers.fill(0); + self.bytes_container.clear(); + + let mut tmp_input_offset = input_offset.position() as u32; + let final_input_offset = tmp_input_offset + this_size - N as u32; + while tmp_input_offset <= final_input_offset { + self.best_bit_from_data(input, tmp_input_offset); + self.bytes_container.put_u8(self.optimal_bits); + self.bytes_container.put_u8(self.exception_count); + if self.exception_count > 0 { + self.bytes_container.put_u8(self.max_bits); + let index = usize::from(self.max_bits - self.optimal_bits); + let needed = self.data_pointers[index] + usize::from(self.exception_count); + if needed > self.exception_buffers[index].len() { + let new_cap = needed.saturating_mul(2).next_multiple_of(32); + self.exception_buffers[index].resize(new_cap, T::ZERO); + } + for k in 0..N as u32 { + let value = input[(k + tmp_input_offset) as usize]; + if !value.shr(self.optimal_bits).is_zero() { + self.bytes_container.put_u8(k as u8); + self.exception_buffers[index][self.data_pointers[index]] = + value.shr(self.optimal_bits); + self.data_pointers[index] += 1; + } + } + } + for k in (0..N as u32).step_by(32) { + T::fast_pack( + input, + (tmp_input_offset + k) as usize, + output, + tmp_output_offset as usize, + self.optimal_bits, + ); + tmp_output_offset += u32::from(self.optimal_bits); + } + tmp_input_offset += N as u32; + } + input_offset.set_position(u64::from(tmp_input_offset)); + output[header_pos] = tmp_output_offset - header_pos as u32; + let byte_size = self.bytes_container.len(); + while (self.bytes_container.len() & 3) != 0 { + self.bytes_container.put_u8(0); + } + output[tmp_output_offset as usize] = byte_size as u32; + tmp_output_offset += 1; + let how_many_ints = self.bytes_container.len() / 4; + let meta_u32s: &[u32] = cast_slice(self.bytes_container.chunk()); + output[tmp_output_offset as usize..][..how_many_ints] + .copy_from_slice(&meta_u32s[..how_many_ints]); + tmp_output_offset += how_many_ints as u32; + + let mut bitmap: u64 = 0; + for k in 2..=usize::from(T::WIDTH) { + if self.data_pointers[k] != 0 { + bitmap |= 1u64 << (k - 1); + } + } + T::write_bitmap(bitmap, &mut output[tmp_output_offset as usize..]); + tmp_output_offset += T::BITMAP_WORDS; + + for k in 2..=usize::from(T::WIDTH) { + if self.data_pointers[k] != 0 { + output[tmp_output_offset as usize] = self.data_pointers[k] as u32; + tmp_output_offset += 1; + let mut j = 0; + while j < self.data_pointers[k] { + T::fast_pack( + &self.exception_buffers[k], + j, + output, + tmp_output_offset as usize, + k as u8, + ); + tmp_output_offset += k as u32; + j += 32; + } + let overflow = j as u32 - self.data_pointers[k] as u32; + tmp_output_offset -= (overflow * k as u32) / 32; + } + } + output_offset.set_position(u64::from(tmp_output_offset)); + } + + fn best_bit_from_data(&mut self, input: &[T], pos: u32) { + self.freqs.fill(0); + let k_end = min(pos + N as u32, input.len() as u32); + for k in pos..k_end { + self.freqs[usize::from(input[k as usize].significant_bits())] += 1; + } + + self.optimal_bits = T::WIDTH; + while self.freqs[self.optimal_bits as usize] == 0 { + self.optimal_bits -= 1; + } + self.max_bits = self.optimal_bits; + + let mut best_cost = u32::from(self.optimal_bits) * N as u32; + let mut num_exceptions: u32 = 0; + self.exception_count = 0; + + for bits in (0..self.optimal_bits).rev() { + num_exceptions += self.freqs[bits as usize + 1]; + if num_exceptions == N as u32 { + break; + } + let diff = u32::from(self.max_bits - bits); + let mut cost = num_exceptions * OVERHEAD_OF_EACH_EXCEPT + + num_exceptions * diff + + u32::from(bits) * N as u32 + + 8; + if diff == 1 { + cost -= num_exceptions; + } + if cost < best_cost { + best_cost = cost; + self.optimal_bits = bits; + self.exception_count = num_exceptions as u8; + } + } + } + + #[expect(clippy::too_many_lines)] + fn decode_page( + &mut self, + input: &[u32], + input_offset: &mut Cursor, + output: &mut [T], + output_offset: &mut Cursor, + this_size: u32, + ) -> FastPForResult<()> { + let n = u32::try_from(input.len()) + .map_err(|_| FastPForError::InvalidInputLength(input.len()))?; + + let init_pos = + u32::try_from(input_offset.position()).map_err(|_| FastPForError::NotEnoughData)?; + let where_meta = input.get_val(init_pos)?; + input_offset.increment(); + let mut inexcept = init_pos + .checked_add(where_meta) + .ok_or(FastPForError::NotEnoughData)?; + let bytesize = input.get_val(inexcept)?; + inexcept = inexcept + .checked_add(1) + .ok_or(FastPForError::NotEnoughData)?; + let input_bytes: &[u8] = cast_slice(input); + let mut byte_pos = (inexcept as usize) + .checked_mul(4) + .filter(|&bp| bp <= input_bytes.len()) + .ok_or(FastPForError::NotEnoughData)?; + let length = bytesize.div_ceil(4); + inexcept = inexcept + .checked_add(length) + .ok_or(FastPForError::NotEnoughData)?; + + let bitmap = T::read_bitmap(input, inexcept)?; + inexcept = inexcept + .checked_add(T::BITMAP_WORDS) + .ok_or(FastPForError::NotEnoughData)?; + + for k in 2..=u32::from(T::WIDTH) { + if (bitmap & (1u64 << (k - 1))) != 0 { + let size = input.get_val(inexcept)?; + inexcept = inexcept + .checked_add(1) + .ok_or(FastPForError::NotEnoughData)?; + if size > self.page_size { + return Err(FastPForError::NotEnoughData); + } + let rounded_up = size.next_multiple_of(32) as usize; + if self.exception_buffers[k as usize].len() < rounded_up { + self.exception_buffers[k as usize].resize(rounded_up, T::ZERO); + } + let mut j: u32 = 0; + while j.checked_add(32).is_some_and(|j32| j32 <= size) + && inexcept.checked_add(k).is_some_and(|ie| ie <= n) + { + T::fast_unpack( + input, + inexcept as usize, + &mut self.exception_buffers[k as usize], + j as usize, + k as u8, + ); + inexcept += k; + j += 32; + } + if j < size { + let words_needed = (size - j).saturating_mul(k).div_ceil(32); + let avail = n - inexcept.min(n); + if avail < words_needed { + return Err(FastPForError::NotEnoughData); + } + let copy_len = words_needed as usize; + let mut tail_buf = [0u32; 64]; + if copy_len == 0 { + return Err(FastPForError::NotEnoughData); + } + let start = inexcept as usize; + let src = input + .get(start..start + copy_len) + .ok_or(FastPForError::NotEnoughData)?; + tail_buf[..copy_len].copy_from_slice(src); + T::fast_unpack( + &tail_buf, + 0, + &mut self.exception_buffers[k as usize], + j as usize, + k as u8, + ); + inexcept += k; + j += 32; + } + let overflow = j - size; + inexcept -= (overflow * k) / 32; + } + } + + self.data_pointers.fill(0); + let mut tmp_output_offset = output_offset.position() as u32; + let mut tmp_input_offset = input_offset.position() as u32; + + let run_end = this_size / N as u32; + for _ in 0..run_end { + let bits = input_bytes.get_val(byte_pos)?; + if bits > T::WIDTH { + return Err(FastPForError::NotEnoughData); + } + byte_pos += 1; + let num_exceptions = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + for k in (0..N as u32).step_by(32) { + let in_start = tmp_input_offset as usize; + let out_start = (tmp_output_offset + k) as usize; + let in_end = in_start + .checked_add(usize::from(bits)) + .ok_or(FastPForError::NotEnoughData)?; + if in_end > input.len() { + return Err(FastPForError::NotEnoughData); + } + let out_end = out_start + .checked_add(32) + .ok_or(FastPForError::OutputBufferTooSmall)?; + if out_end > output.len() { + return Err(FastPForError::OutputBufferTooSmall); + } + T::fast_unpack(input, in_start, output, out_start, bits); + tmp_input_offset += u32::from(bits); + } + if num_exceptions > 0 { + let maxbits = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + let index = maxbits + .checked_sub(bits) + .ok_or(FastPForError::NotEnoughData)?; + if maxbits > T::WIDTH || index == 0 || index > T::WIDTH { + return Err(FastPForError::NotEnoughData); + } + let index = usize::from(index); + if index == 1 { + for _ in 0..num_exceptions { + let pos = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + if u32::from(pos) >= N as u32 { + return Err(FastPForError::NotEnoughData); + } + let out_idx = tmp_output_offset as usize + pos as usize; + if out_idx >= output.len() { + return Err(FastPForError::OutputBufferTooSmall); + } + T::or_shl_assign(&mut output[out_idx], T::one_shl(bits), 0); + } + } else { + for _ in 0..num_exceptions { + let pos = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + if u32::from(pos) >= N as u32 { + return Err(FastPForError::NotEnoughData); + } + let out_idx = tmp_output_offset as usize + pos as usize; + if out_idx >= output.len() { + return Err(FastPForError::OutputBufferTooSmall); + } + let ptr = self.data_pointers[index]; + let except_value = self.exception_buffers[index].get_val(ptr)?; + T::or_shl_assign(&mut output[out_idx], except_value, bits); + self.data_pointers[index] += 1; + } + } + } + tmp_output_offset += N as u32; + } + output_offset.set_position(u64::from(tmp_output_offset)); + input_offset.set_position(u64::from(inexcept)); + Ok(()) + } +} diff --git a/src/rust/integer_compression/mod.rs b/src/rust/integer_compression/mod.rs index 76b6ffe..25fe8cb 100644 --- a/src/rust/integer_compression/mod.rs +++ b/src/rust/integer_compression/mod.rs @@ -3,5 +3,6 @@ pub mod bitpacking_wide; pub mod bitunpacking; pub mod fastpfor; pub mod fastpfor64; +pub mod fastpfor_engine; pub mod just_copy; pub mod variable_byte; From 91d0be59a5a71218878700af37e6e277697e14d3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:09:19 +0000 Subject: [PATCH 08/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/rust/integer_compression/fastpfor_engine.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/rust/integer_compression/fastpfor_engine.rs b/src/rust/integer_compression/fastpfor_engine.rs index b7f4407..3522ff4 100644 --- a/src/rust/integer_compression/fastpfor_engine.rs +++ b/src/rust/integer_compression/fastpfor_engine.rs @@ -58,7 +58,10 @@ pub trait FastPForInt: Copy + 'static { fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult; } -#[allow(clippy::use_self, reason = "u32 literals here are stream words, not the Self element type")] +#[allow( + clippy::use_self, + reason = "u32 literals here are stream words, not the Self element type" +)] impl FastPForInt for u32 { const WIDTH: u8 = 32; const BITMAP_WORDS: u32 = 1; @@ -94,7 +97,10 @@ impl FastPForInt for u32 { } } -#[allow(clippy::use_self, reason = "u32 literals here are stream words, not the Self element type")] +#[allow( + clippy::use_self, + reason = "u32 literals here are stream words, not the Self element type" +)] impl FastPForInt for u64 { const WIDTH: u8 = 64; const BITMAP_WORDS: u32 = 2; From 640866dda5c44f12b8ed60d1ebeb55d5075c43c9 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Fri, 17 Jul 2026 14:39:42 +0200 Subject: [PATCH 09/23] fix: satisfy nightly clippy and rustfmt for CI - Use Vec::clear() instead of truncate(0) in the encode_compare fuzz target (newer clippy flags truncate-to-zero). - Apply rustfmt to the generic-engine allow-attribute and the u64 fuzz target. Co-Authored-By: Claude Opus 4.8 (1M context) --- fuzz/fuzz_targets/encode_compare.rs | 2 +- fuzz/fuzz_targets/fastpfor_u64.rs | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/fuzz/fuzz_targets/encode_compare.rs b/fuzz/fuzz_targets/encode_compare.rs index ad4835f..895f735 100644 --- a/fuzz/fuzz_targets/encode_compare.rs +++ b/fuzz/fuzz_targets/encode_compare.rs @@ -50,7 +50,7 @@ fuzz_target!(|data: FuzzInput| { pair.name, ); - decoded.truncate(0); + decoded.clear(); cpp_codec .decode(&cpp_out, &mut decoded, None) .expect("C++ decode of self-compressed data must not fail"); diff --git a/fuzz/fuzz_targets/fastpfor_u64.rs b/fuzz/fuzz_targets/fastpfor_u64.rs index deaaeff..caab43f 100644 --- a/fuzz/fuzz_targets/fastpfor_u64.rs +++ b/fuzz/fuzz_targets/fastpfor_u64.rs @@ -12,12 +12,17 @@ struct Input { fn check(rust: &mut impl BlockCodec64, cpp: &mut impl BlockCodec64, data: &[u64], name: &str) { let mut rust_enc = Vec::new(); - rust.encode64(data, &mut rust_enc).expect("Rust encode64 failed"); + rust.encode64(data, &mut rust_enc) + .expect("Rust encode64 failed"); let mut cpp_enc = Vec::new(); - cpp.encode64(data, &mut cpp_enc).expect("C++ encode64 failed"); + cpp.encode64(data, &mut cpp_enc) + .expect("C++ encode64 failed"); - assert_eq!(rust_enc, cpp_enc, "{name}: Rust and C++ encode64 bytes differ"); + assert_eq!( + rust_enc, cpp_enc, + "{name}: Rust and C++ encode64 bytes differ" + ); let mut rust_dec = Vec::new(); rust.decode64(&rust_enc, &mut rust_dec) From 620d82473e30df705353ee7a1ceab4dc2f7cb9f1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:08:20 +0000 Subject: [PATCH 10/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/test_utils.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test_utils.rs b/src/test_utils.rs index 1bd2fed..9b2729b 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -11,7 +11,9 @@ #![allow(dead_code, missing_docs, clippy::unwrap_used)] #[allow(unused_imports)] -use fastpfor::{AnyLenCodec, BlockCodec, BlockCodec64, FastPForError, FastPForResult, slice_to_blocks}; +use fastpfor::{ + AnyLenCodec, BlockCodec, BlockCodec64, FastPForError, FastPForResult, slice_to_blocks, +}; #[cfg(feature = "rust")] use fastpfor::{ FastPFor128, FastPFor256, FastPForBlock128, FastPForBlock256, JustCopy, VariableByte, From c6045e0d17ce5e8013aa8e82039193310ee1a728 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 17 Jul 2026 19:16:04 -0400 Subject: [PATCH 11/23] wip --- src/rust/integer_compression/fastpfor.rs | 483 +++++++++++++++- src/rust/integer_compression/fastpfor64.rs | 27 +- .../integer_compression/fastpfor_engine.rs | 521 ------------------ src/rust/integer_compression/fastpfor_int.rs | 126 +++++ src/rust/integer_compression/mod.rs | 2 +- 5 files changed, 590 insertions(+), 569 deletions(-) delete mode 100644 src/rust/integer_compression/fastpfor_engine.rs create mode 100644 src/rust/integer_compression/fastpfor_int.rs diff --git a/src/rust/integer_compression/fastpfor.rs b/src/rust/integer_compression/fastpfor.rs index fabf276..e55aca2 100644 --- a/src/rust/integer_compression/fastpfor.rs +++ b/src/rust/integer_compression/fastpfor.rs @@ -1,37 +1,47 @@ +use std::array; +use std::cmp::min; use std::io::Cursor; use bytemuck::cast_slice; +use bytes::{Buf as _, BufMut as _, BytesMut}; -use crate::helpers::AsUsize; -use crate::rust::integer_compression::fastpfor_engine::FastPForEngine; +use crate::helpers::{AsUsize, GetWithErr, greatest_multiple}; +use crate::rust::cursor::IncrementCursor; +use crate::rust::integer_compression::fastpfor_int::FastPForInt; use crate::{BlockCodec, FastPForError, FastPForResult}; mod sealed { - /// Sealed marker trait: only `[u32; 128]` and `[u32; 256]` are valid `FastPFor` block arrays. + /// Sealed marker trait: only valid `[T; N]` block arrays are accepted by `FastPFor`. /// /// This is intentionally private so that users cannot implement it for other sizes, - /// preventing instantiation of `FastPFor` for unsupported `N` at compile time. + /// preventing instantiation of `FastPFor` for unsupported `N`/`T` at compile time. pub trait BlockSize: bytemuck::Pod {} impl BlockSize for [u32; 128] {} impl BlockSize for [u32; 256] {} + impl BlockSize for [u64; 128] {} + impl BlockSize for [u64; 256] {} } +/// Overhead cost (in bits) for storing each exception's position in the block +const OVERHEAD_OF_EACH_EXCEPT: u32 = 8; + /// Default page size in number of integers. const DEFAULT_PAGE_SIZE: u32 = 65536; -/// Type alias for [`FastPFor`] with 128-element blocks. -pub type FastPForBlock128 = FastPFor<128>; +/// Type alias for [`FastPFor`] with 128-element `u32` blocks. +pub type FastPForBlock128 = FastPFor<128, { u32::BITS as usize + 1 }, u32>; -/// Type alias for [`FastPFor`] with 256-element blocks. -pub type FastPForBlock256 = FastPFor<256>; +/// Type alias for [`FastPFor`] with 256-element `u32` blocks. +pub type FastPForBlock256 = FastPFor<256, { u32::BITS as usize + 1 }, u32>; /// Fast Patched Frame-of-Reference ([FastPFOR](https://github.com/lemire/FastPFor)) codec. /// -/// `N` is the block size (128 or 256 values per block). This struct implements -/// [`BlockCodec`] with `Block = [u32; N]`, giving compile-time guarantees that -/// only correctly-sized blocks are accepted. +/// `N` is the block size (128 or 256 values per block) and `T` the element type +/// ([`u32`] or [`u64`]). This struct implements [`BlockCodec`] with `Block = [u32; N]` +/// for the `u32` element type, giving compile-time guarantees that only correctly-sized +/// blocks are accepted. /// -/// Use [`FastPForBlock128`] or [`FastPForBlock256`] as convenient type aliases. +/// Use [`FastPForBlock128`] or [`FastPForBlock256`] as convenient `u32` type aliases. /// /// To compress arbitrary-length data (including a sub-block remainder), /// wrap this in a [`CompositeCodec`](crate::CompositeCodec): @@ -44,22 +54,39 @@ pub type FastPForBlock256 = FastPFor<256>; /// codec.encode(&data, &mut out).unwrap(); /// ``` #[derive(Debug)] -pub struct FastPFor { - engine: FastPForEngine, +pub struct FastPFor< + const N: usize, + const WIDTHS: usize = { u32::BITS as usize + 1 }, + T: FastPForInt = u32, +> { + /// Exception values indexed by bit width difference + exception_buffers: [Vec; WIDTHS], + /// Metadata buffer for encoding/decoding + bytes_container: BytesMut, + /// Maximum integers per page + page_size: u32, + /// Position trackers for exception arrays + data_pointers: [usize; WIDTHS], + /// Frequency count for each bit width: + /// `freqs[i]` = count of values needing exactly i bits + freqs: [u32; WIDTHS], + /// Optimal number of bits chosen for the current block + optimal_bits: u8, + /// Number of exceptions that don't fit in the optimal bit width + exception_count: u8, + /// Maximum bit width required for any value in the block + max_bits: u8, } -impl Default for FastPFor -where - [u32; N]: sealed::BlockSize, -{ +impl Default for FastPFor { fn default() -> Self { Self::new(DEFAULT_PAGE_SIZE) .expect("DEFAULT_PAGE_SIZE is a multiple of all valid block sizes") } } -impl FastPFor { - /// Creates a new codec with the given page size. +impl FastPFor { + /// Creates a new `FastPForBlock` with a codec with the given page size. /// /// Returns an error if `page_size` is not a multiple of the block size. /// Use [`Default`] for the default page size. @@ -71,9 +98,405 @@ impl FastPFor { }); } Ok(Self { - engine: FastPForEngine::new(page_size), + bytes_container: BytesMut::with_capacity( + (3 * page_size / N as u32 + page_size) as usize, + ), + page_size, + exception_buffers: array::from_fn(|_| Vec::new()), + data_pointers: [0; WIDTHS], + freqs: [0; WIDTHS], + optimal_bits: 0, + exception_count: 0, + max_bits: 0, }) } + + pub(crate) fn compress_blocks( + &mut self, + input: &[T], + input_length: u32, + input_offset: &mut Cursor, + output: &mut [u32], + output_offset: &mut Cursor, + ) { + let inlength = greatest_multiple(input_length, N as u32); + let final_inpos = input_offset.position() as u32 + inlength; + while input_offset.position() as u32 != final_inpos { + let this_size = min(self.page_size, final_inpos - input_offset.position() as u32); + self.encode_page(input, this_size, input_offset, output, output_offset); + } + } + + pub(crate) fn decode_headless_blocks( + &mut self, + input: &[u32], + inlength: u32, + input_offset: &mut Cursor, + output: &mut [T], + output_offset: &mut Cursor, + ) -> FastPForResult<()> { + let mynvalue = greatest_multiple(inlength, N as u32); + let final_out = output_offset.position() as u32 + mynvalue; + while output_offset.position() as u32 != final_out { + let this_size = min(self.page_size, final_out - output_offset.position() as u32); + self.decode_page(input, input_offset, output, output_offset, this_size)?; + } + Ok(()) + } + + /// Encodes a page using optimal bit width per block. + /// + /// For each block: + /// - Determines best bit width, bitpacks regular values, + /// - Stores exceptions with positions. + /// - Writes header, packed data, metadata bytes, and exception values. + /// + /// # Arguments + /// * `this_size` - Must be multiple of `block_size` + /// * `input_offset` - Advanced by `this_size` + /// * `output_offset` - Advanced by compressed size + fn encode_page( + &mut self, + input: &[T], + this_size: u32, + input_offset: &mut Cursor, + output: &mut [u32], + output_offset: &mut Cursor, + ) { + let header_pos = output_offset.position() as usize; + output_offset.increment(); + let mut tmp_output_offset = output_offset.position() as u32; + + // Data pointers to 0 + self.data_pointers.fill(0); + self.bytes_container.clear(); + + let mut tmp_input_offset = input_offset.position() as u32; + let final_input_offset = tmp_input_offset + this_size - N as u32; + while tmp_input_offset <= final_input_offset { + self.best_bit_from_data(input, tmp_input_offset); + self.bytes_container.put_u8(self.optimal_bits); + self.bytes_container.put_u8(self.exception_count); + if self.exception_count > 0 { + self.bytes_container.put_u8(self.max_bits); + let index = usize::from(self.max_bits - self.optimal_bits); + let needed = self.data_pointers[index] + usize::from(self.exception_count); + if needed > self.exception_buffers[index].len() { + // Grow to the next multiple of 32 above 2×needed, to amortize resizes. + let new_cap = needed.saturating_mul(2).next_multiple_of(32); + self.exception_buffers[index].resize(new_cap, T::ZERO); + } + for k in 0..N as u32 { + if !input[(k + tmp_input_offset) as usize] + .shr(self.optimal_bits) + .is_zero() + { + self.bytes_container.put_u8(k as u8); + self.exception_buffers[index][self.data_pointers[index]] = + input[(k + tmp_input_offset) as usize].shr(self.optimal_bits); + self.data_pointers[index] += 1; + } + } + } + for k in (0..N as u32).step_by(32) { + T::fast_pack( + input, + (tmp_input_offset + k) as usize, + output, + tmp_output_offset as usize, + self.optimal_bits, + ); + tmp_output_offset += u32::from(self.optimal_bits); + } + tmp_input_offset += N as u32; + } + input_offset.set_position(u64::from(tmp_input_offset)); + output[header_pos] = tmp_output_offset - header_pos as u32; + let byte_size = self.bytes_container.len(); + while (self.bytes_container.len() & 3) != 0 { + self.bytes_container.put_u8(0); + } + // Output should have 3 position as 4 + output[tmp_output_offset as usize] = byte_size as u32; + tmp_output_offset += 1; + let how_many_ints = self.bytes_container.len() / 4; + // Match C++ memcpy: copy metadata bytes as u32s in one shot (native byte order). + let meta_u32s: &[u32] = cast_slice(self.bytes_container.chunk()); + output[tmp_output_offset as usize..][..how_many_ints] + .copy_from_slice(&meta_u32s[..how_many_ints]); + tmp_output_offset += how_many_ints as u32; + // Exception bitmap: one bit per bit-width bucket, written as `T::BITMAP_WORDS` words. + let mut bitmap: u64 = 0; + for k in 2..=usize::from(T::WIDTH) { + if self.data_pointers[k] != 0 { + bitmap |= 1u64 << (k - 1); + } + } + T::write_bitmap(bitmap, &mut output[tmp_output_offset as usize..]); + tmp_output_offset += T::BITMAP_WORDS; + + for k in 2..=usize::from(T::WIDTH) { + if self.data_pointers[k] != 0 { + output[tmp_output_offset as usize] = self.data_pointers[k] as u32; + tmp_output_offset += 1; + let mut j = 0; + while j < self.data_pointers[k] { + T::fast_pack( + &self.exception_buffers[k], + j, + output, + tmp_output_offset as usize, + k as u8, + ); + tmp_output_offset += k as u32; + j += 32; + } + + // Overflow adjustment + let overflow = j as u32 - self.data_pointers[k] as u32; + tmp_output_offset -= (overflow * k as u32) / 32; + } + } + output_offset.set_position(u64::from(tmp_output_offset)); + } + + /// Computes optimal bit width minimizing total storage cost. + /// + /// Analyzes frequency distribution to balance regular value bits against exception overhead. + fn best_bit_from_data(&mut self, input: &[T], pos: u32) { + self.freqs.fill(0); + let k_end = min(pos + N as u32, input.len() as u32); + for k in pos..k_end { + self.freqs[usize::from(input[k as usize].significant_bits())] += 1; + } + + self.optimal_bits = T::WIDTH; + while self.freqs[self.optimal_bits as usize] == 0 { + self.optimal_bits -= 1; + } + self.max_bits = self.optimal_bits; + + let mut best_cost = u32::from(self.optimal_bits) * N as u32; + let mut num_exceptions: u32 = 0; + self.exception_count = 0; + + for bits in (0..self.optimal_bits).rev() { + num_exceptions += self.freqs[bits as usize + 1]; + if num_exceptions == N as u32 { + break; + } + let diff = u32::from(self.max_bits - bits); + let mut cost = num_exceptions * OVERHEAD_OF_EACH_EXCEPT + + num_exceptions * diff + + u32::from(bits) * N as u32 + + 8; + if diff == 1 { + cost -= num_exceptions; + } + if cost < best_cost { + best_cost = cost; + self.optimal_bits = bits; + self.exception_count = num_exceptions as u8; + } + } + } + + /// Decodes a compressed page. + /// + /// Reads header to locate exception data, loads exceptions by bit width, + /// unpacks regular values per block, patches in exceptions by position. + /// + /// # Arguments + /// * `this_size` - Expected decompressed integer count + /// * `input_offset` - Advanced by bytes read + /// * `output_offset` - Advanced by `this_size` + #[expect(clippy::too_many_lines)] + fn decode_page( + &mut self, + input: &[u32], + input_offset: &mut Cursor, + output: &mut [T], + output_offset: &mut Cursor, + this_size: u32, + ) -> FastPForResult<()> { + let n = u32::try_from(input.len()) + .map_err(|_| FastPForError::InvalidInputLength(input.len()))?; + + let init_pos = + u32::try_from(input_offset.position()).map_err(|_| FastPForError::NotEnoughData)?; + let where_meta = input.get_val(init_pos)?; + input_offset.increment(); + let mut inexcept = init_pos + .checked_add(where_meta) + .ok_or(FastPForError::NotEnoughData)?; + let bytesize = input.get_val(inexcept)?; + inexcept = inexcept + .checked_add(1) + .ok_or(FastPForError::NotEnoughData)?; + // Point a byte cursor directly at the metadata region in `input`, + // mirrors C++ `const uint8_t *bytep = reinterpret_cast(inexcept)`. + // The C++ encoder uses a raw `memcpy` of bytes into the u32 output (no endian + // conversion), and the decoder does a raw reinterpret_cast back -- both native byte + // order. `cast_slice` is the exact Rust equivalent: a safe, zero-copy native view. + let input_bytes: &[u8] = cast_slice(input); + let mut byte_pos = (inexcept as usize) + .checked_mul(4) + .filter(|&bp| bp <= input_bytes.len()) + .ok_or(FastPForError::NotEnoughData)?; + let length = bytesize.div_ceil(4); + inexcept = inexcept + .checked_add(length) + .ok_or(FastPForError::NotEnoughData)?; + + let bitmap = T::read_bitmap(input, inexcept)?; + inexcept = inexcept + .checked_add(T::BITMAP_WORDS) + .ok_or(FastPForError::NotEnoughData)?; + + for k in 2..=u32::from(T::WIDTH) { + if (bitmap & (1u64 << (k - 1))) != 0 { + let size = input.get_val(inexcept)?; + inexcept = inexcept + .checked_add(1) + .ok_or(FastPForError::NotEnoughData)?; + // Reject adversarial inputs: exceptions can't exceed the page size. + if size > self.page_size { + return Err(FastPForError::NotEnoughData); + } + // Ensure the buffer is large enough for `size` values, rounded up + // to the next group of 32 for the bitunpacking calls. + let rounded_up = size.next_multiple_of(32) as usize; + if self.exception_buffers[k as usize].len() < rounded_up { + self.exception_buffers[k as usize].resize(rounded_up, T::ZERO); + } + let mut j: u32 = 0; + // Process full groups directly from input + while j.checked_add(32).is_some_and(|j32| j32 <= size) + && inexcept.checked_add(k).is_some_and(|ie| ie <= n) + { + T::fast_unpack( + input, + inexcept as usize, + &mut self.exception_buffers[k as usize], + j as usize, + k as u8, + ); + inexcept += k; // safe: loop guard checked inexcept + k <= n <= u32::MAX + j += 32; // safe: loop guard checked j + 32 <= size + } + // Handle the final partial group using a stack buffer (mirrors C++ buffer[PACKSIZE*2]) + if j < size { + let words_needed = (size - j) // safe: j < size + .saturating_mul(k) + .div_ceil(32); + let avail = n - inexcept.min(n); + if avail < words_needed { + return Err(FastPForError::NotEnoughData); + } + let copy_len = words_needed as usize; + let mut tail_buf = [0u32; 64]; + if copy_len == 0 { + return Err(FastPForError::NotEnoughData); + } + let start = inexcept as usize; + let src = input + .get(start..start + copy_len) + .ok_or(FastPForError::NotEnoughData)?; + tail_buf[..copy_len].copy_from_slice(src); + let tail_inpos = 0; + T::fast_unpack( + &tail_buf, + tail_inpos, + &mut self.exception_buffers[k as usize], + j as usize, + k as u8, + ); + inexcept += k; + j += 32; + } + let overflow = j - size; + inexcept -= (overflow * k) / 32; + } + } + + self.data_pointers.fill(0); + let mut tmp_output_offset = output_offset.position() as u32; + let mut tmp_input_offset = input_offset.position() as u32; + + let run_end = this_size / N as u32; + for _ in 0..run_end { + let bits = input_bytes.get_val(byte_pos)?; + if bits > T::WIDTH { + return Err(FastPForError::NotEnoughData); + } + byte_pos += 1; + let num_exceptions = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + for k in (0..N as u32).step_by(32) { + let in_start = tmp_input_offset as usize; + let out_start = (tmp_output_offset + k) as usize; + let in_end = in_start + .checked_add(usize::from(bits)) + .ok_or(FastPForError::NotEnoughData)?; + if in_end > input.len() { + return Err(FastPForError::NotEnoughData); + } + let out_end = out_start + .checked_add(32) + .ok_or(FastPForError::OutputBufferTooSmall)?; + if out_end > output.len() { + return Err(FastPForError::OutputBufferTooSmall); + } + T::fast_unpack(input, in_start, output, out_start, bits); + tmp_input_offset += u32::from(bits); + } + if num_exceptions > 0 { + let maxbits = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + let index = maxbits + .checked_sub(bits) + .ok_or(FastPForError::NotEnoughData)?; + if maxbits > T::WIDTH || index == 0 || index > T::WIDTH { + return Err(FastPForError::NotEnoughData); + } + let index = usize::from(index); + if index == 1 { + for _ in 0..num_exceptions { + let pos = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + if u32::from(pos) >= N as u32 { + return Err(FastPForError::NotEnoughData); + } + let out_idx = tmp_output_offset as usize + pos as usize; + if out_idx >= output.len() { + return Err(FastPForError::OutputBufferTooSmall); + } + T::or_shl_assign(&mut output[out_idx], T::one_shl(bits), 0); + } + } else { + for _ in 0..num_exceptions { + let pos = input_bytes.get_val(byte_pos)?; + byte_pos += 1; + if u32::from(pos) >= N as u32 { + return Err(FastPForError::NotEnoughData); + } + let out_idx = tmp_output_offset as usize + pos as usize; + if out_idx >= output.len() { + return Err(FastPForError::OutputBufferTooSmall); + } + let ptr = self.data_pointers[index]; + let except_value = self.exception_buffers[index].get_val(ptr)?; + T::or_shl_assign(&mut output[out_idx], except_value, bits); + self.data_pointers[index] += 1; + } + } + } + tmp_output_offset += N as u32; + } + output_offset.set_position(u64::from(tmp_output_offset)); + input_offset.set_position(u64::from(inexcept)); + Ok(()) + } } impl BlockCodec for FastPFor @@ -100,7 +523,7 @@ where // Write length header then compress. out[start] = n_values; - self.engine.compress_blocks( + self.compress_blocks( flat, n_values, &mut in_off, @@ -148,7 +571,7 @@ where let mut in_off = Cursor::new(0u32); let mut out_off = Cursor::new(0u32); - self.engine.decode_headless_blocks( + self.decode_headless_blocks( rest, block_n_values, &mut in_off, @@ -186,13 +609,14 @@ mod tests { #[test] fn test_empty_blocks_ok() { - // Empty input encodes to length header [0] and decodes cleanly. + // Empty input encodes to length header [0] (matches C++ FastPFor) and decodes cleanly. let enc = block_compress::(&[]).unwrap(); assert_eq!(enc, [0]); let dec = block_decompress::(&enc, Some(0)).unwrap(); assert!(dec.is_empty()); } + // Tests ported from C++ #[test] fn test_constant_sequence() { block_roundtrip::(&vec![42u32; 65536]); @@ -231,16 +655,20 @@ mod tests { block_roundtrip::(&input); } + // ── Error / edge tests not covered by `tests/decode_validation.rs` ───── + // + // `AnyLenCodec::decode` treats an empty slice as tail-only and succeeds; an empty + // `decode_blocks` input is still invalid. Headless decode is internal-only. + #[test] fn uncompress_zero_input_length_err() { - // Truly empty input (no header word at all) is invalid. + // Truly empty input (no header word at all) is invalid — C++ would crash reading *in. block_decompress::(&[], None).unwrap_err(); } #[test] fn headless_uncompress_zero_inlength_128_ok() { FastPForBlock128::default() - .engine .decode_headless_blocks( &[], 0, @@ -253,6 +681,7 @@ mod tests { #[test] fn decode_where_meta_overflow() { + // `decode_headless_blocks` only: no `AnyLenCodec` entry point passes this layout. let data: Vec = (0..256u32) .map(|i| if i % 2 == 0 { 1u32 << 30 } else { 3 }) .collect(); @@ -264,7 +693,6 @@ mod tests { let out_length = padded[1]; assert!( FastPForBlock256::default() - .engine .decode_headless_blocks( &padded, out_length, @@ -286,6 +714,7 @@ mod tests { /// `decode_blocks` with `expected_len: None` and header=0 returns `Ok` with empty output. #[test] fn decode_blocks_header_only_input() { + // Input with just the length header [0]: no blocks to decode. let input = vec![0u32]; let out = block_decompress::(&input, None).unwrap(); assert!(out.is_empty()); diff --git a/src/rust/integer_compression/fastpfor64.rs b/src/rust/integer_compression/fastpfor64.rs index 1ec4e12..1721a62 100644 --- a/src/rust/integer_compression/fastpfor64.rs +++ b/src/rust/integer_compression/fastpfor64.rs @@ -14,28 +14,15 @@ use bytemuck::{cast_slice, cast_slice_mut}; use crate::codec::default_max_decoded_len; use crate::helpers::AsUsize; -use crate::rust::integer_compression::fastpfor_engine::FastPForEngine; +use crate::rust::integer_compression::fastpfor::FastPFor; use crate::{BlockCodec64, FastPForError, FastPForResult}; -/// Default page size in number of integers. -const DEFAULT_PAGE_SIZE: u32 = 65536; - /// 64-bit `FastPFOR` codec: `FastPFOR`-packed blocks plus a variable-byte tail. /// -/// `N` is the block size (128 or 256 values). -/// This is the internal `u64` engine behind [`FastPFor128`](crate::FastPFor128) and [`FastPFor256`](crate::FastPFor256). -#[derive(Debug)] -pub struct FastPForWide { - engine: FastPForEngine, -} - -impl Default for FastPForWide { - fn default() -> Self { - Self { - engine: FastPForEngine::new(DEFAULT_PAGE_SIZE), - } - } -} +/// `N` is the block size (128 or 256 values). This is [`FastPFor`] specialized to the +/// `u64` element type, and is the internal `u64` codec behind +/// [`FastPFor128`](crate::FastPFor128) and [`FastPFor256`](crate::FastPFor256). +pub type FastPForWide = FastPFor; /// Variable-byte encoding of the `u64` tail. /// @@ -117,7 +104,7 @@ impl BlockCodec64 for FastPForWide { let mut in_off = Cursor::new(0u32); let mut out_off = Cursor::new(0u32); - self.engine.compress_blocks( + self.compress_blocks( &input[..rounded], n_values, &mut in_off, @@ -151,7 +138,7 @@ impl BlockCodec64 for FastPForWide { out.resize(start + n_blocks * N, 0); let mut in_off = Cursor::new(0u32); let mut out_off = Cursor::new(0u32); - self.engine.decode_headless_blocks( + self.decode_headless_blocks( rest, block_n_values, &mut in_off, diff --git a/src/rust/integer_compression/fastpfor_engine.rs b/src/rust/integer_compression/fastpfor_engine.rs deleted file mode 100644 index 3522ff4..0000000 --- a/src/rust/integer_compression/fastpfor_engine.rs +++ /dev/null @@ -1,521 +0,0 @@ -//! Width-generic `FastPFOR` page engine shared by the 32- and 64-bit codecs. -//! -//! The block-splitting, best-bit search, exception handling, and metadata layout are -//! identical for `u32` and `u64`; only the element width differs. -//! [`FastPForInt`] abstracts the width-specific pieces so a single [`FastPForEngine`] -//! implements the algorithm once. -//! `u32` keeps its hand-unrolled bit-packing kernels; `u64` uses the generic wide packer. - -use std::array; -use std::cmp::min; -use std::io::Cursor; - -use bytemuck::cast_slice; -use bytes::{Buf as _, BufMut as _, BytesMut}; - -use crate::helpers::{GetWithErr, greatest_multiple}; -use crate::rust::cursor::IncrementCursor; -use crate::rust::integer_compression::{bitpacking, bitpacking_wide, bitunpacking}; -use crate::{FastPForError, FastPForResult}; - -/// Overhead cost (in bits) for storing each exception's position in the block. -const OVERHEAD_OF_EACH_EXCEPT: u32 = 8; - -/// One frequency/exception bucket per possible bit width, up to the widest supported (`u64`). -const WIDTHS: usize = 65; - -/// Element type of a `FastPFOR` stream: [`u32`] or [`u64`]. -/// -/// Implementors supply the width-specific operations the engine needs. -/// The exception bitmap spans [`BITMAP_WORDS`](Self::BITMAP_WORDS) output words. -pub trait FastPForInt: Copy + 'static { - /// Bit width of the element: 32 or 64. - const WIDTH: u8; - /// Output words occupied by the exception bitmap: 1 for `u32`, 2 for `u64`. - const BITMAP_WORDS: u32; - /// The zero value. - const ZERO: Self; - - /// Number of significant bits, i.e. `WIDTH - leading_zeros` (0 for a zero value). - fn significant_bits(self) -> u8; - /// Logical right shift by `n`, where `n < WIDTH`. - fn shr(self, n: u8) -> Self; - /// Whether the value is zero. - fn is_zero(self) -> bool; - /// `*dst |= val << shift`, where `shift < WIDTH`. - fn or_shl_assign(dst: &mut Self, val: Self, shift: u8); - /// `1 << shift`, where `shift < WIDTH`. - fn one_shl(shift: u8) -> Self; - - /// Pack 32 values at `bit` bits each into `out`. - fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8); - /// Unpack 32 values at `bit` bits each from `src`. - fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8); - - /// Write the exception bitmap as [`BITMAP_WORDS`](Self::BITMAP_WORDS) words at `out`. - fn write_bitmap(bitmap: u64, out: &mut [u32]); - /// Read the exception bitmap from [`BITMAP_WORDS`](Self::BITMAP_WORDS) words at `pos`. - fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult; -} - -#[allow( - clippy::use_self, - reason = "u32 literals here are stream words, not the Self element type" -)] -impl FastPForInt for u32 { - const WIDTH: u8 = 32; - const BITMAP_WORDS: u32 = 1; - const ZERO: Self = 0; - - fn significant_bits(self) -> u8 { - (32 - self.leading_zeros()) as u8 - } - fn shr(self, n: u8) -> Self { - self >> n - } - fn is_zero(self) -> bool { - self == 0 - } - fn or_shl_assign(dst: &mut Self, val: Self, shift: u8) { - *dst |= val << shift; - } - fn one_shl(shift: u8) -> Self { - 1 << shift - } - fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { - bitpacking::fast_pack(src, inpos, out, outpos, bit); - } - fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { - bitunpacking::fast_unpack(src, inpos, out, outpos, bit); - } - fn write_bitmap(bitmap: u64, out: &mut [u32]) { - out[0] = bitmap as u32; - } - fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { - let word: u32 = input.get_val(pos)?; - Ok(u64::from(word)) - } -} - -#[allow( - clippy::use_self, - reason = "u32 literals here are stream words, not the Self element type" -)] -impl FastPForInt for u64 { - const WIDTH: u8 = 64; - const BITMAP_WORDS: u32 = 2; - const ZERO: Self = 0; - - fn significant_bits(self) -> u8 { - (64 - self.leading_zeros()) as u8 - } - fn shr(self, n: u8) -> Self { - self >> n - } - fn is_zero(self) -> bool { - self == 0 - } - fn or_shl_assign(dst: &mut Self, val: Self, shift: u8) { - *dst |= val << shift; - } - fn one_shl(shift: u8) -> Self { - 1 << shift - } - fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { - bitpacking_wide::pack_wide(src, inpos, out, outpos, bit); - } - fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { - bitpacking_wide::unpack_wide(src, inpos, out, outpos, bit); - } - fn write_bitmap(bitmap: u64, out: &mut [u32]) { - out[0] = bitmap as u32; - out[1] = (bitmap >> 32) as u32; - } - fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { - let lo: u32 = input.get_val(pos)?; - let hi_pos = pos.checked_add(1).ok_or(FastPForError::NotEnoughData)?; - let hi: u32 = input.get_val(hi_pos)?; - Ok(u64::from(lo) | (u64::from(hi) << 32)) - } -} - -/// Shared `FastPFOR` scratch state and page codec for a block size `N` and element type `T`. -#[derive(Debug)] -pub struct FastPForEngine { - /// Exception values grouped by `max_bits - optimal_bits`. - exception_buffers: [Vec; WIDTHS], - /// Per-block metadata (bit widths, exception counts, positions). - bytes_container: BytesMut, - /// Maximum integers per page. - page_size: u32, - /// Write positions into `exception_buffers`. - data_pointers: [usize; WIDTHS], - /// Count of values needing exactly `i` bits. - freqs: [u32; WIDTHS], - /// Chosen bit width for the current block. - optimal_bits: u8, - /// Exceptions that exceed `optimal_bits`. - exception_count: u8, - /// Widest value in the current block. - max_bits: u8, -} - -impl FastPForEngine { - pub fn new(page_size: u32) -> Self { - Self { - bytes_container: BytesMut::with_capacity( - (3 * page_size / N as u32 + page_size) as usize, - ), - page_size, - exception_buffers: array::from_fn(|_| Vec::new()), - data_pointers: [0; WIDTHS], - freqs: [0; WIDTHS], - optimal_bits: 0, - exception_count: 0, - max_bits: 0, - } - } - - pub fn compress_blocks( - &mut self, - input: &[T], - input_length: u32, - input_offset: &mut Cursor, - output: &mut [u32], - output_offset: &mut Cursor, - ) { - let inlength = greatest_multiple(input_length, N as u32); - let final_inpos = input_offset.position() as u32 + inlength; - while input_offset.position() as u32 != final_inpos { - let this_size = min(self.page_size, final_inpos - input_offset.position() as u32); - self.encode_page(input, this_size, input_offset, output, output_offset); - } - } - - pub fn decode_headless_blocks( - &mut self, - input: &[u32], - inlength: u32, - input_offset: &mut Cursor, - output: &mut [T], - output_offset: &mut Cursor, - ) -> FastPForResult<()> { - let mynvalue = greatest_multiple(inlength, N as u32); - let final_out = output_offset.position() as u32 + mynvalue; - while output_offset.position() as u32 != final_out { - let this_size = min(self.page_size, final_out - output_offset.position() as u32); - self.decode_page(input, input_offset, output, output_offset, this_size)?; - } - Ok(()) - } - - fn encode_page( - &mut self, - input: &[T], - this_size: u32, - input_offset: &mut Cursor, - output: &mut [u32], - output_offset: &mut Cursor, - ) { - let header_pos = output_offset.position() as usize; - output_offset.increment(); - let mut tmp_output_offset = output_offset.position() as u32; - - self.data_pointers.fill(0); - self.bytes_container.clear(); - - let mut tmp_input_offset = input_offset.position() as u32; - let final_input_offset = tmp_input_offset + this_size - N as u32; - while tmp_input_offset <= final_input_offset { - self.best_bit_from_data(input, tmp_input_offset); - self.bytes_container.put_u8(self.optimal_bits); - self.bytes_container.put_u8(self.exception_count); - if self.exception_count > 0 { - self.bytes_container.put_u8(self.max_bits); - let index = usize::from(self.max_bits - self.optimal_bits); - let needed = self.data_pointers[index] + usize::from(self.exception_count); - if needed > self.exception_buffers[index].len() { - let new_cap = needed.saturating_mul(2).next_multiple_of(32); - self.exception_buffers[index].resize(new_cap, T::ZERO); - } - for k in 0..N as u32 { - let value = input[(k + tmp_input_offset) as usize]; - if !value.shr(self.optimal_bits).is_zero() { - self.bytes_container.put_u8(k as u8); - self.exception_buffers[index][self.data_pointers[index]] = - value.shr(self.optimal_bits); - self.data_pointers[index] += 1; - } - } - } - for k in (0..N as u32).step_by(32) { - T::fast_pack( - input, - (tmp_input_offset + k) as usize, - output, - tmp_output_offset as usize, - self.optimal_bits, - ); - tmp_output_offset += u32::from(self.optimal_bits); - } - tmp_input_offset += N as u32; - } - input_offset.set_position(u64::from(tmp_input_offset)); - output[header_pos] = tmp_output_offset - header_pos as u32; - let byte_size = self.bytes_container.len(); - while (self.bytes_container.len() & 3) != 0 { - self.bytes_container.put_u8(0); - } - output[tmp_output_offset as usize] = byte_size as u32; - tmp_output_offset += 1; - let how_many_ints = self.bytes_container.len() / 4; - let meta_u32s: &[u32] = cast_slice(self.bytes_container.chunk()); - output[tmp_output_offset as usize..][..how_many_ints] - .copy_from_slice(&meta_u32s[..how_many_ints]); - tmp_output_offset += how_many_ints as u32; - - let mut bitmap: u64 = 0; - for k in 2..=usize::from(T::WIDTH) { - if self.data_pointers[k] != 0 { - bitmap |= 1u64 << (k - 1); - } - } - T::write_bitmap(bitmap, &mut output[tmp_output_offset as usize..]); - tmp_output_offset += T::BITMAP_WORDS; - - for k in 2..=usize::from(T::WIDTH) { - if self.data_pointers[k] != 0 { - output[tmp_output_offset as usize] = self.data_pointers[k] as u32; - tmp_output_offset += 1; - let mut j = 0; - while j < self.data_pointers[k] { - T::fast_pack( - &self.exception_buffers[k], - j, - output, - tmp_output_offset as usize, - k as u8, - ); - tmp_output_offset += k as u32; - j += 32; - } - let overflow = j as u32 - self.data_pointers[k] as u32; - tmp_output_offset -= (overflow * k as u32) / 32; - } - } - output_offset.set_position(u64::from(tmp_output_offset)); - } - - fn best_bit_from_data(&mut self, input: &[T], pos: u32) { - self.freqs.fill(0); - let k_end = min(pos + N as u32, input.len() as u32); - for k in pos..k_end { - self.freqs[usize::from(input[k as usize].significant_bits())] += 1; - } - - self.optimal_bits = T::WIDTH; - while self.freqs[self.optimal_bits as usize] == 0 { - self.optimal_bits -= 1; - } - self.max_bits = self.optimal_bits; - - let mut best_cost = u32::from(self.optimal_bits) * N as u32; - let mut num_exceptions: u32 = 0; - self.exception_count = 0; - - for bits in (0..self.optimal_bits).rev() { - num_exceptions += self.freqs[bits as usize + 1]; - if num_exceptions == N as u32 { - break; - } - let diff = u32::from(self.max_bits - bits); - let mut cost = num_exceptions * OVERHEAD_OF_EACH_EXCEPT - + num_exceptions * diff - + u32::from(bits) * N as u32 - + 8; - if diff == 1 { - cost -= num_exceptions; - } - if cost < best_cost { - best_cost = cost; - self.optimal_bits = bits; - self.exception_count = num_exceptions as u8; - } - } - } - - #[expect(clippy::too_many_lines)] - fn decode_page( - &mut self, - input: &[u32], - input_offset: &mut Cursor, - output: &mut [T], - output_offset: &mut Cursor, - this_size: u32, - ) -> FastPForResult<()> { - let n = u32::try_from(input.len()) - .map_err(|_| FastPForError::InvalidInputLength(input.len()))?; - - let init_pos = - u32::try_from(input_offset.position()).map_err(|_| FastPForError::NotEnoughData)?; - let where_meta = input.get_val(init_pos)?; - input_offset.increment(); - let mut inexcept = init_pos - .checked_add(where_meta) - .ok_or(FastPForError::NotEnoughData)?; - let bytesize = input.get_val(inexcept)?; - inexcept = inexcept - .checked_add(1) - .ok_or(FastPForError::NotEnoughData)?; - let input_bytes: &[u8] = cast_slice(input); - let mut byte_pos = (inexcept as usize) - .checked_mul(4) - .filter(|&bp| bp <= input_bytes.len()) - .ok_or(FastPForError::NotEnoughData)?; - let length = bytesize.div_ceil(4); - inexcept = inexcept - .checked_add(length) - .ok_or(FastPForError::NotEnoughData)?; - - let bitmap = T::read_bitmap(input, inexcept)?; - inexcept = inexcept - .checked_add(T::BITMAP_WORDS) - .ok_or(FastPForError::NotEnoughData)?; - - for k in 2..=u32::from(T::WIDTH) { - if (bitmap & (1u64 << (k - 1))) != 0 { - let size = input.get_val(inexcept)?; - inexcept = inexcept - .checked_add(1) - .ok_or(FastPForError::NotEnoughData)?; - if size > self.page_size { - return Err(FastPForError::NotEnoughData); - } - let rounded_up = size.next_multiple_of(32) as usize; - if self.exception_buffers[k as usize].len() < rounded_up { - self.exception_buffers[k as usize].resize(rounded_up, T::ZERO); - } - let mut j: u32 = 0; - while j.checked_add(32).is_some_and(|j32| j32 <= size) - && inexcept.checked_add(k).is_some_and(|ie| ie <= n) - { - T::fast_unpack( - input, - inexcept as usize, - &mut self.exception_buffers[k as usize], - j as usize, - k as u8, - ); - inexcept += k; - j += 32; - } - if j < size { - let words_needed = (size - j).saturating_mul(k).div_ceil(32); - let avail = n - inexcept.min(n); - if avail < words_needed { - return Err(FastPForError::NotEnoughData); - } - let copy_len = words_needed as usize; - let mut tail_buf = [0u32; 64]; - if copy_len == 0 { - return Err(FastPForError::NotEnoughData); - } - let start = inexcept as usize; - let src = input - .get(start..start + copy_len) - .ok_or(FastPForError::NotEnoughData)?; - tail_buf[..copy_len].copy_from_slice(src); - T::fast_unpack( - &tail_buf, - 0, - &mut self.exception_buffers[k as usize], - j as usize, - k as u8, - ); - inexcept += k; - j += 32; - } - let overflow = j - size; - inexcept -= (overflow * k) / 32; - } - } - - self.data_pointers.fill(0); - let mut tmp_output_offset = output_offset.position() as u32; - let mut tmp_input_offset = input_offset.position() as u32; - - let run_end = this_size / N as u32; - for _ in 0..run_end { - let bits = input_bytes.get_val(byte_pos)?; - if bits > T::WIDTH { - return Err(FastPForError::NotEnoughData); - } - byte_pos += 1; - let num_exceptions = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - for k in (0..N as u32).step_by(32) { - let in_start = tmp_input_offset as usize; - let out_start = (tmp_output_offset + k) as usize; - let in_end = in_start - .checked_add(usize::from(bits)) - .ok_or(FastPForError::NotEnoughData)?; - if in_end > input.len() { - return Err(FastPForError::NotEnoughData); - } - let out_end = out_start - .checked_add(32) - .ok_or(FastPForError::OutputBufferTooSmall)?; - if out_end > output.len() { - return Err(FastPForError::OutputBufferTooSmall); - } - T::fast_unpack(input, in_start, output, out_start, bits); - tmp_input_offset += u32::from(bits); - } - if num_exceptions > 0 { - let maxbits = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - let index = maxbits - .checked_sub(bits) - .ok_or(FastPForError::NotEnoughData)?; - if maxbits > T::WIDTH || index == 0 || index > T::WIDTH { - return Err(FastPForError::NotEnoughData); - } - let index = usize::from(index); - if index == 1 { - for _ in 0..num_exceptions { - let pos = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - if u32::from(pos) >= N as u32 { - return Err(FastPForError::NotEnoughData); - } - let out_idx = tmp_output_offset as usize + pos as usize; - if out_idx >= output.len() { - return Err(FastPForError::OutputBufferTooSmall); - } - T::or_shl_assign(&mut output[out_idx], T::one_shl(bits), 0); - } - } else { - for _ in 0..num_exceptions { - let pos = input_bytes.get_val(byte_pos)?; - byte_pos += 1; - if u32::from(pos) >= N as u32 { - return Err(FastPForError::NotEnoughData); - } - let out_idx = tmp_output_offset as usize + pos as usize; - if out_idx >= output.len() { - return Err(FastPForError::OutputBufferTooSmall); - } - let ptr = self.data_pointers[index]; - let except_value = self.exception_buffers[index].get_val(ptr)?; - T::or_shl_assign(&mut output[out_idx], except_value, bits); - self.data_pointers[index] += 1; - } - } - } - tmp_output_offset += N as u32; - } - output_offset.set_position(u64::from(tmp_output_offset)); - input_offset.set_position(u64::from(inexcept)); - Ok(()) - } -} diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs new file mode 100644 index 0000000..a613ffc --- /dev/null +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -0,0 +1,126 @@ +//! Element-width abstraction shared by the 32- and 64-bit `FastPFOR` codecs. +//! +//! The block-splitting, best-bit search, exception handling, and metadata layout are +//! identical for `u32` and `u64`; only the element width differs. +//! [`FastPForInt`] abstracts the width-specific pieces so a single [`FastPFor`](super::fastpfor::FastPFor) +//! implements the algorithm once. +//! `u32` keeps its hand-unrolled bit-packing kernels; `u64` uses the generic wide packer. + +use crate::helpers::GetWithErr; +use crate::rust::integer_compression::{bitpacking, bitpacking_wide, bitunpacking}; +use crate::{FastPForError, FastPForResult}; + +/// Element type of a `FastPFOR` stream: [`u32`] or [`u64`]. +/// +/// Implementors supply the width-specific operations the engine needs. +/// The exception bitmap spans [`BITMAP_WORDS`](Self::BITMAP_WORDS) output words. +pub trait FastPForInt: Copy + 'static { + /// Bit width of the element: 32 or 64. + const WIDTH: u8; + /// Output words occupied by the exception bitmap: 1 for `u32`, 2 for `u64`. + const BITMAP_WORDS: u32; + /// The zero value. + const ZERO: Self; + + /// Number of significant bits, i.e. `WIDTH - leading_zeros` (0 for a zero value). + fn significant_bits(self) -> u8; + /// Logical right shift by `n`, where `n < WIDTH`. + fn shr(self, n: u8) -> Self; + /// Whether the value is zero. + fn is_zero(self) -> bool; + /// `*dst |= val << shift`, where `shift < WIDTH`. + fn or_shl_assign(dst: &mut Self, val: Self, shift: u8); + /// `1 << shift`, where `shift < WIDTH`. + fn one_shl(shift: u8) -> Self; + + /// Pack 32 values at `bit` bits each into `out`. + fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8); + /// Unpack 32 values at `bit` bits each from `src`. + fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8); + + /// Write the exception bitmap as [`BITMAP_WORDS`](Self::BITMAP_WORDS) words at `out`. + fn write_bitmap(bitmap: u64, out: &mut [u32]); + /// Read the exception bitmap from [`BITMAP_WORDS`](Self::BITMAP_WORDS) words at `pos`. + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult; +} + +#[allow( + clippy::use_self, + reason = "u32 literals here are stream words, not the Self element type" +)] +impl FastPForInt for u32 { + const WIDTH: u8 = 32; + const BITMAP_WORDS: u32 = 1; + const ZERO: Self = 0; + + fn significant_bits(self) -> u8 { + (32 - self.leading_zeros()) as u8 + } + fn shr(self, n: u8) -> Self { + self >> n + } + fn is_zero(self) -> bool { + self == 0 + } + fn or_shl_assign(dst: &mut Self, val: Self, shift: u8) { + *dst |= val << shift; + } + fn one_shl(shift: u8) -> Self { + 1 << shift + } + fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { + bitpacking::fast_pack(src, inpos, out, outpos, bit); + } + fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { + bitunpacking::fast_unpack(src, inpos, out, outpos, bit); + } + fn write_bitmap(bitmap: u64, out: &mut [u32]) { + out[0] = bitmap as u32; + } + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { + let word: u32 = input.get_val(pos)?; + Ok(u64::from(word)) + } +} + +#[allow( + clippy::use_self, + reason = "u32 literals here are stream words, not the Self element type" +)] +impl FastPForInt for u64 { + const WIDTH: u8 = 64; + const BITMAP_WORDS: u32 = 2; + const ZERO: Self = 0; + + fn significant_bits(self) -> u8 { + (64 - self.leading_zeros()) as u8 + } + fn shr(self, n: u8) -> Self { + self >> n + } + fn is_zero(self) -> bool { + self == 0 + } + fn or_shl_assign(dst: &mut Self, val: Self, shift: u8) { + *dst |= val << shift; + } + fn one_shl(shift: u8) -> Self { + 1 << shift + } + fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { + bitpacking_wide::pack_wide(src, inpos, out, outpos, bit); + } + fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { + bitpacking_wide::unpack_wide(src, inpos, out, outpos, bit); + } + fn write_bitmap(bitmap: u64, out: &mut [u32]) { + out[0] = bitmap as u32; + out[1] = (bitmap >> 32) as u32; + } + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { + let lo: u32 = input.get_val(pos)?; + let hi_pos = pos.checked_add(1).ok_or(FastPForError::NotEnoughData)?; + let hi: u32 = input.get_val(hi_pos)?; + Ok(u64::from(lo) | (u64::from(hi) << 32)) + } +} diff --git a/src/rust/integer_compression/mod.rs b/src/rust/integer_compression/mod.rs index 25fe8cb..476f6e9 100644 --- a/src/rust/integer_compression/mod.rs +++ b/src/rust/integer_compression/mod.rs @@ -3,6 +3,6 @@ pub mod bitpacking_wide; pub mod bitunpacking; pub mod fastpfor; pub mod fastpfor64; -pub mod fastpfor_engine; +pub mod fastpfor_int; pub mod just_copy; pub mod variable_byte; From dcf64f7dd401a5610b1331db864cd4ba95fac624 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 17 Jul 2026 19:37:11 -0400 Subject: [PATCH 12/23] wip --- README.md | 2 +- src/lib.rs | 3 +- src/rust/integer_compression/fastpfor.rs | 43 +++++++------ src/rust/integer_compression/fastpfor64.rs | 2 +- src/rust/integer_compression/fastpfor_int.rs | 64 +++++++++++++++++++- src/test_utils.rs | 3 + 6 files changed, 89 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 98330d1..e62c78c 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ assert_eq!(decoded, input); Enable the `cpp` feature in `Cargo.toml`: ```toml -fastpfor = { version = "0.1", features = ["cpp"] } +fastpfor = { version = "0.9", features = ["cpp"] } ``` All C++ codecs implement the same `AnyLenCodec` trait (`encode` / `decode`), so diff --git a/src/lib.rs b/src/lib.rs index 3a17e00..62325d0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,8 +18,7 @@ pub mod cpp; pub(crate) mod rust; mod codec; -pub use codec::BlockCodec64; -pub use codec::{AnyLenCodec, BlockCodec, slice_to_blocks}; +pub use codec::{AnyLenCodec, BlockCodec, BlockCodec64, slice_to_blocks}; pub(crate) mod helpers; diff --git a/src/rust/integer_compression/fastpfor.rs b/src/rust/integer_compression/fastpfor.rs index e55aca2..b4c9da5 100644 --- a/src/rust/integer_compression/fastpfor.rs +++ b/src/rust/integer_compression/fastpfor.rs @@ -1,4 +1,3 @@ -use std::array; use std::cmp::min; use std::io::Cursor; @@ -29,17 +28,21 @@ const OVERHEAD_OF_EACH_EXCEPT: u32 = 8; const DEFAULT_PAGE_SIZE: u32 = 65536; /// Type alias for [`FastPFor`] with 128-element `u32` blocks. -pub type FastPForBlock128 = FastPFor<128, { u32::BITS as usize + 1 }, u32>; +pub type FastPForBlock128 = FastPFor<128, u32>; /// Type alias for [`FastPFor`] with 256-element `u32` blocks. -pub type FastPForBlock256 = FastPFor<256, { u32::BITS as usize + 1 }, u32>; +pub type FastPForBlock256 = FastPFor<256, u32>; /// Fast Patched Frame-of-Reference ([FastPFOR](https://github.com/lemire/FastPFor)) codec. /// /// `N` is the block size (128 or 256 values per block) and `T` the element type -/// ([`u32`] or [`u64`]). This struct implements [`BlockCodec`] with `Block = [u32; N]` -/// for the `u32` element type, giving compile-time guarantees that only correctly-sized -/// blocks are accepted. +/// ([`u32`] or [`u64`], defaulting to `u32`). This struct implements [`BlockCodec`] with +/// `Block = [u32; N]` for the `u32` element type, giving compile-time guarantees that only +/// correctly-sized blocks are accepted. +/// +/// The per-block scratch buffers are sized exactly for `T` (`T::WIDTH + 1` buckets) via the +/// sealed [`FastPForInt`] trait, so the bucket count is neither wasted nor part of this +/// type's signature. /// /// Use [`FastPForBlock128`] or [`FastPForBlock256`] as convenient `u32` type aliases. /// @@ -54,22 +57,18 @@ pub type FastPForBlock256 = FastPFor<256, { u32::BITS as usize + 1 }, u32>; /// codec.encode(&data, &mut out).unwrap(); /// ``` #[derive(Debug)] -pub struct FastPFor< - const N: usize, - const WIDTHS: usize = { u32::BITS as usize + 1 }, - T: FastPForInt = u32, -> { +pub struct FastPFor { /// Exception values indexed by bit width difference - exception_buffers: [Vec; WIDTHS], + exception_buffers: T::ExceptionBuffers, /// Metadata buffer for encoding/decoding bytes_container: BytesMut, /// Maximum integers per page page_size: u32, /// Position trackers for exception arrays - data_pointers: [usize; WIDTHS], + data_pointers: T::DataPointers, /// Frequency count for each bit width: /// `freqs[i]` = count of values needing exactly i bits - freqs: [u32; WIDTHS], + freqs: T::Freqs, /// Optimal number of bits chosen for the current block optimal_bits: u8, /// Number of exceptions that don't fit in the optimal bit width @@ -78,14 +77,14 @@ pub struct FastPFor< max_bits: u8, } -impl Default for FastPFor { +impl Default for FastPFor { fn default() -> Self { Self::new(DEFAULT_PAGE_SIZE) .expect("DEFAULT_PAGE_SIZE is a multiple of all valid block sizes") } } -impl FastPFor { +impl FastPFor { /// Creates a new `FastPForBlock` with a codec with the given page size. /// /// Returns an error if `page_size` is not a multiple of the block size. @@ -102,9 +101,9 @@ impl FastPFor (3 * page_size / N as u32 + page_size) as usize, ), page_size, - exception_buffers: array::from_fn(|_| Vec::new()), - data_pointers: [0; WIDTHS], - freqs: [0; WIDTHS], + exception_buffers: T::new_exception_buffers(), + data_pointers: T::new_data_pointers(), + freqs: T::new_freqs(), optimal_bits: 0, exception_count: 0, max_bits: 0, @@ -168,7 +167,7 @@ impl FastPFor let mut tmp_output_offset = output_offset.position() as u32; // Data pointers to 0 - self.data_pointers.fill(0); + self.data_pointers.as_mut().fill(0); self.bytes_container.clear(); let mut tmp_input_offset = input_offset.position() as u32; @@ -264,7 +263,7 @@ impl FastPFor /// /// Analyzes frequency distribution to balance regular value bits against exception overhead. fn best_bit_from_data(&mut self, input: &[T], pos: u32) { - self.freqs.fill(0); + self.freqs.as_mut().fill(0); let k_end = min(pos + N as u32, input.len() as u32); for k in pos..k_end { self.freqs[usize::from(input[k as usize].significant_bits())] += 1; @@ -419,7 +418,7 @@ impl FastPFor } } - self.data_pointers.fill(0); + self.data_pointers.as_mut().fill(0); let mut tmp_output_offset = output_offset.position() as u32; let mut tmp_input_offset = input_offset.position() as u32; diff --git a/src/rust/integer_compression/fastpfor64.rs b/src/rust/integer_compression/fastpfor64.rs index 1721a62..1b66f6b 100644 --- a/src/rust/integer_compression/fastpfor64.rs +++ b/src/rust/integer_compression/fastpfor64.rs @@ -22,7 +22,7 @@ use crate::{BlockCodec64, FastPForError, FastPForResult}; /// `N` is the block size (128 or 256 values). This is [`FastPFor`] specialized to the /// `u64` element type, and is the internal `u64` codec behind /// [`FastPFor128`](crate::FastPFor128) and [`FastPFor256`](crate::FastPFor256). -pub type FastPForWide = FastPFor; +pub type FastPForWide = FastPFor; /// Variable-byte encoding of the `u64` tail. /// diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs index a613ffc..8edb220 100644 --- a/src/rust/integer_compression/fastpfor_int.rs +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -5,16 +5,34 @@ //! [`FastPForInt`] abstracts the width-specific pieces so a single [`FastPFor`](super::fastpfor::FastPFor) //! implements the algorithm once. //! `u32` keeps its hand-unrolled bit-packing kernels; `u64` uses the generic wide packer. +//! +//! The trait is **sealed**: only [`u32`] and [`u64`] implement it, so callers cannot plug in +//! an unsupported element type. Each implementor also fixes the exact size of the per-block +//! scratch buffers (`WIDTH + 1` buckets) as associated types, so no space is wasted and the +//! bucket count never leaks into the public [`FastPFor`](super::fastpfor::FastPFor) signature. + +use std::array; +use std::fmt::Debug; +use std::ops::{Index, IndexMut}; use crate::helpers::GetWithErr; use crate::rust::integer_compression::{bitpacking, bitpacking_wide, bitunpacking}; use crate::{FastPForError, FastPForResult}; +mod sealed { + pub trait Sealed {} + impl Sealed for u32 {} + impl Sealed for u64 {} +} + /// Element type of a `FastPFOR` stream: [`u32`] or [`u64`]. /// -/// Implementors supply the width-specific operations the engine needs. +/// Implementors supply the width-specific operations the engine needs, plus the concrete +/// scratch-buffer array types (one bucket per possible bit width, i.e. `WIDTH + 1`). /// The exception bitmap spans [`BITMAP_WORDS`](Self::BITMAP_WORDS) output words. -pub trait FastPForInt: Copy + 'static { +/// +/// This trait is sealed and cannot be implemented outside this crate. +pub trait FastPForInt: Copy + 'static + sealed::Sealed { /// Bit width of the element: 32 or 64. const WIDTH: u8; /// Output words occupied by the exception bitmap: 1 for `u32`, 2 for `u64`. @@ -22,6 +40,20 @@ pub trait FastPForInt: Copy + 'static { /// The zero value. const ZERO: Self; + /// Exception values grouped by bit-width bucket: `[Vec; WIDTH + 1]`. + type ExceptionBuffers: Index> + IndexMut + Debug; + /// Per-bit-width frequency counts: `[u32; WIDTH + 1]`. + type Freqs: Index + IndexMut + AsMut<[u32]> + Debug; + /// Write positions into `ExceptionBuffers`: `[usize; WIDTH + 1]`. + type DataPointers: Index + IndexMut + AsMut<[usize]> + Debug; + + /// Fresh, empty exception buffers. + fn new_exception_buffers() -> Self::ExceptionBuffers; + /// Fresh, zeroed frequency counts. + fn new_freqs() -> Self::Freqs; + /// Fresh, zeroed data pointers. + fn new_data_pointers() -> Self::DataPointers; + /// Number of significant bits, i.e. `WIDTH - leading_zeros` (0 for a zero value). fn significant_bits(self) -> u8; /// Logical right shift by `n`, where `n < WIDTH`. @@ -53,6 +85,20 @@ impl FastPForInt for u32 { const BITMAP_WORDS: u32 = 1; const ZERO: Self = 0; + type ExceptionBuffers = [Vec; u32::BITS as usize + 1]; + type Freqs = [u32; u32::BITS as usize + 1]; + type DataPointers = [usize; u32::BITS as usize + 1]; + + fn new_exception_buffers() -> Self::ExceptionBuffers { + array::from_fn(|_| Vec::new()) + } + fn new_freqs() -> Self::Freqs { + [0; u32::BITS as usize + 1] + } + fn new_data_pointers() -> Self::DataPointers { + [0; u32::BITS as usize + 1] + } + fn significant_bits(self) -> u8 { (32 - self.leading_zeros()) as u8 } @@ -92,6 +138,20 @@ impl FastPForInt for u64 { const BITMAP_WORDS: u32 = 2; const ZERO: Self = 0; + type ExceptionBuffers = [Vec; u64::BITS as usize + 1]; + type Freqs = [u32; u64::BITS as usize + 1]; + type DataPointers = [usize; u64::BITS as usize + 1]; + + fn new_exception_buffers() -> Self::ExceptionBuffers { + array::from_fn(|_| Vec::new()) + } + fn new_freqs() -> Self::Freqs { + [0; u64::BITS as usize + 1] + } + fn new_data_pointers() -> Self::DataPointers { + [0; u64::BITS as usize + 1] + } + fn significant_bits(self) -> u8 { (64 - self.leading_zeros()) as u8 } diff --git a/src/test_utils.rs b/src/test_utils.rs index 9b2729b..9ff6fea 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -48,6 +48,7 @@ pub fn roundtrip_full(data: &[u32], expected_len assert_eq!(decompressed, data); } +#[cfg(feature = "cpp")] pub fn roundtrip64(data: &[u64]) { let mut codec = C::default(); let mut compressed = Vec::new(); @@ -100,12 +101,14 @@ pub fn block_decompress( Ok(out) } +#[cfg(feature = "cpp")] pub fn compress64(data: &[u64]) -> FastPForResult> { let mut compressed = Vec::new(); C::default().encode64(data, &mut compressed)?; Ok(compressed) } +#[cfg(feature = "cpp")] pub fn decompress64(compressed: &[u32]) -> FastPForResult> { let mut out = Vec::new(); C::default().decode64(compressed, &mut out)?; From 2a2e2da9272b30bcc5eaaaf5d5ed6c72a3bbb5b8 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 17 Jul 2026 21:17:52 -0400 Subject: [PATCH 13/23] optimize --- src/rust/integer_compression/fastpfor_int.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs index 8edb220..6913aa9 100644 --- a/src/rust/integer_compression/fastpfor_int.rs +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -99,30 +99,39 @@ impl FastPForInt for u32 { [0; u32::BITS as usize + 1] } + #[inline] fn significant_bits(self) -> u8 { (32 - self.leading_zeros()) as u8 } + #[inline] fn shr(self, n: u8) -> Self { self >> n } + #[inline] fn is_zero(self) -> bool { self == 0 } + #[inline] fn or_shl_assign(dst: &mut Self, val: Self, shift: u8) { *dst |= val << shift; } + #[inline] fn one_shl(shift: u8) -> Self { 1 << shift } + #[inline] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { bitpacking::fast_pack(src, inpos, out, outpos, bit); } + #[inline] fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { bitunpacking::fast_unpack(src, inpos, out, outpos, bit); } + #[inline] fn write_bitmap(bitmap: u64, out: &mut [u32]) { out[0] = bitmap as u32; } + #[inline] fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { let word: u32 = input.get_val(pos)?; Ok(u64::from(word)) @@ -152,31 +161,40 @@ impl FastPForInt for u64 { [0; u64::BITS as usize + 1] } + #[inline] fn significant_bits(self) -> u8 { (64 - self.leading_zeros()) as u8 } + #[inline] fn shr(self, n: u8) -> Self { self >> n } + #[inline] fn is_zero(self) -> bool { self == 0 } + #[inline] fn or_shl_assign(dst: &mut Self, val: Self, shift: u8) { *dst |= val << shift; } + #[inline] fn one_shl(shift: u8) -> Self { 1 << shift } + #[inline] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { bitpacking_wide::pack_wide(src, inpos, out, outpos, bit); } + #[inline] fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { bitpacking_wide::unpack_wide(src, inpos, out, outpos, bit); } + #[inline] fn write_bitmap(bitmap: u64, out: &mut [u32]) { out[0] = bitmap as u32; out[1] = (bitmap >> 32) as u32; } + #[inline] fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { let lo: u32 = input.get_val(pos)?; let hi_pos = pos.checked_add(1).ok_or(FastPForError::NotEnoughData)?; From b6f4da0b156560b383884e638ea66927d1063e19 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 17 Jul 2026 21:34:13 -0400 Subject: [PATCH 14/23] wip --- src/rust/integer_compression/fastpfor.rs | 6 ++--- src/rust/integer_compression/fastpfor_int.rs | 27 +++++++++++++------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/rust/integer_compression/fastpfor.rs b/src/rust/integer_compression/fastpfor.rs index b4c9da5..dd6ac4e 100644 --- a/src/rust/integer_compression/fastpfor.rs +++ b/src/rust/integer_compression/fastpfor.rs @@ -225,10 +225,10 @@ impl FastPFor { .copy_from_slice(&meta_u32s[..how_many_ints]); tmp_output_offset += how_many_ints as u32; // Exception bitmap: one bit per bit-width bucket, written as `T::BITMAP_WORDS` words. - let mut bitmap: u64 = 0; + let mut bitmap = T::ZERO; for k in 2..=usize::from(T::WIDTH) { if self.data_pointers[k] != 0 { - bitmap |= 1u64 << (k - 1); + T::or_shl_assign(&mut bitmap, T::one_shl((k - 1) as u8), 0); } } T::write_bitmap(bitmap, &mut output[tmp_output_offset as usize..]); @@ -353,7 +353,7 @@ impl FastPFor { .ok_or(FastPForError::NotEnoughData)?; for k in 2..=u32::from(T::WIDTH) { - if (bitmap & (1u64 << (k - 1))) != 0 { + if bitmap.nth_bit_set((k - 1) as u8) { let size = input.get_val(inexcept)?; inexcept = inexcept .checked_add(1) diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs index 6913aa9..f9f2a4f 100644 --- a/src/rust/integer_compression/fastpfor_int.rs +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -64,6 +64,8 @@ pub trait FastPForInt: Copy + 'static + sealed::Sealed { fn or_shl_assign(dst: &mut Self, val: Self, shift: u8); /// `1 << shift`, where `shift < WIDTH`. fn one_shl(shift: u8) -> Self; + /// Whether bit `n` is set, where `n < WIDTH`. + fn nth_bit_set(self, n: u8) -> bool; /// Pack 32 values at `bit` bits each into `out`. fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8); @@ -71,9 +73,9 @@ pub trait FastPForInt: Copy + 'static + sealed::Sealed { fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8); /// Write the exception bitmap as [`BITMAP_WORDS`](Self::BITMAP_WORDS) words at `out`. - fn write_bitmap(bitmap: u64, out: &mut [u32]); + fn write_bitmap(bitmap: Self, out: &mut [u32]); /// Read the exception bitmap from [`BITMAP_WORDS`](Self::BITMAP_WORDS) words at `pos`. - fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult; + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult; } #[allow( @@ -120,6 +122,10 @@ impl FastPForInt for u32 { 1 << shift } #[inline] + fn nth_bit_set(self, n: u8) -> bool { + (self >> n) & 1 != 0 + } + #[inline] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { bitpacking::fast_pack(src, inpos, out, outpos, bit); } @@ -128,13 +134,12 @@ impl FastPForInt for u32 { bitunpacking::fast_unpack(src, inpos, out, outpos, bit); } #[inline] - fn write_bitmap(bitmap: u64, out: &mut [u32]) { - out[0] = bitmap as u32; + fn write_bitmap(bitmap: Self, out: &mut [u32]) { + out[0] = bitmap; } #[inline] - fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { - let word: u32 = input.get_val(pos)?; - Ok(u64::from(word)) + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { + input.get_val(pos) } } @@ -182,6 +187,10 @@ impl FastPForInt for u64 { 1 << shift } #[inline] + fn nth_bit_set(self, n: u8) -> bool { + (self >> n) & 1 != 0 + } + #[inline] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { bitpacking_wide::pack_wide(src, inpos, out, outpos, bit); } @@ -190,12 +199,12 @@ impl FastPForInt for u64 { bitpacking_wide::unpack_wide(src, inpos, out, outpos, bit); } #[inline] - fn write_bitmap(bitmap: u64, out: &mut [u32]) { + fn write_bitmap(bitmap: Self, out: &mut [u32]) { out[0] = bitmap as u32; out[1] = (bitmap >> 32) as u32; } #[inline] - fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { + fn read_bitmap(input: &[u32], pos: u32) -> FastPForResult { let lo: u32 = input.get_val(pos)?; let hi_pos = pos.checked_add(1).ok_or(FastPForError::NotEnoughData)?; let hi: u32 = input.get_val(hi_pos)?; From 84d4367661551605baba2457e19766f01d56dbaf Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 17 Jul 2026 21:47:00 -0400 Subject: [PATCH 15/23] wip --- src/rust/integer_compression/fastpfor.rs | 15 ++-- src/rust/integer_compression/fastpfor_int.rs | 94 ++++++-------------- 2 files changed, 34 insertions(+), 75 deletions(-) diff --git a/src/rust/integer_compression/fastpfor.rs b/src/rust/integer_compression/fastpfor.rs index dd6ac4e..a23010a 100644 --- a/src/rust/integer_compression/fastpfor.rs +++ b/src/rust/integer_compression/fastpfor.rs @@ -186,13 +186,10 @@ impl FastPFor { self.exception_buffers[index].resize(new_cap, T::ZERO); } for k in 0..N as u32 { - if !input[(k + tmp_input_offset) as usize] - .shr(self.optimal_bits) - .is_zero() - { + if input[(k + tmp_input_offset) as usize] >> self.optimal_bits != T::ZERO { self.bytes_container.put_u8(k as u8); self.exception_buffers[index][self.data_pointers[index]] = - input[(k + tmp_input_offset) as usize].shr(self.optimal_bits); + input[(k + tmp_input_offset) as usize] >> self.optimal_bits; self.data_pointers[index] += 1; } } @@ -228,7 +225,7 @@ impl FastPFor { let mut bitmap = T::ZERO; for k in 2..=usize::from(T::WIDTH) { if self.data_pointers[k] != 0 { - T::or_shl_assign(&mut bitmap, T::one_shl((k - 1) as u8), 0); + bitmap |= T::ONE << (k - 1) as u8; } } T::write_bitmap(bitmap, &mut output[tmp_output_offset as usize..]); @@ -353,7 +350,7 @@ impl FastPFor { .ok_or(FastPForError::NotEnoughData)?; for k in 2..=u32::from(T::WIDTH) { - if bitmap.nth_bit_set((k - 1) as u8) { + if bitmap & (T::ONE << (k - 1) as u8) != T::ZERO { let size = input.get_val(inexcept)?; inexcept = inexcept .checked_add(1) @@ -470,7 +467,7 @@ impl FastPFor { if out_idx >= output.len() { return Err(FastPForError::OutputBufferTooSmall); } - T::or_shl_assign(&mut output[out_idx], T::one_shl(bits), 0); + output[out_idx] |= T::ONE << bits; } } else { for _ in 0..num_exceptions { @@ -485,7 +482,7 @@ impl FastPFor { } let ptr = self.data_pointers[index]; let except_value = self.exception_buffers[index].get_val(ptr)?; - T::or_shl_assign(&mut output[out_idx], except_value, bits); + output[out_idx] |= except_value << bits; self.data_pointers[index] += 1; } } diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs index f9f2a4f..0932465 100644 --- a/src/rust/integer_compression/fastpfor_int.rs +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -3,7 +3,9 @@ //! The block-splitting, best-bit search, exception handling, and metadata layout are //! identical for `u32` and `u64`; only the element width differs. //! [`FastPForInt`] abstracts the width-specific pieces so a single [`FastPFor`](super::fastpfor::FastPFor) -//! implements the algorithm once. +//! implements the algorithm once. Ordinary arithmetic uses the standard operator traits +//! (`>>`, `<<`, `&`, `|=`) that the trait requires as bounds; only the genuinely +//! width-specific pieces (bit-packing kernels and the exception bitmap layout) are methods. //! `u32` keeps its hand-unrolled bit-packing kernels; `u64` uses the generic wide packer. //! //! The trait is **sealed**: only [`u32`] and [`u64`] implement it, so callers cannot plug in @@ -13,7 +15,7 @@ use std::array; use std::fmt::Debug; -use std::ops::{Index, IndexMut}; +use std::ops::{BitAnd, BitOrAssign, Index, IndexMut, Shl, Shr}; use crate::helpers::GetWithErr; use crate::rust::integer_compression::{bitpacking, bitpacking_wide, bitunpacking}; @@ -27,18 +29,30 @@ mod sealed { /// Element type of a `FastPFOR` stream: [`u32`] or [`u64`]. /// -/// Implementors supply the width-specific operations the engine needs, plus the concrete -/// scratch-buffer array types (one bucket per possible bit width, i.e. `WIDTH + 1`). -/// The exception bitmap spans [`BITMAP_WORDS`](Self::BITMAP_WORDS) output words. +/// The operator bounds (`Shr`/`Shl` by `u8`, `BitAnd`, `BitOrAssign`) let the codec use plain +/// `>>`, `<<`, `&`, and `|=` on values; only the width-specific pieces are methods. Implementors +/// also supply the concrete scratch-buffer array types (one bucket per possible bit width, +/// i.e. `WIDTH + 1`). The exception bitmap spans [`BITMAP_WORDS`](Self::BITMAP_WORDS) output words. /// /// This trait is sealed and cannot be implemented outside this crate. -pub trait FastPForInt: Copy + 'static + sealed::Sealed { +pub trait FastPForInt: + Copy + + 'static + + Eq + + sealed::Sealed + + Shr + + Shl + + BitAnd + + BitOrAssign +{ /// Bit width of the element: 32 or 64. - const WIDTH: u8; + const WIDTH: u8 = (size_of::() * 8) as u8; /// Output words occupied by the exception bitmap: 1 for `u32`, 2 for `u64`. - const BITMAP_WORDS: u32; + const BITMAP_WORDS: u32 = Self::WIDTH as u32 / u32::BITS; /// The zero value. const ZERO: Self; + /// The one value. + const ONE: Self; /// Exception values grouped by bit-width bucket: `[Vec; WIDTH + 1]`. type ExceptionBuffers: Index> + IndexMut + Debug; @@ -56,16 +70,6 @@ pub trait FastPForInt: Copy + 'static + sealed::Sealed { /// Number of significant bits, i.e. `WIDTH - leading_zeros` (0 for a zero value). fn significant_bits(self) -> u8; - /// Logical right shift by `n`, where `n < WIDTH`. - fn shr(self, n: u8) -> Self; - /// Whether the value is zero. - fn is_zero(self) -> bool; - /// `*dst |= val << shift`, where `shift < WIDTH`. - fn or_shl_assign(dst: &mut Self, val: Self, shift: u8); - /// `1 << shift`, where `shift < WIDTH`. - fn one_shl(shift: u8) -> Self; - /// Whether bit `n` is set, where `n < WIDTH`. - fn nth_bit_set(self, n: u8) -> bool; /// Pack 32 values at `bit` bits each into `out`. fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8); @@ -80,12 +84,11 @@ pub trait FastPForInt: Copy + 'static + sealed::Sealed { #[allow( clippy::use_self, - reason = "u32 literals here are stream words, not the Self element type" + reason = "u32 here is the stream word type, not the Self element type" )] impl FastPForInt for u32 { - const WIDTH: u8 = 32; - const BITMAP_WORDS: u32 = 1; const ZERO: Self = 0; + const ONE: Self = 1; type ExceptionBuffers = [Vec; u32::BITS as usize + 1]; type Freqs = [u32; u32::BITS as usize + 1]; @@ -103,27 +106,7 @@ impl FastPForInt for u32 { #[inline] fn significant_bits(self) -> u8 { - (32 - self.leading_zeros()) as u8 - } - #[inline] - fn shr(self, n: u8) -> Self { - self >> n - } - #[inline] - fn is_zero(self) -> bool { - self == 0 - } - #[inline] - fn or_shl_assign(dst: &mut Self, val: Self, shift: u8) { - *dst |= val << shift; - } - #[inline] - fn one_shl(shift: u8) -> Self { - 1 << shift - } - #[inline] - fn nth_bit_set(self, n: u8) -> bool { - (self >> n) & 1 != 0 + Self::WIDTH - self.leading_zeros() as u8 } #[inline] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { @@ -145,12 +128,11 @@ impl FastPForInt for u32 { #[allow( clippy::use_self, - reason = "u32 literals here are stream words, not the Self element type" + reason = "u32 here is the stream word type, not the Self element type" )] impl FastPForInt for u64 { - const WIDTH: u8 = 64; - const BITMAP_WORDS: u32 = 2; const ZERO: Self = 0; + const ONE: Self = 1; type ExceptionBuffers = [Vec; u64::BITS as usize + 1]; type Freqs = [u32; u64::BITS as usize + 1]; @@ -168,27 +150,7 @@ impl FastPForInt for u64 { #[inline] fn significant_bits(self) -> u8 { - (64 - self.leading_zeros()) as u8 - } - #[inline] - fn shr(self, n: u8) -> Self { - self >> n - } - #[inline] - fn is_zero(self) -> bool { - self == 0 - } - #[inline] - fn or_shl_assign(dst: &mut Self, val: Self, shift: u8) { - *dst |= val << shift; - } - #[inline] - fn one_shl(shift: u8) -> Self { - 1 << shift - } - #[inline] - fn nth_bit_set(self, n: u8) -> bool { - (self >> n) & 1 != 0 + Self::WIDTH - self.leading_zeros() as u8 } #[inline] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { From 9b3fbed927405e73ef324ac6652549a01a02c23a Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 17 Jul 2026 22:00:50 -0400 Subject: [PATCH 16/23] wip --- src/rust/integer_compression/fastpfor_int.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs index 0932465..90301cb 100644 --- a/src/rust/integer_compression/fastpfor_int.rs +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -108,11 +108,15 @@ impl FastPForInt for u32 { fn significant_bits(self) -> u8 { Self::WIDTH - self.leading_zeros() as u8 } - #[inline] + // `inline(always)`: this is a thin forwarder; without it the wrapper accumulates the whole + // inlined kernel and then exceeds the inline threshold, so `decode_page`/`encode_page` would + // emit a real call per 32-value group instead of inlining the kernel (as the concrete `u32` + // code on `main` does). See the packing-kernel benchmarks. + #[inline(always)] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { bitpacking::fast_pack(src, inpos, out, outpos, bit); } - #[inline] + #[inline(always)] fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { bitunpacking::fast_unpack(src, inpos, out, outpos, bit); } @@ -152,11 +156,12 @@ impl FastPForInt for u64 { fn significant_bits(self) -> u8 { Self::WIDTH - self.leading_zeros() as u8 } - #[inline] + // `inline(always)`: forward directly to the wide kernel (see the `u32` impl for rationale). + #[inline(always)] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { bitpacking_wide::pack_wide(src, inpos, out, outpos, bit); } - #[inline] + #[inline(always)] fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { bitpacking_wide::unpack_wide(src, inpos, out, outpos, bit); } From f68089f8a13430adeaec848827e6721dc7dc13b7 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 17 Jul 2026 23:00:14 -0400 Subject: [PATCH 17/23] refactor more to per-bit files --- src/lib.rs | 2 +- src/rust/fastpfor_codec.rs | 6 +- .../{bitpacking.rs => bit_pack32.rs} | 2 +- .../{bitpacking_wide.rs => bit_pack64.rs} | 8 +- .../{bitunpacking.rs => bit_unpack32.rs} | 0 src/rust/integer_compression/fastpfor.rs | 236 +----------------- src/rust/integer_compression/fastpfor32.rs | 235 +++++++++++++++++ src/rust/integer_compression/fastpfor64.rs | 25 +- src/rust/integer_compression/fastpfor_int.rs | 10 +- src/rust/integer_compression/mod.rs | 7 +- src/rust/mod.rs | 4 +- 11 files changed, 272 insertions(+), 263 deletions(-) rename src/rust/integer_compression/{bitpacking.rs => bit_pack32.rs} (99%) rename src/rust/integer_compression/{bitpacking_wide.rs => bit_pack64.rs} (93%) rename src/rust/integer_compression/{bitunpacking.rs => bit_unpack32.rs} (100%) create mode 100644 src/rust/integer_compression/fastpfor32.rs diff --git a/src/lib.rs b/src/lib.rs index 62325d0..0133bf2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,7 +29,7 @@ pub use bytemuck::Pod; #[cfg(feature = "rust")] pub use rust::{ CompositeCodec, FastPFor, FastPFor128, FastPFor256, FastPForBlock128, FastPForBlock256, - JustCopy, VariableByte, + FastPForBlockWide128, FastPForBlockWide256, JustCopy, VariableByte, }; // `src/test_utils.rs` uses `fastpfor::...`; alias this crate for unit tests only. diff --git a/src/rust/fastpfor_codec.rs b/src/rust/fastpfor_codec.rs index 85e6c44..4a2f7b0 100644 --- a/src/rust/fastpfor_codec.rs +++ b/src/rust/fastpfor_codec.rs @@ -8,8 +8,8 @@ use crate::FastPForResult; use crate::codec::{AnyLenCodec, BlockCodec64}; use crate::rust::VariableByte; use crate::rust::composite::CompositeCodec; -use crate::rust::integer_compression::fastpfor::{FastPForBlock128, FastPForBlock256}; -use crate::rust::integer_compression::fastpfor64::FastPForWide; +use crate::rust::integer_compression::fastpfor::FastPFor; +use crate::rust::integer_compression::fastpfor32::{FastPForBlock128, FastPForBlock256}; macro_rules! define_fastpfor { ($(#[$meta:meta])* $name:ident, $block:ty, $n:literal) => { @@ -17,7 +17,7 @@ macro_rules! define_fastpfor { #[derive(Debug, Default)] pub struct $name { narrow: CompositeCodec<$block, VariableByte>, - wide: FastPForWide<$n>, + wide: FastPFor<$n, u64>, } impl AnyLenCodec for $name { diff --git a/src/rust/integer_compression/bitpacking.rs b/src/rust/integer_compression/bit_pack32.rs similarity index 99% rename from src/rust/integer_compression/bitpacking.rs rename to src/rust/integer_compression/bit_pack32.rs index d911113..8a7b839 100644 --- a/src/rust/integer_compression/bitpacking.rs +++ b/src/rust/integer_compression/bit_pack32.rs @@ -1305,7 +1305,7 @@ mod tests { use rand::RngExt as _; use super::fast_pack; - use crate::rust::integer_compression::bitunpacking::fast_unpack; + use crate::rust::integer_compression::bit_unpack32::fast_unpack; #[test] fn pack_unpack_roundtrip() { diff --git a/src/rust/integer_compression/bitpacking_wide.rs b/src/rust/integer_compression/bit_pack64.rs similarity index 93% rename from src/rust/integer_compression/bitpacking_wide.rs rename to src/rust/integer_compression/bit_pack64.rs index d3ae71f..597785c 100644 --- a/src/rust/integer_compression/bitpacking_wide.rs +++ b/src/rust/integer_compression/bit_pack64.rs @@ -1,7 +1,7 @@ //! Generic scalar bit-packing for 64-bit values. //! //! Packs and unpacks groups of 32 values at any bit width `0..=64`. -//! The layout is a little-endian bitstream, matching the hand-unrolled 32-bit kernels in [`bitpacking`](super::bitpacking). +//! The layout is a little-endian bitstream, matching the hand-unrolled 32-bit kernels in [`bitpacking`](super::bit_pack32). //! Value `j` occupies bits `[j*bit, (j+1)*bit)` of the concatenated stream. //! Each call moves exactly `bit` `u32` words. @@ -59,7 +59,7 @@ pub fn unpack_wide(input: &[u32], inpos: usize, output: &mut [u64], outpos: usiz #[cfg(test)] mod tests { use super::*; - use crate::rust::integer_compression::{bitpacking, bitunpacking}; + use crate::rust::integer_compression::{bit_pack32, bit_unpack32}; #[test] fn wide_matches_u32_kernels() { @@ -75,7 +75,7 @@ mod tests { let masked64: [u64; 32] = std::array::from_fn(|i| u64::from(masked32[i])); let mut out_ref = vec![0u32; bit as usize]; - bitpacking::fast_pack(&masked32, 0, &mut out_ref, 0, bit); + bit_pack32::fast_pack(&masked32, 0, &mut out_ref, 0, bit); let mut out_wide = vec![0u32; bit as usize]; pack_wide(&masked64, 0, &mut out_wide, 0, bit); @@ -83,7 +83,7 @@ mod tests { assert_eq!(out_ref, out_wide, "pack mismatch at bit={bit}"); let mut back_ref = vec![0u32; 32]; - bitunpacking::fast_unpack(&out_ref, 0, &mut back_ref, 0, bit); + bit_unpack32::fast_unpack(&out_ref, 0, &mut back_ref, 0, bit); let mut back_wide = vec![0u64; 32]; unpack_wide(&out_wide, 0, &mut back_wide, 0, bit); diff --git a/src/rust/integer_compression/bitunpacking.rs b/src/rust/integer_compression/bit_unpack32.rs similarity index 100% rename from src/rust/integer_compression/bitunpacking.rs rename to src/rust/integer_compression/bit_unpack32.rs diff --git a/src/rust/integer_compression/fastpfor.rs b/src/rust/integer_compression/fastpfor.rs index a23010a..e82fe66 100644 --- a/src/rust/integer_compression/fastpfor.rs +++ b/src/rust/integer_compression/fastpfor.rs @@ -4,12 +4,12 @@ use std::io::Cursor; use bytemuck::cast_slice; use bytes::{Buf as _, BufMut as _, BytesMut}; -use crate::helpers::{AsUsize, GetWithErr, greatest_multiple}; +use crate::helpers::{GetWithErr, greatest_multiple}; use crate::rust::cursor::IncrementCursor; use crate::rust::integer_compression::fastpfor_int::FastPForInt; -use crate::{BlockCodec, FastPForError, FastPForResult}; +use crate::{FastPForError, FastPForResult}; -mod sealed { +pub(crate) mod sealed { /// Sealed marker trait: only valid `[T; N]` block arrays are accepted by `FastPFor`. /// /// This is intentionally private so that users cannot implement it for other sizes, @@ -27,12 +27,6 @@ const OVERHEAD_OF_EACH_EXCEPT: u32 = 8; /// Default page size in number of integers. const DEFAULT_PAGE_SIZE: u32 = 65536; -/// Type alias for [`FastPFor`] with 128-element `u32` blocks. -pub type FastPForBlock128 = FastPFor<128, u32>; - -/// Type alias for [`FastPFor`] with 256-element `u32` blocks. -pub type FastPForBlock256 = FastPFor<256, u32>; - /// Fast Patched Frame-of-Reference ([FastPFOR](https://github.com/lemire/FastPFor)) codec. /// /// `N` is the block size (128 or 256 values per block) and `T` the element type @@ -57,7 +51,7 @@ pub type FastPForBlock256 = FastPFor<256, u32>; /// codec.encode(&data, &mut out).unwrap(); /// ``` #[derive(Debug)] -pub struct FastPFor { +pub struct FastPFor { /// Exception values indexed by bit width difference exception_buffers: T::ExceptionBuffers, /// Metadata buffer for encoding/decoding @@ -494,225 +488,3 @@ impl FastPFor { Ok(()) } } - -impl BlockCodec for FastPFor -where - [u32; N]: sealed::BlockSize, -{ - type Block = [u32; N]; - - fn encode_blocks(&mut self, blocks: &[Self::Block], out: &mut Vec) -> FastPForResult<()> { - let n_values = (blocks.len() * N) as u32; - if blocks.is_empty() { - out.push(n_values); - return Ok(()); - } - let flat: &[u32] = cast_slice(blocks); - - let capacity = flat.len() * 2 + 1024; - let start = out.len(); - // Reserve slot for the length header, then space for compressed data. - out.resize(start + 1 + capacity, 0); - - let mut in_off = Cursor::new(0u32); - let mut out_off = Cursor::new(0u32); - - // Write length header then compress. - out[start] = n_values; - self.compress_blocks( - flat, - n_values, - &mut in_off, - &mut out[start + 1..], - &mut out_off, - ); - - let written = 1 + out_off.position() as usize; - out.truncate(start + written); - Ok(()) - } - - fn decode_blocks( - &mut self, - input: &[u32], - expected_len: Option, - out: &mut Vec, - ) -> FastPForResult { - let Some((&block_n_values, rest)) = input.split_first() else { - return Err(FastPForError::NotEnoughData); - }; - if block_n_values % N as u32 != 0 { - return Err(FastPForError::NotEnoughData); - } - if let Some(expected) = expected_len { - if block_n_values != expected { - return Err(FastPForError::DecodedCountMismatch { - actual: block_n_values.as_usize(), - expected: expected.as_usize(), - }); - } - } else { - let max = Self::max_decompressed_len(input.len()); - if block_n_values.as_usize() > max { - return Err(FastPForError::NotEnoughData); - } - } - let n_blocks = block_n_values as usize / N; - if n_blocks == 0 { - return Ok(1); - } - let start = out.len(); - out.resize(start + n_blocks * N, 0); - - let mut in_off = Cursor::new(0u32); - let mut out_off = Cursor::new(0u32); - - self.decode_headless_blocks( - rest, - block_n_values, - &mut in_off, - &mut out[start..], - &mut out_off, - )?; - - let written = out_off.position() as usize; - if written != n_blocks * N { - out.truncate(start + written); - } - // +1 for the header word (block_n_values) that precedes `rest`. - Ok(1 + in_off.position() as usize) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{block_compress, block_decompress, block_roundtrip}; - - #[test] - fn fastpfor_test() { - let mut data = vec![0u32; 256]; - data[126] = u32::MAX; - block_roundtrip::(&data); - } - - #[test] - fn fastpfor_test_128() { - let mut data = vec![0u32; 128]; - data[126] = u32::MAX; - block_roundtrip::(&data); - } - - #[test] - fn test_empty_blocks_ok() { - // Empty input encodes to length header [0] (matches C++ FastPFor) and decodes cleanly. - let enc = block_compress::(&[]).unwrap(); - assert_eq!(enc, [0]); - let dec = block_decompress::(&enc, Some(0)).unwrap(); - assert!(dec.is_empty()); - } - - // Tests ported from C++ - #[test] - fn test_constant_sequence() { - block_roundtrip::(&vec![42u32; 65536]); - } - - #[test] - fn test_alternating_sequence() { - let data: Vec<_> = (0..65536u32).map(|i| u32::from(i % 2 != 0)).collect(); - block_roundtrip::(&data); - } - - #[test] - fn test_large_numbers() { - let data: Vec = (0..65536u32).map(|i| i + (1u32 << 30)).collect(); - block_roundtrip::(&data); - } - - #[test] - fn cursor_api_roundtrip() { - block_roundtrip::(&vec![42u32; 256]); - } - - #[test] - fn headless_compress_unfit_pagesize() { - // 640 values with 128-block codec spans two pages (512 + 128), exercising the loop. - let input: Vec = (0..640u32).collect(); - block_roundtrip::(&input); - } - - #[test] - fn exception_value_vector_resizes() { - // Alternating large/small values trigger exception-buffer resizing across pages. - let input: Vec = (0..1024u32) - .map(|i| if i % 2 == 0 { 1 << 30 } else { 3 }) - .collect(); - block_roundtrip::(&input); - } - - // ── Error / edge tests not covered by `tests/decode_validation.rs` ───── - // - // `AnyLenCodec::decode` treats an empty slice as tail-only and succeeds; an empty - // `decode_blocks` input is still invalid. Headless decode is internal-only. - - #[test] - fn uncompress_zero_input_length_err() { - // Truly empty input (no header word at all) is invalid — C++ would crash reading *in. - block_decompress::(&[], None).unwrap_err(); - } - - #[test] - fn headless_uncompress_zero_inlength_128_ok() { - FastPForBlock128::default() - .decode_headless_blocks( - &[], - 0, - &mut Cursor::new(0u32), - &mut [], - &mut Cursor::new(0u32), - ) - .expect("zero-length decompress must succeed"); - } - - #[test] - fn decode_where_meta_overflow() { - // `decode_headless_blocks` only: no `AnyLenCodec` entry point passes this layout. - let data: Vec = (0..256u32) - .map(|i| if i % 2 == 0 { 1u32 << 30 } else { 3 }) - .collect(); - let compressed = block_compress::(&data).unwrap(); - - let mut padded = vec![0u32]; - padded.extend_from_slice(&compressed); - padded[2] = u32::MAX; - let out_length = padded[1]; - assert!( - FastPForBlock256::default() - .decode_headless_blocks( - &padded, - out_length, - &mut Cursor::new(1u32), - &mut vec![0u32; 320], - &mut Cursor::new(0u32), - ) - .is_err() - ); - } - - #[test] - fn decode_index1_branch_valid() { - let mut data = vec![1u32; 256]; - data[0] = 3; - block_roundtrip::(&data); - } - - /// `decode_blocks` with `expected_len: None` and header=0 returns `Ok` with empty output. - #[test] - fn decode_blocks_header_only_input() { - // Input with just the length header [0]: no blocks to decode. - let input = vec![0u32]; - let out = block_decompress::(&input, None).unwrap(); - assert!(out.is_empty()); - } -} diff --git a/src/rust/integer_compression/fastpfor32.rs b/src/rust/integer_compression/fastpfor32.rs new file mode 100644 index 0000000..9fe4672 --- /dev/null +++ b/src/rust/integer_compression/fastpfor32.rs @@ -0,0 +1,235 @@ +use std::io::Cursor; + +use bytemuck::cast_slice; + +use crate::helpers::AsUsize; +use crate::rust::integer_compression::fastpfor::sealed; +use crate::{BlockCodec, FastPFor, FastPForError, FastPForResult}; + +/// Type alias for [`FastPFor`] with 128-element `u32` blocks. +pub type FastPForBlock128 = FastPFor<128, u32>; + +/// Type alias for [`FastPFor`] with 256-element `u32` blocks. +pub type FastPForBlock256 = FastPFor<256, u32>; + +impl BlockCodec for FastPFor +where + [u32; N]: sealed::BlockSize, +{ + type Block = [u32; N]; + + fn encode_blocks(&mut self, blocks: &[Self::Block], out: &mut Vec) -> FastPForResult<()> { + let n_values = (blocks.len() * N) as u32; + if blocks.is_empty() { + out.push(n_values); + return Ok(()); + } + let flat: &[u32] = cast_slice(blocks); + + let capacity = flat.len() * 2 + 1024; + let start = out.len(); + // Reserve slot for the length header, then space for compressed data. + out.resize(start + 1 + capacity, 0); + + let mut in_off = Cursor::new(0u32); + let mut out_off = Cursor::new(0u32); + + // Write length header then compress. + out[start] = n_values; + self.compress_blocks( + flat, + n_values, + &mut in_off, + &mut out[start + 1..], + &mut out_off, + ); + + let written = 1 + out_off.position() as usize; + out.truncate(start + written); + Ok(()) + } + + fn decode_blocks( + &mut self, + input: &[u32], + expected_len: Option, + out: &mut Vec, + ) -> FastPForResult { + let Some((&block_n_values, rest)) = input.split_first() else { + return Err(FastPForError::NotEnoughData); + }; + if block_n_values % N as u32 != 0 { + return Err(FastPForError::NotEnoughData); + } + if let Some(expected) = expected_len { + if block_n_values != expected { + return Err(FastPForError::DecodedCountMismatch { + actual: block_n_values.as_usize(), + expected: expected.as_usize(), + }); + } + } else { + let max = Self::max_decompressed_len(input.len()); + if block_n_values.as_usize() > max { + return Err(FastPForError::NotEnoughData); + } + } + let n_blocks = block_n_values as usize / N; + if n_blocks == 0 { + return Ok(1); + } + let start = out.len(); + out.resize(start + n_blocks * N, 0); + + let mut in_off = Cursor::new(0u32); + let mut out_off = Cursor::new(0u32); + + self.decode_headless_blocks( + rest, + block_n_values, + &mut in_off, + &mut out[start..], + &mut out_off, + )?; + + let written = out_off.position() as usize; + if written != n_blocks * N { + out.truncate(start + written); + } + // +1 for the header word (block_n_values) that precedes `rest`. + Ok(1 + in_off.position() as usize) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{block_compress, block_decompress, block_roundtrip}; + + #[test] + fn fastpfor_test() { + let mut data = vec![0u32; 256]; + data[126] = u32::MAX; + block_roundtrip::(&data); + } + + #[test] + fn fastpfor_test_128() { + let mut data = vec![0u32; 128]; + data[126] = u32::MAX; + block_roundtrip::(&data); + } + + #[test] + fn test_empty_blocks_ok() { + // Empty input encodes to length header [0] (matches C++ FastPFor) and decodes cleanly. + let enc = block_compress::(&[]).unwrap(); + assert_eq!(enc, [0]); + let dec = block_decompress::(&enc, Some(0)).unwrap(); + assert!(dec.is_empty()); + } + + // Tests ported from C++ + #[test] + fn test_constant_sequence() { + block_roundtrip::(&vec![42u32; 65536]); + } + + #[test] + fn test_alternating_sequence() { + let data: Vec<_> = (0..65536u32).map(|i| u32::from(i % 2 != 0)).collect(); + block_roundtrip::(&data); + } + + #[test] + fn test_large_numbers() { + let data: Vec = (0..65536u32).map(|i| i + (1u32 << 30)).collect(); + block_roundtrip::(&data); + } + + #[test] + fn cursor_api_roundtrip() { + block_roundtrip::(&vec![42u32; 256]); + } + + #[test] + fn headless_compress_unfit_pagesize() { + // 640 values with 128-block codec spans two pages (512 + 128), exercising the loop. + let input: Vec = (0..640u32).collect(); + block_roundtrip::(&input); + } + + #[test] + fn exception_value_vector_resizes() { + // Alternating large/small values trigger exception-buffer resizing across pages. + let input: Vec = (0..1024u32) + .map(|i| if i % 2 == 0 { 1 << 30 } else { 3 }) + .collect(); + block_roundtrip::(&input); + } + + // ── Error / edge tests not covered by `tests/decode_validation.rs` ───── + // + // `AnyLenCodec::decode` treats an empty slice as tail-only and succeeds; an empty + // `decode_blocks` input is still invalid. Headless decode is internal-only. + + #[test] + fn uncompress_zero_input_length_err() { + // Truly empty input (no header word at all) is invalid — C++ would crash reading *in. + block_decompress::(&[], None).unwrap_err(); + } + + #[test] + fn headless_uncompress_zero_inlength_128_ok() { + FastPForBlock128::default() + .decode_headless_blocks( + &[], + 0, + &mut Cursor::new(0u32), + &mut [], + &mut Cursor::new(0u32), + ) + .expect("zero-length decompress must succeed"); + } + + #[test] + fn decode_where_meta_overflow() { + // `decode_headless_blocks` only: no `AnyLenCodec` entry point passes this layout. + let data: Vec = (0..256u32) + .map(|i| if i % 2 == 0 { 1u32 << 30 } else { 3 }) + .collect(); + let compressed = block_compress::(&data).unwrap(); + + let mut padded = vec![0u32]; + padded.extend_from_slice(&compressed); + padded[2] = u32::MAX; + let out_length = padded[1]; + assert!( + FastPForBlock256::default() + .decode_headless_blocks( + &padded, + out_length, + &mut Cursor::new(1u32), + &mut vec![0u32; 320], + &mut Cursor::new(0u32), + ) + .is_err() + ); + } + + #[test] + fn decode_index1_branch_valid() { + let mut data = vec![1u32; 256]; + data[0] = 3; + block_roundtrip::(&data); + } + + /// `decode_blocks` with `expected_len: None` and header=0 returns `Ok` with empty output. + #[test] + fn decode_blocks_header_only_input() { + // Input with just the length header [0]: no blocks to decode. + let input = vec![0u32]; + let out = block_decompress::(&input, None).unwrap(); + assert!(out.is_empty()); + } +} diff --git a/src/rust/integer_compression/fastpfor64.rs b/src/rust/integer_compression/fastpfor64.rs index 1b66f6b..411b6c6 100644 --- a/src/rust/integer_compression/fastpfor64.rs +++ b/src/rust/integer_compression/fastpfor64.rs @@ -4,8 +4,8 @@ //! Values, exceptions, and the exception bitmap are 64 bits wide instead of 32. //! The output is byte-compatible with the C++ `CppFastPFor128` / `CppFastPFor256` 64-bit paths. //! -//! [`FastPForWide`] is the 64-bit half of the public [`FastPFor128`](crate::FastPFor128) / -//! [`FastPFor256`](crate::FastPFor256) codecs and is not exported on its own. +//! [`FastPForBlockWide128`]/[`FastPForBlockWide256`] are the 64-bit half of the public [`FastPFor128`](crate::FastPFor128) / +//! [`FastPFor256`](crate::FastPFor256) codecs. //! It handles complete blocks, then a [`VariableByte`] tail encodes the sub-block remainder. use std::io::Cursor; @@ -17,12 +17,11 @@ use crate::helpers::AsUsize; use crate::rust::integer_compression::fastpfor::FastPFor; use crate::{BlockCodec64, FastPForError, FastPForResult}; -/// 64-bit `FastPFOR` codec: `FastPFOR`-packed blocks plus a variable-byte tail. -/// -/// `N` is the block size (128 or 256 values). This is [`FastPFor`] specialized to the -/// `u64` element type, and is the internal `u64` codec behind -/// [`FastPFor128`](crate::FastPFor128) and [`FastPFor256`](crate::FastPFor256). -pub type FastPForWide = FastPFor; +/// Type alias for [`FastPFor`] with 128-element `u64` blocks. +pub type FastPForBlockWide128 = FastPFor<128, u64>; + +/// Type alias for [`FastPFor`] with 256-element `u64` blocks. +pub type FastPForBlockWide256 = FastPFor<256, u64>; /// Variable-byte encoding of the `u64` tail. /// @@ -89,7 +88,7 @@ fn vbyte_decode64(input: &[u32], out: &mut Vec) -> FastPForResult<()> { Ok(()) } -impl BlockCodec64 for FastPForWide { +impl BlockCodec64 for FastPFor { fn encode64(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()> { let rounded = (input.len() / N) * N; let n_values = rounded as u32; @@ -158,7 +157,7 @@ mod tests { use super::*; fn roundtrip(input: &[u64]) { - let mut codec = FastPForWide::::default(); + let mut codec = FastPFor::::default(); let mut encoded = Vec::new(); codec.encode64(input, &mut encoded).unwrap(); let mut decoded = Vec::new(); @@ -239,7 +238,7 @@ mod tests { ] } - fn assert_parity(rust: &mut FastPForWide, cpp: &mut impl BlockCodec64) { + fn assert_parity(rust: &mut FastPFor, cpp: &mut impl BlockCodec64) { for data in cases() { let mut rust_enc = Vec::new(); rust.encode64(&data, &mut rust_enc).unwrap(); @@ -260,7 +259,7 @@ mod tests { #[test] fn parity_128() { assert_parity( - &mut FastPForWide::<128>::default(), + &mut FastPFor::<128, u64>::default(), &mut CppFastPFor128::default(), ); } @@ -268,7 +267,7 @@ mod tests { #[test] fn parity_256() { assert_parity( - &mut FastPForWide::<256>::default(), + &mut FastPFor::<256, u64>::default(), &mut CppFastPFor256::default(), ); } diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs index 90301cb..7877314 100644 --- a/src/rust/integer_compression/fastpfor_int.rs +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -18,7 +18,7 @@ use std::fmt::Debug; use std::ops::{BitAnd, BitOrAssign, Index, IndexMut, Shl, Shr}; use crate::helpers::GetWithErr; -use crate::rust::integer_compression::{bitpacking, bitpacking_wide, bitunpacking}; +use crate::rust::integer_compression::{bit_pack32, bit_pack64, bit_unpack32}; use crate::{FastPForError, FastPForResult}; mod sealed { @@ -114,11 +114,11 @@ impl FastPForInt for u32 { // code on `main` does). See the packing-kernel benchmarks. #[inline(always)] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { - bitpacking::fast_pack(src, inpos, out, outpos, bit); + bit_pack32::fast_pack(src, inpos, out, outpos, bit); } #[inline(always)] fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { - bitunpacking::fast_unpack(src, inpos, out, outpos, bit); + bit_unpack32::fast_unpack(src, inpos, out, outpos, bit); } #[inline] fn write_bitmap(bitmap: Self, out: &mut [u32]) { @@ -159,11 +159,11 @@ impl FastPForInt for u64 { // `inline(always)`: forward directly to the wide kernel (see the `u32` impl for rationale). #[inline(always)] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { - bitpacking_wide::pack_wide(src, inpos, out, outpos, bit); + bit_pack64::pack_wide(src, inpos, out, outpos, bit); } #[inline(always)] fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { - bitpacking_wide::unpack_wide(src, inpos, out, outpos, bit); + bit_pack64::unpack_wide(src, inpos, out, outpos, bit); } #[inline] fn write_bitmap(bitmap: Self, out: &mut [u32]) { diff --git a/src/rust/integer_compression/mod.rs b/src/rust/integer_compression/mod.rs index 476f6e9..7de1cc6 100644 --- a/src/rust/integer_compression/mod.rs +++ b/src/rust/integer_compression/mod.rs @@ -1,7 +1,8 @@ -pub mod bitpacking; -pub mod bitpacking_wide; -pub mod bitunpacking; +pub mod bit_pack32; +pub mod bit_pack64; +pub mod bit_unpack32; pub mod fastpfor; +pub mod fastpfor32; pub mod fastpfor64; pub mod fastpfor_int; pub mod just_copy; diff --git a/src/rust/mod.rs b/src/rust/mod.rs index b889a19..ad1b448 100644 --- a/src/rust/mod.rs +++ b/src/rust/mod.rs @@ -7,7 +7,9 @@ pub use composite::CompositeCodec; /// Any-length `FastPFOR` codecs supporting both `u32` and `u64`. pub use fastpfor_codec::{FastPFor128, FastPFor256}; /// Type-safe block codec with block size encoded in the type. -pub use integer_compression::fastpfor::{FastPFor, FastPForBlock128, FastPForBlock256}; +pub use integer_compression::fastpfor::FastPFor; +pub use integer_compression::fastpfor32::{FastPForBlock128, FastPForBlock256}; +pub use integer_compression::fastpfor64::{FastPForBlockWide128, FastPForBlockWide256}; /// Pass-through codec — implements [`AnyLenCodec`](crate::codec::AnyLenCodec). pub use integer_compression::just_copy::JustCopy; /// Variable-byte codec — implements [`AnyLenCodec`](crate::codec::AnyLenCodec). From 1df5c8c82743730743e05282af2defb0a6930844 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sat, 18 Jul 2026 13:07:02 +0200 Subject: [PATCH 18/23] fix: allow inline_always on fast_pack/fast_unpack forwarders The pedantic `clippy::inline_always` lint was failing CI on the thin forwarders in the `FastPForInt` impls. The `#[inline(always)]` is intentional (documented above each fn); add an explicit `#[allow]` with reason so clippy stops flagging it. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/rust/integer_compression/fastpfor_int.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs index 7877314..77c27d0 100644 --- a/src/rust/integer_compression/fastpfor_int.rs +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -113,10 +113,12 @@ impl FastPForInt for u32 { // emit a real call per 32-value group instead of inlining the kernel (as the concrete `u32` // code on `main` does). See the packing-kernel benchmarks. #[inline(always)] + #[allow(clippy::inline_always, reason = "thin forwarder; see comment above")] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { bit_pack32::fast_pack(src, inpos, out, outpos, bit); } #[inline(always)] + #[allow(clippy::inline_always, reason = "thin forwarder; see comment above")] fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { bit_unpack32::fast_unpack(src, inpos, out, outpos, bit); } @@ -158,10 +160,12 @@ impl FastPForInt for u64 { } // `inline(always)`: forward directly to the wide kernel (see the `u32` impl for rationale). #[inline(always)] + #[allow(clippy::inline_always, reason = "thin forwarder; see comment above")] fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8) { bit_pack64::pack_wide(src, inpos, out, outpos, bit); } #[inline(always)] + #[allow(clippy::inline_always, reason = "thin forwarder; see comment above")] fn fast_unpack(src: &[u32], inpos: usize, out: &mut [Self], outpos: usize, bit: u8) { bit_pack64::unpack_wide(src, inpos, out, outpos, bit); } From dadcb057e0148dc909af7c5956832f756475edfa Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sat, 18 Jul 2026 13:14:29 +0200 Subject: [PATCH 19/23] fix: repair broken rustdoc intra-doc links on FastPFor CI's `just docs` denies rustdoc warnings. Fix the four broken links on the `FastPFor` doc comment: - `BlockCodec`, `FastPForBlock128`, `FastPForBlock256` now use explicit `crate::` paths (they're re-exported at the crate root, not in scope in this module). - `FastPForInt` is a private sealed trait, so drop the intra-doc link and leave it as plain code. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/rust/integer_compression/fastpfor.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/rust/integer_compression/fastpfor.rs b/src/rust/integer_compression/fastpfor.rs index e82fe66..999da15 100644 --- a/src/rust/integer_compression/fastpfor.rs +++ b/src/rust/integer_compression/fastpfor.rs @@ -30,15 +30,16 @@ const DEFAULT_PAGE_SIZE: u32 = 65536; /// Fast Patched Frame-of-Reference ([FastPFOR](https://github.com/lemire/FastPFor)) codec. /// /// `N` is the block size (128 or 256 values per block) and `T` the element type -/// ([`u32`] or [`u64`], defaulting to `u32`). This struct implements [`BlockCodec`] with -/// `Block = [u32; N]` for the `u32` element type, giving compile-time guarantees that only +/// ([`u32`] or [`u64`], defaulting to `u32`). This struct implements [`BlockCodec`](crate::BlockCodec) +/// with `Block = [u32; N]` for the `u32` element type, giving compile-time guarantees that only /// correctly-sized blocks are accepted. /// /// The per-block scratch buffers are sized exactly for `T` (`T::WIDTH + 1` buckets) via the -/// sealed [`FastPForInt`] trait, so the bucket count is neither wasted nor part of this +/// sealed `FastPForInt` trait, so the bucket count is neither wasted nor part of this /// type's signature. /// -/// Use [`FastPForBlock128`] or [`FastPForBlock256`] as convenient `u32` type aliases. +/// Use [`FastPForBlock128`](crate::FastPForBlock128) or [`FastPForBlock256`](crate::FastPForBlock256) +/// as convenient `u32` type aliases. /// /// To compress arbitrary-length data (including a sub-block remainder), /// wrap this in a [`CompositeCodec`](crate::CompositeCodec): From 6290d3d3682e2fc558e99d558d111eb503bc7456 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sat, 18 Jul 2026 15:12:14 +0200 Subject: [PATCH 20/23] refactor: width-generic codec family, split u32/u64 by type Collapse the two-field FastPFor codec into a single generic `FastPForCodec` with one `inner: CompositeCodec, VariableByte>`. - Give `BlockCodec`/`AnyLenCodec` an associated `Elem` so the block engine and composite are one width-generic implementation instead of duplicated u32/u64 paths. - Unify `VariableByte` into a generic `VariableByte` with u32 and u64 tail impls; drop the free `vbyte_encode64`/`vbyte_decode64` functions and `VariableByteWide`. - Split the public codec by width: `FastPFor128`/`FastPFor256` are u32; add `FastPForWide128`/`FastPForWide256` for u64. Each type serves one width, so it holds one codec. - Delete the redundant `BlockCodec64 for FastPFor`; the u64 type keeps a delegating `BlockCodec64` impl for C++ parity comparison. Wire format unchanged: C++ u64 parity tests pass. All tests, clippy, rustfmt, and rustdoc clean under both default and `cpp` features. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EqJPh46kx6YzhwUjX6EPDn --- README.md | 37 ++-- fuzz/fuzz_targets/fastpfor_u64.rs | 10 +- src/codec.rs | 43 +++-- src/cpp/codecs.rs | 2 + src/lib.rs | 3 +- src/rust/composite.rs | 18 +- src/rust/fastpfor_codec.rs | 150 +++++++++------ src/rust/integer_compression/fastpfor32.rs | 16 +- src/rust/integer_compression/fastpfor64.rs | 176 +++--------------- src/rust/integer_compression/fastpfor_int.rs | 1 + src/rust/integer_compression/just_copy.rs | 2 + src/rust/integer_compression/variable_byte.rs | 104 +++++++++-- src/rust/mod.rs | 4 +- src/test_utils.rs | 33 ++-- 14 files changed, 305 insertions(+), 294 deletions(-) diff --git a/README.md b/README.md index e62c78c..809d81f 100644 --- a/README.md +++ b/README.md @@ -60,21 +60,22 @@ assert_eq!(decoded, input); ### 64-bit integers (`u64`) -The same `FastPFor128` / `FastPFor256` codecs also compress `u64` values via the -`BlockCodec64` trait (`encode64` / `decode64`). The wire format is byte-compatible -with the C++ `CppFastPFor128` / `CppFastPFor256` 64-bit paths. +The `FastPForWide128` / `FastPForWide256` codecs compress `u64` values. +They implement `AnyLenCodec` (with `Elem = u64`) for native use, and `BlockCodec64` +(`encode64` / `decode64`) for comparison against the C++ codecs. +The wire format is byte-compatible with the C++ `CppFastPFor128` / `CppFastPFor256` 64-bit paths. ```rust -use fastpfor::{BlockCodec64, FastPFor256}; +use fastpfor::{AnyLenCodec, FastPForWide256}; -let mut codec = FastPFor256::default(); +let mut codec = FastPForWide256::default(); let input: Vec = (0..600).map(|i| i * 1_000_000_000).collect(); let mut encoded = Vec::new(); -codec.encode64(&input, &mut encoded).unwrap(); +codec.encode(&input, &mut encoded).unwrap(); let mut decoded = Vec::new(); -codec.decode64(&encoded, &mut decoded).unwrap(); +codec.decode(&encoded, &mut decoded, None).unwrap(); assert_eq!(decoded, input); ``` @@ -111,16 +112,18 @@ The `FASTPFOR_SIMD_MODE` environment variable (`portable` or `native`) can overr ### Rust (`rust` feature) -Rust block codecs require block-aligned input. `CompositeCodec` chains a block codec with a tail codec (e.g. `VariableByte`) to handle arbitrary-length input. `FastPFor256` and `FastPFor128` are type aliases for such composites. - -| Codec | Description | -|--------------------|--------------------------------------------------------------| -| `FastPFor256` | `CompositeCodec` of `FastPForBlock256` + `VariableByte` | -| `FastPFor128` | `CompositeCodec` of `FastPForBlock128` + `VariableByte` | -| `VariableByte` | Variable-byte encoding, MSB is opposite to protobuf's varint | -| `JustCopy` | No compression; useful as a baseline | -| `FastPForBlock256` | `FastPFor` with 256-element blocks; block-aligned input only | -| `FastPForBlock128` | `FastPFor` with 128-element blocks; block-aligned input only | +Rust block codecs require block-aligned input. `CompositeCodec` chains a block codec with a tail codec (e.g. `VariableByte`) to handle arbitrary-length input. `FastPFor256`/`FastPFor128` (for `u32`) and `FastPForWide256`/`FastPForWide128` (for `u64`) are type aliases for such composites. + +| Codec | Description | +|--------------------|-----------------------------------------------------------------| +| `FastPFor256` | `CompositeCodec` of `FastPForBlock256` + `VariableByte` (`u32`) | +| `FastPFor128` | `CompositeCodec` of `FastPForBlock128` + `VariableByte` (`u32`) | +| `FastPForWide256` | `CompositeCodec` of `FastPForBlockWide256` + `VariableByte` (`u64`) | +| `FastPForWide128` | `CompositeCodec` of `FastPForBlockWide128` + `VariableByte` (`u64`) | +| `VariableByte` | Variable-byte encoding, MSB is opposite to protobuf's varint | +| `JustCopy` | No compression; useful as a baseline | +| `FastPForBlock256` | `FastPFor` with 256-element `u32` blocks; block-aligned input only | +| `FastPForBlock128` | `FastPFor` with 128-element `u32` blocks; block-aligned input only | ### C++ (`cpp` feature) diff --git a/fuzz/fuzz_targets/fastpfor_u64.rs b/fuzz/fuzz_targets/fastpfor_u64.rs index caab43f..37a2eaf 100644 --- a/fuzz/fuzz_targets/fastpfor_u64.rs +++ b/fuzz/fuzz_targets/fastpfor_u64.rs @@ -1,7 +1,7 @@ #![no_main] use fastpfor::cpp::{CppFastPFor128, CppFastPFor256}; -use fastpfor::{BlockCodec64, FastPFor128, FastPFor256}; +use fastpfor::{BlockCodec64, FastPForWide128, FastPForWide256}; use libfuzzer_sys::fuzz_target; #[derive(arbitrary::Arbitrary, Debug)] @@ -43,17 +43,17 @@ fn check(rust: &mut impl BlockCodec64, cpp: &mut impl BlockCodec64, data: &[u64] fuzz_target!(|input: Input| { if input.use_256 { check( - &mut FastPFor256::default(), + &mut FastPForWide256::default(), &mut CppFastPFor256::default(), &input.data, - "FastPFor256", + "FastPForWide256", ); } else { check( - &mut FastPFor128::default(), + &mut FastPForWide128::default(), &mut CppFastPFor128::default(), &input.data, - "FastPFor128", + "FastPForWide128", ); } }); diff --git a/src/codec.rs b/src/codec.rs index 52e1ea7..087eddb 100644 --- a/src/codec.rs +++ b/src/codec.rs @@ -28,6 +28,7 @@ pub(crate) fn default_max_decoded_len(compressed_words: usize) -> usize { /// #[derive(Default)] /// struct MyCodec; /// impl BlockCodec for MyCodec { +/// type Elem = u32; /// type Block = [u32; 256]; /// fn encode_blocks(&mut self, blocks: &[[u32; 256]], out: &mut Vec) /// -> FastPForResult<()> { todo!() } @@ -36,21 +37,25 @@ pub(crate) fn default_max_decoded_len(compressed_words: usize) -> usize { /// } /// ``` pub trait BlockCodec: Default { - /// The fixed-size block type. Must be plain-old-data (`Pod`). - /// In practice this will be `[u32; 128]` or `[u32; 256]`. + /// The unpacked element type: [`u32`] or [`u64`]. + /// The compressed stream is always `Vec`; only decoded values use this width. + type Elem: Pod; + + /// The fixed-size block type, which must be plain-old-data (`Pod`). + /// In practice `[u32; 128]`, `[u32; 256]`, `[u64; 128]`, or `[u64; 256]`. type Block: Pod; - /// Number of `u32` elements in one block. + /// Number of [`Elem`](BlockCodec::Elem) values in one block. /// - /// Equal to `size_of::() / 4`. Use this when computing - /// element counts from block counts, e.g. `n_blocks * codec.elements_per_block()`. + /// Equal to `size_of::() / size_of::()`. + /// Use this when computing element counts from block counts. #[inline] #[must_use] fn size() -> usize where Self: Sized, { - size_of::() / size_of::() + size_of::() / size_of::() } /// Compress a slice of complete, fixed-size blocks. @@ -76,7 +81,7 @@ pub trait BlockCodec: Default { &mut self, input: &[u32], expected_len: Option, - out: &mut Vec, + out: &mut Vec, ) -> FastPForResult; /// Maximum decompressed element count for a given compressed input length. @@ -93,10 +98,12 @@ pub trait BlockCodec: Default { /// Codec that supports compressing 64-bit integers into a 32-bit word stream. /// -/// Implemented by the pure-Rust [`FastPFor128`](crate::FastPFor128) and [`FastPFor256`](crate::FastPFor256) codecs. +/// Implemented by the pure-Rust [`FastPForWide128`](crate::FastPForWide128) and +/// [`FastPForWide256`](crate::FastPForWide256) codecs. /// With the `cpp` feature, `CppFastPFor128`, `CppFastPFor256`, and `CppVarInt` also implement it. -/// For simple use, call `encode64` / `decode64` directly on the struct. /// +/// This is a shared interface for cross-codec comparison; for native Rust use, +/// [`FastPForWide128`](crate::FastPForWide128) also implements [`AnyLenCodec`] with `Elem = u64`. /// Import `BlockCodec64` only when writing generic code over several 64-bit codecs. pub trait BlockCodec64 { /// Compress 64-bit integers into a 32-bit word stream. @@ -112,8 +119,12 @@ pub trait BlockCodec64 { /// trait directly. Block-oriented codecs are wrapped in `CompositeCodec` /// to produce an `AnyLenCodec`. pub trait AnyLenCodec: Default { - /// Compress an arbitrary-length slice of `u32` values. - fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()>; + /// The unpacked element type: [`u32`] or [`u64`]. + /// The compressed stream is always `Vec`; only decoded values use this width. + type Elem; + + /// Compress an arbitrary-length slice of [`Elem`](AnyLenCodec::Elem) values. + fn encode(&mut self, input: &[Self::Elem], out: &mut Vec) -> FastPForResult<()>; /// Maximum decompressed element count for a given compressed input length. /// Reject `expected_len` values exceeding this to avoid allocation from bad data. @@ -138,7 +149,7 @@ pub trait AnyLenCodec: Default { fn decode( &mut self, input: &[u32], - out: &mut Vec, + out: &mut Vec, expected_len: Option, ) -> FastPForResult<()>; } @@ -162,9 +173,11 @@ pub trait AnyLenCodec: Default { /// assert_eq!(remainder.len(), 88); /// ``` #[must_use] -pub fn slice_to_blocks(input: &[u32]) -> (&[Blocks::Block], &[u32]) { - let block_u32s = Blocks::size(); - let aligned_down = (input.len() / block_u32s) * block_u32s; +pub fn slice_to_blocks( + input: &[Blocks::Elem], +) -> (&[Blocks::Block], &[Blocks::Elem]) { + let block_elems = Blocks::size(); + let aligned_down = (input.len() / block_elems) * block_elems; let (aligned, remainder) = input.split_at(aligned_down); let blocks: &[Blocks::Block] = cast_slice(aligned); // must not panic (blocks, remainder) diff --git a/src/cpp/codecs.rs b/src/cpp/codecs.rs index be18dcd..24bf91b 100644 --- a/src/cpp/codecs.rs +++ b/src/cpp/codecs.rs @@ -37,6 +37,8 @@ macro_rules! implement_cpp_codecs { } impl AnyLenCodec for $name { + type Elem = u32; + fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { encode32_to_vec_ffi(&self.0, input, out) } diff --git a/src/lib.rs b/src/lib.rs index 0133bf2..b7bd39e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,7 +29,8 @@ pub use bytemuck::Pod; #[cfg(feature = "rust")] pub use rust::{ CompositeCodec, FastPFor, FastPFor128, FastPFor256, FastPForBlock128, FastPForBlock256, - FastPForBlockWide128, FastPForBlockWide256, JustCopy, VariableByte, + FastPForBlockWide128, FastPForBlockWide256, FastPForWide128, FastPForWide256, JustCopy, + VariableByte, }; // `src/test_utils.rs` uses `fastpfor::...`; alias this crate for unit tests only. diff --git a/src/rust/composite.rs b/src/rust/composite.rs index 9bcd2fd..2501d69 100644 --- a/src/rust/composite.rs +++ b/src/rust/composite.rs @@ -41,26 +41,32 @@ use crate::helpers::AsUsize; /// assert_eq!(decoded, data); /// ``` #[derive(Debug)] -pub struct CompositeCodec { +pub struct CompositeCodec> { block: Blocks, tail: Tail, } -impl Default for CompositeCodec { +impl> Default + for CompositeCodec +{ fn default() -> Self { Self::new(Blocks::default(), Tail::default()) } } -impl CompositeCodec { +impl> CompositeCodec { /// Creates a new `CompositeCodec` from a block codec and a tail codec. pub fn new(block: Blocks, tail: Tail) -> Self { Self { block, tail } } } -impl AnyLenCodec for CompositeCodec { - fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { +impl> AnyLenCodec + for CompositeCodec +{ + type Elem = Blocks::Elem; + + fn encode(&mut self, input: &[Self::Elem], out: &mut Vec) -> FastPForResult<()> { let (blocks, remainder) = slice_to_blocks::(input); // C++ CompositeCodec: concatenate block + tail. Block codec writes length header (0 when empty). self.block.encode_blocks(blocks, out)?; @@ -71,7 +77,7 @@ impl AnyLenCodec for CompositeCodec, + out: &mut Vec, expected_len: Option, ) -> FastPForResult<()> { let start_len = out.len(); diff --git a/src/rust/fastpfor_codec.rs b/src/rust/fastpfor_codec.rs index 4a2f7b0..d2feb2f 100644 --- a/src/rust/fastpfor_codec.rs +++ b/src/rust/fastpfor_codec.rs @@ -1,86 +1,114 @@ -//! Public any-length `FastPFOR` codecs supporting both 32- and 64-bit integers. +//! Public any-length `FastPFOR` codecs. //! -//! [`FastPFor128`] and [`FastPFor256`] are the primary entry points. -//! Each implements [`AnyLenCodec`] for `u32` and [`BlockCodec64`] for `u64`. -//! Aligned blocks are coded with `FastPFOR` and the sub-block remainder with variable-byte coding. +//! [`FastPFor128`]/[`FastPFor256`] compress `u32`; [`FastPForWide128`]/[`FastPForWide256`] compress `u64`. +//! Each is one [`CompositeCodec`]: the width-generic block engine plus a variable-byte tail for the remainder. use crate::FastPForResult; use crate::codec::{AnyLenCodec, BlockCodec64}; use crate::rust::VariableByte; use crate::rust::composite::CompositeCodec; -use crate::rust::integer_compression::fastpfor::FastPFor; -use crate::rust::integer_compression::fastpfor32::{FastPForBlock128, FastPForBlock256}; +use crate::rust::integer_compression::fastpfor::{FastPFor, sealed}; +use crate::rust::integer_compression::fastpfor_int::FastPForInt; -macro_rules! define_fastpfor { - ($(#[$meta:meta])* $name:ident, $block:ty, $n:literal) => { - $(#[$meta])* - #[derive(Debug, Default)] - pub struct $name { - narrow: CompositeCodec<$block, VariableByte>, - wide: FastPFor<$n, u64>, - } - - impl AnyLenCodec for $name { - fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { - self.narrow.encode(input, out) - } +/// Any-length `FastPFOR` codec over `N`-value blocks of width `T` ([`u32`] or [`u64`]). +/// +/// A single [`CompositeCodec`] pairing the width-generic block engine with a [`VariableByte`] tail. +/// Instantiate through the [`FastPFor128`]/[`FastPForWide128`] aliases. +#[derive(Debug)] +pub struct FastPForCodec +where + [T; N]: sealed::BlockSize, + VariableByte: AnyLenCodec, +{ + inner: CompositeCodec, VariableByte>, +} - fn decode( - &mut self, - input: &[u32], - out: &mut Vec, - expected_len: Option, - ) -> FastPForResult<()> { - self.narrow.decode(input, out, expected_len) - } +// Hand-written (not derived) so `default()` needs no `T: Default` bound; +// the tail's `AnyLenCodec: Default` supertrait already guarantees it. +impl Default for FastPForCodec +where + [T; N]: sealed::BlockSize, + VariableByte: AnyLenCodec, +{ + fn default() -> Self { + Self { + inner: CompositeCodec::default(), } + } +} - impl BlockCodec64 for $name { - fn encode64(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()> { - self.wide.encode64(input, out) - } +impl AnyLenCodec for FastPForCodec +where + [T; N]: sealed::BlockSize, + VariableByte: AnyLenCodec, +{ + type Elem = T; - fn decode64(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { - self.wide.decode64(input, out) - } - } - }; -} + fn encode(&mut self, input: &[T], out: &mut Vec) -> FastPForResult<()> { + self.inner.encode(input, out) + } -define_fastpfor! { - /// Any-length `FastPFOR` codec with 128-value blocks. - /// - /// Compresses `u32` via [`AnyLenCodec`] and `u64` via [`BlockCodec64`]. - FastPFor128, FastPForBlock128, 128 + fn decode( + &mut self, + input: &[u32], + out: &mut Vec, + expected_len: Option, + ) -> FastPForResult<()> { + self.inner.decode(input, out, expected_len) + } } -define_fastpfor! { - /// Any-length `FastPFOR` codec with 256-value blocks. - /// - /// Compresses `u32` via [`AnyLenCodec`] and `u64` via [`BlockCodec64`]. - FastPFor256, FastPForBlock256, 256 +/// Compresses 64-bit integers through the shared [`BlockCodec64`] interface. +/// +/// Lets the `u64` codecs be compared against the C++ codecs, which expose `u64` the same way. +impl BlockCodec64 for FastPForCodec +where + [u64; N]: sealed::BlockSize, +{ + fn encode64(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()> { + self.inner.encode(input, out) + } + + fn decode64(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { + self.inner.decode(input, out, None) + } } +/// Any-length `u32` `FastPFOR` codec with 128-value blocks. +pub type FastPFor128 = FastPForCodec<128, u32>; + +/// Any-length `u32` `FastPFOR` codec with 256-value blocks. +pub type FastPFor256 = FastPForCodec<256, u32>; + +/// Any-length `u64` `FastPFOR` codec with 128-value blocks. +pub type FastPForWide128 = FastPForCodec<128, u64>; + +/// Any-length `u64` `FastPFOR` codec with 256-value blocks. +pub type FastPForWide256 = FastPForCodec<256, u64>; + #[cfg(test)] mod tests { use super::*; #[test] - fn one_codec_handles_both_widths() { + fn narrow_codec_roundtrips_u32() { let mut codec = FastPFor256::default(); + let data: Vec = (0..600).collect(); + let mut enc = Vec::new(); + codec.encode(&data, &mut enc).unwrap(); + let mut dec = Vec::new(); + codec.decode(&enc, &mut dec, None).unwrap(); + assert_eq!(dec, data); + } - let data32: Vec = (0..600).collect(); - let mut enc32 = Vec::new(); - codec.encode(&data32, &mut enc32).unwrap(); - let mut dec32 = Vec::new(); - codec.decode(&enc32, &mut dec32, None).unwrap(); - assert_eq!(dec32, data32); - - let data64: Vec = (0..600).map(|i| i * 1_000_000_000).collect(); - let mut enc64 = Vec::new(); - codec.encode64(&data64, &mut enc64).unwrap(); - let mut dec64 = Vec::new(); - codec.decode64(&enc64, &mut dec64).unwrap(); - assert_eq!(dec64, data64); + #[test] + fn wide_codec_roundtrips_u64() { + let mut codec = FastPForWide256::default(); + let data: Vec = (0..600).map(|i| i * 1_000_000_000).collect(); + let mut enc = Vec::new(); + codec.encode(&data, &mut enc).unwrap(); + let mut dec = Vec::new(); + codec.decode(&enc, &mut dec, None).unwrap(); + assert_eq!(dec, data); } } diff --git a/src/rust/integer_compression/fastpfor32.rs b/src/rust/integer_compression/fastpfor32.rs index 9fe4672..db44814 100644 --- a/src/rust/integer_compression/fastpfor32.rs +++ b/src/rust/integer_compression/fastpfor32.rs @@ -4,6 +4,7 @@ use bytemuck::cast_slice; use crate::helpers::AsUsize; use crate::rust::integer_compression::fastpfor::sealed; +use crate::rust::integer_compression::fastpfor_int::FastPForInt; use crate::{BlockCodec, FastPFor, FastPForError, FastPForResult}; /// Type alias for [`FastPFor`] with 128-element `u32` blocks. @@ -12,11 +13,12 @@ pub type FastPForBlock128 = FastPFor<128, u32>; /// Type alias for [`FastPFor`] with 256-element `u32` blocks. pub type FastPForBlock256 = FastPFor<256, u32>; -impl BlockCodec for FastPFor +impl BlockCodec for FastPFor where - [u32; N]: sealed::BlockSize, + [T; N]: sealed::BlockSize, { - type Block = [u32; N]; + type Elem = T; + type Block = [T; N]; fn encode_blocks(&mut self, blocks: &[Self::Block], out: &mut Vec) -> FastPForResult<()> { let n_values = (blocks.len() * N) as u32; @@ -24,9 +26,9 @@ where out.push(n_values); return Ok(()); } - let flat: &[u32] = cast_slice(blocks); + let flat: &[T] = cast_slice(blocks); - let capacity = flat.len() * 2 + 1024; + let capacity = flat.len() * 3 + 1024; let start = out.len(); // Reserve slot for the length header, then space for compressed data. out.resize(start + 1 + capacity, 0); @@ -53,7 +55,7 @@ where &mut self, input: &[u32], expected_len: Option, - out: &mut Vec, + out: &mut Vec, ) -> FastPForResult { let Some((&block_n_values, rest)) = input.split_first() else { return Err(FastPForError::NotEnoughData); @@ -79,7 +81,7 @@ where return Ok(1); } let start = out.len(); - out.resize(start + n_blocks * N, 0); + out.resize(start + n_blocks * N, T::ZERO); let mut in_off = Cursor::new(0u32); let mut out_off = Cursor::new(0u32); diff --git a/src/rust/integer_compression/fastpfor64.rs b/src/rust/integer_compression/fastpfor64.rs index 411b6c6..2338fb5 100644 --- a/src/rust/integer_compression/fastpfor64.rs +++ b/src/rust/integer_compression/fastpfor64.rs @@ -1,21 +1,13 @@ -//! 64-bit ([`u64`]) `FastPFOR` codec. +//! 64-bit ([`u64`]) `FastPFOR` block-codec aliases. //! -//! This is the widened counterpart of the 32-bit [`FastPFor`](super::fastpfor::FastPFor). -//! Values, exceptions, and the exception bitmap are 64 bits wide instead of 32. -//! The output is byte-compatible with the C++ `CppFastPFor128` / `CppFastPFor256` 64-bit paths. -//! -//! [`FastPForBlockWide128`]/[`FastPForBlockWide256`] are the 64-bit half of the public [`FastPFor128`](crate::FastPFor128) / -//! [`FastPFor256`](crate::FastPFor256) codecs. -//! It handles complete blocks, then a [`VariableByte`] tail encodes the sub-block remainder. - -use std::io::Cursor; +//! [`FastPForBlockWide128`]/[`FastPForBlockWide256`] are [`FastPFor`] specialised to `u64` blocks. +//! They implement the block-only [`BlockCodec`](crate::BlockCodec); the width-generic engine and +//! exception bitmap are 64 bits wide instead of 32. +//! The public any-length `u64` codecs [`FastPForWide128`](crate::FastPForWide128) / +//! [`FastPForWide256`](crate::FastPForWide256) pair these with a [`VariableByte`](crate::VariableByte) tail. +//! The block wire format is byte-compatible with the C++ `CppFastPFor128` / `CppFastPFor256` 64-bit paths. -use bytemuck::{cast_slice, cast_slice_mut}; - -use crate::codec::default_max_decoded_len; -use crate::helpers::AsUsize; use crate::rust::integer_compression::fastpfor::FastPFor; -use crate::{BlockCodec64, FastPForError, FastPForResult}; /// Type alias for [`FastPFor`] with 128-element `u64` blocks. pub type FastPForBlockWide128 = FastPFor<128, u64>; @@ -23,141 +15,17 @@ pub type FastPForBlockWide128 = FastPFor<128, u64>; /// Type alias for [`FastPFor`] with 256-element `u64` blocks. pub type FastPForBlockWide256 = FastPFor<256, u64>; -/// Variable-byte encoding of the `u64` tail. -/// -/// Each value is emitted little-endian in 7-bit groups. -/// Every byte but the last has its high bit clear. -/// The final byte sets its high bit as a terminator. -/// The stream is zero-padded to a whole number of `u32` words. -fn vbyte_encode64(input: &[u64], out: &mut Vec) { - if input.is_empty() { - return; - } - let start = out.len(); - let capacity = input.len() * 3 + 4; - out.resize(start + capacity, 0); - let bytes: &mut [u8] = cast_slice_mut(&mut out[start..]); - let mut byte_pos = 0; - for &value in input { - let mut v = value; - while v >= 0x80 { - bytes[byte_pos] = (v as u8) & 0x7F; - byte_pos += 1; - v >>= 7; - } - bytes[byte_pos] = (v as u8) | 0x80; - byte_pos += 1; - } - while byte_pos % 4 != 0 { - bytes[byte_pos] = 0; - byte_pos += 1; - } - out.truncate(start + byte_pos / 4); -} - -/// Inverse of [`vbyte_encode64`]. -/// Trailing zero padding decodes to no value, since a padding byte never sets the terminator bit. -fn vbyte_decode64(input: &[u32], out: &mut Vec) -> FastPForResult<()> { - if input.is_empty() { - return Ok(()); - } - let bytes: &[u8] = cast_slice(input); - let byte_len = bytes.len(); - let mut byte_pos = 0; - while byte_pos < byte_len { - let mut v: u64 = 0; - let mut shift = 0u32; - loop { - if byte_pos >= byte_len { - return Ok(()); - } - let c = bytes[byte_pos]; - byte_pos += 1; - if shift >= 64 { - return Err(FastPForError::NotEnoughData); - } - if c >= 0x80 { - v |= u64::from(c & 0x7F) << shift; - out.push(v); - break; - } - v |= u64::from(c) << shift; - shift += 7; - } - } - Ok(()) -} - -impl BlockCodec64 for FastPFor { - fn encode64(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()> { - let rounded = (input.len() / N) * N; - let n_values = rounded as u32; - - let start = out.len(); - if rounded == 0 { - out.push(0); - } else { - let capacity = rounded * 3 + 1024; - out.resize(start + 1 + capacity, 0); - out[start] = n_values; - - let mut in_off = Cursor::new(0u32); - let mut out_off = Cursor::new(0u32); - self.compress_blocks( - &input[..rounded], - n_values, - &mut in_off, - &mut out[start + 1..], - &mut out_off, - ); - let written = 1 + out_off.position() as usize; - out.truncate(start + written); - } - - vbyte_encode64(&input[rounded..], out); - Ok(()) - } - - fn decode64(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { - let Some((&block_n_values, rest)) = input.split_first() else { - return Ok(()); - }; - if block_n_values % N as u32 != 0 { - return Err(FastPForError::NotEnoughData); - } - if block_n_values.as_usize() > default_max_decoded_len(input.len()) { - return Err(FastPForError::NotEnoughData); - } - let n_blocks = block_n_values.as_usize() / N; - - let consumed = if n_blocks == 0 { - 1 - } else { - let start = out.len(); - out.resize(start + n_blocks * N, 0); - let mut in_off = Cursor::new(0u32); - let mut out_off = Cursor::new(0u32); - self.decode_headless_blocks( - rest, - block_n_values, - &mut in_off, - &mut out[start..], - &mut out_off, - )?; - 1 + in_off.position() as usize - }; - - let tail_input = input.get(consumed..).ok_or(FastPForError::NotEnoughData)?; - vbyte_decode64(tail_input, out) - } -} - #[cfg(test)] mod tests { - use super::*; - - fn roundtrip(input: &[u64]) { - let mut codec = FastPFor::::default(); + use crate::codec::BlockCodec64; + use crate::rust::fastpfor_codec::FastPForCodec; + use crate::rust::integer_compression::fastpfor::sealed; + + fn roundtrip(input: &[u64]) + where + [u64; N]: sealed::BlockSize, + { + let mut codec = FastPForCodec::::default(); let mut encoded = Vec::new(); codec.encode64(input, &mut encoded).unwrap(); let mut decoded = Vec::new(); @@ -218,7 +86,6 @@ mod tests { #[cfg(feature = "cpp")] mod cpp_parity { use super::*; - use crate::BlockCodec64; use crate::cpp::{CppFastPFor128, CppFastPFor256}; fn cases() -> Vec> { @@ -238,7 +105,12 @@ mod tests { ] } - fn assert_parity(rust: &mut FastPFor, cpp: &mut impl BlockCodec64) { + fn assert_parity( + rust: &mut FastPForCodec, + cpp: &mut impl BlockCodec64, + ) where + [u64; N]: sealed::BlockSize, + { for data in cases() { let mut rust_enc = Vec::new(); rust.encode64(&data, &mut rust_enc).unwrap(); @@ -259,7 +131,7 @@ mod tests { #[test] fn parity_128() { assert_parity( - &mut FastPFor::<128, u64>::default(), + &mut FastPForCodec::<128, u64>::default(), &mut CppFastPFor128::default(), ); } @@ -267,7 +139,7 @@ mod tests { #[test] fn parity_256() { assert_parity( - &mut FastPFor::<256, u64>::default(), + &mut FastPForCodec::<256, u64>::default(), &mut CppFastPFor256::default(), ); } diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs index 77c27d0..4f49238 100644 --- a/src/rust/integer_compression/fastpfor_int.rs +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -39,6 +39,7 @@ pub trait FastPForInt: Copy + 'static + Eq + + bytemuck::Pod + sealed::Sealed + Shr + Shl diff --git a/src/rust/integer_compression/just_copy.rs b/src/rust/integer_compression/just_copy.rs index 767ad11..64137ef 100644 --- a/src/rust/integer_compression/just_copy.rs +++ b/src/rust/integer_compression/just_copy.rs @@ -17,6 +17,8 @@ impl JustCopy { } impl AnyLenCodec for JustCopy { + type Elem = u32; + fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { out.extend_from_slice(input); Ok(()) diff --git a/src/rust/integer_compression/variable_byte.rs b/src/rust/integer_compression/variable_byte.rs index 8c6b2f2..6737c97 100644 --- a/src/rust/integer_compression/variable_byte.rs +++ b/src/rust/integer_compression/variable_byte.rs @@ -1,4 +1,5 @@ use std::io::Cursor; +use std::marker::PhantomData; use bytemuck::{cast_slice, cast_slice_mut}; @@ -7,12 +8,27 @@ use crate::helpers::AsUsize; use crate::rust::cursor::IncrementCursor; use crate::{FastPForError, FastPForResult}; -/// Variable-byte encoding codec for integer compression. -#[derive(Debug, Default)] -pub struct VariableByte; +/// Variable-byte encoding codec, generic over element width `T` ([`u32`] or [`u64`]). +#[derive(Debug)] +pub struct VariableByte(PhantomData); + +// Hand-written (not derived) so no `T: Default` bound leaks onto every user of the codec. +impl Default for VariableByte { + fn default() -> Self { + Self(PhantomData) + } +} + +impl VariableByte { + /// Creates a new instance. + #[must_use] + pub fn new() -> Self { + Self(PhantomData) + } +} // Helper functions with const generics for extracting 7-bit chunks -impl VariableByte { +impl VariableByte { /// Extract 7 bits from position i (with masking) const fn extract_7bits(val: u32) -> u8 { ((val >> (7 * I)) & ((1 << 7) - 1)) as u8 @@ -22,15 +38,6 @@ impl VariableByte { const fn extract_7bits_maskless(val: u32) -> u8 { (val >> (7 * I)) as u8 } -} - -// Implemented for consistency with other codecs -impl VariableByte { - /// Creates a new instance - #[must_use] - pub fn new() -> Self { - Self - } /// Compress `input_length` u32 values from `input[input_offset..]` into /// `output[output_offset..]` as packed variable-byte u8 values (stored in @@ -301,7 +308,9 @@ impl VariableByte { } } -impl AnyLenCodec for VariableByte { +impl AnyLenCodec for VariableByte { + type Elem = u32; + fn encode(&mut self, input: &[u32], out: &mut Vec) -> FastPForResult<()> { let capacity = input.len() * 2 + 4; let start = out.len(); @@ -352,6 +361,73 @@ impl AnyLenCodec for VariableByte { } } +impl AnyLenCodec for VariableByte { + type Elem = u64; + + fn encode(&mut self, input: &[u64], out: &mut Vec) -> FastPForResult<()> { + if input.is_empty() { + return Ok(()); + } + let start = out.len(); + let capacity = input.len() * 3 + 4; + out.resize(start + capacity, 0); + let bytes: &mut [u8] = cast_slice_mut(&mut out[start..]); + let mut byte_pos = 0; + for &value in input { + let mut v = value; + while v >= 0x80 { + bytes[byte_pos] = (v as u8) & 0x7F; + byte_pos += 1; + v >>= 7; + } + bytes[byte_pos] = (v as u8) | 0x80; + byte_pos += 1; + } + while byte_pos % 4 != 0 { + bytes[byte_pos] = 0; + byte_pos += 1; + } + out.truncate(start + byte_pos / 4); + Ok(()) + } + + fn decode( + &mut self, + input: &[u32], + out: &mut Vec, + _expected_len: Option, + ) -> FastPForResult<()> { + if input.is_empty() { + return Ok(()); + } + let bytes: &[u8] = cast_slice(input); + let byte_len = bytes.len(); + let mut byte_pos = 0; + while byte_pos < byte_len { + let mut v: u64 = 0; + let mut shift = 0u32; + loop { + if byte_pos >= byte_len { + return Ok(()); + } + let c = bytes[byte_pos]; + byte_pos += 1; + if shift >= 64 { + return Err(FastPForError::NotEnoughData); + } + if c >= 0x80 { + v |= u64::from(c & 0x7F) << shift; + out.push(v); + break; + } + v |= u64::from(c) << shift; + shift += 7; + } + } + Ok(()) + } +} + #[cfg(test)] mod tests { use std::collections::hash_map::RandomState; diff --git a/src/rust/mod.rs b/src/rust/mod.rs index ad1b448..fe9fbc5 100644 --- a/src/rust/mod.rs +++ b/src/rust/mod.rs @@ -4,8 +4,8 @@ mod fastpfor_codec; mod integer_compression; pub use composite::CompositeCodec; -/// Any-length `FastPFOR` codecs supporting both `u32` and `u64`. -pub use fastpfor_codec::{FastPFor128, FastPFor256}; +/// Any-length `FastPFOR` codecs: `FastPFor*` for `u32`, `FastPForWide*` for `u64`. +pub use fastpfor_codec::{FastPFor128, FastPFor256, FastPForWide128, FastPForWide256}; /// Type-safe block codec with block size encoded in the type. pub use integer_compression::fastpfor::FastPFor; pub use integer_compression::fastpfor32::{FastPForBlock128, FastPForBlock256}; diff --git a/src/test_utils.rs b/src/test_utils.rs index 9ff6fea..0c5c704 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -25,17 +25,20 @@ pub const RNG_SEED: u64 = 456; // Generic codec helpers // --------------------------------------------------------------------------- -pub fn roundtrip(data: &[u32]) { +pub fn roundtrip>(data: &[u32]) { roundtrip_expected::(data, Some(data.len().try_into().unwrap())); } /// Encode `data` with a caller-owned codec, decode with `expected_len: None`, assert round-trip. -pub fn roundtrip_expected(data: &[u32], expected_len: Option) { +pub fn roundtrip_expected>(data: &[u32], expected_len: Option) { roundtrip_full::(data, expected_len); } /// Encode `data` with a caller-owned codec, decode with `expected_len: None`, assert round-trip. -pub fn roundtrip_full(data: &[u32], expected_len: Option) { +pub fn roundtrip_full, D: AnyLenCodec>( + data: &[u32], + expected_len: Option, +) { let mut encoder = E::default(); let mut compressed = Vec::new(); encoder.encode(data, &mut compressed).unwrap(); @@ -58,19 +61,19 @@ pub fn roundtrip64(data: &[u64]) { assert_eq!(decoded, data); } -pub fn block_roundtrip(data: &[u32]) { +pub fn block_roundtrip>(data: &[u32]) { let compressed = block_compress::(data).unwrap(); let decompressed = block_decompress::(&compressed, Some(data.len() as u32)).unwrap(); assert_eq!(decompressed, data); } -pub fn compress(data: &[u32]) -> FastPForResult> { +pub fn compress>(data: &[u32]) -> FastPForResult> { let mut compressed = Vec::new(); C::default().encode(data, &mut compressed)?; Ok(compressed) } -pub fn decompress( +pub fn decompress>( compressed: &[u32], expected_len: Option, ) -> FastPForResult> { @@ -79,7 +82,7 @@ pub fn decompress( Ok(decompressed) } -pub fn block_compress(data: &[u32]) -> FastPForResult> { +pub fn block_compress>(data: &[u32]) -> FastPForResult> { let (blocks, remainder) = slice_to_blocks::(data); if !remainder.is_empty() { return Err(FastPForError::InputMustBeMultipleOfBlockSize { @@ -92,7 +95,7 @@ pub fn block_compress(data: &[u32]) -> FastPForResult> { Ok(out) } -pub fn block_decompress( +pub fn block_decompress>( compressed: &[u32], expected_len: Option, ) -> FastPForResult> { @@ -142,8 +145,8 @@ pub fn block_roundtrip_all(data: &[u32]) { #[cfg(feature = "rust")] pub fn roundtrip_composite(data: &[u32]) where - B: BlockCodec, - T: AnyLenCodec, + B: BlockCodec, + T: AnyLenCodec, { roundtrip::>(data); } @@ -315,7 +318,7 @@ mod rust_bench { _codec: PhantomData, } - impl CompressFixture { + impl> CompressFixture { fn new(name: &'static str, generator: DataGeneratorFn, block_count: usize) -> Self { let original = generator(block_count * C::size()); Self { @@ -328,7 +331,7 @@ mod rust_bench { } } - impl BlockSizeFixture { + impl> BlockSizeFixture { pub fn new(block_count: usize) -> Self { let original = generate_uniform_data_small_value_distribution(block_count * C::size()); Self { @@ -340,7 +343,7 @@ mod rust_bench { } } - pub fn compress_fixtures( + pub fn compress_fixtures>( block_counts: &[usize], ) -> Vec<(usize, CompressFixture)> { block_counts @@ -353,7 +356,9 @@ mod rust_bench { .collect() } - pub fn ratio_fixtures(block_count: usize) -> Vec> { + pub fn ratio_fixtures>( + block_count: usize, + ) -> Vec> { ALL_PATTERNS .iter() .map(|&(name, generator)| CompressFixture::::new(name, generator, block_count)) From b267b64b8da3c9237056474cb8529ab3dfa1297b Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sat, 18 Jul 2026 15:49:09 +0200 Subject: [PATCH 21/23] refactor: source generic integer ops from num_traits::PrimInt Replace the hand-written operator bounds, ZERO/ONE constants, and per-type significant_bits impls on FastPForInt with a PrimInt supertrait bound. Only the width-specific members (scratch buffers, packing/bitmap kernels) remain. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01EqJPh46kx6YzhwUjX6EPDn --- Cargo.toml | 3 +- src/rust/integer_compression/fastpfor.rs | 18 ++++---- src/rust/integer_compression/fastpfor32.rs | 2 +- src/rust/integer_compression/fastpfor_int.rs | 48 ++++---------------- 4 files changed, 21 insertions(+), 50 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 211801b..cc5f88f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,12 +30,13 @@ cpp_portable = ["cpp"] # Optimize FastPFOR for the current CPU. cpp_native = ["cpp"] cpp = ["dep:cmake", "dep:cxx", "dep:cxx-build"] -rust = ["dep:bytes"] +rust = ["dep:bytes", "dep:num-traits"] [dependencies] bytemuck = { version = "1.25.0", features = ["min_const_generics"] } bytes = { version = "1.11", optional = true } cxx = { version = "1.0.194", optional = true } +num-traits = { version = "0.2", optional = true } thiserror = "2.0.18" [build-dependencies] diff --git a/src/rust/integer_compression/fastpfor.rs b/src/rust/integer_compression/fastpfor.rs index 999da15..7ab5936 100644 --- a/src/rust/integer_compression/fastpfor.rs +++ b/src/rust/integer_compression/fastpfor.rs @@ -178,13 +178,13 @@ impl FastPFor { if needed > self.exception_buffers[index].len() { // Grow to the next multiple of 32 above 2×needed, to amortize resizes. let new_cap = needed.saturating_mul(2).next_multiple_of(32); - self.exception_buffers[index].resize(new_cap, T::ZERO); + self.exception_buffers[index].resize(new_cap, T::zero()); } for k in 0..N as u32 { - if input[(k + tmp_input_offset) as usize] >> self.optimal_bits != T::ZERO { + if input[(k + tmp_input_offset) as usize] >> usize::from(self.optimal_bits) != T::zero() { self.bytes_container.put_u8(k as u8); self.exception_buffers[index][self.data_pointers[index]] = - input[(k + tmp_input_offset) as usize] >> self.optimal_bits; + input[(k + tmp_input_offset) as usize] >> usize::from(self.optimal_bits); self.data_pointers[index] += 1; } } @@ -217,10 +217,10 @@ impl FastPFor { .copy_from_slice(&meta_u32s[..how_many_ints]); tmp_output_offset += how_many_ints as u32; // Exception bitmap: one bit per bit-width bucket, written as `T::BITMAP_WORDS` words. - let mut bitmap = T::ZERO; + let mut bitmap = T::zero(); for k in 2..=usize::from(T::WIDTH) { if self.data_pointers[k] != 0 { - bitmap |= T::ONE << (k - 1) as u8; + bitmap |= T::one() << (k - 1); } } T::write_bitmap(bitmap, &mut output[tmp_output_offset as usize..]); @@ -345,7 +345,7 @@ impl FastPFor { .ok_or(FastPForError::NotEnoughData)?; for k in 2..=u32::from(T::WIDTH) { - if bitmap & (T::ONE << (k - 1) as u8) != T::ZERO { + if bitmap & (T::one() << (k - 1) as usize) != T::zero() { let size = input.get_val(inexcept)?; inexcept = inexcept .checked_add(1) @@ -358,7 +358,7 @@ impl FastPFor { // to the next group of 32 for the bitunpacking calls. let rounded_up = size.next_multiple_of(32) as usize; if self.exception_buffers[k as usize].len() < rounded_up { - self.exception_buffers[k as usize].resize(rounded_up, T::ZERO); + self.exception_buffers[k as usize].resize(rounded_up, T::zero()); } let mut j: u32 = 0; // Process full groups directly from input @@ -462,7 +462,7 @@ impl FastPFor { if out_idx >= output.len() { return Err(FastPForError::OutputBufferTooSmall); } - output[out_idx] |= T::ONE << bits; + output[out_idx] |= T::one() << usize::from(bits); } } else { for _ in 0..num_exceptions { @@ -477,7 +477,7 @@ impl FastPFor { } let ptr = self.data_pointers[index]; let except_value = self.exception_buffers[index].get_val(ptr)?; - output[out_idx] |= except_value << bits; + output[out_idx] |= except_value << usize::from(bits); self.data_pointers[index] += 1; } } diff --git a/src/rust/integer_compression/fastpfor32.rs b/src/rust/integer_compression/fastpfor32.rs index db44814..c2f1315 100644 --- a/src/rust/integer_compression/fastpfor32.rs +++ b/src/rust/integer_compression/fastpfor32.rs @@ -81,7 +81,7 @@ where return Ok(1); } let start = out.len(); - out.resize(start + n_blocks * N, T::ZERO); + out.resize(start + n_blocks * N, T::zero()); let mut in_off = Cursor::new(0u32); let mut out_off = Cursor::new(0u32); diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs index 4f49238..c84e8b1 100644 --- a/src/rust/integer_compression/fastpfor_int.rs +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -15,7 +15,9 @@ use std::array; use std::fmt::Debug; -use std::ops::{BitAnd, BitOrAssign, Index, IndexMut, Shl, Shr}; +use std::ops::{BitOrAssign, Index, IndexMut}; + +use num_traits::PrimInt; use crate::helpers::GetWithErr; use crate::rust::integer_compression::{bit_pack32, bit_pack64, bit_unpack32}; @@ -27,33 +29,12 @@ mod sealed { impl Sealed for u64 {} } -/// Element type of a `FastPFOR` stream: [`u32`] or [`u64`]. -/// -/// The operator bounds (`Shr`/`Shl` by `u8`, `BitAnd`, `BitOrAssign`) let the codec use plain -/// `>>`, `<<`, `&`, and `|=` on values; only the width-specific pieces are methods. Implementors -/// also supply the concrete scratch-buffer array types (one bucket per possible bit width, -/// i.e. `WIDTH + 1`). The exception bitmap spans [`BITMAP_WORDS`](Self::BITMAP_WORDS) output words. -/// -/// This trait is sealed and cannot be implemented outside this crate. -pub trait FastPForInt: - Copy - + 'static - + Eq - + bytemuck::Pod - + sealed::Sealed - + Shr - + Shl - + BitAnd - + BitOrAssign -{ +/// Sealed element type of a `FastPFOR` stream: [`u32`] or [`u64`]. +pub trait FastPForInt: PrimInt + 'static + bytemuck::Pod + sealed::Sealed + BitOrAssign { /// Bit width of the element: 32 or 64. const WIDTH: u8 = (size_of::() * 8) as u8; /// Output words occupied by the exception bitmap: 1 for `u32`, 2 for `u64`. const BITMAP_WORDS: u32 = Self::WIDTH as u32 / u32::BITS; - /// The zero value. - const ZERO: Self; - /// The one value. - const ONE: Self; /// Exception values grouped by bit-width bucket: `[Vec; WIDTH + 1]`. type ExceptionBuffers: Index> + IndexMut + Debug; @@ -70,7 +51,10 @@ pub trait FastPForInt: fn new_data_pointers() -> Self::DataPointers; /// Number of significant bits, i.e. `WIDTH - leading_zeros` (0 for a zero value). - fn significant_bits(self) -> u8; + #[inline] + fn significant_bits(self) -> u8 { + Self::WIDTH - self.leading_zeros() as u8 + } /// Pack 32 values at `bit` bits each into `out`. fn fast_pack(src: &[Self], inpos: usize, out: &mut [u32], outpos: usize, bit: u8); @@ -88,9 +72,6 @@ pub trait FastPForInt: reason = "u32 here is the stream word type, not the Self element type" )] impl FastPForInt for u32 { - const ZERO: Self = 0; - const ONE: Self = 1; - type ExceptionBuffers = [Vec; u32::BITS as usize + 1]; type Freqs = [u32; u32::BITS as usize + 1]; type DataPointers = [usize; u32::BITS as usize + 1]; @@ -105,10 +86,6 @@ impl FastPForInt for u32 { [0; u32::BITS as usize + 1] } - #[inline] - fn significant_bits(self) -> u8 { - Self::WIDTH - self.leading_zeros() as u8 - } // `inline(always)`: this is a thin forwarder; without it the wrapper accumulates the whole // inlined kernel and then exceeds the inline threshold, so `decode_page`/`encode_page` would // emit a real call per 32-value group instead of inlining the kernel (as the concrete `u32` @@ -138,9 +115,6 @@ impl FastPForInt for u32 { reason = "u32 here is the stream word type, not the Self element type" )] impl FastPForInt for u64 { - const ZERO: Self = 0; - const ONE: Self = 1; - type ExceptionBuffers = [Vec; u64::BITS as usize + 1]; type Freqs = [u32; u64::BITS as usize + 1]; type DataPointers = [usize; u64::BITS as usize + 1]; @@ -155,10 +129,6 @@ impl FastPForInt for u64 { [0; u64::BITS as usize + 1] } - #[inline] - fn significant_bits(self) -> u8 { - Self::WIDTH - self.leading_zeros() as u8 - } // `inline(always)`: forward directly to the wide kernel (see the `u32` impl for rationale). #[inline(always)] #[allow(clippy::inline_always, reason = "thin forwarder; see comment above")] From 2f5fc2c66ef711f77514898fd1390227736c888a Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sat, 18 Jul 2026 15:50:14 +0200 Subject: [PATCH 22/23] Apply suggestion from @CommanderStorm --- src/rust/integer_compression/fastpfor_int.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/rust/integer_compression/fastpfor_int.rs b/src/rust/integer_compression/fastpfor_int.rs index c84e8b1..a74f8d1 100644 --- a/src/rust/integer_compression/fastpfor_int.rs +++ b/src/rust/integer_compression/fastpfor_int.rs @@ -1,17 +1,4 @@ //! Element-width abstraction shared by the 32- and 64-bit `FastPFOR` codecs. -//! -//! The block-splitting, best-bit search, exception handling, and metadata layout are -//! identical for `u32` and `u64`; only the element width differs. -//! [`FastPForInt`] abstracts the width-specific pieces so a single [`FastPFor`](super::fastpfor::FastPFor) -//! implements the algorithm once. Ordinary arithmetic uses the standard operator traits -//! (`>>`, `<<`, `&`, `|=`) that the trait requires as bounds; only the genuinely -//! width-specific pieces (bit-packing kernels and the exception bitmap layout) are methods. -//! `u32` keeps its hand-unrolled bit-packing kernels; `u64` uses the generic wide packer. -//! -//! The trait is **sealed**: only [`u32`] and [`u64`] implement it, so callers cannot plug in -//! an unsupported element type. Each implementor also fixes the exact size of the per-block -//! scratch buffers (`WIDTH + 1` buckets) as associated types, so no space is wasted and the -//! bucket count never leaks into the public [`FastPFor`](super::fastpfor::FastPFor) signature. use std::array; use std::fmt::Debug; From 056725d03c1a8d3b89ef36ca77ff7fe029aff3e2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 18 Jul 2026 13:51:35 +0000 Subject: [PATCH 23/23] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/rust/integer_compression/fastpfor.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/rust/integer_compression/fastpfor.rs b/src/rust/integer_compression/fastpfor.rs index 7ab5936..43b69ec 100644 --- a/src/rust/integer_compression/fastpfor.rs +++ b/src/rust/integer_compression/fastpfor.rs @@ -181,10 +181,13 @@ impl FastPFor { self.exception_buffers[index].resize(new_cap, T::zero()); } for k in 0..N as u32 { - if input[(k + tmp_input_offset) as usize] >> usize::from(self.optimal_bits) != T::zero() { + if input[(k + tmp_input_offset) as usize] >> usize::from(self.optimal_bits) + != T::zero() + { self.bytes_container.put_u8(k as u8); - self.exception_buffers[index][self.data_pointers[index]] = - input[(k + tmp_input_offset) as usize] >> usize::from(self.optimal_bits); + self.exception_buffers[index][self.data_pointers[index]] = input + [(k + tmp_input_offset) as usize] + >> usize::from(self.optimal_bits); self.data_pointers[index] += 1; } }