From 1f7f12096ec1faed0fbba6b7a77ca05ad0c057f3 Mon Sep 17 00:00:00 2001 From: jryancarr Date: Mon, 3 Aug 2026 11:44:45 -0400 Subject: [PATCH 01/11] Add bit-packing serialization mode (compr_mode_type::bitpack). Ciphertext and key data consist of 64-bit words storing integers modulo primes much smaller than the word size. The significant bits are close to uniformly random and hence essentially incompressible; only the always-zero high bits are redundant, and a general-purpose compressor cannot remove partial bytes of them. The new mode splits the serialized stream into 4096-byte blocks and packs each block's run of 64-bit words using only as many bits per word as the largest word in the block requires, so each word begins immediately after the last significant bit of the previous one. A per-block phase (0-7 verbatim leading bytes) aligns the packed run with the data's natural word grid, since serialized metadata is not always a multiple of eight bytes. On a BFV ciphertext at poly_modulus_degree 8192 this saves 17% over Zstandard (31% over uncompressed). Loading is streamed one block at a time through a pull-based streambuf, mirroring the zlib/zstd paths, so hostile size claims cannot drive unbounded allocation. The default compression mode and the wire format of all existing modes are unchanged; the new mode requires no external dependency and is always available. Co-Authored-By: Claude Fable 5 --- dotnet/src/Serialization.cs | 13 +- dotnet/tests/CiphertextTests.cs | 38 +++ dotnet/tests/SerializationTests.cs | 4 +- native/src/seal/serialization.cpp | 66 +++++ native/src/seal/serialization.h | 13 + native/src/seal/util/CMakeLists.txt | 2 + native/src/seal/util/bitpack.cpp | 359 ++++++++++++++++++++++++++++ native/src/seal/util/bitpack.h | 156 ++++++++++++ native/tests/seal/ciphertext.cpp | 46 ++++ native/tests/seal/serialization.cpp | 238 +++++++++++++++++- 10 files changed, 931 insertions(+), 4 deletions(-) create mode 100644 native/src/seal/util/bitpack.cpp create mode 100644 native/src/seal/util/bitpack.h diff --git a/dotnet/src/Serialization.cs b/dotnet/src/Serialization.cs index 90e37b4b1..3b1936fcc 100644 --- a/dotnet/src/Serialization.cs +++ b/dotnet/src/Serialization.cs @@ -15,6 +15,14 @@ namespace Microsoft.Research.SEAL /// a large number of zero bytes in the output. Any compression algorithm should /// be able to clean up these zero bytes and hence compress both ciphertext and /// key data. + /// + /// Alternatively, ComprModeType.BitPack re-encodes each block of 64-bit words + /// using only as many bits per word as the largest word in the block requires, + /// discarding exactly the always-zero high bits. The significant bits of + /// ciphertext and key data are close to uniformly random and hence essentially + /// incompressible, so bit-packing typically produces smaller output than a + /// general-purpose compressor, which cannot remove partial bytes. Unlike ZLIB + /// and Zstandard, bit-packing performs no integrity checking of the data. /// public enum ComprModeType : byte { @@ -25,7 +33,10 @@ public enum ComprModeType : byte ZLIB = 1, /// Use Zstandard compression. - ZSTD = 2 + ZSTD = 2, + + /// Use bit-packing of 64-bit words. + BitPack = 3 } /// Class to provide functionality for serialization. diff --git a/dotnet/tests/CiphertextTests.cs b/dotnet/tests/CiphertextTests.cs index de143d0ea..80114af9b 100644 --- a/dotnet/tests/CiphertextTests.cs +++ b/dotnet/tests/CiphertextTests.cs @@ -221,6 +221,44 @@ public void BFVSaveLoadTest() } } + [TestMethod] + public void BFVBitPackSaveLoadTest() + { + SEALContext context = GlobalContext.BFVContext; + KeyGenerator keygen = new KeyGenerator(context); + keygen.CreatePublicKey(out PublicKey publicKey); + + Encryptor encryptor = new Encryptor(context, publicKey); + Plaintext plain = new Plaintext("2x^3 + 4x^2 + 5x^1 + 6"); + Ciphertext cipher = new Ciphertext(); + + encryptor.Encrypt(plain, cipher); + + Ciphertext loaded = new Ciphertext(); + long saveSize = 0; + + using (MemoryStream mem = new MemoryStream()) + { + saveSize = cipher.Save(mem, ComprModeType.BitPack); + + mem.Seek(offset: 0, loc: SeekOrigin.Begin); + + loaded.Load(context, mem); + } + + Assert.IsTrue(ValCheck.IsValidFor(loaded, context)); + + ulong ulongCount = cipher.Size * cipher.PolyModulusDegree * cipher.CoeffModulusSize; + for (ulong i = 0; i < ulongCount; i++) + { + Assert.AreEqual(cipher[i], loaded[i]); + } + + // The coefficients are uniformly random modulo the coefficient modulus primes, which are well under + // 64 bits, so packing each 64-bit word to its significant bits must beat the unpacked size. + Assert.IsTrue(saveSize < cipher.SaveSize(ComprModeType.None)); + } + [TestMethod] public void BGVSaveLoadTest() { diff --git a/dotnet/tests/SerializationTests.cs b/dotnet/tests/SerializationTests.cs index e0a219765..7bf3654f2 100644 --- a/dotnet/tests/SerializationTests.cs +++ b/dotnet/tests/SerializationTests.cs @@ -28,7 +28,9 @@ public void IsValidHeader() invalidHeader.VersionMajor = 0x02; Assert.IsFalse(Serialization.IsValidHeader(invalidHeader)); invalidHeader.VersionMajor = SEALVersion.Major; - invalidHeader.ComprMode = (ComprModeType)0x03; + invalidHeader.ComprMode = ComprModeType.BitPack; + Assert.IsTrue(Serialization.IsValidHeader(invalidHeader)); + invalidHeader.ComprMode = (ComprModeType)0x04; Assert.IsFalse(Serialization.IsValidHeader(invalidHeader)); } diff --git a/native/src/seal/serialization.cpp b/native/src/seal/serialization.cpp index 477584d48..c4f348a2d 100644 --- a/native/src/seal/serialization.cpp +++ b/native/src/seal/serialization.cpp @@ -4,6 +4,7 @@ #include "seal/dynarray.h" #include "seal/memorymanager.h" #include "seal/serialization.h" +#include "seal/util/bitpack.h" #include "seal/util/common.h" #include "seal/util/streambuf.h" #include "seal/util/ztools.h" @@ -99,6 +100,9 @@ namespace seal case compr_mode_type::zlib: return ztools::zlib_deflate_size_bound(in_size); #endif + case compr_mode_type::bitpack: + return bitpack::bitpack_size_bound(in_size); + case compr_mode_type::none: // No compression return in_size; @@ -315,6 +319,29 @@ namespace seal break; } #endif + case compr_mode_type::bitpack: + { + // First save_members to a temporary byte stream; set the size of the temporary stream to be right from + // the start to avoid extra reallocs. + SafeByteBuffer safe_buffer( + bitpack::bitpack_size_bound(raw_size - static_cast(sizeof(SEALHeader))), clear_buffers); + iostream temp_stream(&safe_buffer); + temp_stream.exceptions(ios_base::badbit | ios_base::failbit); + save_members(temp_stream); + + auto safe_pool(MemoryManager::GetPool(mm_prof_opt::mm_force_new, clear_buffers)); + + // Create temporary aliasing DynArray to wrap safe_buffer + DynArray safe_buffer_array( + Pointer::Aliasing(safe_buffer.data()), safe_buffer.size(), + static_cast(temp_stream.tellp()), false, safe_pool); + + // After bit-packing, write_header_pack_buffer will write the final size to the given header and + // write the header to stream, before writing the bit-packed output. + bitpack::bitpack_write_header_pack_buffer( + safe_buffer_array, reinterpret_cast(&header), stream, safe_pool); + break; + } default: throw invalid_argument("unsupported compression mode"); } @@ -514,6 +541,45 @@ namespace seal break; } #endif + case compr_mode_type::bitpack: + { + // header.size counts the whole object including the header, so the packed payload is exactly + // header.size - sizeof(SEALHeader). Computing it directly keeps it correct on non-seekable streams, + // where tellg() returns -1 and cannot be used to measure the header. + auto packed_size = header.size - static_cast(sizeof(SEALHeader)); + + auto safe_pool = MemoryManager::GetPool(mm_prof_opt::mm_force_new, clear_buffers); + + // Unpack on demand directly into the parser rather than decoding the entire payload up front. This + // bounds memory use during loading to what the parser actually reads, so a hostile size claim in the + // packed data cannot drive an unbounded allocation. + streamoff packed_remaining = 0; + { + auto unpack_buffer = + bitpack::make_bitpack_unpack_buffer(stream, safe_cast(packed_size), safe_pool); + istream temp_stream(unpack_buffer.get()); + temp_stream.exceptions(ios_base::badbit | ios_base::failbit); + + load_members(temp_stream, version); + + if (unpack_buffer->failed()) + { + throw logic_error("stream decompression failed"); + } + packed_remaining = unpack_buffer->remaining(); + } + + // The parser may not have pulled the whole packed payload. On a seekable stream, where header.size + // was confirmed to fit the available input, skip any remainder so the stream ends exactly at + // stream_start_pos + header.size, keeping concatenated objects loadable. On a non-seekable stream + // header.size is unverified, so skipping is gated off: an inflated header.size must not drive + // stream.ignore() past the real data into an over-read or an indefinite block. + if (size_verified && packed_remaining > 0) + { + stream.ignore(packed_remaining); + } + break; + } default: throw invalid_argument("unsupported compression mode"); } diff --git a/native/src/seal/serialization.h b/native/src/seal/serialization.h index 1ea6730ca..61195ba5d 100644 --- a/native/src/seal/serialization.h +++ b/native/src/seal/serialization.h @@ -19,6 +19,15 @@ namespace seal a large number of zero bytes in the output. Any compression algorithm should be able to clean up these zero bytes and hence compress both ciphertext and key data. + + Alternatively, compr_mode_type::bitpack re-encodes each block of 64-bit + words using only as many bits per word as the largest word in the block + requires, discarding exactly the always-zero high bits. The significant + bits of ciphertext and key data are close to uniformly random and hence + essentially incompressible, so bit-packing typically produces smaller + output than a general-purpose compressor, which cannot remove partial + bytes. Unlike ZLIB and Zstandard, bit-packing performs no integrity + checking of the data. */ enum class compr_mode_type : std::uint8_t { @@ -32,6 +41,8 @@ namespace seal // Use Zstandard compression zstd = 2, #endif + // Use bit-packing of 64-bit words + bitpack = 3, }; /** @@ -109,7 +120,9 @@ namespace seal #endif #ifdef SEAL_USE_ZSTD case static_cast(compr_mode_type::zstd): + /* fall through */ #endif + case static_cast(compr_mode_type::bitpack): return true; } return false; diff --git a/native/src/seal/util/CMakeLists.txt b/native/src/seal/util/CMakeLists.txt index 7863325f9..7651034e5 100644 --- a/native/src/seal/util/CMakeLists.txt +++ b/native/src/seal/util/CMakeLists.txt @@ -3,6 +3,7 @@ # Source files in this directory set(SEAL_SOURCE_FILES ${SEAL_SOURCE_FILES} + ${CMAKE_CURRENT_LIST_DIR}/bitpack.cpp ${CMAKE_CURRENT_LIST_DIR}/blake2b.c ${CMAKE_CURRENT_LIST_DIR}/blake2xb.c ${CMAKE_CURRENT_LIST_DIR}/clipnormal.cpp @@ -31,6 +32,7 @@ set(SEAL_SOURCE_FILES ${SEAL_SOURCE_FILES} # Add header files for installation install( FILES + ${CMAKE_CURRENT_LIST_DIR}/bitpack.h ${CMAKE_CURRENT_LIST_DIR}/blake2.h ${CMAKE_CURRENT_LIST_DIR}/blake2-impl.h ${CMAKE_CURRENT_LIST_DIR}/clang.h diff --git a/native/src/seal/util/bitpack.cpp b/native/src/seal/util/bitpack.cpp new file mode 100644 index 000000000..0affed9bf --- /dev/null +++ b/native/src/seal/util/bitpack.cpp @@ -0,0 +1,359 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#include "seal/serialization.h" +#include "seal/util/bitpack.h" +#include "seal/util/common.h" +#include +#include + +using namespace std; + +namespace seal +{ + namespace util + { + namespace bitpack + { + namespace + { + // Size in bytes of the 64-bit words being packed. + constexpr size_t bytes_per_word = sizeof(uint64_t); + + // Slack allowing the packing and unpacking loops to address words through whole-uint64_t reads and + // writes near the end of a block without stepping out of bounds. + constexpr size_t block_slack = bytes_per_word; + + // Number of whole 64-bit words in a block of block_len bytes read at the given phase. + SEAL_NODISCARD inline size_t block_word_count(size_t block_len, size_t phase) noexcept + { + return (block_len - phase) / bytes_per_word; + } + + // Encoded size in bytes of the body of a block of block_len bytes read at the given phase: the + // verbatim phase bytes, the packed words, and the verbatim tail bytes. + SEAL_NODISCARD inline size_t block_body_size(size_t block_len, size_t phase, int width) noexcept + { + size_t words = block_word_count(block_len, phase); + size_t packed_bytes = (words * static_cast(width) + size_t(7)) >> 3; + return block_len - words * bytes_per_word + packed_bytes; + } + } // namespace + + void bitpack_write_header_pack_buffer( + const DynArray &in, void *header_ptr, ostream &out_stream, MemoryPoolHandle pool) + { + if (!pool) + { + throw invalid_argument("pool is uninitialized"); + } + + Serialization::SEALHeader &header = *reinterpret_cast(header_ptr); + + size_t in_size = in.size(); + const unsigned char *in_data = reinterpret_cast(in.cbegin()); + + // The whole-uint64_t writes below rely on the output being zero-filled (DynArray zero-fills) and on + // block_slack bytes of headroom past the size bound. + DynArray out(add_safe(bitpack_size_bound(in_size), block_slack), pool); + unsigned char *out_data = reinterpret_cast(out.begin()); + + // Write the original byte count + uint64_t in_size64 = static_cast(in_size); + memcpy(out_data, &in_size64, bytes_per_word); + size_t out_pos = bytes_per_word; + + for (size_t block_start = 0; block_start < in_size;) + { + size_t block_len = min(bitpack_block_bytes, in_size - block_start); + const unsigned char *block_in = in_data + block_start; + + // The word data in the stream need not fall on the stream's own word grid (serialized metadata + // is not always a multiple of eight bytes), so choose the phase that minimizes the encoded size + // of the block. + size_t phase = 0; + int width = 0; + size_t body_size = block_len; + for (size_t p = 0; p <= min(size_t(7), block_len); p++) + { + size_t words = block_word_count(block_len, p); + uint64_t block_or = 0; + for (size_t i = 0; i < words; i++) + { + uint64_t word = 0; + memcpy(&word, block_in + p + i * bytes_per_word, bytes_per_word); + block_or |= word; + } + int p_width = get_significant_bit_count(block_or); + size_t p_body_size = block_body_size(block_len, p, p_width); + if (p == 0 || p_body_size < body_size) + { + phase = p; + width = p_width; + body_size = p_body_size; + } + } + size_t words = block_word_count(block_len, phase); + + out_data[out_pos++] = static_cast(width); + out_data[out_pos++] = static_cast(phase); + + // Verbatim phase bytes + memcpy(out_data + out_pos, block_in, phase); + out_pos += phase; + + // Pack the words consecutively starting from the least significant bit. Each word carries at + // most width significant bits, so nothing is lost; the read-modify-write below only ever ORs + // significant bits into the zero-filled output. + unsigned char *packed_out = out_data + out_pos; + size_t bit_pos = 0; + for (size_t i = 0; i < words; i++) + { + uint64_t word = 0; + memcpy(&word, block_in + phase + i * bytes_per_word, bytes_per_word); + size_t byte_index = bit_pos >> 3; + int shift = static_cast(bit_pos & size_t(7)); + uint64_t low_word = 0; + memcpy(&low_word, packed_out + byte_index, bytes_per_word); + low_word |= word << shift; + memcpy(packed_out + byte_index, &low_word, bytes_per_word); + if (shift && width > bits_per_uint64 - shift) + { + packed_out[byte_index + bytes_per_word] = + static_cast(word >> (bits_per_uint64 - shift)); + } + bit_pos += static_cast(width); + } + out_pos += (words * static_cast(width) + size_t(7)) >> 3; + + // Verbatim tail bytes + size_t tail = block_len - phase - words * bytes_per_word; + memcpy(out_data + out_pos, block_in + block_len - tail, tail); + out_pos += tail; + + block_start += block_len; + } + + // Populate the header + header.compr_mode = compr_mode_type::bitpack; + header.size = static_cast(add_safe(sizeof(Serialization::SEALHeader), out_pos)); + + auto old_except_mask = out_stream.exceptions(); + try + { + // Throw exceptions on ios_base::badbit and ios_base::failbit + out_stream.exceptions(ios_base::badbit | ios_base::failbit); + + // Write the header and the data + out_stream.write(reinterpret_cast(&header), sizeof(Serialization::SEALHeader)); + out_stream.write(reinterpret_cast(out_data), safe_cast(out_pos)); + } + catch (...) + { + out_stream.exceptions(old_except_mask); + throw; + } + + out_stream.exceptions(old_except_mask); + } + + BitUnpackGetBuffer::BitUnpackGetBuffer(istream &in_stream, streamoff in_size, MemoryPoolHandle pool) + : in_buf_(allocate(bitpack_block_bytes + block_slack, pool)), + out_buf_(allocate(bitpack_block_bytes, pool)), in_stream_(in_stream), + in_remaining_(in_size), in_stream_except_mask_(in_stream.exceptions()) + { + // Unpacking reports failure through failed_ rather than stream exceptions, so clear the mask while + // we read; it is restored in the destructor. + in_stream_.exceptions(ios_base::goodbit); + + // Start with an empty get area so that the first read triggers underflow(). + char_type *base = reinterpret_cast(out_buf_.get()); + setg(base, base, base); + } + + BitUnpackGetBuffer::~BitUnpackGetBuffer() + { + in_stream_.exceptions(in_stream_except_mask_); + } + + streamsize BitUnpackGetBuffer::read_packed(unsigned char *dst, streamsize count) + { + streamsize to_read = min(count, in_remaining_); + if (to_read <= 0) + { + return 0; + } + in_stream_.read(reinterpret_cast(dst), to_read); + streamsize got = in_stream_.gcount(); + in_remaining_ -= got; + return got; + } + + size_t BitUnpackGetBuffer::unpack_block() + { + if (finished_) + { + return 0; + } + + if (!started_) + { + // The packed data begins with the original byte count + unsigned char size_bytes[bytes_per_word]; + if (read_packed(size_bytes, static_cast(bytes_per_word)) != + static_cast(bytes_per_word)) + { + failed_ = true; + return 0; + } + memcpy(&raw_remaining_, size_bytes, bytes_per_word); + started_ = true; + if (!raw_remaining_) + { + finished_ = true; + return 0; + } + } + + size_t block_len = + static_cast(min(static_cast(bitpack_block_bytes), raw_remaining_)); + + unsigned char block_header[2]; + if (read_packed(block_header, 2) != 2) + { + failed_ = true; + return 0; + } + int width = static_cast(block_header[0]); + size_t phase = static_cast(block_header[1]); + if (width > bits_per_uint64 || phase > min(size_t(7), block_len)) + { + failed_ = true; + return 0; + } + size_t words = block_word_count(block_len, phase); + size_t packed_bytes = (words * static_cast(width) + size_t(7)) >> 3; + size_t tail = block_len - phase - words * bytes_per_word; + + // Verbatim phase bytes + if (read_packed(out_buf_.get(), safe_cast(phase)) != safe_cast(phase)) + { + failed_ = true; + return 0; + } + + if (read_packed(in_buf_.get(), safe_cast(packed_bytes)) != + safe_cast(packed_bytes)) + { + failed_ = true; + return 0; + } + + // Unpack the words. The whole-uint64_t reads may pick up bits past the packed data (block_slack + // bytes of headroom make them safe); the mask discards everything above the significant bits. + uint64_t mask = (width == bits_per_uint64) ? ~uint64_t(0) : ((uint64_t(1) << width) - 1); + size_t bit_pos = 0; + for (size_t i = 0; i < words; i++) + { + size_t byte_index = bit_pos >> 3; + int shift = static_cast(bit_pos & size_t(7)); + uint64_t low_word = 0; + memcpy(&low_word, in_buf_.get() + byte_index, bytes_per_word); + low_word >>= shift; + if (shift && width > bits_per_uint64 - shift) + { + uint64_t high_byte = in_buf_.get()[byte_index + bytes_per_word]; + low_word |= high_byte << (bits_per_uint64 - shift); + } + uint64_t word = low_word & mask; + memcpy(out_buf_.get() + phase + i * bytes_per_word, &word, bytes_per_word); + bit_pos += static_cast(width); + } + + // Verbatim tail bytes + if (read_packed(out_buf_.get() + block_len - tail, safe_cast(tail)) != + safe_cast(tail)) + { + failed_ = true; + return 0; + } + + raw_remaining_ -= block_len; + if (!raw_remaining_) + { + finished_ = true; + } + return block_len; + } + + BitUnpackGetBuffer::int_type BitUnpackGetBuffer::underflow() + { + if (gptr() < egptr()) + { + return traits_type::to_int_type(*gptr()); + } + + // Pull and unpack blocks until we produce output, reach the end of the data, or fail. unpack_block() + // guarantees progress: whenever it makes none it sets failed_ or finished_, so this loop always + // terminates. + while (!failed_) + { + size_t produced = unpack_block(); + if (failed_) + { + break; + } + if (produced) + { + char_type *base = reinterpret_cast(out_buf_.get()); + setg(base, base, base + produced); + total_produced_ += static_cast(produced); + return traits_type::to_int_type(*gptr()); + } + if (finished_) + { + return traits_type::eof(); + } + } + return traits_type::eof(); + } + + streamsize BitUnpackGetBuffer::xsgetn(char_type *s, streamsize count) + { + streamsize total = 0; + while (total < count) + { + if (gptr() == egptr() && traits_type::eq_int_type(underflow(), traits_type::eof())) + { + break; + } + streamsize avail = min(count - total, static_cast(egptr() - gptr())); + copy_n(gptr(), avail, s + total); + + // avail is at most bitpack_block_bytes, which is well within the range of int. + gbump(static_cast(avail)); + total += avail; + } + return total; + } + + BitUnpackGetBuffer::pos_type BitUnpackGetBuffer::seekoff( + off_type off, ios_base::seekdir dir, ios_base::openmode which) + { + // Only a no-op seek to the current input position is supported, i.e. tellg(). The position is the + // number of unpacked bytes already consumed from the get area. + if (off == 0 && dir == ios_base::cur && (which & ios_base::in)) + { + return pos_type(total_produced_ - static_cast(egptr() - gptr())); + } + return pos_type(off_type(-1)); + } + + unique_ptr make_bitpack_unpack_buffer( + istream &in_stream, streamoff in_size, MemoryPoolHandle pool) + { + return make_unique(in_stream, in_size, std::move(pool)); + } + } // namespace bitpack + } // namespace util +} // namespace seal diff --git a/native/src/seal/util/bitpack.h b/native/src/seal/util/bitpack.h new file mode 100644 index 000000000..b51d5b58a --- /dev/null +++ b/native/src/seal/util/bitpack.h @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#pragma once + +#include "seal/dynarray.h" +#include "seal/memorymanager.h" +#include "seal/util/common.h" +#include "seal/util/defines.h" +#include "seal/util/pointer.h" +#include +#include +#include +#include +#include +#include + +namespace seal +{ + namespace util + { + namespace bitpack + { + /** + The functions in this namespace implement the compr_mode_type::bitpack encoding of a serialized byte + stream. The stream is split into blocks of bitpack_block_bytes bytes; in each block, a run of 64-bit + words is re-encoded using only as many bits per word as the largest word in the run requires, so each + word begins immediately after the last significant bit of the previous one, even in the middle of a + byte. Since ciphertext and key data consist of words storing integers modulo primes much smaller than + the word size, and the significant bits are high-entropy, this discards exactly the always-zero high + bits that a general-purpose compressor cannot improve upon. + + Serialized metadata is not always a multiple of eight bytes, so the word data in the stream need not + fall on the stream's own word grid. Each block therefore carries a phase: the number of initial bytes + (0 to 7) stored verbatim before the packed run, chosen by the encoder to minimize the encoded size of + the block. Bytes after the last whole word in the block are likewise stored verbatim. + + The encoded format is, in order: + + 1. the size in bytes of the original byte stream (8 bytes) + 2. for each block of block_len = min(bitpack_block_bytes, bytes remaining) original bytes: + a. the bit width used for the packed words (1 byte, at most 64) + b. the phase (1 byte, at most min(7, block_len)) + c. phase verbatim bytes + d. the (block_len - phase) / 8 words packed consecutively starting from the least significant bit + e. the remaining (block_len - phase) % 8 bytes verbatim + + A width of zero denotes words that are all zero, packed into no bytes at all. Like the rest of the + serialized data, the encoding is in the byte order of the host. + */ + + // Number of original bytes encoded in each bit-packed block. + constexpr std::size_t bitpack_block_bytes = 4096; + + /** + Bit-packs data in the given buffer, completes the given SEALHeader by writing in the size of the output + and setting the compression mode to compr_mode_type::bitpack and finally writes the SEALHeader followed + by the bit-packed data in the given stream. + + @param[in] in The buffer to bit-pack + @param[out] header_ptr A pointer to a SEALHeader instance matching the output of the encoding + @param[out] out_stream The stream to write to + @param[in] pool The MemoryPoolHandle pointing to a valid memory pool + @throws std::invalid_argument if pool is uninitialized + @throws std::runtime_error if I/O operations failed + */ + void bitpack_write_header_pack_buffer( + const DynArray &in, void *header_ptr, std::ostream &out_stream, MemoryPoolHandle pool); + + /** + A get-only stream buffer that unpacks bit-packed data from an underlying input stream on demand, one + block at a time, instead of decoding the entire payload up front into a single growing buffer. Because + the parser only pulls as many bytes as its (validated) parameters require, this bounds the memory used + during deserialization to a small constant, defending against hostile size claims in the encoded data. + The buffer is forward-only: it reports the current read position (so tellg() works, which nested loads + rely on) but does not support repositioning. + */ + class BitUnpackGetBuffer final : public std::streambuf + { + public: + BitUnpackGetBuffer(std::istream &in_stream, std::streamoff in_size, MemoryPoolHandle pool); + + ~BitUnpackGetBuffer() override; + + BitUnpackGetBuffer(const BitUnpackGetBuffer ©) = delete; + + BitUnpackGetBuffer &operator=(const BitUnpackGetBuffer &assign) = delete; + + // True if malformed or truncated input was encountered while pulling data. + SEAL_NODISCARD bool failed() const noexcept + { + return failed_; + } + + // Number of packed bytes from the bound that have not yet been read from the underlying stream. + SEAL_NODISCARD std::streamoff remaining() const noexcept + { + return in_remaining_; + } + + private: + // Unpacks the next block into out_buf_, returning the number of bytes produced. Sets finished_ when + // all of the original bytes have been produced and failed_ on malformed or truncated input. + std::size_t unpack_block(); + + // Reads up to count packed bytes from the underlying stream, capped by the remaining bound. Returns + // the number of bytes actually read. + std::streamsize read_packed(unsigned char *dst, std::streamsize count); + + int_type underflow() override; + + std::streamsize xsgetn(char_type *s, std::streamsize count) override; + + // Supports only tellg() (a no-op seek to the current input position); any other seek fails. This is + // enough for the nested loads that verify their size via tellg(). + pos_type seekoff( + off_type off, std::ios_base::seekdir dir, + std::ios_base::openmode which = std::ios_base::in | std::ios_base::out) override; + + Pointer in_buf_; + + Pointer out_buf_; + + std::istream &in_stream_; + + std::streamoff in_remaining_; + + std::ios_base::iostate in_stream_except_mask_; + + // Number of original bytes that remain to be produced; valid once started_ is set. + std::uint64_t raw_remaining_ = 0; + + bool started_ = false; + + bool failed_ = false; + + bool finished_ = false; + + // Total unpacked bytes handed to the get area so far; used to report the read position. + std::streamoff total_produced_ = 0; + }; + + // Creates a streambuf that unpacks in_size bytes of bit-packed data from in_stream on demand. + std::unique_ptr make_bitpack_unpack_buffer( + std::istream &in_stream, std::streamoff in_size, MemoryPoolHandle pool); + + template + SEAL_NODISCARD SizeT bitpack_size_bound(SizeT in_size) + { + // 8 bytes for the original size and a width and a phase byte per block of bitpack_block_bytes = + // 4096 original bytes (plus rounding up); the blocks themselves never exceed their original size. + return util::add_safe(in_size, in_size >> 11, SizeT(17)); + } + } // namespace bitpack + } // namespace util +} // namespace seal diff --git a/native/tests/seal/ciphertext.cpp b/native/tests/seal/ciphertext.cpp index 032afbb23..4434cc105 100644 --- a/native/tests/seal/ciphertext.cpp +++ b/native/tests/seal/ciphertext.cpp @@ -127,6 +127,52 @@ namespace sealtest ASSERT_TRUE(ctxt.data() != ctxt2.data()); } + TEST(CiphertextTest, BFVBitPackSaveLoadCiphertext) + { + stringstream stream; + EncryptionParameters parms(scheme_type::bfv); + parms.set_poly_modulus_degree(1024); + parms.set_coeff_modulus(CoeffModulus::BFVDefault(1024)); + parms.set_plain_modulus(0xF0F0); + + SEALContext context(parms, false); + KeyGenerator keygen(context); + PublicKey pk; + keygen.create_public_key(pk); + Encryptor encryptor(context, pk); + + Ciphertext ctxt; + encryptor.encrypt(Plaintext("Ax^10 + 9x^9 + 8x^8 + 7x^7 + 6x^6 + 5x^5 + 4x^4 + 3x^3 + 2x^2 + 1"), ctxt); + + // A bit-packed round-trip preserves the ciphertext exactly + auto bitpack_size = ctxt.save(stream, compr_mode_type::bitpack); + Ciphertext ctxt2; + ctxt2.load(context, stream); + ASSERT_TRUE(ctxt.parms_id() == ctxt2.parms_id()); + ASSERT_TRUE( + is_equal_uint(ctxt.data(), ctxt2.data(), parms.poly_modulus_degree() * parms.coeff_modulus().size() * 2)); + ASSERT_TRUE(ctxt.data() != ctxt2.data()); + + // The first block contains the ciphertext metadata (among it full-width parms_id hash words) and packs at + // up to the full 64 bits, but every later block holds only coefficients smaller than the coefficient + // modulus primes and hence packs at their bit width, so the total is guaranteed to beat the unpacked size. + ASSERT_LT(bitpack_size, ctxt.save_size(compr_mode_type::none)); + + // Seeded ciphertexts bit-pack too: the same seeded object saved with and without bit-packing must load to + // identical data + Encryptor sym_encryptor(context, keygen.secret_key()); + auto seeded = sym_encryptor.encrypt_symmetric(Plaintext("3x^7 + 2")); + stringstream seeded_stream; + seeded.save(seeded_stream, compr_mode_type::bitpack); + seeded.save(seeded_stream, compr_mode_type::none); + Ciphertext from_bitpack, from_none; + from_bitpack.load(context, seeded_stream); + from_none.load(context, seeded_stream); + ASSERT_TRUE(from_bitpack.parms_id() == from_none.parms_id()); + ASSERT_TRUE(is_equal_uint( + from_bitpack.data(), from_none.data(), parms.poly_modulus_degree() * parms.coeff_modulus().size() * 2)); + } + TEST(CiphertextTest, LoadZeroSizeRejectsOversizedDynArray) { EncryptionParameters parms(scheme_type::bfv); diff --git a/native/tests/seal/serialization.cpp b/native/tests/seal/serialization.cpp index 702794d4d..86c4ff4e9 100644 --- a/native/tests/seal/serialization.cpp +++ b/native/tests/seal/serialization.cpp @@ -79,6 +79,35 @@ namespace sealtest } }; + // A serializable object whose payload is a sequence of 64-bit words, mirroring how real SEAL objects store + // coefficient data; used to pin down the exact bit-packed output size. + struct word_struct + { + std::vector words; + + void save_members(ostream &stream) + { + uint64_t n = static_cast(words.size()); + stream.write(reinterpret_cast(&n), sizeof(uint64_t)); + stream.write(reinterpret_cast(words.data()), static_cast(words.size() * 8)); + } + + void load_members(istream &stream) + { + uint64_t n = 0; + stream.read(reinterpret_cast(&n), sizeof(uint64_t)); + words.resize(static_cast(n)); + stream.read(reinterpret_cast(words.data()), static_cast(n * 8)); + } + + streamoff save_size(compr_mode_type compr_mode) const + { + size_t raw = sizeof(uint64_t) + words.size() * 8; + return static_cast( + sizeof(Serialization::SEALHeader) + Serialization::ComprSizeEstimate(raw, compr_mode)); + } + }; + // A serializable object that, on save, writes a small prefix followed by a large filler, but on load reads // only the prefix. Modeling a hostile/oversized payload: the loader must not need to inflate the unread filler // (the decompression-bomb defense), and must leave the stream positioned at the end of the object. @@ -237,6 +266,22 @@ namespace sealtest #ifdef SEAL_USE_ZLIB modes.push_back(compr_mode_type::zlib); #endif +#ifdef SEAL_USE_ZSTD + modes.push_back(compr_mode_type::zstd); +#endif + modes.push_back(compr_mode_type::bitpack); + return modes; + } + + // The compression modes that verify the integrity of the compressed data on load. Bit-packing performs no + // integrity checking: corrupted packed bits decode to wrong values rather than a detected error, like + // compr_mode_type::none. + std::vector checksummed_compr_modes() + { + std::vector modes; +#ifdef SEAL_USE_ZLIB + modes.push_back(compr_mode_type::zlib); +#endif #ifdef SEAL_USE_ZSTD modes.push_back(compr_mode_type::zstd); #endif @@ -269,7 +314,9 @@ namespace sealtest invalid_header.version_major = 0x02; ASSERT_FALSE(Serialization::IsValidHeader(invalid_header)); invalid_header.version_major = SEAL_VERSION_MAJOR; - invalid_header.compr_mode = (compr_mode_type)0x03; + invalid_header.compr_mode = compr_mode_type::bitpack; + ASSERT_TRUE(Serialization::IsValidHeader(invalid_header)); + invalid_header.compr_mode = (compr_mode_type)0x04; ASSERT_FALSE(Serialization::IsValidHeader(invalid_header)); } @@ -430,6 +477,17 @@ namespace sealtest ASSERT_EQ(st.c, st3.c); } #endif + { + test_struct st3; + out_size = Serialization::Save( + bind(&test_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), stream, + compr_mode_type::bitpack, false); + in_size = Serialization::Load(bind(&test_struct::load_members, &st3, _1), stream, false); + ASSERT_EQ(out_size, in_size); + ASSERT_EQ(st.a, st3.a); + ASSERT_EQ(st.b, st3.b); + ASSERT_EQ(st.c, st3.c); + } } TEST(SerializationTest, SaveLoadToBuffer) @@ -510,6 +568,30 @@ namespace sealtest ASSERT_EQ(st.c, st3.c); } #endif + { + // Reset buffer back to zero + memset(buffer, 0, arr_size); + + test_struct st3; + ss.seekp(0); + test_out_size = Serialization::Save( + bind(&test_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), ss, + compr_mode_type::bitpack, false); + out_size = Serialization::Save( + bind(&test_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), buffer, arr_size, + compr_mode_type::bitpack, false); + ASSERT_EQ(test_out_size, out_size); + for (size_t i = static_cast(out_size); i < arr_size; i++) + { + ASSERT_EQ(seal_byte{}, buffer[i]); + } + + in_size = Serialization::Load(bind(&test_struct::load_members, &st3, _1), buffer, arr_size, false); + ASSERT_EQ(out_size, in_size); + ASSERT_EQ(st.a, st3.a); + ASSERT_EQ(st.b, st3.b); + ASSERT_EQ(st.c, st3.c); + } } // Round-trips a payload larger than the 256 KB internal decompression buffer to exercise multi-chunk streaming @@ -624,7 +706,7 @@ namespace sealtest st.data[i] = static_cast((i * 2654435761ULL) >> 24); } - for (auto mode : available_compr_modes()) + for (auto mode : checksummed_compr_modes()) { stringstream stream; Serialization::Save(bind(&large_struct::save_members, &st, _1), st.save_size(mode), stream, mode, false); @@ -803,4 +885,156 @@ namespace sealtest ASSERT_LT(buf.consumed(), streamsize(1) << 20); } } + + // Bit-packing an all-word payload of bounded-width values must produce exactly the size the format prescribes + // (8 bytes for the original size, then per block a width byte, a phase byte, and the packed words) and must + // round-trip. + TEST(SerializationTest, BitPackSizeAndRoundTrip) + { + using namespace placeholders; + + // 1023 values of at most 36 significant bits; with the 8-byte count in front, the serialized stream is + // exactly 8192 bytes, i.e. two full blocks of 512 word-aligned words each (phase 0, no verbatim bytes). + word_struct st; + st.words.resize(1023); + uint64_t state = 1; + for (size_t i = 0; i < st.words.size(); i++) + { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + st.words[i] = state & ((uint64_t(1) << 36) - 1); + } + + // Pin the width of both blocks to exactly 36 bits. The stream words are the count followed by the values, + // so the second block starts at value index 511. + st.words[0] |= uint64_t(1) << 35; + st.words[511] |= uint64_t(1) << 35; + + stringstream stream; + auto out_size = Serialization::Save( + bind(&word_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), stream, + compr_mode_type::bitpack, false); + + // 16 (SEALHeader) + 8 (original size) + 2 * (1 width byte + 1 phase byte + 512 * 36 / 8) + ASSERT_EQ(16 + 8 + 2 * (2 + 2304), out_size); + + word_struct st2; + auto in_size = Serialization::Load(bind(&word_struct::load_members, &st2, _1), stream, false); + ASSERT_EQ(out_size, in_size); + ASSERT_TRUE(st.words == st2.words); + } + + // The encoder must find word data that does not fall on the stream's own word grid: values shifted off the + // grid by a 1-byte prefix (as a seal_byte member does in real objects) must still pack at their bit width, + // costing only the per-block phase bytes relative to the aligned encoding. + TEST(SerializationTest, BitPackMisalignedWords) + { + using namespace placeholders; + + word_struct aligned; + aligned.words.resize(1023); + uint64_t state = 12345; + for (size_t i = 0; i < aligned.words.size(); i++) + { + state = state * 6364136223846793005ULL + 1442695040888963407ULL; + aligned.words[i] = state & ((uint64_t(1) << 36) - 1); + } + + struct prefixed_word_struct + { + word_struct inner; + + void save_members(ostream &stream) + { + seal_byte prefix{}; + stream.write(reinterpret_cast(&prefix), 1); + inner.save_members(stream); + } + + streamoff save_size(compr_mode_type compr_mode) const + { + size_t raw = 1 + sizeof(uint64_t) + inner.words.size() * 8; + return static_cast( + sizeof(Serialization::SEALHeader) + Serialization::ComprSizeEstimate(raw, compr_mode)); + } + }; + prefixed_word_struct st; + st.inner = aligned; + + stringstream aligned_stream; + auto aligned_size = Serialization::Save( + bind(&word_struct::save_members, &aligned, _1), aligned.save_size(compr_mode_type::bitpack), aligned_stream, + compr_mode_type::bitpack, false); + + stringstream prefixed_stream; + auto prefixed_size = Serialization::Save( + bind(&prefixed_word_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), prefixed_stream, + compr_mode_type::bitpack, false); + + // The prefixed stream is 1 original byte longer and spans 3 blocks instead of 2; the packed words must + // not grow beyond the extra verbatim and block-header bytes. + ASSERT_LE(prefixed_size, aligned_size + 16); + } + + // A width byte exceeding 64 is malformed and must be rejected cleanly. + TEST(SerializationTest, BitPackTamperedWidthThrows) + { + using namespace placeholders; + + test_struct st{ 3, ~0, 3.14159 }; + stringstream ss; + Serialization::Save( + bind(&test_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), ss, + compr_mode_type::bitpack, false); + + // The first block's width byte follows the SEALHeader (16 bytes) and the original size (8 bytes). + string bytes = ss.str(); + bytes[24] = static_cast(65); + + stringstream tampered(bytes); + test_struct st2; + ASSERT_ANY_THROW(Serialization::Load(bind(&test_struct::load_members, &st2, _1), tampered, false)); + } + + // A phase byte exceeding 7 is malformed and must be rejected cleanly. + TEST(SerializationTest, BitPackTamperedPhaseThrows) + { + using namespace placeholders; + + test_struct st{ 3, ~0, 3.14159 }; + stringstream ss; + Serialization::Save( + bind(&test_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), ss, + compr_mode_type::bitpack, false); + + // The first block's phase byte follows the SEALHeader (16 bytes), the original size (8 bytes), and the + // width byte. + string bytes = ss.str(); + bytes[25] = static_cast(8); + + stringstream tampered(bytes); + test_struct st2; + ASSERT_ANY_THROW(Serialization::Load(bind(&test_struct::load_members, &st2, _1), tampered, false)); + } + + // An understated original size makes the parser read past the end of the unpacked data and must be rejected + // cleanly. + TEST(SerializationTest, BitPackTamperedSizeThrows) + { + using namespace placeholders; + + test_struct st{ 3, ~0, 3.14159 }; + stringstream ss; + Serialization::Save( + bind(&test_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), ss, + compr_mode_type::bitpack, false); + + // The original size is the 8 bytes following the SEALHeader; understate it below what load_members reads. + string bytes = ss.str(); + uint64_t small_size = 8; + memcpy(&bytes[16], &small_size, sizeof(uint64_t)); + + stringstream tampered(bytes); + test_struct st2; + ASSERT_ANY_THROW(Serialization::Load(bind(&test_struct::load_members, &st2, _1), tampered, false)); + } } // namespace sealtest From a02e9a8f8144b90a6fa92adfd400d5af1e7d19f9 Mon Sep 17 00:00:00 2001 From: jryancarr Date: Mon, 3 Aug 2026 15:04:52 -0400 Subject: [PATCH 02/11] Replace bitpack format description with ASCII diagrams. Co-Authored-By: Claude Fable 5 --- native/src/seal/util/bitpack.h | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/native/src/seal/util/bitpack.h b/native/src/seal/util/bitpack.h index b51d5b58a..86b012f02 100644 --- a/native/src/seal/util/bitpack.h +++ b/native/src/seal/util/bitpack.h @@ -35,15 +35,30 @@ namespace seal (0 to 7) stored verbatim before the packed run, chosen by the encoder to minimize the encoded size of the block. Bytes after the last whole word in the block are likewise stored verbatim. - The encoded format is, in order: - - 1. the size in bytes of the original byte stream (8 bytes) - 2. for each block of block_len = min(bitpack_block_bytes, bytes remaining) original bytes: - a. the bit width used for the packed words (1 byte, at most 64) - b. the phase (1 byte, at most min(7, block_len)) - c. phase verbatim bytes - d. the (block_len - phase) / 8 words packed consecutively starting from the least significant bit - e. the remaining (block_len - phase) % 8 bytes verbatim + The encoded format is the size of the original byte stream followed by one encoded block per + block_len = min(bitpack_block_bytes, bytes remaining) original bytes: + + +---------------+---------+---------+-- --+---------+ + | original size | block 0 | block 1 | ... | block k | + | (8 bytes) | | | | | + +---------------+---------+---------+-- --+---------+ + + Each block encodes its block_len original bytes as + + +-----------+-----------+~~~~~~~~~~~~~+--------------------------------+~~~~~~~~~~~~~+ + | width | phase | verbatim | packed words | verbatim | + | (1 byte, | (1 byte, | (phase | (ceil(words * width / 8) | (tail | + | max 64) | max 7) | bytes) | bytes) | bytes) | + +-----------+-----------+~~~~~~~~~~~~~+--------------------------------+~~~~~~~~~~~~~+ + + where words = (block_len - phase) / 8 and tail = (block_len - phase) % 8; the phase never exceeds + block_len. In the packed words area, word i occupies bits [i * width, (i + 1) * width), least + significant bit first, with no regard for byte boundaries: + + bit: 0 width 2 * width + +---------+---------+---------+-- + | word 0 | word 1 | word 2 | ... + +---------+---------+---------+-- A width of zero denotes words that are all zero, packed into no bytes at all. Like the rest of the serialized data, the encoding is in the byte order of the host. From 33d0e4c2ed34ee94a6e48bc7ffb18a397897f7a4 Mon Sep 17 00:00:00 2001 From: jryancarr Date: Mon, 3 Aug 2026 16:26:22 -0400 Subject: [PATCH 03/11] Make the bitpack block size part of the encoded stream; default to 1024 bytes. A block-size sweep across real objects shows the optimum scales with sqrt(object size / number of unpackable runs): roughly 512 bytes for a small ciphertext, 2-4 KB for multi-megabyte keys. No single constant is optimal, and the previous 4096 was baked into the decoder, silently making it part of the wire format. The block size (as its base-2 logarithm, validated to 64 B - 64 KB on load) is now recorded in the encoded stream, so future encoders can tune it per object without a format change. The encoder default moves to 1024 bytes, which is within 0.3% of the per-object optimum on large objects and about 15% smaller than 4096-byte blocks on small ones. Co-Authored-By: Claude Fable 5 --- native/src/seal/util/bitpack.cpp | 42 ++++++++++++------- native/src/seal/util/bitpack.h | 46 ++++++++++++++++----- native/tests/seal/serialization.cpp | 62 +++++++++++++++++++++-------- 3 files changed, 109 insertions(+), 41 deletions(-) diff --git a/native/src/seal/util/bitpack.cpp b/native/src/seal/util/bitpack.cpp index 0affed9bf..ac6d803b0 100644 --- a/native/src/seal/util/bitpack.cpp +++ b/native/src/seal/util/bitpack.cpp @@ -58,10 +58,11 @@ namespace seal DynArray out(add_safe(bitpack_size_bound(in_size), block_slack), pool); unsigned char *out_data = reinterpret_cast(out.begin()); - // Write the original byte count + // Write the original byte count and the block size uint64_t in_size64 = static_cast(in_size); memcpy(out_data, &in_size64, bytes_per_word); size_t out_pos = bytes_per_word; + out_data[out_pos++] = static_cast(get_significant_bit_count(bitpack_block_bytes) - 1); for (size_t block_start = 0; block_start < in_size;) { @@ -158,17 +159,21 @@ namespace seal } BitUnpackGetBuffer::BitUnpackGetBuffer(istream &in_stream, streamoff in_size, MemoryPoolHandle pool) - : in_buf_(allocate(bitpack_block_bytes + block_slack, pool)), - out_buf_(allocate(bitpack_block_bytes, pool)), in_stream_(in_stream), - in_remaining_(in_size), in_stream_except_mask_(in_stream.exceptions()) + : pool_(std::move(pool)), in_stream_(in_stream), in_remaining_(in_size), + in_stream_except_mask_(in_stream.exceptions()) { + if (!pool_) + { + throw invalid_argument("pool is uninitialized"); + } + // Unpacking reports failure through failed_ rather than stream exceptions, so clear the mask while // we read; it is restored in the destructor. in_stream_.exceptions(ios_base::goodbit); - // Start with an empty get area so that the first read triggers underflow(). - char_type *base = reinterpret_cast(out_buf_.get()); - setg(base, base, base); + // Start with an empty get area so that the first read triggers underflow(); the buffers are + // allocated once the block size has been read from the packed data. + setg(nullptr, nullptr, nullptr); } BitUnpackGetBuffer::~BitUnpackGetBuffer() @@ -198,15 +203,24 @@ namespace seal if (!started_) { - // The packed data begins with the original byte count - unsigned char size_bytes[bytes_per_word]; - if (read_packed(size_bytes, static_cast(bytes_per_word)) != - static_cast(bytes_per_word)) + // The packed data begins with the original byte count and the block size + unsigned char prologue[bytes_per_word + 1]; + if (read_packed(prologue, static_cast(sizeof(prologue))) != + static_cast(sizeof(prologue))) + { + failed_ = true; + return 0; + } + memcpy(&raw_remaining_, prologue, bytes_per_word); + int block_log2 = static_cast(prologue[bytes_per_word]); + if (block_log2 < bitpack_block_log2_min || block_log2 > bitpack_block_log2_max) { failed_ = true; return 0; } - memcpy(&raw_remaining_, size_bytes, bytes_per_word); + block_bytes_ = size_t(1) << block_log2; + in_buf_ = allocate(block_bytes_ + block_slack, pool_); + out_buf_ = allocate(block_bytes_, pool_); started_ = true; if (!raw_remaining_) { @@ -216,7 +230,7 @@ namespace seal } size_t block_len = - static_cast(min(static_cast(bitpack_block_bytes), raw_remaining_)); + static_cast(min(static_cast(block_bytes_), raw_remaining_)); unsigned char block_header[2]; if (read_packed(block_header, 2) != 2) @@ -330,7 +344,7 @@ namespace seal streamsize avail = min(count - total, static_cast(egptr() - gptr())); copy_n(gptr(), avail, s + total); - // avail is at most bitpack_block_bytes, which is well within the range of int. + // avail is at most the block size (at most 64 KB), which is well within the range of int. gbump(static_cast(avail)); total += avail; } diff --git a/native/src/seal/util/bitpack.h b/native/src/seal/util/bitpack.h index 86b012f02..0998fb6cb 100644 --- a/native/src/seal/util/bitpack.h +++ b/native/src/seal/util/bitpack.h @@ -35,13 +35,19 @@ namespace seal (0 to 7) stored verbatim before the packed run, chosen by the encoder to minimize the encoded size of the block. Bytes after the last whole word in the block are likewise stored verbatim. - The encoded format is the size of the original byte stream followed by one encoded block per - block_len = min(bitpack_block_bytes, bytes remaining) original bytes: + The encoded format is the size of the original byte stream and the block size, followed by one + encoded block per block_len = min(block size, bytes remaining) original bytes: - +---------------+---------+---------+-- --+---------+ - | original size | block 0 | block 1 | ... | block k | - | (8 bytes) | | | | | - +---------------+---------+---------+-- --+---------+ + +---------------+------------+---------+---------+-- --+---------+ + | original size | block size | block 0 | block 1 | ... | block k | + | (8 bytes) | (log2, | | | | | + | | 1 byte) | | | | | + +---------------+------------+---------+---------+-- --+---------+ + + The block size is a power of two between 64 and 65536 bytes, recorded as its base-2 logarithm; it + trades the two per-block header bytes (favoring large blocks) against how much data shares a block + with unpackable bytes such as serialized metadata (favoring small blocks). This encoder always + writes bitpack_block_bytes, a good compromise across object sizes; decoders accept the full range. Each block encodes its block_len original bytes as @@ -64,8 +70,19 @@ namespace seal serialized data, the encoding is in the byte order of the host. */ - // Number of original bytes encoded in each bit-packed block. - constexpr std::size_t bitpack_block_bytes = 4096; + // Bounds for the base-2 logarithm of the block size accepted when unpacking. + constexpr int bitpack_block_log2_min = 6; + + constexpr int bitpack_block_log2_max = 16; + + // Number of original bytes encoded in each bit-packed block by this encoder. + constexpr std::size_t bitpack_block_bytes = 1024; + + static_assert( + (bitpack_block_bytes & (bitpack_block_bytes - 1)) == 0 && + bitpack_block_bytes >= (std::size_t(1) << bitpack_block_log2_min) && + bitpack_block_bytes <= (std::size_t(1) << bitpack_block_log2_max), + "bitpack_block_bytes must be a power of two within the accepted range"); /** Bit-packs data in the given buffer, completes the given SEALHeader by writing in the size of the output @@ -132,6 +149,9 @@ namespace seal off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which = std::ios_base::in | std::ios_base::out) override; + MemoryPoolHandle pool_; + + // Allocated once the block size has been read from the packed data. Pointer in_buf_; Pointer out_buf_; @@ -145,6 +165,9 @@ namespace seal // Number of original bytes that remain to be produced; valid once started_ is set. std::uint64_t raw_remaining_ = 0; + // Block size read from the packed data; valid once started_ is set. + std::size_t block_bytes_ = 0; + bool started_ = false; bool failed_ = false; @@ -162,9 +185,10 @@ namespace seal template SEAL_NODISCARD SizeT bitpack_size_bound(SizeT in_size) { - // 8 bytes for the original size and a width and a phase byte per block of bitpack_block_bytes = - // 4096 original bytes (plus rounding up); the blocks themselves never exceed their original size. - return util::add_safe(in_size, in_size >> 11, SizeT(17)); + // 9 bytes for the original size and the block size, and a width and a phase byte per block of + // bitpack_block_bytes = 1024 original bytes (plus rounding up); the blocks themselves never + // exceed their original size. + return util::add_safe(in_size, in_size >> 9, SizeT(17)); } } // namespace bitpack } // namespace util diff --git a/native/tests/seal/serialization.cpp b/native/tests/seal/serialization.cpp index 86c4ff4e9..f3b08f1e6 100644 --- a/native/tests/seal/serialization.cpp +++ b/native/tests/seal/serialization.cpp @@ -887,14 +887,15 @@ namespace sealtest } // Bit-packing an all-word payload of bounded-width values must produce exactly the size the format prescribes - // (8 bytes for the original size, then per block a width byte, a phase byte, and the packed words) and must - // round-trip. + // (the original size and the block size, then per block a width byte, a phase byte, and the packed words) and + // must round-trip. TEST(SerializationTest, BitPackSizeAndRoundTrip) { using namespace placeholders; // 1023 values of at most 36 significant bits; with the 8-byte count in front, the serialized stream is - // exactly 8192 bytes, i.e. two full blocks of 512 word-aligned words each (phase 0, no verbatim bytes). + // exactly 8192 bytes, i.e. eight full 1024-byte blocks of 128 word-aligned words each (phase 0, no + // verbatim bytes). word_struct st; st.words.resize(1023); uint64_t state = 1; @@ -904,18 +905,21 @@ namespace sealtest st.words[i] = state & ((uint64_t(1) << 36) - 1); } - // Pin the width of both blocks to exactly 36 bits. The stream words are the count followed by the values, - // so the second block starts at value index 511. + // Pin the width of every block to exactly 36 bits. The stream words are the count followed by the values, + // so block i (of 128 stream words each) starts at value index 128 * i - 1. st.words[0] |= uint64_t(1) << 35; - st.words[511] |= uint64_t(1) << 35; + for (size_t block = 1; block < 8; block++) + { + st.words[128 * block - 1] |= uint64_t(1) << 35; + } stringstream stream; auto out_size = Serialization::Save( bind(&word_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), stream, compr_mode_type::bitpack, false); - // 16 (SEALHeader) + 8 (original size) + 2 * (1 width byte + 1 phase byte + 512 * 36 / 8) - ASSERT_EQ(16 + 8 + 2 * (2 + 2304), out_size); + // 16 (SEALHeader) + 8 (original size) + 1 (block size) + 8 * (1 width byte + 1 phase byte + 128 * 36 / 8) + ASSERT_EQ(16 + 8 + 1 + 8 * (2 + 576), out_size); word_struct st2; auto in_size = Serialization::Load(bind(&word_struct::load_members, &st2, _1), stream, false); @@ -970,9 +974,10 @@ namespace sealtest bind(&prefixed_word_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), prefixed_stream, compr_mode_type::bitpack, false); - // The prefixed stream is 1 original byte longer and spans 3 blocks instead of 2; the packed words must - // not grow beyond the extra verbatim and block-header bytes. - ASSERT_LE(prefixed_size, aligned_size + 16); + // The prefixed stream is 1 original byte longer and spans one more block; realignment costs at most the + // per-block phase and tail verbatim bytes plus one extra block header, far below the 8 bits per word + // (over 4,000 bytes here) that losing alignment would cost. + ASSERT_LE(prefixed_size, aligned_size + 80); } // A width byte exceeding 64 is malformed and must be rejected cleanly. @@ -986,9 +991,10 @@ namespace sealtest bind(&test_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), ss, compr_mode_type::bitpack, false); - // The first block's width byte follows the SEALHeader (16 bytes) and the original size (8 bytes). + // The first block's width byte follows the SEALHeader (16 bytes), the original size (8 bytes), and the + // block size (1 byte). string bytes = ss.str(); - bytes[24] = static_cast(65); + bytes[25] = static_cast(65); stringstream tampered(bytes); test_struct st2; @@ -1006,16 +1012,40 @@ namespace sealtest bind(&test_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), ss, compr_mode_type::bitpack, false); - // The first block's phase byte follows the SEALHeader (16 bytes), the original size (8 bytes), and the - // width byte. + // The first block's phase byte follows the SEALHeader (16 bytes), the original size (8 bytes), the block + // size (1 byte), and the width byte. string bytes = ss.str(); - bytes[25] = static_cast(8); + bytes[26] = static_cast(8); stringstream tampered(bytes); test_struct st2; ASSERT_ANY_THROW(Serialization::Load(bind(&test_struct::load_members, &st2, _1), tampered, false)); } + // A block size outside the accepted power-of-two range is malformed and must be rejected cleanly. + TEST(SerializationTest, BitPackTamperedBlockSizeThrows) + { + using namespace placeholders; + + test_struct st{ 3, ~0, 3.14159 }; + stringstream ss; + Serialization::Save( + bind(&test_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), ss, + compr_mode_type::bitpack, false); + + // The block size byte follows the SEALHeader (16 bytes) and the original size (8 bytes). + string bytes = ss.str(); + bytes[24] = static_cast(5); + + stringstream tampered(bytes); + test_struct st2; + ASSERT_ANY_THROW(Serialization::Load(bind(&test_struct::load_members, &st2, _1), tampered, false)); + + bytes[24] = static_cast(17); + stringstream tampered2(bytes); + ASSERT_ANY_THROW(Serialization::Load(bind(&test_struct::load_members, &st2, _1), tampered2, false)); + } + // An understated original size makes the parser read past the end of the unpacked data and must be rejected // cleanly. TEST(SerializationTest, BitPackTamperedSizeThrows) From 0474ef81965c227a7dda840192efc8f5ac892635 Mon Sep 17 00:00:00 2001 From: jryancarr Date: Tue, 4 Aug 2026 11:38:51 -0400 Subject: [PATCH 04/11] Express the phase bound as bytes_per_word - 1 instead of a literal 7. The maximum useful phase is one byte less than a word: a phase of bytes_per_word reproduces the alignment of phase zero while wasting a verbatim word. Spelling the bound as bytes_per_word - 1 also distinguishes it from the bit-within-a-byte masks in the packing loops, whose 7s are bits-per-byte quantities and intentionally unchanged. Co-Authored-By: Claude Fable 5 --- native/src/seal/util/bitpack.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/native/src/seal/util/bitpack.cpp b/native/src/seal/util/bitpack.cpp index ac6d803b0..966143edf 100644 --- a/native/src/seal/util/bitpack.cpp +++ b/native/src/seal/util/bitpack.cpp @@ -53,8 +53,7 @@ namespace seal size_t in_size = in.size(); const unsigned char *in_data = reinterpret_cast(in.cbegin()); - // The whole-uint64_t writes below rely on the output being zero-filled (DynArray zero-fills) and on - // block_slack bytes of headroom past the size bound. + // Allocates a zero-filled array with block_slack bytes of extra headroom past the size bound. DynArray out(add_safe(bitpack_size_bound(in_size), block_slack), pool); unsigned char *out_data = reinterpret_cast(out.begin()); @@ -71,11 +70,12 @@ namespace seal // The word data in the stream need not fall on the stream's own word grid (serialized metadata // is not always a multiple of eight bytes), so choose the phase that minimizes the encoded size - // of the block. + // of the block. A phase of bytes_per_word would reproduce the alignment of phase zero, so only + // smaller values need to be considered. size_t phase = 0; int width = 0; size_t body_size = block_len; - for (size_t p = 0; p <= min(size_t(7), block_len); p++) + for (size_t p = 0; p <= min(bytes_per_word - 1, block_len); p++) { size_t words = block_word_count(block_len, p); uint64_t block_or = 0; @@ -240,7 +240,7 @@ namespace seal } int width = static_cast(block_header[0]); size_t phase = static_cast(block_header[1]); - if (width > bits_per_uint64 || phase > min(size_t(7), block_len)) + if (width > bits_per_uint64 || phase > min(bytes_per_word - 1, block_len)) { failed_ = true; return 0; From 346b72f5703960dc5e4ac297202d0433d2c9e652 Mon Sep 17 00:00:00 2001 From: jryancarr Date: Tue, 4 Aug 2026 11:42:33 -0400 Subject: [PATCH 05/11] Document and test tiny-block phase handling. For a final block shorter than a word, every phase up to the block length encodes zero packed words at identical cost, splitting the bytes between the verbatim phase prefix and tail; the encoder's tie-break settles on phase zero. The decoder must nevertheless accept all of the equivalent encodings and reject a phase beyond the block length, which would underflow the word count. A new test pins both behaviors with hand-crafted streams. Co-Authored-By: Claude Fable 5 --- native/src/seal/util/bitpack.cpp | 4 ++- native/tests/seal/serialization.cpp | 43 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/native/src/seal/util/bitpack.cpp b/native/src/seal/util/bitpack.cpp index 966143edf..c00bcba7b 100644 --- a/native/src/seal/util/bitpack.cpp +++ b/native/src/seal/util/bitpack.cpp @@ -71,7 +71,9 @@ namespace seal // The word data in the stream need not fall on the stream's own word grid (serialized metadata // is not always a multiple of eight bytes), so choose the phase that minimizes the encoded size // of the block. A phase of bytes_per_word would reproduce the alignment of phase zero, so only - // smaller values need to be considered. + // smaller values need to be considered. The clamp to block_len keeps the word count from + // underflowing on a block shorter than a word; for such a block every phase encodes zero words + // at the same size and the tie-break below settles on phase zero. size_t phase = 0; int width = 0; size_t body_size = block_len; diff --git a/native/tests/seal/serialization.cpp b/native/tests/seal/serialization.cpp index f3b08f1e6..64b1b1905 100644 --- a/native/tests/seal/serialization.cpp +++ b/native/tests/seal/serialization.cpp @@ -1022,6 +1022,49 @@ namespace sealtest ASSERT_ANY_THROW(Serialization::Load(bind(&test_struct::load_members, &st2, _1), tampered, false)); } + // A block shorter than a word admits several equivalent encodings: any phase up to the block length splits + // the bytes between the verbatim phase prefix and the verbatim tail, with zero packed words. The decoder must + // accept all of them (an encoder is free to emit any) and must reject a phase beyond the block length, which + // would underflow the word count. + TEST(SerializationTest, BitPackTinyBlockPhases) + { + using namespace placeholders; + + for (unsigned phase = 0; phase <= 4; phase++) + { + // Hand-craft a stream holding the 3 original bytes { 0xAA, 0xBB, 0xCC } in a single tiny block + Serialization::SEALHeader header; + header.compr_mode = compr_mode_type::bitpack; + header.size = 16 + 8 + 1 + 2 + 3; + string blob(reinterpret_cast(&header), sizeof(Serialization::SEALHeader)); + uint64_t original_size = 3; + blob.append(reinterpret_cast(&original_size), sizeof(uint64_t)); + blob.push_back(static_cast(10)); // block size: 2^10 bytes + blob.push_back(static_cast(0)); // width + blob.push_back(static_cast(phase)); + blob.push_back(static_cast(0xAA)); + blob.push_back(static_cast(0xBB)); + blob.push_back(static_cast(0xCC)); + + stringstream stream(blob); + unsigned char loaded[3]{}; + auto load_fn = [&](istream &in_stream, SEALVersion) { + in_stream.read(reinterpret_cast(loaded), 3); + }; + if (phase <= 3) + { + Serialization::Load(load_fn, stream, false); + ASSERT_EQ(0xAA, loaded[0]); + ASSERT_EQ(0xBB, loaded[1]); + ASSERT_EQ(0xCC, loaded[2]); + } + else + { + ASSERT_ANY_THROW(Serialization::Load(load_fn, stream, false)); + } + } + } + // A block size outside the accepted power-of-two range is malformed and must be rejected cleanly. TEST(SerializationTest, BitPackTamperedBlockSizeThrows) { From a8845f0e778619c8d761ba9b93cad8b4617bebcb Mon Sep 17 00:00:00 2001 From: jryancarr Date: Thu, 6 Aug 2026 14:27:04 -0400 Subject: [PATCH 06/11] rewrite bitpack overview --- native/src/seal/util/bitpack.h | 130 +++++++++++++++++++++------------ 1 file changed, 85 insertions(+), 45 deletions(-) diff --git a/native/src/seal/util/bitpack.h b/native/src/seal/util/bitpack.h index 0998fb6cb..b786db324 100644 --- a/native/src/seal/util/bitpack.h +++ b/native/src/seal/util/bitpack.h @@ -23,51 +23,91 @@ namespace seal { /** The functions in this namespace implement the compr_mode_type::bitpack encoding of a serialized byte - stream. The stream is split into blocks of bitpack_block_bytes bytes; in each block, a run of 64-bit - words is re-encoded using only as many bits per word as the largest word in the run requires, so each - word begins immediately after the last significant bit of the previous one, even in the middle of a - byte. Since ciphertext and key data consist of words storing integers modulo primes much smaller than - the word size, and the significant bits are high-entropy, this discards exactly the always-zero high - bits that a general-purpose compressor cannot improve upon. - - Serialized metadata is not always a multiple of eight bytes, so the word data in the stream need not - fall on the stream's own word grid. Each block therefore carries a phase: the number of initial bytes - (0 to 7) stored verbatim before the packed run, chosen by the encoder to minimize the encoded size of - the block. Bytes after the last whole word in the block are likewise stored verbatim. - - The encoded format is the size of the original byte stream and the block size, followed by one - encoded block per block_len = min(block size, bytes remaining) original bytes: - - +---------------+------------+---------+---------+-- --+---------+ - | original size | block size | block 0 | block 1 | ... | block k | - | (8 bytes) | (log2, | | | | | - | | 1 byte) | | | | | - +---------------+------------+---------+---------+-- --+---------+ - - The block size is a power of two between 64 and 65536 bytes, recorded as its base-2 logarithm; it - trades the two per-block header bytes (favoring large blocks) against how much data shares a block - with unpackable bytes such as serialized metadata (favoring small blocks). This encoder always - writes bitpack_block_bytes, a good compromise across object sizes; decoders accept the full range. - - Each block encodes its block_len original bytes as - - +-----------+-----------+~~~~~~~~~~~~~+--------------------------------+~~~~~~~~~~~~~+ - | width | phase | verbatim | packed words | verbatim | - | (1 byte, | (1 byte, | (phase | (ceil(words * width / 8) | (tail | - | max 64) | max 7) | bytes) | bytes) | bytes) | - +-----------+-----------+~~~~~~~~~~~~~+--------------------------------+~~~~~~~~~~~~~+ - - where words = (block_len - phase) / 8 and tail = (block_len - phase) % 8; the phase never exceeds - block_len. In the packed words area, word i occupies bits [i * width, (i + 1) * width), least - significant bit first, with no regard for byte boundaries: - - bit: 0 width 2 * width - +---------+---------+---------+-- - | word 0 | word 1 | word 2 | ... - +---------+---------+---------+-- - - A width of zero denotes words that are all zero, packed into no bytes at all. Like the rest of the - serialized data, the encoding is in the byte order of the host. + stream. This can be thought of as a compression algorithm designed specifically for the types of byte + streams SEAL will most often be serializing. + + The vast majority of data serialized / deserialized by SEAL will be arrays of uint64_t representing + polynomial coefficients in RNS form. Each of the uint64_ts in a single array will be residues modulo + some prime q, where q <= 2^60. Thus, some number of high-order bits of these values will always be zero, + while the lower-order bits will (when serializing any cryptographic data) have high entropy. Standard + compression algorithms like zlib are designed to operate on whole bytes, so they will fail to compress + any zero bits that do not appear as a whole byte, and the high-entropy cryptographic bits are + incompressible. + + The optimal way to compress these arrays of polynomial coefficients would be to simply use knowledge + of q to append one coefficient after another with no zero-bits inbetween. However, SEAL's serialization + API deliberately hides details of the payload's schema from the serializer, which just sees a raw byte + stream. While the majority of that byte stream will usually contain these uint64_t arrays, each array + may have a different value of q, and the stream may be interspersed with other small data structures such + as metadata, parms_ids, etc. + + The bitpack compression scheme works by breaking the byte stream up into blocks of size + bitpack_block_bytes (currently 1024), and attempting to compress each block as if it contains a section + of a uint64_t array of polynomial coefficients modulo some q, using the strategy described below. + On blocks that really do contain such data, it achieves near-optimal compression; on blocks that contain + any other kind of data it will probably not compress the data at all, but such blocks are rare enough + that the overall performance when applied to real SEAL payloads is closer to optimal than standard + compression algorithms can achieve. + + The structure of the output of bitpack compression is as follows. It begins with a 9-byte header, + consisting of the original size of the byte array pre-compression (8 bytes), and the log of + bitpack_block_bytes used for the stream (1 byte). The remainder of the output is a series of compressed + blocks. + + +---------------+------------+---------+---------+-- --+---------+ + | original size | block size | block 0 | block 1 | ... | block k | + | (8 bytes) | (log2, | | | | | + | | 1 byte) | | | | | + +---------------+------------+---------+---------+-- --+---------+ + + The compression algorithm for each block works independently, as follows. The algorithm assumes that + the input block contains a series of uint64_t values with some number of high bits that are consistently 0, + however the start of the first uint64_t value may be offset from the start of the stream by some unknown + number of bytes between 0 and 7 (this misalignment can be caused by interleaved metadata earlier in the + stream, for example). This offset is called the "phase," and the compression algorithm simply tries all + possible phase values between 0 and 7 and proceeds with the one that yields the smallest compressed block. + + An example input stream might look like the following. (The annotations below the figure represent where + the data came from.) + + 00 00 00 00 00 25 EB 79 2B 00 00 00 00 14 AC 2D 26 00 00 00 ... + +-------------------+-------------------------------+---------------------------+ + | metadata tail | coefficient 0 | coefficient 1 | + | (from prev block) | 0x2B79EB25 -> 30 bits | 0x272DAC14 -> 29 bits | + +-------------------+-------------------------------+---------------------------+ + + Here we can see that the optimal value of "phase" will be 5, since this stream does actually contain an + array of uint64_t values and they start on the sixth byte. The algorithm will then cast the remainder of the + stream as uint64_ts, bitwise-OR those values together, and record the position of the highest-order 1 bit. + This is the "width," and the number of uint64_ts in the array is the value of "words". (The width is the + algorithm's best guess at the bit size of the coefficient modulus.) The final output of compression is + then: + + +-----------+-----------+~~~~~~~~~~~~~+--------------------------------+~~~~~~~~~~~~~+ + | width | phase | verbatim | packed words | verbatim | + | (1 byte, | (1 byte, | (phase | (ceil(words * width / 8) | (tail | + | max 64) | max 7) | bytes) | bytes) | bytes) | + +-----------+-----------+~~~~~~~~~~~~~+--------------------------------+~~~~~~~~~~~~~+ + + Here, the "verbatim" blocks before and after the packed words simply contain the raw data from the array + before the packed stream started and after it ends, if the phase value was not 0. + + For the example stream above, this would look like: + + 1C 05 00 00 00 00 00 25 EB 79 2B 05 6B CB ... 00 00 00 + +-------+-------+~~~~~~~~~~~~~~~~+----------------------------------+~~~~~~~~~~~+ + | width | phase | verbatim | packed words | verbatim | + | 1 B | 1 B | 5 B | 127 words x 30 bits -> 477 B | 3 B | + +-------+-------+~~~~~~~~~~~~~~~~+----------------------------------+~~~~~~~~~~~+ + + The total size of the compressed block in this example would be 487 bytes. + + Two other scenarios are worth mentioning. First, when the input stream contains something other than the + expected content (e.g. metadata), generally this algorithm will not find any value for "width" below 64, + and phase will default to 0, meaning the "compressed" output will be the raw input block plus two + additional bytes carrying the width (64) and phase (0). Second, if the input contains all 0-bits for + whatever reason, the computed width and phase will both be 0, so the compressed output will simply be two + bytes containing 0s. */ // Bounds for the base-2 logarithm of the block size accepted when unpacking. From fc3dff80ee95f9b28ed3dc795c9dc434fc07e733 Mon Sep 17 00:00:00 2001 From: jryancarr Date: Thu, 6 Aug 2026 15:05:07 -0400 Subject: [PATCH 07/11] minor doc edits --- native/src/seal/util/bitpack.h | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/native/src/seal/util/bitpack.h b/native/src/seal/util/bitpack.h index b786db324..0101c8332 100644 --- a/native/src/seal/util/bitpack.h +++ b/native/src/seal/util/bitpack.h @@ -28,14 +28,14 @@ namespace seal The vast majority of data serialized / deserialized by SEAL will be arrays of uint64_t representing polynomial coefficients in RNS form. Each of the uint64_ts in a single array will be residues modulo - some prime q, where q <= 2^60. Thus, some number of high-order bits of these values will always be zero, + some prime q, where q < 2^61. Thus, some number of high-order bits of these values will always be zero, while the lower-order bits will (when serializing any cryptographic data) have high entropy. Standard compression algorithms like zlib are designed to operate on whole bytes, so they will fail to compress any zero bits that do not appear as a whole byte, and the high-entropy cryptographic bits are incompressible. The optimal way to compress these arrays of polynomial coefficients would be to simply use knowledge - of q to append one coefficient after another with no zero-bits inbetween. However, SEAL's serialization + of q to append one coefficient after another with no padding bits in between. However, SEAL's serialization API deliberately hides details of the payload's schema from the serializer, which just sees a raw byte stream. While the majority of that byte stream will usually contain these uint64_t arrays, each array may have a different value of q, and the stream may be interspersed with other small data structures such @@ -61,8 +61,8 @@ namespace seal +---------------+------------+---------+---------+-- --+---------+ The compression algorithm for each block works independently, as follows. The algorithm assumes that - the input block contains a series of uint64_t values with some number of high bits that are consistently 0, - however the start of the first uint64_t value may be offset from the start of the stream by some unknown + the input block contains a series of uint64_t values with some number of high bits that are consistently + 0; however, the start of the first uint64_t value may be offset from the start of the stream by some unknown number of bytes between 0 and 7 (this misalignment can be caused by interleaved metadata earlier in the stream, for example). This offset is called the "phase," and the compression algorithm simply tries all possible phase values between 0 and 7 and proceeds with the one that yields the smallest compressed block. @@ -70,15 +70,15 @@ namespace seal An example input stream might look like the following. (The annotations below the figure represent where the data came from.) - 00 00 00 00 00 25 EB 79 2B 00 00 00 00 14 AC 2D 26 00 00 00 ... + 00 00 00 00 00 25 EB 79 2B 00 00 00 00 14 AC 2D 27 00 00 00 ... +-------------------+-------------------------------+---------------------------+ | metadata tail | coefficient 0 | coefficient 1 | - | (from prev block) | 0x2B79EB25 -> 30 bits | 0x272DAC14 -> 29 bits | + | (from prev block) | 0x2B79EB25 -> 30 bits | 0x272DAC14 -> 30 bits | +-------------------+-------------------------------+---------------------------+ Here we can see that the optimal value of "phase" will be 5, since this stream does actually contain an array of uint64_t values and they start on the sixth byte. The algorithm will then cast the remainder of the - stream as uint64_ts, bitwise-OR those values together, and record the position of the highest-order 1 bit. + block as uint64_ts, bitwise-OR those values together, and record the position of the highest-order 1 bit. This is the "width," and the number of uint64_ts in the array is the value of "words". (The width is the algorithm's best guess at the bit size of the coefficient modulus.) The final output of compression is then: @@ -94,20 +94,23 @@ namespace seal For the example stream above, this would look like: - 1C 05 00 00 00 00 00 25 EB 79 2B 05 6B CB ... 00 00 00 + 1E 05 00 00 00 00 00 25 EB 79 2B 05 6B CB ... xx xx xx +-------+-------+~~~~~~~~~~~~~~~~+----------------------------------+~~~~~~~~~~~+ | width | phase | verbatim | packed words | verbatim | | 1 B | 1 B | 5 B | 127 words x 30 bits -> 477 B | 3 B | +-------+-------+~~~~~~~~~~~~~~~~+----------------------------------+~~~~~~~~~~~+ - The total size of the compressed block in this example would be 487 bytes. + The trailing xx bytes are the start of a coefficient that no longer fit in a whole uint64_t before the + block ended; they are carried verbatim, and being the low-order bytes of a coefficient they are + high-entropy rather than zero. The total size of the compressed block in this example would be 487 + bytes. Two other scenarios are worth mentioning. First, when the input stream contains something other than the expected content (e.g. metadata), generally this algorithm will not find any value for "width" below 64, - and phase will default to 0, meaning the "compressed" output will be the raw input block plus two - additional bytes carrying the width (64) and phase (0). Second, if the input contains all 0-bits for - whatever reason, the computed width and phase will both be 0, so the compressed output will simply be two - bytes containing 0s. + and phase will default to 0, meaning the "compressed" output will two bytes carrying the width (64) and + phase (0), followed by the raw input block. Second, if the input contains all 0-bits for whatever reason, + the computed width and phase will both be 0, so the compressed output will simply be two bytes containing + 0s. */ // Bounds for the base-2 logarithm of the block size accepted when unpacking. From 742e5714be9152d419ba4910167bb20361789fda Mon Sep 17 00:00:00 2001 From: jryancarr Date: Thu, 6 Aug 2026 16:18:46 -0400 Subject: [PATCH 08/11] More tests, continued to edit docs --- dotnet/src/Serialization.cs | 8 -------- native/src/seal/serialization.h | 8 -------- native/src/seal/util/bitpack.cpp | 11 ++++++++-- native/src/seal/util/bitpack.h | 8 ++++---- native/tests/seal/ciphertext.cpp | 7 ++++--- native/tests/seal/serialization.cpp | 31 +++++++++++++++++++++++++++++ 6 files changed, 48 insertions(+), 25 deletions(-) diff --git a/dotnet/src/Serialization.cs b/dotnet/src/Serialization.cs index 3b1936fcc..604f07e9c 100644 --- a/dotnet/src/Serialization.cs +++ b/dotnet/src/Serialization.cs @@ -15,14 +15,6 @@ namespace Microsoft.Research.SEAL /// a large number of zero bytes in the output. Any compression algorithm should /// be able to clean up these zero bytes and hence compress both ciphertext and /// key data. - /// - /// Alternatively, ComprModeType.BitPack re-encodes each block of 64-bit words - /// using only as many bits per word as the largest word in the block requires, - /// discarding exactly the always-zero high bits. The significant bits of - /// ciphertext and key data are close to uniformly random and hence essentially - /// incompressible, so bit-packing typically produces smaller output than a - /// general-purpose compressor, which cannot remove partial bytes. Unlike ZLIB - /// and Zstandard, bit-packing performs no integrity checking of the data. /// public enum ComprModeType : byte { diff --git a/native/src/seal/serialization.h b/native/src/seal/serialization.h index 61195ba5d..8ba8a5d13 100644 --- a/native/src/seal/serialization.h +++ b/native/src/seal/serialization.h @@ -20,14 +20,6 @@ namespace seal be able to clean up these zero bytes and hence compress both ciphertext and key data. - Alternatively, compr_mode_type::bitpack re-encodes each block of 64-bit - words using only as many bits per word as the largest word in the block - requires, discarding exactly the always-zero high bits. The significant - bits of ciphertext and key data are close to uniformly random and hence - essentially incompressible, so bit-packing typically produces smaller - output than a general-purpose compressor, which cannot remove partial - bytes. Unlike ZLIB and Zstandard, bit-packing performs no integrity - checking of the data. */ enum class compr_mode_type : std::uint8_t { diff --git a/native/src/seal/util/bitpack.cpp b/native/src/seal/util/bitpack.cpp index c00bcba7b..39f1f2dde 100644 --- a/native/src/seal/util/bitpack.cpp +++ b/native/src/seal/util/bitpack.cpp @@ -57,7 +57,7 @@ namespace seal DynArray out(add_safe(bitpack_size_bound(in_size), block_slack), pool); unsigned char *out_data = reinterpret_cast(out.begin()); - // Write the original byte count and the block size + // Write the 9-byte stream header: the original byte count and the base-2 log of the block size uint64_t in_size64 = static_cast(in_size); memcpy(out_data, &in_size64, bytes_per_word); size_t out_pos = bytes_per_word; @@ -74,6 +74,12 @@ namespace seal // smaller values need to be considered. The clamp to block_len keeps the word count from // underflowing on a block shorter than a word; for such a block every phase encodes zero words // at the same size and the tie-break below settles on phase zero. + // + // (Note: The current approach scans through the block eight times, one for each possible phase. + // An optimization is possible that cuts this down to one scan, using the fact that we could OR + // all (phase 0) uint64_ts together and infer the phase from where the zeroes ended up. However, + // this has more complicated bookkeeping around the start and end of the block and leads to only a + // modest speedup for an already very-fast routine, so it's been left for future work.) size_t phase = 0; int width = 0; size_t body_size = block_len; @@ -205,7 +211,8 @@ namespace seal if (!started_) { - // The packed data begins with the original byte count and the block size + // The compressed stream begins with its 9-byte header: the original byte count and the base-2 + // log of the block size unsigned char prologue[bytes_per_word + 1]; if (read_packed(prologue, static_cast(sizeof(prologue))) != static_cast(sizeof(prologue))) diff --git a/native/src/seal/util/bitpack.h b/native/src/seal/util/bitpack.h index 0101c8332..bf5b2c7d7 100644 --- a/native/src/seal/util/bitpack.h +++ b/native/src/seal/util/bitpack.h @@ -107,10 +107,10 @@ namespace seal Two other scenarios are worth mentioning. First, when the input stream contains something other than the expected content (e.g. metadata), generally this algorithm will not find any value for "width" below 64, - and phase will default to 0, meaning the "compressed" output will two bytes carrying the width (64) and - phase (0), followed by the raw input block. Second, if the input contains all 0-bits for whatever reason, - the computed width and phase will both be 0, so the compressed output will simply be two bytes containing - 0s. + and phase will default to 0, meaning the "compressed" output will be two bytes carrying the width (64) + and phase (0), followed by the raw input block. Second, if the input contains all 0-bits for whatever + reason, the computed width and phase will both be 0, so the compressed output will simply be two bytes + containing 0s. */ // Bounds for the base-2 logarithm of the block size accepted when unpacking. diff --git a/native/tests/seal/ciphertext.cpp b/native/tests/seal/ciphertext.cpp index 4434cc105..e8323e929 100644 --- a/native/tests/seal/ciphertext.cpp +++ b/native/tests/seal/ciphertext.cpp @@ -153,9 +153,10 @@ namespace sealtest is_equal_uint(ctxt.data(), ctxt2.data(), parms.poly_modulus_degree() * parms.coeff_modulus().size() * 2)); ASSERT_TRUE(ctxt.data() != ctxt2.data()); - // The first block contains the ciphertext metadata (among it full-width parms_id hash words) and packs at - // up to the full 64 bits, but every later block holds only coefficients smaller than the coefficient - // modulus primes and hence packs at their bit width, so the total is guaranteed to beat the unpacked size. + // The first block contains the ciphertext metadata (among them the full-width parms_id hash words) and + // packs at up to the full 64 bits, but every later block holds only coefficients smaller than the + // coefficient modulus primes and hence packs at their bit width, so the total is guaranteed to beat the + // unpacked size. ASSERT_LT(bitpack_size, ctxt.save_size(compr_mode_type::none)); // Seeded ciphertexts bit-pack too: the same seeded object saved with and without bit-packing must load to diff --git a/native/tests/seal/serialization.cpp b/native/tests/seal/serialization.cpp index 64b1b1905..ff003873c 100644 --- a/native/tests/seal/serialization.cpp +++ b/native/tests/seal/serialization.cpp @@ -1110,4 +1110,35 @@ namespace sealtest test_struct st2; ASSERT_ANY_THROW(Serialization::Load(bind(&test_struct::load_members, &st2, _1), tampered, false)); } + + // An overstated original size promises far more data than the (unmodified) SEALHeader.size can back. The + // claimed size must act only as a bound on production -- never an allocation -- and the shortfall must be + // rejected cleanly once the parser reads past what the real input provides. + TEST(SerializationTest, BitPackOverstatedSizeThrows) + { + using namespace placeholders; + + // Three blocks (1024, 1024, 960 bytes); a decoder believing the overstated size parses the short final + // block as a full one and runs out of input partway through it. + large_struct st; + st.data.resize(3000 - sizeof(uint64_t)); + for (size_t i = 0; i < st.data.size(); i++) + { + st.data[i] = static_cast((i * 2654435761ULL) >> 16); + } + + stringstream ss; + Serialization::Save( + bind(&large_struct::save_members, &st, _1), st.save_size(compr_mode_type::bitpack), ss, + compr_mode_type::bitpack, false); + + // The original size is the 8 bytes following the SEALHeader; overstate it to 2^63. + string bytes = ss.str(); + uint64_t huge_size = uint64_t(1) << 63; + memcpy(&bytes[16], &huge_size, sizeof(uint64_t)); + + stringstream tampered(bytes); + large_struct st2; + ASSERT_ANY_THROW(Serialization::Load(bind(&large_struct::load_members, &st2, _1), tampered, false)); + } } // namespace sealtest From 3655d11cac72721a443e81662b6fa9f4e295afb5 Mon Sep 17 00:00:00 2001 From: jryancarr Date: Thu, 6 Aug 2026 16:59:39 -0400 Subject: [PATCH 09/11] New benchmarks --- native/bench/CMakeLists.txt | 1 + native/bench/bench.cpp | 143 ++++++++++++++++++++++++ native/bench/bench.h | 35 ++++++ native/bench/serialize.cpp | 209 ++++++++++++++++++++++++++++++++++++ 4 files changed, 388 insertions(+) create mode 100644 native/bench/serialize.cpp diff --git a/native/bench/CMakeLists.txt b/native/bench/CMakeLists.txt index 0b0c99cb3..663db826a 100644 --- a/native/bench/CMakeLists.txt +++ b/native/bench/CMakeLists.txt @@ -58,6 +58,7 @@ if(SEAL_BUILD_BENCH) ${CMAKE_CURRENT_LIST_DIR}/bench.cpp ${CMAKE_CURRENT_LIST_DIR}/keygen.cpp ${CMAKE_CURRENT_LIST_DIR}/ntt.cpp + ${CMAKE_CURRENT_LIST_DIR}/serialize.cpp ${CMAKE_CURRENT_LIST_DIR}/bfv.cpp ${CMAKE_CURRENT_LIST_DIR}/bgv.cpp ${CMAKE_CURRENT_LIST_DIR}/ckks.cpp diff --git a/native/bench/bench.cpp b/native/bench/bench.cpp index 89dbb0dc9..04423552b 100644 --- a/native/bench/bench.cpp +++ b/native/bench/bench.cpp @@ -129,6 +129,7 @@ namespace sealbench // 3. BGV // 4. CKKS // 5. Util + // 6. Serialize int n = static_cast(parms.first); int log_q = static_cast( bm_env_map.find(parms_ckks)->second->context().key_context_data()->total_coeff_modulus_bit_count()); @@ -224,6 +225,148 @@ namespace sealbench SEAL_BENCHMARK_REGISTER(UTIL, n, 0, NTTInverseLowLevel, bm_util_ntt_inverse_low_level, bm_env_bfv); SEAL_BENCHMARK_REGISTER(UTIL, n, 0, NTTForwardLowLevelLazy, bm_util_ntt_forward_low_level_lazy, bm_env_bfv); SEAL_BENCHMARK_REGISTER(UTIL, n, 0, NTTInverseLowLevelLazy, bm_util_ntt_inverse_low_level_lazy, bm_env_bfv); + + // Serialization cases save and load a ciphertext (per scheme) and the keys under each supported + // compression mode, reporting the serialized size in bytes as the counter "size". Keys have the same + // size and content shape in every scheme, so they are registered once, under the BFV environment. Note + // that the Galois keys here contain two elements; full rotation key sets scale the numbers linearly. +#define SEAL_BENCHMARK_REGISTER_SERIALIZE(name, func, env, mode) \ + SEAL_BENCHMARK_REGISTER(SERIALIZE, n, log_q, name, func, env, seal::compr_mode_type::mode) + + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextBFVNone, bm_serialize_save_ct, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextBFVNone, bm_serialize_load_ct, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextBGVNone, bm_serialize_save_ct, bm_env_bgv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextBGVNone, bm_serialize_load_ct, bm_env_bgv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextCKKSNone, bm_serialize_save_ct, bm_env_ckks, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextCKKSNone, bm_serialize_load_ct, bm_env_ckks, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SavePublicKeyNone, bm_serialize_save_pk, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadPublicKeyNone, bm_serialize_load_pk, bm_env_bfv, none); +#ifdef SEAL_USE_ZLIB + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextBFVZlib, bm_serialize_save_ct, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextBFVZlib, bm_serialize_load_ct, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextBGVZlib, bm_serialize_save_ct, bm_env_bgv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextBGVZlib, bm_serialize_load_ct, bm_env_bgv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextCKKSZlib, bm_serialize_save_ct, bm_env_ckks, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextCKKSZlib, bm_serialize_load_ct, bm_env_ckks, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SavePublicKeyZlib, bm_serialize_save_pk, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadPublicKeyZlib, bm_serialize_load_pk, bm_env_bfv, zlib); +#endif +#ifdef SEAL_USE_ZSTD + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextBFVZstd, bm_serialize_save_ct, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextBFVZstd, bm_serialize_load_ct, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextBGVZstd, bm_serialize_save_ct, bm_env_bgv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextBGVZstd, bm_serialize_load_ct, bm_env_bgv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextCKKSZstd, bm_serialize_save_ct, bm_env_ckks, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextCKKSZstd, bm_serialize_load_ct, bm_env_ckks, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SavePublicKeyZstd, bm_serialize_save_pk, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadPublicKeyZstd, bm_serialize_load_pk, bm_env_bfv, zstd); +#endif + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextBFVBitPack, bm_serialize_save_ct, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextBFVBitPack, bm_serialize_load_ct, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextBGVBitPack, bm_serialize_save_ct, bm_env_bgv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextBGVBitPack, bm_serialize_load_ct, bm_env_bgv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveCiphertextCKKSBitPack, bm_serialize_save_ct, bm_env_ckks, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadCiphertextCKKSBitPack, bm_serialize_load_ct, bm_env_ckks, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SavePublicKeyBitPack, bm_serialize_save_pk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadPublicKeyBitPack, bm_serialize_load_pk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSecretKeyNone, bm_serialize_save_sk, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSecretKeyNone, bm_serialize_load_sk, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededCiphertextBFVNone, bm_serialize_save_seeded_ct, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededCiphertextBFVNone, bm_serialize_load_seeded_ct, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededCiphertextBGVNone, bm_serialize_save_seeded_ct, bm_env_bgv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededCiphertextBGVNone, bm_serialize_load_seeded_ct, bm_env_bgv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededCiphertextCKKSNone, bm_serialize_save_seeded_ct, bm_env_ckks, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededCiphertextCKKSNone, bm_serialize_load_seeded_ct, bm_env_ckks, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededPublicKeyNone, bm_serialize_save_seeded_pk, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededPublicKeyNone, bm_serialize_load_seeded_pk, bm_env_bfv, none); +#ifdef SEAL_USE_ZLIB + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSecretKeyZlib, bm_serialize_save_sk, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSecretKeyZlib, bm_serialize_load_sk, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededCiphertextBFVZlib, bm_serialize_save_seeded_ct, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededCiphertextBFVZlib, bm_serialize_load_seeded_ct, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededCiphertextBGVZlib, bm_serialize_save_seeded_ct, bm_env_bgv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededCiphertextBGVZlib, bm_serialize_load_seeded_ct, bm_env_bgv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededCiphertextCKKSZlib, bm_serialize_save_seeded_ct, bm_env_ckks, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededCiphertextCKKSZlib, bm_serialize_load_seeded_ct, bm_env_ckks, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededPublicKeyZlib, bm_serialize_save_seeded_pk, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededPublicKeyZlib, bm_serialize_load_seeded_pk, bm_env_bfv, zlib); +#endif +#ifdef SEAL_USE_ZSTD + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSecretKeyZstd, bm_serialize_save_sk, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSecretKeyZstd, bm_serialize_load_sk, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededCiphertextBFVZstd, bm_serialize_save_seeded_ct, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededCiphertextBFVZstd, bm_serialize_load_seeded_ct, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededCiphertextBGVZstd, bm_serialize_save_seeded_ct, bm_env_bgv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededCiphertextBGVZstd, bm_serialize_load_seeded_ct, bm_env_bgv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededCiphertextCKKSZstd, bm_serialize_save_seeded_ct, bm_env_ckks, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededCiphertextCKKSZstd, bm_serialize_load_seeded_ct, bm_env_ckks, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededPublicKeyZstd, bm_serialize_save_seeded_pk, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededPublicKeyZstd, bm_serialize_load_seeded_pk, bm_env_bfv, zstd); +#endif + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSecretKeyBitPack, bm_serialize_save_sk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSecretKeyBitPack, bm_serialize_load_sk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE( + SaveSeededCiphertextBFVBitPack, bm_serialize_save_seeded_ct, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE( + LoadSeededCiphertextBFVBitPack, bm_serialize_load_seeded_ct, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE( + SaveSeededCiphertextBGVBitPack, bm_serialize_save_seeded_ct, bm_env_bgv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE( + LoadSeededCiphertextBGVBitPack, bm_serialize_load_seeded_ct, bm_env_bgv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE( + SaveSeededCiphertextCKKSBitPack, bm_serialize_save_seeded_ct, bm_env_ckks, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE( + LoadSeededCiphertextCKKSBitPack, bm_serialize_load_seeded_ct, bm_env_ckks, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededPublicKeyBitPack, bm_serialize_save_seeded_pk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededPublicKeyBitPack, bm_serialize_load_seeded_pk, bm_env_bfv, bitpack); + if (bm_env_bfv->context().using_keyswitching()) + { + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveRelinKeysNone, bm_serialize_save_rlk, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadRelinKeysNone, bm_serialize_load_rlk, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveGaloisKeysNone, bm_serialize_save_glk, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadGaloisKeysNone, bm_serialize_load_glk, bm_env_bfv, none); +#ifdef SEAL_USE_ZLIB + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveRelinKeysZlib, bm_serialize_save_rlk, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadRelinKeysZlib, bm_serialize_load_rlk, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveGaloisKeysZlib, bm_serialize_save_glk, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadGaloisKeysZlib, bm_serialize_load_glk, bm_env_bfv, zlib); +#endif +#ifdef SEAL_USE_ZSTD + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveRelinKeysZstd, bm_serialize_save_rlk, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadRelinKeysZstd, bm_serialize_load_rlk, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveGaloisKeysZstd, bm_serialize_save_glk, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadGaloisKeysZstd, bm_serialize_load_glk, bm_env_bfv, zstd); +#endif + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveRelinKeysBitPack, bm_serialize_save_rlk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadRelinKeysBitPack, bm_serialize_load_rlk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveGaloisKeysBitPack, bm_serialize_save_glk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadGaloisKeysBitPack, bm_serialize_load_glk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededRelinKeysNone, bm_serialize_save_seeded_rlk, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededRelinKeysNone, bm_serialize_load_seeded_rlk, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededGaloisKeysNone, bm_serialize_save_seeded_glk, bm_env_bfv, none); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededGaloisKeysNone, bm_serialize_load_seeded_glk, bm_env_bfv, none); +#ifdef SEAL_USE_ZLIB + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededRelinKeysZlib, bm_serialize_save_seeded_rlk, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededRelinKeysZlib, bm_serialize_load_seeded_rlk, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededGaloisKeysZlib, bm_serialize_save_seeded_glk, bm_env_bfv, zlib); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededGaloisKeysZlib, bm_serialize_load_seeded_glk, bm_env_bfv, zlib); +#endif +#ifdef SEAL_USE_ZSTD + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededRelinKeysZstd, bm_serialize_save_seeded_rlk, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededRelinKeysZstd, bm_serialize_load_seeded_rlk, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(SaveSeededGaloisKeysZstd, bm_serialize_save_seeded_glk, bm_env_bfv, zstd); + SEAL_BENCHMARK_REGISTER_SERIALIZE(LoadSeededGaloisKeysZstd, bm_serialize_load_seeded_glk, bm_env_bfv, zstd); +#endif + SEAL_BENCHMARK_REGISTER_SERIALIZE( + SaveSeededRelinKeysBitPack, bm_serialize_save_seeded_rlk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE( + LoadSeededRelinKeysBitPack, bm_serialize_load_seeded_rlk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE( + SaveSeededGaloisKeysBitPack, bm_serialize_save_seeded_glk, bm_env_bfv, bitpack); + SEAL_BENCHMARK_REGISTER_SERIALIZE( + LoadSeededGaloisKeysBitPack, bm_serialize_load_seeded_glk, bm_env_bfv, bitpack); + } +#undef SEAL_BENCHMARK_REGISTER_SERIALIZE } } // namespace sealbench diff --git a/native/bench/bench.h b/native/bench/bench.h index bb801098b..6a4f7cb5a 100644 --- a/native/bench/bench.h +++ b/native/bench/bench.h @@ -425,4 +425,39 @@ namespace sealbench void bm_ckks_rescale_inplace(benchmark::State &state, std::shared_ptr bm_env); void bm_ckks_relin_inplace(benchmark::State &state, std::shared_ptr bm_env); void bm_ckks_rotate(benchmark::State &state, std::shared_ptr bm_env); + + /** + Benchmark cases for serialization save/load under a given compression mode. Each case reports the + serialized size in bytes as the counter "size". + */ + void bm_serialize_save_ct(benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_load_ct(benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_save_pk(benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_load_pk(benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_save_rlk( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_load_rlk( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_save_glk( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_load_glk( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_save_sk(benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_load_sk(benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_save_seeded_ct( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_load_seeded_ct( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_save_seeded_pk( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_load_seeded_pk( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_save_seeded_rlk( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_load_seeded_rlk( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_save_seeded_glk( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); + void bm_serialize_load_seeded_glk( + benchmark::State &state, std::shared_ptr bm_env, seal::compr_mode_type compr_mode); } // namespace sealbench diff --git a/native/bench/serialize.cpp b/native/bench/serialize.cpp new file mode 100644 index 000000000..16a17202a --- /dev/null +++ b/native/bench/serialize.cpp @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#include "seal/seal.h" +#include "bench.h" + +using namespace benchmark; +using namespace sealbench; +using namespace seal; +using namespace std; + +/** +This file defines benchmarks for serialization: saving and loading ciphertexts and keys under each supported +compression mode. Every case reports the serialized object size in bytes as the counter "size". Loading is +measured through the load() member functions and therefore includes the validity checks they perform. +*/ + +namespace sealbench +{ + namespace + { + template + void bm_serialize_save(State &state, const T &in, compr_mode_type compr_mode) + { + vector buf(static_cast(in.save_size(compr_mode))); + streamoff size = 0; + for (auto _ : state) + { + size = in.save(buf.data(), buf.size(), compr_mode); + } + state.counters["size"] = static_cast(size); + } + + template + void bm_serialize_load_from(State &state, const SEALContext &context, const vector &buf, size_t size) + { + T out; + for (auto _ : state) + { + out.load(context, buf.data(), size); + } + state.counters["size"] = static_cast(size); + } + + template + void bm_serialize_load(State &state, const SEALContext &context, const T &in, compr_mode_type compr_mode) + { + vector buf(static_cast(in.save_size(compr_mode))); + size_t size = static_cast(in.save(buf.data(), buf.size(), compr_mode)); + bm_serialize_load_from(state, context, buf, size); + } + + // Loading a seeded object expands the seed into the full object, so the times below include the PRNG + // sampling that regenerates the seeded polynomials. + template + void bm_serialize_load_seeded( + State &state, const SEALContext &context, const Serializable &in, compr_mode_type compr_mode) + { + vector buf(static_cast(in.save_size(compr_mode))); + size_t size = static_cast(in.save(buf.data(), buf.size(), compr_mode)); + bm_serialize_load_from(state, context, buf, size); + } + + void randomize_ct(shared_ptr &bm_env, Ciphertext &ct) + { + switch (bm_env->parms().scheme()) + { + case scheme_type::bfv: + bm_env->randomize_ct_bfv(ct); + break; + case scheme_type::bgv: + bm_env->randomize_ct_bgv(ct); + break; + case scheme_type::ckks: + bm_env->randomize_ct_ckks(ct); + break; + default: + break; + } + } + + void randomize_pt(shared_ptr &bm_env, Plaintext &pt) + { + switch (bm_env->parms().scheme()) + { + case scheme_type::bfv: + bm_env->randomize_pt_bfv(pt); + break; + case scheme_type::bgv: + bm_env->randomize_pt_bgv(pt); + break; + case scheme_type::ckks: + bm_env->randomize_pt_ckks(pt); + break; + default: + break; + } + } + + Serializable make_seeded_ct(shared_ptr &bm_env) + { + Plaintext &pt = bm_env->pt()[0]; + randomize_pt(bm_env, pt); + return bm_env->encryptor()->encrypt_symmetric(pt); + } + } // namespace + + void bm_serialize_save_ct(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + Ciphertext &ct = bm_env->ct()[0]; + randomize_ct(bm_env, ct); + bm_serialize_save(state, ct, compr_mode); + } + + void bm_serialize_load_ct(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + Ciphertext &ct = bm_env->ct()[0]; + randomize_ct(bm_env, ct); + bm_serialize_load(state, bm_env->context(), ct, compr_mode); + } + + void bm_serialize_save_pk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + bm_serialize_save(state, bm_env->pk(), compr_mode); + } + + void bm_serialize_load_pk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + bm_serialize_load(state, bm_env->context(), bm_env->pk(), compr_mode); + } + + void bm_serialize_save_rlk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + bm_serialize_save(state, bm_env->rlk(), compr_mode); + } + + void bm_serialize_load_rlk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + bm_serialize_load(state, bm_env->context(), bm_env->rlk(), compr_mode); + } + + void bm_serialize_save_glk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + bm_serialize_save(state, bm_env->glk(), compr_mode); + } + + void bm_serialize_load_glk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + bm_serialize_load(state, bm_env->context(), bm_env->glk(), compr_mode); + } + + void bm_serialize_save_sk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + bm_serialize_save(state, bm_env->sk(), compr_mode); + } + + void bm_serialize_load_sk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + bm_serialize_load(state, bm_env->context(), bm_env->sk(), compr_mode); + } + + void bm_serialize_save_seeded_ct(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + auto seeded = make_seeded_ct(bm_env); + bm_serialize_save(state, seeded, compr_mode); + } + + void bm_serialize_load_seeded_ct(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + auto seeded = make_seeded_ct(bm_env); + bm_serialize_load_seeded(state, bm_env->context(), seeded, compr_mode); + } + + void bm_serialize_save_seeded_pk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + auto seeded = bm_env->keygen()->create_public_key(); + bm_serialize_save(state, seeded, compr_mode); + } + + void bm_serialize_load_seeded_pk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + auto seeded = bm_env->keygen()->create_public_key(); + bm_serialize_load_seeded(state, bm_env->context(), seeded, compr_mode); + } + + void bm_serialize_save_seeded_rlk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + auto seeded = bm_env->keygen()->create_relin_keys(); + bm_serialize_save(state, seeded, compr_mode); + } + + void bm_serialize_load_seeded_rlk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + auto seeded = bm_env->keygen()->create_relin_keys(); + bm_serialize_load_seeded(state, bm_env->context(), seeded, compr_mode); + } + + void bm_serialize_save_seeded_glk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + auto seeded = bm_env->keygen()->create_galois_keys(bm_env->galois_elts_all()); + bm_serialize_save(state, seeded, compr_mode); + } + + void bm_serialize_load_seeded_glk(State &state, shared_ptr bm_env, compr_mode_type compr_mode) + { + auto seeded = bm_env->keygen()->create_galois_keys(bm_env->galois_elts_all()); + bm_serialize_load_seeded(state, bm_env->context(), seeded, compr_mode); + } +} // namespace sealbench From b44780641f9eceb602960e0d71a1db0288838627 Mon Sep 17 00:00:00 2001 From: jryancarr Date: Thu, 6 Aug 2026 17:25:28 -0400 Subject: [PATCH 10/11] Address review: document no-integrity-checking, add key round-trip tests. The compr_mode_type::bitpack enumerator (C++ and .NET) now states that bit-packing, unlike ZLIB and Zstandard, performs no integrity checking of the data; users migrating from zlib would otherwise silently lose Adler-32's accidental-corruption detection. Removed a stray blank line left in the enum documentation by an earlier edit. Added GaloisKeys and RelinKeys bit-packed round-trip tests covering both expanded keys (data equality and a size win over the unpacked form) and seeded keys, whose nested seeded frames must load to keys identical to the same seeded object saved uncompressed. Co-Authored-By: Claude Fable 5 --- dotnet/src/Serialization.cs | 3 +- native/src/seal/serialization.h | 4 +-- native/tests/seal/galoiskeys.cpp | 55 +++++++++++++++++++++++++++++++ native/tests/seal/relinkeys.cpp | 56 ++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 3 deletions(-) diff --git a/dotnet/src/Serialization.cs b/dotnet/src/Serialization.cs index 604f07e9c..ac36b828d 100644 --- a/dotnet/src/Serialization.cs +++ b/dotnet/src/Serialization.cs @@ -27,7 +27,8 @@ public enum ComprModeType : byte /// Use Zstandard compression. ZSTD = 2, - /// Use bit-packing of 64-bit words. + /// Use bit-packing of 64-bit words. Unlike ZLIB and Zstandard, bit-packing performs no + /// integrity checking of the data. BitPack = 3 } diff --git a/native/src/seal/serialization.h b/native/src/seal/serialization.h index 8ba8a5d13..6820ce5ed 100644 --- a/native/src/seal/serialization.h +++ b/native/src/seal/serialization.h @@ -19,7 +19,6 @@ namespace seal a large number of zero bytes in the output. Any compression algorithm should be able to clean up these zero bytes and hence compress both ciphertext and key data. - */ enum class compr_mode_type : std::uint8_t { @@ -33,7 +32,8 @@ namespace seal // Use Zstandard compression zstd = 2, #endif - // Use bit-packing of 64-bit words + // Use bit-packing of 64-bit words. Unlike ZLIB and Zstandard, bit-packing performs no integrity + // checking of the data. bitpack = 3, }; diff --git a/native/tests/seal/galoiskeys.cpp b/native/tests/seal/galoiskeys.cpp index 0d931af55..de7858bbd 100644 --- a/native/tests/seal/galoiskeys.cpp +++ b/native/tests/seal/galoiskeys.cpp @@ -98,6 +98,61 @@ namespace sealtest galoiskey_save_load(scheme_type::bgv); } + TEST(GaloisKeysTest, GaloisKeysBitPackSaveLoad) + { + auto galoiskey_bitpack_save_load = [](scheme_type scheme) { + EncryptionParameters parms(scheme); + parms.set_poly_modulus_degree(256); + parms.set_plain_modulus(65537); + parms.set_coeff_modulus(CoeffModulus::Create(256, { 60, 50 })); + SEALContext context(parms, false, sec_level_type::none); + KeyGenerator keygen(context); + + auto compare_keys = [](const GaloisKeys &a, const GaloisKeys &b) { + ASSERT_TRUE(a.parms_id() == b.parms_id()); + ASSERT_EQ(a.data().size(), b.data().size()); + for (size_t j = 0; j < a.data().size(); j++) + { + ASSERT_EQ(a.data()[j].size(), b.data()[j].size()); + for (size_t i = 0; i < a.data()[j].size(); i++) + { + ASSERT_EQ(a.data()[j][i].data().dyn_array().size(), b.data()[j][i].data().dyn_array().size()); + ASSERT_TRUE(is_equal_uint( + a.data()[j][i].data().data(), b.data()[j][i].data().data(), + a.data()[j][i].data().dyn_array().size())); + } + } + }; + + // Expanded keys round-trip bit-packed + stringstream stream; + GaloisKeys keys; + GaloisKeys test_keys; + keygen.create_galois_keys(keys); + auto bitpack_size = keys.save(stream, compr_mode_type::bitpack); + test_keys.load(context, stream); + compare_keys(keys, test_keys); + + // The key data is uniformly random modulo the coefficient modulus primes, so bit-packing must beat + // the unpacked size + ASSERT_LT(bitpack_size, keys.save_size(compr_mode_type::none)); + + // Seeded keys bit-pack too, with the seeded polynomials regenerated on load: the same seeded object + // saved with and without bit-packing must load to identical keys + stringstream seeded_stream; + auto seeded = keygen.create_galois_keys(); + seeded.save(seeded_stream, compr_mode_type::bitpack); + seeded.save(seeded_stream, compr_mode_type::none); + GaloisKeys from_bitpack; + GaloisKeys from_none; + from_bitpack.load(context, seeded_stream); + from_none.load(context, seeded_stream); + compare_keys(from_bitpack, from_none); + }; + galoiskey_bitpack_save_load(scheme_type::bfv); + galoiskey_bitpack_save_load(scheme_type::bgv); + } + TEST(GaloisKeysTest, GaloisKeysSeededSaveLoad) { auto galoiskey_seeded_save_load = [](scheme_type scheme) { diff --git a/native/tests/seal/relinkeys.cpp b/native/tests/seal/relinkeys.cpp index 4ccb56757..99002d0bc 100644 --- a/native/tests/seal/relinkeys.cpp +++ b/native/tests/seal/relinkeys.cpp @@ -83,6 +83,62 @@ namespace sealtest relin_keys_save_load(scheme_type::bfv); relin_keys_save_load(scheme_type::bgv); } + + TEST(RelinKeysTest, RelinKeysBitPackSaveLoad) + { + auto relin_keys_bitpack_save_load = [](scheme_type scheme) { + EncryptionParameters parms(scheme); + parms.set_poly_modulus_degree(256); + parms.set_plain_modulus(65537); + parms.set_coeff_modulus(CoeffModulus::Create(256, { 60, 50 })); + SEALContext context(parms, false, sec_level_type::none); + KeyGenerator keygen(context); + + auto compare_keys = [](const RelinKeys &a, const RelinKeys &b) { + ASSERT_TRUE(a.parms_id() == b.parms_id()); + ASSERT_EQ(a.data().size(), b.data().size()); + for (size_t j = 0; j < a.data().size(); j++) + { + ASSERT_EQ(a.data()[j].size(), b.data()[j].size()); + for (size_t i = 0; i < a.data()[j].size(); i++) + { + ASSERT_EQ(a.data()[j][i].data().dyn_array().size(), b.data()[j][i].data().dyn_array().size()); + ASSERT_TRUE(is_equal_uint( + a.data()[j][i].data().data(), b.data()[j][i].data().data(), + a.data()[j][i].data().dyn_array().size())); + } + } + }; + + // Expanded keys round-trip bit-packed + stringstream stream; + RelinKeys keys; + RelinKeys test_keys; + keygen.create_relin_keys(keys); + auto bitpack_size = keys.save(stream, compr_mode_type::bitpack); + test_keys.load(context, stream); + compare_keys(keys, test_keys); + + // The key data is uniformly random modulo the coefficient modulus primes, so bit-packing must beat + // the unpacked size + ASSERT_LT(bitpack_size, keys.save_size(compr_mode_type::none)); + + // Seeded keys bit-pack too, with the seeded polynomials regenerated on load: the same seeded object + // saved with and without bit-packing must load to identical keys + stringstream seeded_stream; + auto seeded = keygen.create_relin_keys(); + seeded.save(seeded_stream, compr_mode_type::bitpack); + seeded.save(seeded_stream, compr_mode_type::none); + RelinKeys from_bitpack; + RelinKeys from_none; + from_bitpack.load(context, seeded_stream); + from_none.load(context, seeded_stream); + compare_keys(from_bitpack, from_none); + }; + relin_keys_bitpack_save_load(scheme_type::bfv); + relin_keys_bitpack_save_load(scheme_type::bgv); + } + TEST(RelinKeysTest, RelinKeysSeededSaveLoad) { auto relin_keys_seeded_save_load = [](scheme_type scheme) { From 26e961ff88bc37601ea729eb3051fde705e02dcf Mon Sep 17 00:00:00 2001 From: jryancarr Date: Fri, 7 Aug 2026 09:55:04 -0400 Subject: [PATCH 11/11] More small doc changes --- dotnet/src/Serialization.cs | 2 +- native/src/seal/serialization.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dotnet/src/Serialization.cs b/dotnet/src/Serialization.cs index ac36b828d..45dc7f1f8 100644 --- a/dotnet/src/Serialization.cs +++ b/dotnet/src/Serialization.cs @@ -27,7 +27,7 @@ public enum ComprModeType : byte /// Use Zstandard compression. ZSTD = 2, - /// Use bit-packing of 64-bit words. Unlike ZLIB and Zstandard, bit-packing performs no + /// Use bit-packing compression. Unlike ZLIB and Zstandard, bit-packing performs no /// integrity checking of the data. BitPack = 3 } diff --git a/native/src/seal/serialization.h b/native/src/seal/serialization.h index 6820ce5ed..273a6fab9 100644 --- a/native/src/seal/serialization.h +++ b/native/src/seal/serialization.h @@ -32,7 +32,7 @@ namespace seal // Use Zstandard compression zstd = 2, #endif - // Use bit-packing of 64-bit words. Unlike ZLIB and Zstandard, bit-packing performs no integrity + // Use bit-packing compression. Unlike ZLIB and Zstandard, bit-packing performs no integrity // checking of the data. bitpack = 3, };