Direct Pivco-Huffman port #66
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| # Compiled Apache-2.0 PivCo-Huffman backend. | ||
| # | ||
| # The codec is compiled once per available architecture tier. The facade in | ||
| # include/pixie/huffman/pivco_huffman.h remains a thin C++ CRTP adapter. | ||
|
|
||
| set(PIXIE_PIVCO_ROOT | ||
| "${CMAKE_CURRENT_SOURCE_DIR}/third_party/pivco") | ||
|
|
||
| if (NOT EXISTS "${PIXIE_PIVCO_ROOT}/LICENSE") | ||
| message(FATAL_ERROR | ||
| "The PivCo-Huffman sources and Apache-2.0 LICENSE are required") | ||
| endif () | ||
|
|
||
| set(PIXIE_PIVCO_INCLUDE | ||
| "${PIXIE_PIVCO_ROOT}/include") | ||
| set(PIXIE_PIVCO_SRC | ||
| "${PIXIE_PIVCO_ROOT}/src") | ||
|
|
||
| set(PIXIE_PIVCO_COMMON_SOURCES | ||
| "${PIXIE_PIVCO_SRC}/huffman_table.c" | ||
| "${PIXIE_PIVCO_SRC}/joint_lengths.c" | ||
| "${PIXIE_PIVCO_SRC}/pivco_huffman.c" | ||
| "${PIXIE_PIVCO_SRC}/pivcohuf_file.c") | ||
|
|
||
| add_library(pixie_pivco_scalar OBJECT | ||
| "${PIXIE_PIVCO_SRC}/pivco_huffman_codec.c") | ||
| target_compile_definitions(pixie_pivco_scalar | ||
| PRIVATE PIVCO_BACKEND_SCALAR=1) | ||
|
|
||
| set(PIXIE_PIVCO_OBJECTS | ||
| $<TARGET_OBJECTS:pixie_pivco_scalar>) | ||
| set(PIXIE_PIVCO_DEFINITIONS) | ||
| set(PIXIE_PIVCO_OPTIONS) | ||
| string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" PIXIE_PIVCO_PROCESSOR) | ||
|
|
||
| if (PIXIE_PIVCO_PROCESSOR MATCHES "^(x86_64|amd64)$") | ||
| list(APPEND PIXIE_PIVCO_DEFINITIONS | ||
| PIVCO_HAS_AVX2=1 | ||
| PIVCO_HAS_SSE4=1) | ||
| list(APPEND PIXIE_PIVCO_OPTIONS | ||
| -mavx2 | ||
| -mbmi2 | ||
| -msse4.1 | ||
| -mpopcnt) | ||
|
|
||
| add_library(pixie_pivco_x86 OBJECT | ||
| "${PIXIE_PIVCO_SRC}/pivco_huffman_codec.c") | ||
| target_compile_definitions(pixie_pivco_x86 | ||
| PRIVATE PIVCO_BACKEND_X86=1) | ||
| list(APPEND PIXIE_PIVCO_OBJECTS | ||
| $<TARGET_OBJECTS:pixie_pivco_x86>) | ||
| list(APPEND PIXIE_PIVCO_COMMON_SOURCES | ||
| "${PIXIE_PIVCO_SRC}/pivco_huffman_x86_tables.c") | ||
|
|
||
| if (NOT DISABLE_AVX512 AND EXISTS "/proc/cpuinfo") | ||
| file(READ "/proc/cpuinfo" PIXIE_PIVCO_CPUINFO) | ||
| string(FIND "${PIXIE_PIVCO_CPUINFO}" "avx512_vbmi2" | ||
| PIXIE_PIVCO_AVX512_POSITION) | ||
| include(CheckCCompilerFlag) | ||
| check_c_compiler_flag("-mavx512vbmi2" PIXIE_PIVCO_COMPILER_HAS_AVX512) | ||
| if (PIXIE_PIVCO_COMPILER_HAS_AVX512 AND | ||
| NOT PIXIE_PIVCO_AVX512_POSITION EQUAL -1) | ||
| list(APPEND PIXIE_PIVCO_DEFINITIONS | ||
| PIVCO_HAS_AVX512=1) | ||
| list(APPEND PIXIE_PIVCO_OPTIONS | ||
| -mavx512f | ||
| -mavx512bw | ||
| -mavx512vl | ||
| -mavx512vbmi | ||
| -mavx512vbmi2 | ||
| -mavx512vpopcntdq) | ||
| add_library(pixie_pivco_avx512 OBJECT | ||
| "${PIXIE_PIVCO_SRC}/pivco_huffman_codec.c") | ||
| target_compile_definitions(pixie_pivco_avx512 | ||
| PRIVATE PIVCO_BACKEND_AVX512=1) | ||
| list(APPEND PIXIE_PIVCO_OBJECTS | ||
| $<TARGET_OBJECTS:pixie_pivco_avx512>) | ||
| endif () | ||
| endif () | ||
| elseif (PIXIE_PIVCO_PROCESSOR MATCHES "^(aarch64|arm64)$") | ||
| list(APPEND PIXIE_PIVCO_DEFINITIONS PIVCO_HAS_NEON=1) | ||
| add_library(pixie_pivco_neon OBJECT | ||
| "${PIXIE_PIVCO_SRC}/pivco_huffman_codec.c") | ||
| target_compile_definitions(pixie_pivco_neon | ||
| PRIVATE PIVCO_BACKEND_NEON=1) | ||
| list(APPEND PIXIE_PIVCO_OBJECTS | ||
| $<TARGET_OBJECTS:pixie_pivco_neon>) | ||
| list(APPEND PIXIE_PIVCO_COMMON_SOURCES | ||
| "${PIXIE_PIVCO_SRC}/pivco_huffman_neon_tables.c") | ||
| endif () | ||
|
|
||
| set(PIXIE_PIVCO_CODEC_TARGETS pixie_pivco_scalar) | ||
| if (TARGET pixie_pivco_x86) | ||
| list(APPEND PIXIE_PIVCO_CODEC_TARGETS pixie_pivco_x86) | ||
| endif () | ||
| if (TARGET pixie_pivco_avx512) | ||
| list(APPEND PIXIE_PIVCO_CODEC_TARGETS pixie_pivco_avx512) | ||
| endif () | ||
| if (TARGET pixie_pivco_neon) | ||
| list(APPEND PIXIE_PIVCO_CODEC_TARGETS pixie_pivco_neon) | ||
| endif () | ||
|
|
||
| foreach (codec_target IN LISTS PIXIE_PIVCO_CODEC_TARGETS) | ||
| target_include_directories(${codec_target} | ||
| PRIVATE | ||
| "${PIXIE_PIVCO_INCLUDE}" | ||
| "${PIXIE_PIVCO_SRC}") | ||
| target_compile_definitions(${codec_target} | ||
| PRIVATE ${PIXIE_PIVCO_DEFINITIONS}) | ||
| target_compile_options(${codec_target} | ||
| PRIVATE -O3 ${PIXIE_PIVCO_OPTIONS}) | ||
|
Comment on lines
+110
to
+111
There was a problem hiding this comment.
When the repository's AGENTS.md reference: AGENTS.md:L356-L357 Useful? React with 👍 / 👎. |
||
| endforeach () | ||
|
|
||
| add_library(pixie_pivco STATIC | ||
| ${PIXIE_PIVCO_COMMON_SOURCES} | ||
| ${PIXIE_PIVCO_OBJECTS}) | ||
| add_library(pixie::pivco ALIAS pixie_pivco) | ||
| target_include_directories(pixie_pivco | ||
| PUBLIC | ||
| "${PIXIE_PIVCO_INCLUDE}" | ||
| PRIVATE | ||
| "${PIXIE_PIVCO_SRC}") | ||
| target_compile_definitions(pixie_pivco | ||
| PRIVATE ${PIXIE_PIVCO_DEFINITIONS}) | ||
| target_compile_options(pixie_pivco | ||
| PRIVATE -O3 ${PIXIE_PIVCO_OPTIONS}) | ||
| if (UNIX AND NOT APPLE) | ||
| target_link_libraries(pixie_pivco PUBLIC m) | ||
| endif () | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| #pragma once | ||
|
|
||
| /** | ||
| * @file huffman.h | ||
| * @brief Common CRTP contract for Huffman entropy codecs. | ||
| * | ||
| * PivCo-Huffman ("Pivot-Coded Huffman") reuses the wavelet-tree "tree of | ||
| * bitmaps" layout to turn sequential, bit-by-bit Huffman-tree traversals into | ||
| * vectorizable operations. A codec encodes a byte sequence into a compressed | ||
| * stream by building a Huffman-shaped tree of per-node bitmaps, and decodes it | ||
| * back by traversing that tree. Concrete implementations live under | ||
| * `<pixie/huffman/implementations.h>`. | ||
| * | ||
| * @see Marcin Zukowski, "PivCo-Huffman", v1.0 (2026). | ||
| */ | ||
|
|
||
| #include <cstddef> | ||
| #include <cstdint> | ||
| #include <span> | ||
| #include <vector> | ||
|
|
||
| namespace pixie { | ||
|
|
||
| /** | ||
| * @brief CRTP facade for Huffman entropy codecs. | ||
| * | ||
| * The contract is the source of truth for observable codec semantics. Each | ||
| * concrete implementation inherits `HuffmanBase<Impl>` and supplies the | ||
| * required `*_impl()` extension points. There is no virtual dispatch: the | ||
| * facade delegates statically through CRTP, mirroring the other Pixie families. | ||
| * | ||
| * Range and ownership conventions: | ||
| * - Symbol sequences are zero-based byte streams (`symbol_type`). | ||
| * - Compressed streams are byte-oriented views (`std::byte`). | ||
| * - The caller keeps any non-owning view alive for the codec lifetime. | ||
| * | ||
| * @tparam Impl Concrete codec type implementing the `*_impl()` contract. | ||
| * | ||
| * @see `<pixie/huffman/implementations.h>` for the available concrete | ||
| * implementations. | ||
| */ | ||
| template <class Impl> | ||
| class HuffmanBase { | ||
| public: | ||
| /** @brief Symbol type handled by the codec: one byte per symbol. */ | ||
| using symbol_type = std::uint8_t; | ||
|
|
||
| /** | ||
| * @brief Number of symbols in the uncompressed input stream. | ||
| * @return Logical uncompressed symbol count. | ||
| */ | ||
| std::size_t uncompressed_size() const { | ||
| return impl().uncompressed_size_impl(); | ||
| } | ||
|
|
||
| /** | ||
| * @brief Number of bytes in the compressed representation. | ||
| * @return Compressed stream size in bytes. | ||
| */ | ||
| std::size_t compressed_size() const { return impl().compressed_size_impl(); } | ||
|
|
||
| /** | ||
| * @brief Check whether the codec holds no data. | ||
| * @return `true` when `uncompressed_size() == 0`. | ||
| */ | ||
| bool empty() const { return uncompressed_size() == 0; } | ||
|
|
||
| /** | ||
| * @brief Read-only view of the compressed byte stream. | ||
| * @return Span over the serialized representation. | ||
| * | ||
| * @note The returned view is invalidated by codec destruction. | ||
| */ | ||
| std::span<const std::byte> compressed_data() const { | ||
| return impl().compressed_data_impl(); | ||
| } | ||
|
|
||
| /** | ||
| * @brief Reconstruct the input symbol sequence. | ||
| * @return Decoded symbols of length `uncompressed_size()`. | ||
| */ | ||
| std::vector<symbol_type> decode() const { return impl().decode_impl(); } | ||
|
|
||
| private: | ||
| /** @brief Return this facade as its concrete CRTP implementation. */ | ||
| const Impl& impl() const { return static_cast<const Impl&>(*this); } | ||
| }; | ||
|
|
||
| } // namespace pixie |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| #pragma once | ||
|
|
||
| /** | ||
| * @file implementations.h | ||
| * @brief All Huffman codec implementations provided by Pixie. | ||
| * | ||
| * - `PivCoHuffman`: PivCo-Huffman file codec with scalar and | ||
| * architecture-specific SIMD kernels. | ||
| */ | ||
|
|
||
| #include <pixie/huffman.h> | ||
| #include <pixie/huffman/pivco_huffman.h> |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
On every
x86_64/amd64build, this unconditionally defines the AVX2 backend and compiles all codec and common sources with AVX2, BMI2, SSE4.1, and POPCNT. Sincepivco_encodeandpivco_decodestatically select the x86 worker whenever these definitions exist, binaries run on older x86-64 CPUs—or builds configured withPIXIE_DISABLE_BMI2=ON—can execute unsupported instructions instead of using the included scalar fallback. Detect the required features or honor the fallback options before enabling this tier.AGENTS.md reference: AGENTS.md:L348-L353
Useful? React with 👍 / 👎.