diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 4444654..b993255 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -19,7 +19,8 @@ jobs: - name: Run clang-format check run: | # Find all C/C++ files and check formatting - FILES=$(find . -regex '.*\.\(cpp\|hpp\|cc\|c\|h\)') + FILES=$(find . -path './third_party' -prune -o \ + -regex '.*\.\(cpp\|hpp\|cc\|c\|h\)' -print) echo "Checking formatting on:" echo "$FILES" clang-format --version diff --git a/CMakeLists.txt b/CMakeLists.txt index fff9bf9..f919e75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -131,12 +131,17 @@ if (PIXIE_TESTS) include(GoogleTest) endif () -# Downstream consumers link this target; Pixie itself remains header-only. +# PivCo-Huffman is Pixie's compiled codec backend. +include(cmake/PivCo.cmake) + +# Downstream consumers link this target. Most Pixie implementations remain +# header-only; the Huffman codec is carried as a transitive static dependency. add_library(pixie INTERFACE) add_library(pixie::pixie ALIAS pixie) target_compile_features(pixie INTERFACE cxx_std_20) target_include_directories(pixie INTERFACE $) +target_link_libraries(pixie INTERFACE pixie::pivco) # Keep Pixie's own targets warning-clean without imposing -Werror on consumers. function (pixie_enable_project_warnings target) @@ -300,6 +305,15 @@ if (PIXIE_TESTS) PRIVATE ${sdsl_lite_SOURCE_DIR}/include) endif () + add_executable(huffman_tests + src/tests/huffman_tests.cpp) + target_link_libraries(huffman_tests + PRIVATE + pixie::pixie + gtest + gtest_main + ${PIXIE_DIAGNOSTICS_LIBS}) + set(PIXIE_TEST_TARGETS bit_algorithms_unittests rank_select_unittests @@ -314,7 +328,8 @@ if (PIXIE_TESTS) excess_positions_tests select512_experimental_tests excess_record_lows_tests - rmq_tests) + rmq_tests + huffman_tests) foreach (test_target IN LISTS PIXIE_TEST_TARGETS) pixie_enable_project_warnings(${test_target}) gtest_discover_tests(${test_target} @@ -475,6 +490,14 @@ if (PIXIE_BENCHMARKS) benchmark_main ${PIXIE_DIAGNOSTICS_LIBS}) + add_executable(huffman_file_benchmarks + src/benchmarks/huffman_file_benchmarks.cpp) + target_link_libraries(huffman_file_benchmarks + PRIVATE + pixie::pixie + benchmark + ${PIXIE_DIAGNOSTICS_LIBS}) + set(PIXIE_BENCHMARK_TARGETS rank_select_benchmarks rmm_benchmarks @@ -488,7 +511,8 @@ if (PIXIE_BENCHMARKS) dfuds_tree_benchmarks alignment_comparison_benchmarks excess_positions_benchmarks - select512_benchmarks) + select512_benchmarks + huffman_file_benchmarks) if (PIXIE_THIRD_PARTY_BACKENDS) list(APPEND PIXIE_BENCHMARK_TARGETS rmm_sdsl_benchmarks) diff --git a/README.md b/README.md index 6a7b622..a420b7a 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,8 @@ - **Range min-max tree**: fundamental primitive for efficient computation of excess queries mainly for navigation in balanced parenthesis sequences. - **Succinct trees**: static variants of $2$-bit per entry trees, i.e. LOUDS, DFUDS, BP (Based on Euler tour and Ferrada-Navarro style). - **Wavelet tree**, i.e. static structure that supposts rank/select on arbitrary finite alphabets, supports building a Huffman archieve with fast extraction of arbitrary segment. +- **PivCo-Huffman codec** with scalar, AVX2, AVX-512, and NEON kernels selected + for the build host. - Succinct **cartesian tree** and a state of the art solution to static **RMQ** (array is immutable, queries are not known in advance). --- @@ -143,5 +145,5 @@ int main() { Copyright 2026 Pixie contributors. -Pixie is licensed under the [Apache License 2.0](LICENSE). Optional -third-party benchmark and backend integrations retain their own licenses. +Pixie is licensed under the [Apache License 2.0](LICENSE). Vendored components +and optional backend integrations retain their own license notices. diff --git a/cmake/PivCo.cmake b/cmake/PivCo.cmake new file mode 100644 index 0000000..7f2f88d --- /dev/null +++ b/cmake/PivCo.cmake @@ -0,0 +1,129 @@ +# Compiled Apache-2.0 PivCo-Huffman backend. +# +# The codec is compiled once per available architecture tier. The facade in +# include/pixie/huffman/pivco_huffman.h remains a thin C++ CRTP adapter. + +set(PIXIE_PIVCO_ROOT + "${CMAKE_CURRENT_SOURCE_DIR}/third_party/pivco") + +if (NOT EXISTS "${PIXIE_PIVCO_ROOT}/LICENSE") + message(FATAL_ERROR + "The PivCo-Huffman sources and Apache-2.0 LICENSE are required") +endif () + +set(PIXIE_PIVCO_INCLUDE + "${PIXIE_PIVCO_ROOT}/include") +set(PIXIE_PIVCO_SRC + "${PIXIE_PIVCO_ROOT}/src") + +set(PIXIE_PIVCO_COMMON_SOURCES + "${PIXIE_PIVCO_SRC}/huffman_table.c" + "${PIXIE_PIVCO_SRC}/joint_lengths.c" + "${PIXIE_PIVCO_SRC}/pivco_huffman.c" + "${PIXIE_PIVCO_SRC}/pivcohuf_file.c") + +add_library(pixie_pivco_scalar OBJECT + "${PIXIE_PIVCO_SRC}/pivco_huffman_codec.c") +target_compile_definitions(pixie_pivco_scalar + PRIVATE PIVCO_BACKEND_SCALAR=1) + +set(PIXIE_PIVCO_OBJECTS + $) +set(PIXIE_PIVCO_DEFINITIONS) +set(PIXIE_PIVCO_OPTIONS) +string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" PIXIE_PIVCO_PROCESSOR) + +if (PIXIE_PIVCO_PROCESSOR MATCHES "^(x86_64|amd64)$") + list(APPEND PIXIE_PIVCO_DEFINITIONS + PIVCO_HAS_AVX2=1 + PIVCO_HAS_SSE4=1) + list(APPEND PIXIE_PIVCO_OPTIONS + -mavx2 + -mbmi2 + -msse4.1 + -mpopcnt) + + add_library(pixie_pivco_x86 OBJECT + "${PIXIE_PIVCO_SRC}/pivco_huffman_codec.c") + target_compile_definitions(pixie_pivco_x86 + PRIVATE PIVCO_BACKEND_X86=1) + list(APPEND PIXIE_PIVCO_OBJECTS + $) + list(APPEND PIXIE_PIVCO_COMMON_SOURCES + "${PIXIE_PIVCO_SRC}/pivco_huffman_x86_tables.c") + + if (NOT DISABLE_AVX512 AND EXISTS "/proc/cpuinfo") + file(READ "/proc/cpuinfo" PIXIE_PIVCO_CPUINFO) + string(FIND "${PIXIE_PIVCO_CPUINFO}" "avx512_vbmi2" + PIXIE_PIVCO_AVX512_POSITION) + include(CheckCCompilerFlag) + check_c_compiler_flag("-mavx512vbmi2" PIXIE_PIVCO_COMPILER_HAS_AVX512) + if (PIXIE_PIVCO_COMPILER_HAS_AVX512 AND + NOT PIXIE_PIVCO_AVX512_POSITION EQUAL -1) + list(APPEND PIXIE_PIVCO_DEFINITIONS + PIVCO_HAS_AVX512=1) + list(APPEND PIXIE_PIVCO_OPTIONS + -mavx512f + -mavx512bw + -mavx512vl + -mavx512vbmi + -mavx512vbmi2 + -mavx512vpopcntdq) + add_library(pixie_pivco_avx512 OBJECT + "${PIXIE_PIVCO_SRC}/pivco_huffman_codec.c") + target_compile_definitions(pixie_pivco_avx512 + PRIVATE PIVCO_BACKEND_AVX512=1) + list(APPEND PIXIE_PIVCO_OBJECTS + $) + endif () + endif () +elseif (PIXIE_PIVCO_PROCESSOR MATCHES "^(aarch64|arm64)$") + list(APPEND PIXIE_PIVCO_DEFINITIONS PIVCO_HAS_NEON=1) + add_library(pixie_pivco_neon OBJECT + "${PIXIE_PIVCO_SRC}/pivco_huffman_codec.c") + target_compile_definitions(pixie_pivco_neon + PRIVATE PIVCO_BACKEND_NEON=1) + list(APPEND PIXIE_PIVCO_OBJECTS + $) + list(APPEND PIXIE_PIVCO_COMMON_SOURCES + "${PIXIE_PIVCO_SRC}/pivco_huffman_neon_tables.c") +endif () + +set(PIXIE_PIVCO_CODEC_TARGETS pixie_pivco_scalar) +if (TARGET pixie_pivco_x86) + list(APPEND PIXIE_PIVCO_CODEC_TARGETS pixie_pivco_x86) +endif () +if (TARGET pixie_pivco_avx512) + list(APPEND PIXIE_PIVCO_CODEC_TARGETS pixie_pivco_avx512) +endif () +if (TARGET pixie_pivco_neon) + list(APPEND PIXIE_PIVCO_CODEC_TARGETS pixie_pivco_neon) +endif () + +foreach (codec_target IN LISTS PIXIE_PIVCO_CODEC_TARGETS) + target_include_directories(${codec_target} + PRIVATE + "${PIXIE_PIVCO_INCLUDE}" + "${PIXIE_PIVCO_SRC}") + target_compile_definitions(${codec_target} + PRIVATE ${PIXIE_PIVCO_DEFINITIONS}) + target_compile_options(${codec_target} + PRIVATE -O3 ${PIXIE_PIVCO_OPTIONS}) +endforeach () + +add_library(pixie_pivco STATIC + ${PIXIE_PIVCO_COMMON_SOURCES} + ${PIXIE_PIVCO_OBJECTS}) +add_library(pixie::pivco ALIAS pixie_pivco) +target_include_directories(pixie_pivco + PUBLIC + "${PIXIE_PIVCO_INCLUDE}" + PRIVATE + "${PIXIE_PIVCO_SRC}") +target_compile_definitions(pixie_pivco + PRIVATE ${PIXIE_PIVCO_DEFINITIONS}) +target_compile_options(pixie_pivco + PRIVATE -O3 ${PIXIE_PIVCO_OPTIONS}) +if (UNIX AND NOT APPLE) + target_link_libraries(pixie_pivco PUBLIC m) +endif () diff --git a/include/pixie/huffman.h b/include/pixie/huffman.h new file mode 100644 index 0000000..137b698 --- /dev/null +++ b/include/pixie/huffman.h @@ -0,0 +1,89 @@ +#pragma once + +/** + * @file huffman.h + * @brief Common CRTP contract for Huffman entropy codecs. + * + * PivCo-Huffman ("Pivot-Coded Huffman") reuses the wavelet-tree "tree of + * bitmaps" layout to turn sequential, bit-by-bit Huffman-tree traversals into + * vectorizable operations. A codec encodes a byte sequence into a compressed + * stream by building a Huffman-shaped tree of per-node bitmaps, and decodes it + * back by traversing that tree. Concrete implementations live under + * ``. + * + * @see Marcin Zukowski, "PivCo-Huffman", v1.0 (2026). + */ + +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief CRTP facade for Huffman entropy codecs. + * + * The contract is the source of truth for observable codec semantics. Each + * concrete implementation inherits `HuffmanBase` and supplies the + * required `*_impl()` extension points. There is no virtual dispatch: the + * facade delegates statically through CRTP, mirroring the other Pixie families. + * + * Range and ownership conventions: + * - Symbol sequences are zero-based byte streams (`symbol_type`). + * - Compressed streams are byte-oriented views (`std::byte`). + * - The caller keeps any non-owning view alive for the codec lifetime. + * + * @tparam Impl Concrete codec type implementing the `*_impl()` contract. + * + * @see `` for the available concrete + * implementations. + */ +template +class HuffmanBase { + public: + /** @brief Symbol type handled by the codec: one byte per symbol. */ + using symbol_type = std::uint8_t; + + /** + * @brief Number of symbols in the uncompressed input stream. + * @return Logical uncompressed symbol count. + */ + std::size_t uncompressed_size() const { + return impl().uncompressed_size_impl(); + } + + /** + * @brief Number of bytes in the compressed representation. + * @return Compressed stream size in bytes. + */ + std::size_t compressed_size() const { return impl().compressed_size_impl(); } + + /** + * @brief Check whether the codec holds no data. + * @return `true` when `uncompressed_size() == 0`. + */ + bool empty() const { return uncompressed_size() == 0; } + + /** + * @brief Read-only view of the compressed byte stream. + * @return Span over the serialized representation. + * + * @note The returned view is invalidated by codec destruction. + */ + std::span compressed_data() const { + return impl().compressed_data_impl(); + } + + /** + * @brief Reconstruct the input symbol sequence. + * @return Decoded symbols of length `uncompressed_size()`. + */ + std::vector decode() const { return impl().decode_impl(); } + + private: + /** @brief Return this facade as its concrete CRTP implementation. */ + const Impl& impl() const { return static_cast(*this); } +}; + +} // namespace pixie diff --git a/include/pixie/huffman/implementations.h b/include/pixie/huffman/implementations.h new file mode 100644 index 0000000..0f1a2de --- /dev/null +++ b/include/pixie/huffman/implementations.h @@ -0,0 +1,12 @@ +#pragma once + +/** + * @file implementations.h + * @brief All Huffman codec implementations provided by Pixie. + * + * - `PivCoHuffman`: PivCo-Huffman file codec with scalar and + * architecture-specific SIMD kernels. + */ + +#include +#include diff --git a/include/pixie/huffman/pivco_huffman.h b/include/pixie/huffman/pivco_huffman.h new file mode 100644 index 0000000..961e1f4 --- /dev/null +++ b/include/pixie/huffman/pivco_huffman.h @@ -0,0 +1,135 @@ +#pragma once + +/** + * @file pivco_huffman.h + * @brief Pixie adapter for the PivCo-Huffman file codec. + * + * The codec implementation is Apache-2.0-licensed and lives in + * `third_party/pivco`. Its compiled C implementation preserves the PivCo wire + * format, block layout, and architecture-specific kernels while this header + * exposes the standard Pixie Huffman CRTP facade. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace pixie { + +/** + * @brief Owning PivCo-Huffman codec. + * + * Encoding uses 32 KiB file blocks, plain PivCo-Huffman bitmaps (FSE disabled), + * the hybrid vertical flat layout, and the SIMD backend selected at build + * time. Construction from compressed bytes copies the serialized stream, so + * returned views remain valid for this object's lifetime. + * + * @note Consumers linking `pixie::pixie` receive the required compiled backend + * transitively. + */ +class PivCoHuffman : public HuffmanBase { + public: + /** @brief Symbol type handled by the codec: one byte per symbol. */ + using symbol_type = std::uint8_t; + + /** @brief Number of input symbols encoded in each full block. */ + static constexpr std::size_t kBlockSize = 32 * 1024; + + /** + * @brief Compress @p input into an owned PivCo stream. + * @param input Byte sequence to copy into the compressed representation; an + * empty span creates an empty codec. + * @throws std::runtime_error if the backend cannot encode the input. + */ + explicit PivCoHuffman(std::span input) { encode(input); } + + /** + * @brief Load an owning copy of a serialized PivCo stream. + * @param compressed Serialized bytes to copy; an empty span represents an + * empty codec. + * @throws std::runtime_error if a non-empty stream has an invalid header. + */ + explicit PivCoHuffman(std::span compressed) + : compressed_(compressed.begin(), compressed.end()) { + if (compressed_.empty()) { + uncompressed_size_ = 0; + return; + } + const int status = pivcohuf_peek_uncompressed_size( + bytes(compressed_.data()), compressed_.size(), &uncompressed_size_); + check(status, "inspect compressed stream"); + } + + /** @brief Return the logical uncompressed symbol count. */ + std::size_t uncompressed_size_impl() const { return uncompressed_size_; } + + /** @brief Return the serialized compressed size in bytes. */ + std::size_t compressed_size_impl() const { return compressed_.size(); } + + /** @brief Return a view over the owned PivCo stream. */ + std::span compressed_data_impl() const { + return compressed_; + } + + /** + * @brief Decode and return the complete input symbol sequence. + * @throws std::runtime_error if the serialized stream is malformed or its + * decoded size does not match its header. + */ + std::vector decode_impl() const { + if (uncompressed_size_ == 0) { + return {}; + } + std::vector output(uncompressed_size_); + std::size_t output_size = output.size(); + const int status = + pivcohuf_decompress(bytes(compressed_.data()), compressed_.size(), + output.data(), &output_size); + check(status, "decode compressed stream"); + if (output_size != uncompressed_size_) { + throw std::runtime_error( + "PivCo decoder returned an unexpected output size"); + } + return output; + } + + private: + static std::uint8_t* bytes(std::byte* ptr) { + return reinterpret_cast(ptr); + } + + static const std::uint8_t* bytes(const std::byte* ptr) { + return reinterpret_cast(ptr); + } + + static void check(int status, const char* operation) { + if (status != PIVCOHUF_OK) { + throw std::runtime_error(std::string("failed to ") + operation + + " with PivCo status " + std::to_string(status)); + } + } + + void encode(std::span input) { + uncompressed_size_ = input.size(); + const std::size_t bound = + pivcohuf_compress_bound_blk(input.size(), kBlockSize); + compressed_.resize(bound); + std::size_t compressed_size = compressed_.size(); + const int status = pivcohuf_compress_blk( + input.data(), input.size(), bytes(compressed_.data()), &compressed_size, + 0, kBlockSize, nullptr); + check(status, "encode input"); + compressed_.resize(compressed_size); + } + + std::size_t uncompressed_size_ = 0; + std::vector compressed_; +}; + +} // namespace pixie diff --git a/src/benchmarks/huffman_file_benchmarks.cpp b/src/benchmarks/huffman_file_benchmarks.cpp new file mode 100644 index 0000000..63bd91e --- /dev/null +++ b/src/benchmarks/huffman_file_benchmarks.cpp @@ -0,0 +1,118 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using pixie::PivCoHuffman; + +std::vector read_file(const std::string& path) { + std::ifstream file(path, std::ios::binary | std::ios::ate); + if (!file) { + std::fprintf(stderr, "Cannot open '%s'\n", path.c_str()); + std::exit(1); + } + const std::streamsize size = file.tellg(); + file.seekg(0); + std::vector data(static_cast(size)); + if (size > 0) { + file.read(reinterpret_cast(data.data()), size); + } + return data; +} + +const std::string& benchmark_file() { + static const std::string path = [] { + if (const char* value = std::getenv("PIVCO_BENCH_FILE")) { + return std::string(value); + } + return std::string("prose_pride.txt"); + }(); + return path; +} + +double entropy(std::span data) { + if (data.empty()) { + return 0.0; + } + std::array frequencies{}; + for (std::uint8_t symbol : data) { + ++frequencies[symbol]; + } + double result = 0.0; + const double count = static_cast(data.size()); + for (std::size_t frequency : frequencies) { + if (frequency != 0) { + const double probability = static_cast(frequency) / count; + result -= probability * std::log2(probability); + } + } + return result; +} + +double bits_per_symbol(std::size_t compressed_size, + std::size_t uncompressed_size) { + if (uncompressed_size == 0) { + return 0.0; + } + return static_cast(compressed_size) * 8.0 / + static_cast(uncompressed_size); +} + +template +void encode_file(benchmark::State& state) { + const std::vector data = read_file(benchmark_file()); + state.counters["entropy_bpb"] = entropy(data); + std::size_t compressed_size = 0; + for (auto _ : state) { + Codec codec(data); + compressed_size = codec.compressed_size(); + benchmark::DoNotOptimize(codec.compressed_data().data()); + benchmark::ClobberMemory(); + } + state.counters["bpb"] = bits_per_symbol(compressed_size, data.size()); + state.SetBytesProcessed( + static_cast(state.iterations() * data.size())); +} + +template +void decode_file(benchmark::State& state) { + const std::vector data = read_file(benchmark_file()); + const Codec codec(data); + state.counters["entropy_bpb"] = entropy(data); + state.counters["bpb"] = bits_per_symbol(codec.compressed_size(), data.size()); + for (auto _ : state) { + std::vector decoded = codec.decode(); + benchmark::DoNotOptimize(decoded.data()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed( + static_cast(state.iterations() * data.size())); +} + +BENCHMARK_TEMPLATE(encode_file, PivCoHuffman) + ->Name("PivCoHuffman/EncodeFile") + ->Unit(benchmark::kMillisecond) + ->UseRealTime() + ->Repetitions(10) + ->ReportAggregatesOnly(true); +BENCHMARK_TEMPLATE(decode_file, PivCoHuffman) + ->Name("PivCoHuffman/DecodeFile") + ->Unit(benchmark::kMillisecond) + ->UseRealTime() + ->Repetitions(10) + ->ReportAggregatesOnly(true); + +} // namespace + +BENCHMARK_MAIN(); diff --git a/src/tests/huffman_tests.cpp b/src/tests/huffman_tests.cpp new file mode 100644 index 0000000..ae9b195 --- /dev/null +++ b/src/tests/huffman_tests.cpp @@ -0,0 +1,73 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +template +class HuffmanTest : public testing::Test {}; + +using HuffmanImplementations = testing::Types; +TYPED_TEST_SUITE(HuffmanTest, HuffmanImplementations); + +static_assert(std::is_base_of_v, + pixie::PivCoHuffman>); + +template +void expect_round_trip(const std::vector& input) { + const Codec codec(input); + EXPECT_EQ(codec.uncompressed_size(), input.size()); + EXPECT_EQ(codec.empty(), input.empty()); + EXPECT_EQ(codec.compressed_data().size(), codec.compressed_size()); + EXPECT_EQ(codec.decode(), input); + + const Codec loaded(codec.compressed_data()); + EXPECT_EQ(loaded.uncompressed_size(), input.size()); + EXPECT_EQ(loaded.empty(), input.empty()); + EXPECT_EQ(loaded.decode(), input); +} + +TYPED_TEST(HuffmanTest, EmptyInputRoundTrips) { + expect_round_trip({}); +} + +TYPED_TEST(HuffmanTest, KnownInputRoundTrips) { + const std::vector input{'p', 'i', 'v', 'c', 'o', ' ', 'h', + 'u', 'f', 'f', 'm', 'a', 'n'}; + expect_round_trip(input); +} + +TYPED_TEST(HuffmanTest, FullAlphabetRoundTrips) { + std::vector input; + for (std::size_t repeat = 0; repeat < 32; ++repeat) { + for (std::size_t symbol = 0; symbol < 256; ++symbol) { + input.push_back(static_cast(symbol)); + } + } + expect_round_trip(input); +} + +TYPED_TEST(HuffmanTest, MultipleBlocksRoundTrip) { + std::mt19937_64 rng(42); + std::vector input(TypeParam::kBlockSize * 3 + 97); + for (std::uint8_t& symbol : input) { + symbol = static_cast(rng()); + } + expect_round_trip(input); +} + +TEST(PivCoHuffmanTest, RejectsMalformedSerializedStream) { + const std::array malformed{}; + EXPECT_THROW(pixie::PivCoHuffman(std::span(malformed)), + std::runtime_error); +} + +} // namespace diff --git a/third_party/pivco/LICENSE b/third_party/pivco/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/third_party/pivco/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/pivco/README.pixie.md b/third_party/pivco/README.pixie.md new file mode 100644 index 0000000..d261e90 --- /dev/null +++ b/third_party/pivco/README.pixie.md @@ -0,0 +1,12 @@ +# PivCo-Huffman backend + +This directory contains the Apache-2.0 PivCo-Huffman C implementation used by +Pixie's Huffman codec. Its `LICENSE` file is retained alongside the sources. +Pixie builds the plain PivCo-Huffman file codec and the applicable scalar/SIMD +backend. The optional FSE integration, standalone tools, tests, paper sources, +and historical results are not included. + +Pixie-specific build logic lives in `cmake/PivCo.cmake`. The public C++ adapter +is `include/pixie/huffman/pivco_huffman.h` and is available through the Huffman +implementation catalog. Changes to the vendored source files should carry the +notices required by the Apache-2.0 license. diff --git a/third_party/pivco/include/pivco_huffman.h b/third_party/pivco/include/pivco_huffman.h new file mode 100644 index 0000000..c387421 --- /dev/null +++ b/third_party/pivco/include/pivco_huffman.h @@ -0,0 +1,528 @@ +#ifndef PIVCO_HUFFMAN_H +#define PIVCO_HUFFMAN_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* ---------- Constants ---------- */ + +/* PIVCO_BLOCK_SIZE is the *default* per-block symbol count chosen by the + * file codec / CLI / benchmarks. It is no longer a hard codec limit: the + * codec sizes its scratch dynamically off the runtime N (carried in the + * per-block uint16 wire header), so any block size in [1, PIVCO_WIRE_MAX_N] + * works without a recompile. Defaults are the per-arch sweet spots measured + * across M4 / Granite Rapids / Zen5 (see issue #2): bigger blocks amortise + * per-block table/tree reload, which dominates on the smaller-L1 x86 parts. */ +#ifndef PIVCO_BLOCK_SIZE +/* 32K is the cross-arch sweet spot measured across the full fleet (12 EC2 + * parts + M4): every uarch peaks at or near 32K, and the fast modern AVX-512 + * parts regress past it (cache cliff). See docs/BLOCK_SIZE.md. + * + * Apple M-series is the exception: 32K regresses its text dists (its wide + * L1/L2 already absorbs the per-block cost at 16K, after which the larger + * working set only hurts), so it defaults to 16K. Gated compile-time on + * macOS/arm64 — a macOS arm64 binary is always Apple Silicon, and a macOS + * binary's ISA is fixed at build time, so the gate is exact. An explicit + * -DPIVCO_BLOCK_SIZE still wins (this whole block is #ifndef-guarded). + * Only M4 was measured; M1–M3 are assumed to share the wide-L1 behaviour. + * A principled runtime gate keyed on cache size (which is the real cause) + * could supersede this later. */ +#if defined(__APPLE__) && defined(__aarch64__) +#define PIVCO_BLOCK_SIZE 16384 +#else +#define PIVCO_BLOCK_SIZE 32768 +#endif +#endif + +/* Hard upper bound on a block's symbol count: the per-block wire header + * stores N as a uint16 little-endian field, so N must fit in 16 bits. */ +#define PIVCO_WIRE_MAX_N 65535 + +#define PIVCO_MAX_SYMBOLS 256 + +/* Maximum Huffman code length (length-limited Huffman, like huf0). + * Capping at 11 matches zstd's huf0 max and bounds the canonical + * decode_sym/decode_len tables at 2KB each (fits L1 cleanly). Trees + * for distributions with naturally deeper Huffman (e.g. prose_pride at + * natural max 16) get reshaped: rare deep leaves are pulled up and + * other leaves get longer codes per Kraft. Compression cost is small + * (~0.5-1% on text-like data, 0% on most distributions whose natural + * max is <=11 anyway). Decode is slightly faster on text-like + * distributions, slightly slower on geometric — net wash to small win + * on real workloads. Override at build time if you need + * different trade-off: -DPIVCO_MAX_CODE_LEN=15. */ +#ifndef PIVCO_MAX_CODE_LEN +#define PIVCO_MAX_CODE_LEN 11 +#endif + +/* Maximum encoded size for one block (generous upper bound): + Sum of code bits across all symbols. Worst case: all 8-bit codes + => N bytes. Plus rounding overhead per tree node. */ +#define PIVCO_MAX_ENCODED_SIZE (PIVCO_BLOCK_SIZE * 2) + +/* ---------- Error codes ---------- */ + +#define PIVCO_OK 0 +#define PIVCO_ERR_NULL (-1) +#define PIVCO_ERR_OVERFLOW (-2) +#define PIVCO_ERR_CORRUPT (-3) +#define PIVCO_ERR_EMPTY (-4) +#define PIVCO_ERR_BAD_CFG (-5) + +/* ---------- Huffman tree node (for PIVCO tree-walk) ---------- */ + +/* Compact tree: nodes stored in array, indexed by node ID. + Max nodes = 2 * MAX_SYMBOLS - 1 = 511. + Leaf: symbol >= 0. Internal: symbol = -1, left/right are children. */ +#define PIVCO_MAX_TREE_NODES (2 * PIVCO_MAX_SYMBOLS - 1) + +typedef struct { + int16_t symbol; /* >= 0 for leaf, -1 for internal */ + int16_t left; /* child node index (bit=0) */ + int16_t right; /* child node index (bit=1) */ +} pivco_tree_node_t; + +/* ---------- Per-node decode dispatch ---------- + * + * Classifies each tree node at build_table time so the decoder can + * dispatch via a single switch on table->node_type[node_id] instead of + * per-call conditional chains. Classification is by "leafness" of the + * children alone: bottom-up merges consume a leaf child's symbol + * directly, so a leaf node itself is never dispatched — the parent's + * merge materializes it. + * + * Same classification applies to all backends (scalar, NEON, AVX-512, + * SSE). + */ +typedef enum { + PIVCO_NODE_INTERNAL_FULL = 0, /* both children internal — general partition/merge */ + PIVCO_NODE_INTERNAL_FLAT, /* flat_depth[i] >= 2 — flat-subtree fast path */ + PIVCO_NODE_BOTH_LEAVES, /* both children leaves — merge_cst_cst, partition_none */ + PIVCO_NODE_LEAF_LEFT, /* left child leaf, right internal — merge_cst_vec, partition_right */ + PIVCO_NODE_LEAF, /* leaf — consumed by the parent merge, never dispatched */ +} pivco_node_type_t; + +/* ---------- Huffman table ---------- */ + +/* Arch-specific precomputed gather tables for prim_enc_init. Every pointer is + * NULL unless the host arch fills it (only x86 SSE/AVX2 today, for the 4tab + * no-shift merge). Backends that don't need it ignore the struct; a backend + * that does asserts the fields it uses are non-NULL. The struct type is + * arch-invariant (always these fields) — only the backing storage in the table + * is arch-gated — so it never degenerates to an empty struct. */ +typedef struct { + const uint16_t *s2r_hi; /* sym_to_rank[s] << 8 (u16) — x86 2tab merge */ +} pivco_enc_init_aux_t; + +typedef struct { + /* Per-symbol encode info */ + uint16_t code[PIVCO_MAX_SYMBOLS]; /* canonical Huffman code */ + uint8_t code_len[PIVCO_MAX_SYMBOLS]; /* code length (0 = unused) */ + + /* "partbyrank" encode: a subtree's leaves are a contiguous rank range, so + * per-node routing is `rank > split_rank` (8-bit, vs a 16-bit code bit-test) + * and a flat subtree's local code is `rank - flat_base_rank`. Filled by + * pivco_build_table; byte-identical wire output. */ + uint8_t sym_to_rank[PIVCO_MAX_SYMBOLS]; /* in-order leaf rank per symbol */ +#if defined(__x86_64__) || defined(__i386__) + /* Backing storage for enc_init_aux — the x86 2tab merge hi table (sym_to_rank + * << 8). Other arches don't allocate it. Filled by pivco_build_table. */ + uint16_t enc_init_hi[PIVCO_MAX_SYMBOLS]; +#endif + /* Aux gather tables: pointers into the arch-gated storage above (x86) or all + * NULL (other arches). Self-referential — rebuild, don't bitwise-copy, a + * table after pivco_build_table. */ + pivco_enc_init_aux_t enc_init_aux; + uint8_t split_rank[PIVCO_MAX_TREE_NODES]; /* max rank in node's left subtree */ + uint8_t flat_base_rank[PIVCO_MAX_TREE_NODES]; /* min rank in a flat subtree */ + + /* Tree for PIVCO tree-walk encode/decode */ + pivco_tree_node_t tree[PIVCO_MAX_TREE_NODES]; + int16_t tree_root; + int16_t tree_node_count; + + /* Canonical decode info (for traditional decoder) */ + uint16_t first_code[PIVCO_MAX_CODE_LEN + 1]; + uint16_t first_sym_idx[PIVCO_MAX_CODE_LEN + 1]; + uint16_t sym_count[PIVCO_MAX_CODE_LEN + 1]; + uint8_t sorted_symbols[PIVCO_MAX_SYMBOLS]; + + /* Flat decode table: 2^MAX_CODE_LEN entries (for traditional decoder) */ + uint8_t decode_sym[1 << PIVCO_MAX_CODE_LEN]; + uint8_t decode_len[1 << PIVCO_MAX_CODE_LEN]; + + uint8_t max_len; + uint8_t min_len; + uint16_t num_symbols; + uint8_t fse_enabled; /* baked from pivco_cfg_t at build */ + uint8_t flat_layout; /* baked from pivco_cfg_t at build */ + + /* Flat-subtree fast path: per-node, if flat_depth[i] >= 2 then node i + is the root of a MAXIMAL flat subtree of depth D = flat_depth[i] + (all 2^D leaves at the same relative depth). Encoder emits N*D + packed bits at this node instead of D levels of bitmaps; decoder + reads N*D bits and uses flat_code_to_sym[flat_offset[i] + code] + per element. Pool sum of 2^D across flat subtrees <= num_symbols. */ + uint8_t flat_depth[PIVCO_MAX_TREE_NODES]; + uint16_t flat_offset[PIVCO_MAX_TREE_NODES]; + uint8_t flat_code_to_sym[PIVCO_MAX_SYMBOLS]; + + /* Max leaf depth in the subtree rooted at this node, relative to + * the global tree. At runtime, the encoder checks + * `max_leaf_depth[node] - depth <= 8` to decide whether to repack + * codes_la from uint16 to uint8 and run subsequent partitions on + * byte-wide SIMD. */ + uint8_t max_leaf_depth[PIVCO_MAX_TREE_NODES]; + + /* Decode dispatch type per node — see pivco_node_type_t. Set by + * build_table after tree and flat_depth are finalized. Decoders + * switch on this instead of running per-call conditional chains. */ + uint8_t node_type[PIVCO_MAX_TREE_NODES]; +} pivco_table_t; + +/* ---------- Tree-shape mode (build-time) ---------- + * + * Experimental knob for paper-style ablations. Changes how the chunks + * are decomposed inside pivco_build_table; the codec downstream + * picks up the resulting table->flat_depth/flat_offset/code[] uniformly. + * + * OPTIMIZED production: per length L, decompose c_L by its set + * bits. Produces non-canonical codes that maximize + * flat-D>=2 subtree coverage. + * NAIVE every symbol is a D=0 singleton. Tree shape == + * pure canonical Huffman; no leaf fusion, no flat + * subtrees. Slowest decode; best baseline for "ph + * without any tree optimizations vs Huff0". + * FUSED allow D=1 sibling pairs but no D>=2 flats. Tree + * shape == canonical with `scatter_two` / `merge_two` + * leaf fusion only. + * CANONICAL_FLAT chunks are derived from canonical code positions: + * greedy peel the largest 2^k chunk such that the + * canonical start code is 2^k-aligned and 2^k <= + * remaining. Produces canonical codes that happen + * to contain flat subtrees; isolates the gain from + * the OPTIMIZED non-canonical reorganization. + * + * Set via pivco_cfg_t.tree_mode at table build; both encoder and + * decoder side must build with the same value (the wire format + * carries only code lengths, not tree shape). Default = OPTIMIZED. */ +typedef enum { + PIVCO_TREE_MODE_OPTIMIZED = 0, + PIVCO_TREE_MODE_NAIVE = 1, + PIVCO_TREE_MODE_FUSED = 2, + PIVCO_TREE_MODE_CANONICAL_FLAT = 3, +} pivco_tree_mode_t; + +/* ---------- Compression effort (encoder-side, build-time) ---------- + * + * How much table-build time pivco_build_table spends shaping + * the code lengths for DECOMPRESSION speed (the joint length/shape + * pass in src/joint_lengths.c). More shaping: slower table build, + * faster decompression, ~same compressed size -- an adoption guard + * only accepts shapes whose modeled bits stay within 1.5% of the + * Huffman baseline AND whose modeled decode time improves by at least + * 10%; on any reject the plain Huffman lengths are kept. Encoder + * side only: the wire carries plain code lengths, so ANY decoder + * reads the output and both sides rebuild identical tables. + * + * The cost is per build_table CALL -- the file codec builds one table + * per file -- so it only matters for small inputs or high call rates. + * The superlatives are the extremes; most callers want the middle. + * Set via pivco_cfg_t.effort at table build. Default = PLAIN + * (shaping is opt-in until the lambda/guard tuning settles; see + * issue #20). */ +typedef enum { + PIVCO_EFFORT_PLAIN = 0, /* plain Huffman lengths: no + shaping, no shaping time -- + the default */ + PIVCO_EFFORT_BALANCED = 1, /* a coarse grouped solve buys + most of the decompress win */ + PIVCO_EFFORT_FASTER_DECOMPRESS = 2, /* auto-tier solve: nearly all + of the win */ + PIVCO_EFFORT_FASTEST_DECOMPRESS = 3, /* exact DP, provably optimal + shape in-model: encode-once- + decode-forever data */ + PIVCO_EFFORT_FASTEST_COMPRESS = 4, /* PLAIN below 256 KiB of + input, BALANCED above -- + resolved by input size in the + pivcohuf file codec; a bare + build_table (no size known) + treats it as BALANCED */ +} pivco_effort_t; + +/* ---------- Flat-region wire layout (build-time) ---------- + * + * How a flat subtree's N*D packed bits are laid out in the stream. + * + * NATURAL codes packed sequentially, LSB-first: element i's D + * bits at bit offset i*D. + * VERTICAL hybrid byte-column-major blocks: pivco_vert_n512(n) + * elements in 512-value/64-lane blocks, then + * pivco_vert_n of the remainder in 128-value/16-lane + * blocks, natural tail (see + * src/pivco_huffman_vertical.h). The default: best or + * near-best on every x86 tier and Apple Silicon, and + * never last anywhere. + * VERTICAL_128 128-value/16-lane blocks only + natural tail (no + * 512 span). The fastest layout on ARM servers + * (Graviton 2..5: decode +2..5% median over VERTICAL), + * at the price of large decode losses on wide x86. + * + * Set via pivco_cfg_t.flat_layout at table build. Raw block API: both + * sides must build tables with the same value (like tree_mode, the + * per-block wire does not carry it). The pivcohuf file container + * records it in its FLAGS byte, so files decode with any layout. */ +typedef enum { + PIVCO_FLAT_NATURAL = 0, + PIVCO_FLAT_VERTICAL = 1, + PIVCO_FLAT_VERTICAL_128 = 2, +} pivco_flat_layout_t; + +/* ---------- Build configuration ---------- + * + * The one user-settable configuration object. Consumed only by the + * table builds (pivco_build_table / _from_code_lens); every + * field bakes into the resulting table, so config plays no role after + * build -- encode/decode read everything they need from the table. + * Pass NULL to a build to get pivco_cfg_default. */ +typedef struct { + pivco_tree_mode_t tree_mode; /* default PIVCO_TREE_MODE_OPTIMIZED */ + pivco_effort_t effort; /* default PIVCO_EFFORT_PLAIN */ + int fse_enabled; /* default 1: per-node FSE attempts */ + pivco_flat_layout_t flat_layout; /* default PIVCO_FLAT_VERTICAL */ +} pivco_cfg_t; + +extern const pivco_cfg_t pivco_cfg_default; + +/* ---------- Encoder / decoder contexts ---------- + * + * A context owns the scratch memory one encode (or decode) stream + * needs, plus running stats. Single-threaded objects: create one per + * thread, reuse it across blocks; create/free are malloc-priced, + * everything between is allocation-free (scratch is preallocated for + * PIVCO_WIRE_MAX_N at create and only grows on a larger need). Holds + * no config -- everything user-settable lives in pivco_cfg_t + * at table build. + * + * `stats` accumulate over successful pivco_encode/_decode + * calls (blocks, payload bytes in, stream bytes out -- and the mirror + * for decode); the caller may zero the struct at any time. `internal` + * is the opaque scratch arena. */ +typedef struct { + uint64_t blocks; + uint64_t bytes_in; + uint64_t bytes_out; +} pivco_ctx_stats_t; + +typedef struct { + pivco_ctx_stats_t stats; + void *internal; +} pivco_encoder_t; + +typedef struct { + pivco_ctx_stats_t stats; + void *internal; +} pivco_decoder_t; + +pivco_encoder_t *pivco_encoder_create(void); +pivco_decoder_t *pivco_decoder_create(void); +void pivco_encoder_free(pivco_encoder_t *enc); +void pivco_decoder_free(pivco_decoder_t *dec); + +/* Byte histogram: ADDS counts of in[0..n) into freq[256] (caller + * zeroes; accumulate across buffers/chunks freely). SIMD on capable + * hosts; scratch from the encoder context. */ +int pivco_histogram(pivco_encoder_t *enc, const uint8_t *in, size_t n, + uint64_t freq[PIVCO_MAX_SYMBOLS]); + +/* The pass itself (called by pivco_build_table between length + * derivation and table construction; exposed for tests/benchmarks). + * Rewrites lengths[] -- Huffman lengths for freq[], already limited to + * PIVCO_MAX_CODE_LEN -- in place. cfg as in the builds (NULL means + * pivco_cfg_default). Returns 0 if a shaped set was adopted, -1 if + * the baseline was kept (PLAIN effort, non-OPTIMIZED tree mode, guard + * reject, or internal failure such as malloc). */ +int pivco_joint_optimize_lengths(const uint64_t freq[PIVCO_MAX_SYMBOLS], + uint8_t lengths[PIVCO_MAX_SYMBOLS], + const pivco_cfg_t *cfg); + +/* ---------- FSE table-usage stats (debug instrumentation) ---------- + * + * Per-table-id counters incremented inside the encoder every time an + * FSE-coded bitmap is committed. Slot 0 = "FSE attempted but did not + * commit"; slots 1..PIVCO_FSE_NUM_TABLES = pivco_fse_freq[] table picked. + * MUST be >= PIVCO_FSE_NUM_TABLES + 1 (static-asserted in pivco_fse.c). + * Not thread-safe; intended for single-threaded analysis runs. */ +#define PIVCO_FSE_STATS_SLOTS 51 +void pivco_fse_stats_reset(void); +void pivco_fse_stats_get(uint64_t commit_count[PIVCO_FSE_STATS_SLOTS], + uint64_t attempt_count[PIVCO_FSE_STATS_SLOTS], + uint64_t bytes_in[PIVCO_FSE_STATS_SLOTS], + uint64_t bytes_out[PIVCO_FSE_STATS_SLOTS]); + +/* Per-root-event log: one entry per block's root-node visit. + * Captures table_id chosen (0 if below threshold / no table), the + * observed p_major, whether the FSE commit succeeded, and byte counts. + * Useful for showing how a single tree position (the root) adapts + * across blocks of the same file. */ +typedef struct { + int table_id; /* 0 if below MIN_THRESHOLD / no FSE attempt */ + double p_major; /* observed max(n_left,n_right)/n */ + int committed; /* 1 if FSE emitted, 0 otherwise */ + int nbytes_in; /* raw bitmap byte count */ + int nbytes_out; /* fse_len if committed; nbytes_in if not */ +} pivco_fse_root_event_t; + +int pivco_fse_root_count(void); +void pivco_fse_root_get(int idx, pivco_fse_root_event_t *out); + +/* ---------- Table construction ---------- */ + +int pivco_build_table(const pivco_cfg_t *cfg, + const uint64_t freq[PIVCO_MAX_SYMBOLS], + pivco_table_t *table); + +/* Build a Huffman table from already-known code lengths (the path used by + * decoders that recovered code_lens from a wire format). The tree is fully + * determined by the lengths -- within-tier order is symbol-value ascending -- + * so encoder and decoder reconstruct identical tables with no extra wire info. + * + * Internally synthesises power-of-two frequencies that reproduce the lengths + * and runs the same build pipeline as pivco_build_table. + * + * (Through wire v0.3 this also took a rank_within_tier array to reproduce a + * frequency-based within-tier order; that ordering was dropped in v0.4 -- it + * required extra wire bytes and only masked a frequency-blind FSE commit + * policy. See the reshape note in huffman_table.c.) */ +int pivco_build_table_from_code_lens( + const pivco_cfg_t *cfg, + const uint8_t code_lens[PIVCO_MAX_SYMBOLS], + pivco_table_t *table); + +/* Fill the 2^MAX_CODE_LEN flat decode table (decode_sym/decode_len) used only + * by the traditional flat-table decoder (trad_huffman_decode*). Call after + * building the table; pivco_build_table no longer fills it (the + * production tree-walk decoder does not need it). */ +void pivco_build_traditional_table(pivco_table_t *table); + +/* ---------- PIVCO Huffman encode/decode (variable-N blocks, N ≤ PIVCO_BLOCK_SIZE) ---------- + * + * The production entry points. One block per call: + * + * pivco_encode — encodes symbols[0..n) with `table` into + * `out`, sets *out_len. `out` must hold PIVCO_MAX_ENCODED_SIZE + * bytes. Writes a 2-byte LE N header at the start of the stream + * (see pivco_huffman_wire.h). N must satisfy + * 1 ≤ n ≤ PIVCO_BLOCK_SIZE; values outside that range return error. + * + * pivco_decode — decodes one block from in[0..in_len) with + * `table` into `symbols`, sets *consumed to the stream bytes read. + * N comes from the wire — no `n` parameter — and `symbols` must + * have room for the worst case, typically PIVCO_BLOCK_SIZE. + * + * Both compile-time-dispatch to the best backend built into this + * binary (the workers below) and accumulate ctx->stats on success; + * that is their only difference from the workers. */ + +int pivco_encode(pivco_encoder_t *enc, const pivco_table_t *table, + const uint8_t *symbols, size_t n, + uint8_t *out, size_t *out_len); +int pivco_decode(pivco_decoder_t *dec, const pivco_table_t *table, + const uint8_t *in, size_t in_len, + uint8_t *symbols, size_t *consumed); + +/* Per-backend workers: same contract as pivco_encode/_decode + * minus the stats accounting. Exposed for benches and tests; normal + * callers use the dispatching pair above. */ + +int pivco_encode_scalar(pivco_encoder_t *enc, const pivco_table_t *table, + const uint8_t *symbols, size_t n, + uint8_t *out, size_t *out_len); + + +int pivco_decode_scalar(pivco_decoder_t *dec, const pivco_table_t *table, + const uint8_t *in, size_t in_len, + uint8_t *symbols, size_t *consumed); + +#ifdef PIVCO_HAS_NEON +int pivco_encode_neon(pivco_encoder_t *enc, const pivco_table_t *table, + const uint8_t *symbols, size_t n, + uint8_t *out, size_t *out_len); + +/* Bottom-up merge decode (NEON). */ +int pivco_decode_bu_neon(pivco_decoder_t *dec, const pivco_table_t *table, + const uint8_t *in, size_t in_len, + uint8_t *symbols, size_t *consumed); +#endif + +#ifdef PIVCO_HAS_SSE4 +int pivco_encode_x86(pivco_encoder_t *enc, const pivco_table_t *table, + const uint8_t *symbols, size_t n, + uint8_t *out, size_t *out_len); + +/* Bottom-up merge decode (x86 SSE4.1 / AVX-512 VBMI2). */ +int pivco_decode_bu_x86(pivco_decoder_t *dec, const pivco_table_t *table, + const uint8_t *in, size_t in_len, + uint8_t *symbols, size_t *consumed); +#endif + +/* Prior experimental NEON variants (neon2, neon2b, neon_fused_1leaf) + * are preserved under extras/ as negative results. See extras/README_* + * files for writeups. */ + +/* Prefix-radix research backend retired to extras/pivco_huffman_neon_prefix.c + * (alongside its bench_prefix_profile.c and pivco_huffman_neon_common.h). + * BU on the standard 2-way wire format beats it on all 29 distributions + * across all 7 EC2 test hosts; no production caller remained. See + * docs/PREFIX_RADIX.md for the historical design record. */ + +#ifdef PIVCO_HAS_AVX512 +int pivco_encode_avx512(pivco_encoder_t *enc, const pivco_table_t *table, + const uint8_t *symbols, size_t n, + uint8_t *out, size_t *out_len); + +/* Bottom-up merge decode (AVX-512 VBMI2). */ +int pivco_decode_bu_avx512(pivco_decoder_t *dec, const pivco_table_t *table, + const uint8_t *in, size_t in_len, + uint8_t *symbols, size_t *consumed); +#endif + +/* Top-down (TD) decode entry points have been retired (2026-05-14). + * BU is the production decoder on every platform. TD implementations + * still live in the legacy .c files as now-unreachable static functions; + * step 3.8 of the unify-framework refactor retires them along with the + * legacy .c files when codec.c takes over the encode/decode entries. + * See extras/legacy_td/README.md for the git-archaeology pointer. */ + +/* ---------- Traditional Huffman encode/decode (for comparison) ---------- */ + +int trad_huffman_encode(const uint8_t *symbols, size_t n_symbols, + const pivco_table_t *table, + uint8_t *out, size_t *out_len, size_t *out_bits); + +int trad_huffman_decode(const uint8_t *in, size_t in_bits, + const pivco_table_t *table, + uint8_t *symbols, size_t n_symbols); + +/* SotA 4-stream encode/decode (huff0-style) */ +int trad_huffman_encode_4s(const uint8_t *symbols, size_t n_symbols, + const pivco_table_t *table, + uint8_t *out, size_t *out_len); + +int trad_huffman_decode_4s(const uint8_t *in, size_t in_len, + const pivco_table_t *table, + uint8_t *symbols, size_t n_symbols); + +/* ---------- Instrumentation ---------- */ +void pivco_instrument_node_size(int n); +void pivco_dump_node_size_hist(void); + +#ifdef __cplusplus +} +#endif + +#endif /* PIVCO_HUFFMAN_H */ diff --git a/third_party/pivco/include/pivco_prof.h b/third_party/pivco/include/pivco_prof.h new file mode 100644 index 0000000..0014e10 --- /dev/null +++ b/third_party/pivco/include/pivco_prof.h @@ -0,0 +1,179 @@ +/* pivco_prof.h — lightweight per-primitive instrumentation. + * + * Counts calls and elements unconditionally; times a subset of larger + * primitives via a userspace cycle counter (cntvct_el0 on aarch64, + * rdtsc on x86). On Apple Silicon cntvct_el0 runs at 24 MHz nominal + * but Apple's userspace timer reports as if 1 GHz (1 ns/tick); on + * Linux aarch64 cntvct_el0 is typically 1 GHz; on x86 rdtsc runs at + * the base CPU frequency. Use pivco_prof_probe_tick_freq() to convert + * ticks to ns. + * + * Disabled by default (zero cost). Enable with -DPIVCO_PROF=1. + * + * Per-call-site partition loops in decode_node_neon / + * decode_node_avx512 are extracted as named static functions that + * each have their own counter, so the dump can attribute time per + * exact call site (general partition vs one-leaf vs root etc.) + * without conflating them. + */ +#pragma once +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + /* Per-call-site partition loops. Element count = n elements + * processed by the loop on that invocation. */ + + /* decode_node_neon (interior recursion) — same names reused for + * decode_node_avx512 since they semantically do the same thing + * (just different SIMD width). */ + PROF_NODE_FULL = 0, /* general partition, both children non-leaf */ + PROF_NODE_HALF_RIGHT, /* half-partition right (skip_node = left) */ + PROF_NODE_HALF_LEFT, /* half-partition left (skip_node = right) */ + + /* Root-level partition (entry function, identity-base indices). */ + PROF_ROOT_FULL, + PROF_ROOT_HALF_RIGHT, + PROF_ROOT_HALF_LEFT, + + /* Leaf primitives (timed). */ + PROF_SCATTER_SYM, + PROF_SCATTER_BOTH_LEAVES, + PROF_FLAT_DECODE_SCATTER, + PROF_FLAT_DECODE_DIRECT, + + /* Bottom-up decoder (pivco_bu_{neon,x86}.c) per-primitive + * timings. Elements = bytes processed at this call. */ + PROF_BU_MERGE_VEC_VEC, /* general 2-buffer merge */ + PROF_BU_MERGE_CST_VEC, /* left side broadcast constant */ + PROF_BU_MERGE_CST_CST, /* BOTH_LEAVES / both-leaf collapse */ + PROF_BU_MERGE_FLAT, /* INTERNAL_FLAT direct-to-buffer */ + PROF_BU_POPCOUNT_K, /* compute K_right from bitmap */ + PROF_BU_LEAF_MEMSET, /* LEAF / SKIP: write K copies of sym */ + + /* Wire-format decode reads (shared TD/BU; charged per node). */ + PROF_WIRE_KR, /* read K_right:u16 header */ + PROF_WIRE_BITMAP_RAW, /* marker==0: raw bitmap, pointer + advance */ + PROF_WIRE_BITMAP_FSE, /* marker!=0: FSE-decompress bitmap body */ + + /* Encoder (pivco_encode_neon / encode_node_neon). Mirrors + * the decode side: per-primitive timing of the work done inside a + * node body (not the recursion itself), plus the per-block setup. */ + PROF_ENC_INIT, /* codes[]/lens[]/indices[] setup, per block */ + PROF_ENC_NODE_FULL, /* non-flat internal node: mask build + partition_8 */ + PROF_ENC_FLAT, /* flat-subtree node: pack_D_bits */ + PROF_ENC_FLAT_SIMD_ELEMS, /* count-only: elems handled by SIMD path */ + PROF_ENC_FLAT_TAIL_ELEMS, /* count-only: elems handled by scalar tail */ + PROF_ENC_REPACK_U8, /* uint16→uint8 repack at u8-subtree dispatch */ + PROF_ENC_NODE_FULL_U8, /* uint8-path partition body (mirrors NODE_FULL) */ + PROF_ENC_FLAT_U8, /* uint8-path flat pack (mirrors ENC_FLAT) */ + + /* Recursion + entry call counts (count-only; recursive timing + * would double-count). */ + PROF_DECODE_NODE, + PROF_DECODE_ENTRY, + PROF_ENC_NODE_VISIT, /* count-only: calls to encode_node_neon */ + PROF_ENC_ENTRY, /* count-only: calls to pivco_encode_neon */ + + /* File-codec layer (pivcohuf_file.c). Wraps the entire file-level + * pipeline so the CLI can show where time goes outside the + * block-codec inner loops. */ + PROF_FILE_HISTOGRAM, /* per-input histogram scan (compress only) */ + PROF_FILE_BUILD_TABLE_REAL, /* first build_table from real freqs (compress) */ + PROF_FILE_BUILD_TABLE_SYN, /* second build_table from synth freqs (both) */ + PROF_FILE_BODY_CSUM, /* XXH32 over body (currently disabled) */ + PROF_FILE_HDR, /* header parse / write */ + PROF_FILE_PAD, /* trailing-block prep (memcpy + memset) */ + PROF_FILE_BLOCK_ENCODE, /* per-block pivco_encode call */ + PROF_FILE_BLOCK_DECODE, /* per-block pivco_decode call */ + PROF_FILE_BLOCK_PROLOGUE, /* per-block length prefix + offset math */ + + /* FSE per-node entropy coding (v0.2 wire format). */ + PROF_FSE_ENC, /* time spent in pivco_fse_compress, per node */ + PROF_FSE_DEC, /* time spent in pivco_fse_decompress, per node */ + PROF_FSE_HIT_COUNT, /* count-only: nodes where FSE was actually emitted */ + PROF_FSE_RAW_COUNT, /* count-only: nodes that stayed raw (p +#include +#include "pivco_huffman.h" /* pivco_cfg_t */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define PIVCOHUF_MAGIC "PIVCOHUF" +#define PIVCOHUF_VERSION_MAJOR 0 +#define PIVCOHUF_VERSION_MINOR 9 +#define PIVCOHUF_HEADER_SIZE 26 + +/* BODY FLAGS byte (v0.9+). Bits0-1 carry the pivco_flat_layout_t value + * (3 is reserved); any bit outside the layout field (including the + * reserved QUAD_NODES bit) makes the decoder return BAD_VERSION. */ +#define PIVCOHUF_FLAGS_LAYOUT_MASK 0x03u /* bits0-1: flat layout */ +#define PIVCOHUF_FLAG_QUAD_NODES 0x04u /* bit2: reserved, must be 0 */ + +typedef enum { + PIVCOHUF_OK = 0, + PIVCOHUF_ERR_NULL = -1, + PIVCOHUF_ERR_TOO_SHORT = -2, + PIVCOHUF_ERR_BAD_MAGIC = -3, + PIVCOHUF_ERR_BAD_VERSION = -4, + PIVCOHUF_ERR_BAD_HEADER_CHECKSUM = -5, + PIVCOHUF_ERR_BAD_BODY_CHECKSUM = -6, + PIVCOHUF_ERR_BAD_BLOCK_SIZE = -7, + PIVCOHUF_ERR_OUTPUT_TOO_SMALL = -8, + PIVCOHUF_ERR_INTERNAL = -9, +} pivcohuf_status_t; + +/* Worst-case output size given input size. Overestimates; never lies low. + * Uses the default block size (PIVCO_BLOCK_SIZE). */ +size_t pivcohuf_compress_bound(size_t in_len); + +/* As pivcohuf_compress_bound, but for a specific block size. Smaller blocks + * carry more per-block overhead and need a larger bound, so callers of + * pivcohuf_compress_blk must size the output buffer with this. */ +size_t pivcohuf_compress_bound_blk(size_t in_len, size_t block_size); + +/* Compress in[0..in_len) into out (capacity *out_len). On success, + * sets *out_len to the actual encoded length and returns PIVCOHUF_OK. + * Plain Huffman (#PH). */ +int pivcohuf_compress(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len); + +/* As pivcohuf_compress, but `use_ans != 0` selects #PHA: per-block partition + * bitmaps may be ANS(FSE)-coded for a better ratio on skewed data, at some + * decode cost. Same wire format and decoder — pivcohuf_decompress auto-detects + * the ANS-coded blocks, so pha and ph streams decompress identically. */ +int pivcohuf_compress_ex(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, int use_ans); + +/* Decompress in[0..in_len) into out (capacity *out_len). Verifies + * header and body checksums. On success, sets *out_len to the actual + * uncompressed length and returns PIVCOHUF_OK. */ +int pivcohuf_decompress(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len); + +/* Peek the uncompressed size from a compressed stream's header. + * Used to allocate the output buffer before calling decompress. */ +int pivcohuf_peek_uncompressed_size(const uint8_t *in, size_t in_len, + size_t *uncompressed_size); + +/* Per-phase wall-clock breakdown (nanoseconds) filled by the *_timed + * variants. Phases not relevant to the call stay 0 (e.g. freq_ns on + * decompress). freq_ns and build_ns are distinct: a caller who already + * has symbol frequencies can skip the histogram (freq_ns) and build the + * table directly via the block API in pivco_huffman.h. Timing is coarse + * (never inside hot inner loops); pass NULL to skip it entirely. */ +typedef struct { + double freq_ns; /* build frequencies (symbol histogram) -- compress only */ + double build_ns; /* build codes/tree (Huffman table) */ + double codec_ns; /* encode (compress) or decode (decompress) block loop */ + double malloc_ns; /* internal scratch allocations */ +} pivcohuf_timing_t; + +/* Full-parameter compress: cfg (NULL = defaults; fse_enabled selects + * #PHA), explicit block size, optional timing. */ +int pivcohuf_compress_cfg(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + const pivco_cfg_t *cfg, size_t block_size, + pivcohuf_timing_t *timing); + + +/* As pivcohuf_compress_ex / pivcohuf_decompress, but fill *timing (nullable) + * with the per-phase breakdown above. The struct is zeroed on entry. */ +int pivcohuf_compress_timed(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + int use_ans, pivcohuf_timing_t *timing); + +/* As pivcohuf_compress_timed, but with a caller-chosen block size (symbol + * count per block, 1..PIVCO_WIRE_MAX_N). The block size is recorded in the + * stream header, so pivcohuf_decompress reads it back automatically — no + * matching build flag required. Larger blocks amortise per-block table/tree + * reload (a big decode win on small-L1 x86; see issue #2). Size the output + * buffer with pivcohuf_compress_bound_blk(in_len, block_size). timing may be + * NULL. */ +int pivcohuf_compress_blk(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + int use_ans, size_t block_size, + pivcohuf_timing_t *timing); +int pivcohuf_decompress_timed(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + pivcohuf_timing_t *timing); + +#ifdef __cplusplus +} +#endif + +#endif /* PIVCOHUF_FILE_H */ diff --git a/third_party/pivco/src/huffman_table.c b/third_party/pivco/src/huffman_table.c new file mode 100644 index 0000000..e24e3b6 --- /dev/null +++ b/third_party/pivco/src/huffman_table.c @@ -0,0 +1,851 @@ +#include "pivco_huffman.h" +#include +#include +#include +#include "pivco_check.h" + +/* ---------- Code lengths via van Leeuwen's two-queue method ---------- + * Replaces the index-indirected binary min-heap. The heap spent ~⅔ of the + * whole table build pointer-chasing (nodes[indices[i]].freq is two dependent + * loads per compare) for a trivial <=256-leaf tree. The two-queue method is + * O(n) after one sort: leaves pre-sorted ascending by frequency go in one + * queue, internal nodes (whose frequencies are generated monotonically) in a + * second FIFO, so each of the two minima per merge is an O(1) front compare. + * + * Tie discipline reproduces the heap's exactly, giving byte-identical lengths: + * the heap broke ties by node index, and leaves (indices 0..n-1) always sort + * before internals (indices >=n), so on an equal frequency a leaf wins -- the + * `<=` below -- and within a queue the front already holds the lowest index + * (leaves sorted by (freq,sym); internals in creation order). */ +typedef struct { uint64_t freq; uint16_t sym; } leaf_t; + +/* Stable LSD radix sort of leaf[0..n) by frequency ascending, over only the + * bytes the max frequency needs (typically 2-3 for per-window counts). Beats + * qsort here: no indirect compare per element, and stability over the + * symbol-ordered seed keeps the (freq,sym) tie discipline the heap relied on. */ +static void sort_leaves_by_freq(leaf_t *leaf, int n) +{ + uint64_t mx = 0; + for (int i = 0; i < n; i++) if (leaf[i].freq > mx) mx = leaf[i].freq; + int nbytes = 0; + while (mx) { nbytes++; mx >>= 8; } /* freq>0 for every leaf => nbytes>=1 */ + + leaf_t tmp[PIVCO_MAX_SYMBOLS]; + leaf_t *src = leaf, *dst = tmp; + for (int b = 0; b < nbytes; b++) { + int shift = b * 8; + int cnt[256] = {0}; + for (int i = 0; i < n; i++) cnt[(src[i].freq >> shift) & 0xFF]++; + int sum = 0; + for (int c = 0; c < 256; c++) { int t = cnt[c]; cnt[c] = sum; sum += t; } + for (int i = 0; i < n; i++) { int k = (src[i].freq >> shift) & 0xFF; dst[cnt[k]++] = src[i]; } + leaf_t *t = src; src = dst; dst = t; + } + if (src != leaf) memcpy(leaf, src, (size_t)n * sizeof(leaf_t)); +} + +/* Derives code lengths for n_used (>=2) symbols into lengths[] (indexed by + * symbol; untouched entries stay 0). No length limiting -- the caller applies + * limit_code_lengths afterwards, same as the heap path did. */ +static void build_lengths_twoqueue(const uint64_t freq[PIVCO_MAX_SYMBOLS], + int n_used, const int used[PIVCO_MAX_SYMBOLS], + uint8_t lengths[PIVCO_MAX_SYMBOLS]) +{ + leaf_t leaf[PIVCO_MAX_SYMBOLS]; + for (int i = 0; i < n_used; i++) { + leaf[i].freq = freq[used[i]]; + leaf[i].sym = (uint16_t)used[i]; + } + sort_leaves_by_freq(leaf, n_used); + + /* nodes 0..n_used-1 = leaves (sorted order); n_used.. = internals. + * The internal queue is the contiguous index range [ih, it). */ + const int N = n_used; + uint64_t nfreq[PIVCO_MAX_SYMBOLS * 2]; + int parent[PIVCO_MAX_SYMBOLS * 2]; + for (int i = 0; i < N; i++) nfreq[i] = leaf[i].freq; + + int li = 0; /* next unconsumed leaf */ + int ih = N; /* internal-queue head (oldest) */ + int ni = N; /* next internal node index to create (== queue tail) */ + for (int remaining = N; remaining > 1; remaining--) { + int a, b; + if (li < N && (ih == ni || nfreq[li] <= nfreq[ih])) a = li++; else a = ih++; + if (li < N && (ih == ni || nfreq[li] <= nfreq[ih])) b = li++; else b = ih++; + nfreq[ni] = nfreq[a] + nfreq[b]; + parent[a] = ni; + parent[b] = ni; + ni++; /* extends the internal queue tail */ + } + + const int root = ni - 1; /* == 2N-2 */ + uint8_t depth[PIVCO_MAX_SYMBOLS * 2]; + depth[root] = 0; + for (int i = root - 1; i >= 0; i--) /* parent index always > child index */ + depth[i] = (uint8_t)(depth[parent[i]] + 1); + for (int i = 0; i < N; i++) + lengths[leaf[i].sym] = depth[i] > 0 ? depth[i] : 1; +} + +/* ---------- Code length limiting (DEFLATE-style, RFC 1951) ---------- */ + +static void limit_code_lengths(uint8_t *lengths, int n_symbols, int max_len) +{ + /* Count symbols at each length */ + int count[64] = {0}; /* support original lengths up to 63 */ + int max_orig = 0; + for (int i = 0; i < n_symbols; i++) { + if (lengths[i] > 0) { + count[lengths[i]]++; + if (lengths[i] > max_orig) max_orig = lengths[i]; + } + } + if (max_orig <= max_len) return; /* nothing to do */ + + /* Move all symbols longer than max_len down to max_len */ + for (int i = max_orig; i > max_len; i--) { + count[max_len] += count[i]; + count[i] = 0; + } + + /* Now Kraft sum may exceed 1.0. Fix by moving symbols from max_len + to shorter lengths. Each time we move one symbol from length L + to length L-1, the Kraft delta is: 2^(max-L+1) - 2^(max-L) = 2^(max-L). + But that creates a "debt" at length L-1 which may also overflow. + + Work bottom-up: for each length from max_len down, if we have + overflow, push pairs up to parent (length-1). */ + + /* Compute Kraft sum in units of 2^(-max_len) */ + uint64_t kraft = 0; + for (int i = 1; i <= max_len; i++) { + kraft += (uint64_t)count[i] << (max_len - i); + } + uint64_t target = (uint64_t)1 << max_len; + + /* While over-full, increase the longest codes */ + while (kraft > target) { + /* Find a symbol at a length < max_len and increase it by 1. + This reduces Kraft by 2^(max_len - len) - 2^(max_len - len - 1) + = 2^(max_len - len - 1). Pick the longest such length to + minimize Kraft reduction per step. */ + int best = -1; + for (int len = max_len - 1; len >= 1; len--) { + if (count[len] > 0) { + best = len; + break; + } + } + if (best < 0) break; /* shouldn't happen */ + + count[best]--; + count[best + 1]++; + kraft -= (uint64_t)1 << (max_len - best - 1); + } + + /* While under-full, decrease some max_len codes to shorter lengths. + This fills unused Kraft capacity. */ + while (kraft < target && count[max_len] > 0) { + /* Find the shortest length where we can add capacity */ + for (int len = max_len - 1; len >= 1; len--) { + /* Moving one code from max_len to len changes kraft by: + +2^(max_len-len) - 2^(max_len-max_len) = 2^(max_len-len) - 1 */ + uint64_t delta = ((uint64_t)1 << (max_len - len)) - 1; + if (kraft + delta <= target && count[max_len] > 0) { + count[max_len]--; + count[len]++; + kraft += delta; + break; + } + } + /* If we couldn't shorten anything, done */ + if (kraft < target) { + /* Try filling one slot at max_len-1 at a time */ + uint64_t delta = ((uint64_t)1 << 1) - 1; /* moving max_len to max_len-1 */ + if (kraft + delta <= target && count[max_len] >= 2) { + /* Move one from max_len to max_len-1: net = +2 - 1 = +1 */ + count[max_len]--; + count[max_len - 1]++; + kraft += 1; + } else { + break; + } + } + } + + /* Reassign lengths based on new counts. + Sort symbols by original length (as proxy for frequency), + assign shortest new lengths to the most frequent symbols. */ + /* Build sorted list of (original_length, symbol_index) */ + typedef struct { uint8_t len; uint8_t sym; } ls_t; + ls_t sorted[PIVCO_MAX_SYMBOLS]; + int ns = 0; + for (int i = 0; i < n_symbols; i++) { + if (lengths[i] > 0) { + sorted[ns].len = lengths[i] > max_len ? (uint8_t)max_len : lengths[i]; + sorted[ns].sym = (uint8_t)i; + ns++; + } + } + /* Sort by original length (shorter = more frequent = should get shorter code) */ + for (int i = 1; i < ns; i++) { + ls_t tmp = sorted[i]; + int j = i - 1; + while (j >= 0 && sorted[j].len > tmp.len) { + sorted[j + 1] = sorted[j]; + j--; + } + sorted[j + 1] = tmp; + } + + /* Assign new lengths from count array */ + int si = 0; + for (int len = 1; len <= max_len && si < ns; len++) { + for (int c = 0; c < count[len] && si < ns; c++) { + lengths[sorted[si].sym] = (uint8_t)len; + si++; + } + } +} + +/* ---------- Canonical Huffman code assignment ---------- */ + +/* Builds everything downstream of the code lengths (canonical assignment, + * tree, flat-subtree detection, aux tables). Shared by the encode path + * (after the min-heap derives lengths from frequencies) and the decode path + * (lengths come straight off the wire -- no heap needed). Assumes `table` is + * already zeroed and table->num_symbols is set; caller handles n_used <= 1. */ +static int build_table_finish(const uint8_t lengths[PIVCO_MAX_SYMBOLS], + pivco_table_t *table, + const pivco_cfg_t *cfg); + +/* Fill enc_init_aux from sym_to_rank: on x86 the 2tab merge hi table (rank<<8) + * and point the aux at it; elsewhere leave the aux NULL. Called by every build + * path that finalizes sym_to_rank (build_table_finish and the single-symbol fast + * path), so the x86 prim_enc_init never sees a NULL aux. */ +static void fill_enc_init_aux(pivco_table_t *table) +{ +#if defined(__x86_64__) || defined(__i386__) + for (int s = 0; s < PIVCO_MAX_SYMBOLS; s++) + table->enc_init_hi[s] = (uint16_t)((unsigned)table->sym_to_rank[s] << 8); + table->enc_init_aux.s2r_hi = table->enc_init_hi; +#else + table->enc_init_aux.s2r_hi = NULL; +#endif +} + +/* Single-symbol degenerate tree: root -> two leaves of the same symbol. + * Assumes `table` is zeroed. */ +static void build_single_symbol_table(int sym, pivco_table_t *table) +{ + table->code[sym] = 0; + table->code_len[sym] = 1; + table->max_len = 1; + table->min_len = 1; + table->sym_count[1] = 1; + table->first_code[1] = 0; + table->first_sym_idx[1] = 0; + table->sorted_symbols[0] = (uint8_t)sym; + table->tree[0].symbol = -1; + table->tree[0].left = 1; + table->tree[0].right = 2; + table->tree[1].symbol = (int16_t)sym; + table->tree[1].left = -1; + table->tree[1].right = -1; + table->tree[2].symbol = (int16_t)sym; /* both children = same symbol */ + table->tree[2].left = -1; + table->tree[2].right = -1; + table->tree_root = 0; + table->tree_node_count = 3; + /* node 0 (root): both children leaves -> BOTH_LEAVES (the decode + * entry's root fast path handles it); nodes 1, 2: LEAF. */ + table->node_type[0] = PIVCO_NODE_BOTH_LEAVES; + table->node_type[1] = PIVCO_NODE_LEAF; + table->node_type[2] = PIVCO_NODE_LEAF; + fill_enc_init_aux(table); /* sym_to_rank is all-zero (rank 0) here; aux must not stay NULL */ +} + +int pivco_build_table(const pivco_cfg_t *cfg, + const uint64_t freq[PIVCO_MAX_SYMBOLS], + pivco_table_t *table) +{ + if (!freq || !table) return PIVCO_ERR_NULL; + if (!cfg) cfg = &pivco_cfg_default; + if ((unsigned)cfg->flat_layout > PIVCO_FLAT_VERTICAL_128) + return PIVCO_ERR_BAD_CFG; + + memset(table, 0, sizeof(*table)); + + /* Count symbols with nonzero frequency */ + int n_used = 0; + int used[PIVCO_MAX_SYMBOLS]; + for (int i = 0; i < PIVCO_MAX_SYMBOLS; i++) { + if (freq[i] > 0) { + used[n_used++] = i; + } + } + + if (n_used == 0) return PIVCO_ERR_EMPTY; + + table->num_symbols = (uint16_t)n_used; + + if (n_used == 1) { + build_single_symbol_table(used[0], table); + table->fse_enabled = (uint8_t)(cfg->fse_enabled ? 1 : 0); + table->flat_layout = (uint8_t)cfg->flat_layout; + return PIVCO_OK; + } + + /* Derive code lengths from frequencies (two-queue, no heap) */ + uint8_t lengths[PIVCO_MAX_SYMBOLS]; + memset(lengths, 0, sizeof(lengths)); + build_lengths_twoqueue(freq, n_used, used, lengths); + + /* Limit code lengths to PIVCO_MAX_CODE_LEN */ + limit_code_lengths(lengths, PIVCO_MAX_SYMBOLS, PIVCO_MAX_CODE_LEN); + + /* Optional joint length/shape pass (encoder side only; the decoder + rebuilds identically from the transmitted lengths). Any internal + reject keeps the plain Huffman lengths above. */ + if (cfg->effort != PIVCO_EFFORT_PLAIN) + (void)pivco_joint_optimize_lengths(freq, lengths, cfg); + + return build_table_finish(lengths, table, cfg); +} + +/* partbyrank: assign each leaf its in-order rank (left-to-right leaf + * position) in a single in-order pass, returning the next free rank. A + * subtree's leaves are a contiguous rank range, so routing by code-bit is + * equivalent to routing by `rank > split_rank`. Per node, everything is known + * once its left subtree has been visited: + * flat_base_rank[node] = rank on enter (min rank of the subtree) + * split_rank[node] = rank after left - 1 (max rank of the left subtree) + * A flat subtree's leaves are enumerated in code order (== in-order) via + * flat_code_to_sym, not the tree structure, so it does not recurse. */ +static uint16_t assign_inorder_ranks(pivco_table_t *table, + int16_t id, uint16_t rank) +{ + const pivco_tree_node_t *n = &table->tree[id]; + if (n->symbol >= 0) { /* leaf */ + table->sym_to_rank[n->symbol] = (uint8_t)rank; + return (uint16_t)(rank + 1); + } + if (table->flat_depth[id] >= 2) { /* flat subtree */ + table->flat_base_rank[id] = (uint8_t)rank; + int cnt = 1 << table->flat_depth[id]; + for (int i = 0; i < cnt; i++) { + uint8_t sym = table->flat_code_to_sym[table->flat_offset[id] + i]; + table->sym_to_rank[sym] = (uint8_t)(rank + i); + } + return (uint16_t)(rank + cnt); + } + rank = assign_inorder_ranks(table, n->left, rank); + table->split_rank[id] = (uint8_t)(rank - 1); /* max rank of the left subtree */ + return assign_inorder_ranks(table, n->right, rank); +} + +static int build_table_finish(const uint8_t lengths[PIVCO_MAX_SYMBOLS], + pivco_table_t *table, + const pivco_cfg_t *cfg) +{ + table->fse_enabled = (uint8_t)(cfg->fse_enabled ? 1 : 0); + table->flat_layout = (uint8_t)cfg->flat_layout; + /* Copy lengths to table */ + for (int i = 0; i < PIVCO_MAX_SYMBOLS; i++) { + table->code_len[i] = lengths[i]; + } + + /* Histogram code lengths. The len>0 guard isn't about correctness + (sym_count[0] is an unread scratch bin) -- it keeps the unused symbols + from all piling onto bin 0, whose serial store-to-load-forward chain + was 5x slower than the (well-predicted) branch on sparse alphabets. + min/max are derived from the bins below, not inline here. */ + for (int i = 0; i < PIVCO_MAX_SYMBOLS; i++) { + if (lengths[i] > 0) + table->sym_count[lengths[i]]++; + } + + /* Derive min/max code length from the (<=11) length bins. */ + uint8_t max_len = 0, min_len = PIVCO_MAX_CODE_LEN + 1; + for (int L = 1; L <= PIVCO_MAX_CODE_LEN; L++) { + if (table->sym_count[L]) { + if (L < min_len) min_len = (uint8_t)L; + max_len = (uint8_t)L; + } + } + table->max_len = max_len; + table->min_len = min_len; + + /* ---------- Flat-aware code assignment ---------- + * + * Goal: give each symbol a code of its assigned length such that the + * resulting binary tree has as many large flat-D>=2 subtrees as + * possible (consolidates the partition path during tree-walk decode). + * Compression is unaffected — code lengths match the Huffman result. + * + * Algorithm: per length L, decompose c_L by its binary representation + * into "chunks": bits >= 2 form D>=2 flat subtrees of size 2^D rooted + * at depth L-D; bit 1 forms a D=1 sibling pair (handled by stage + * fusion at decode); bit 0 is a singleton. Sort chunks by their + * tree-depth asc (depth = L-D for D>=2 chunks, L-1 for D=1, L for + * singletons), then canonical-assign codes to chunks. Within each + * chunk, top-freq-first symbols of length L are assigned to its + * 2^bit suffix slots (highest freqs go to the largest-D chunk per + * length, where the partition-path savings are deepest). + * + * See IDEAS.md "Flat-aware Huffman tree restructurer" for the gap + * analysis (extras/bench/bench_flat_optimal.c). + */ + + /* Per-length: collect symbols in symbol-value order. + * + * We used to sort within a tier by frequency-desc so the heaviest + * symbols landed in the largest flat chunk. That was dropped: it is + * the only thing that made the tree depend on within-tier frequency + * order, which in turn forced a within-tier ordering onto the wire + * (the v0.3 ORDERING section + rank_within_tier) so the decoder could + * reproduce it. On FSE-coded blocks the freq-order "win" only *masked* + * a bad FSE commit policy (the gate ignores FSE decode cost). Plain + * symbol-value order is deterministic from the code lengths alone, so + * encoder and decoder agree with no rank info transmitted. */ + typedef struct { + uint8_t sym; + } sf_t; + sf_t flat_items[PIVCO_MAX_SYMBOLS]; + int per_len_start[PIVCO_MAX_CODE_LEN + 2]; + { + /* Counting sort by length: prefix-sum the per-length counts, then a + single symbol-order pass places each symbol. Equivalent to the + old nested for-L/for-s scan but O(256) instead of O(max_len*256). */ + int acc = 0; + int cursor[PIVCO_MAX_CODE_LEN + 2]; + for (int L = 1; L <= max_len; L++) { + per_len_start[L] = acc; + cursor[L] = acc; + acc += table->sym_count[L]; + } + per_len_start[max_len + 1] = acc; + for (int s = 0; s < PIVCO_MAX_SYMBOLS; s++) { + uint8_t L = lengths[s]; + if (L) flat_items[cursor[L]++].sym = (uint8_t)s; + } + } + + /* Decompose each c_L into chunks. Strategy depends on tree mode -- + see pivco_cfg_t.tree_mode. Default OPTIMIZED matches the + original production behavior (decompose c_L by its set bits). */ + typedef struct { + uint16_t L; + uint16_t bit; /* 0..PIVCO_MAX_CODE_LEN */ + uint16_t depth; /* tree-depth of chunk root */ + uint16_t n_syms; /* 1 << bit */ + uint16_t root_code; /* canonical code of the chunk root (depth bits) */ + int sym_idx; /* index into flat_items */ + } chunk_t; + chunk_t chunks[PIVCO_MAX_SYMBOLS]; /* upper bound: one chunk per symbol */ + int n_chunks = 0; + pivco_tree_mode_t tree_mode = cfg->tree_mode; + + if (tree_mode == PIVCO_TREE_MODE_NAIVE) { + /* Every symbol is its own D=0 chunk at depth L. */ + for (int L = 1; L <= max_len; L++) { + int c = table->sym_count[L]; + int cur = per_len_start[L]; + for (int i = 0; i < c; i++) { + chunks[n_chunks].L = (uint16_t)L; + chunks[n_chunks].bit = 0; + chunks[n_chunks].depth = (uint16_t)L; + chunks[n_chunks].n_syms = 1; + chunks[n_chunks].sym_idx = cur + i; + n_chunks++; + } + } + } else if (tree_mode == PIVCO_TREE_MODE_FUSED) { + /* D=1 sibling pairs first within each length, then a D=0 singleton + for the odd-tail symbol. Sequential reassign in the standard + depth-sort step gives canonical Huffman codes. */ + for (int L = 1; L <= max_len; L++) { + int c = table->sym_count[L]; + int cur = per_len_start[L]; + int n_pairs = c / 2; + int n_singletons = c & 1; + for (int i = 0; i < n_pairs; i++) { + chunks[n_chunks].L = (uint16_t)L; + chunks[n_chunks].bit = 1; + chunks[n_chunks].depth = (uint16_t)(L - 1); + chunks[n_chunks].n_syms = 2; + chunks[n_chunks].sym_idx = cur; + cur += 2; + n_chunks++; + } + for (int i = 0; i < n_singletons; i++) { + chunks[n_chunks].L = (uint16_t)L; + chunks[n_chunks].bit = 0; + chunks[n_chunks].depth = (uint16_t)L; + chunks[n_chunks].n_syms = 1; + chunks[n_chunks].sym_idx = cur; + cur++; + n_chunks++; + } + } + } else if (tree_mode == PIVCO_TREE_MODE_CANONICAL_FLAT) { + /* Compute canonical first_code[L] = (first_code[L-1] + c_{L-1}) << 1 + starting from min_len. For each length, greedy-peel the largest + 2^k chunk such that the canonical start code C is 2^k-aligned and + 2^k <= remaining. root_code = C >> k. */ + uint32_t fc[PIVCO_MAX_CODE_LEN + 2] = {0}; + uint32_t code = 0; + int last_L = 0; + for (int L = 1; L <= max_len; L++) { + if (table->sym_count[L]) { + if (last_L) code = (code + (uint32_t)table->sym_count[last_L]) << (L - last_L); + fc[L] = code; + last_L = L; + } + } + for (int L = 1; L <= max_len; L++) { + int c = table->sym_count[L]; + if (c == 0) continue; + int cur = per_len_start[L]; + uint32_t C = fc[L]; + int remaining = c; + while (remaining > 0) { + int max_k_align = (C == 0) ? PIVCO_MAX_CODE_LEN : __builtin_ctz(C); + int max_k_count = (remaining > 1) ? (31 - __builtin_clz((unsigned)remaining)) : 0; + int k = max_k_align < max_k_count ? max_k_align : max_k_count; + /* Safety: chunk depth = L-k must be >= 0; since k <= log2(remaining) <= log2(c) <= L-1 + under any valid Kraft length distribution, this is always true. */ + int n = 1 << k; + chunks[n_chunks].L = (uint16_t)L; + chunks[n_chunks].bit = (uint16_t)k; + chunks[n_chunks].depth = (uint16_t)(L - k); + chunks[n_chunks].n_syms = (uint16_t)n; + chunks[n_chunks].root_code = (uint16_t)(C >> k); + chunks[n_chunks].sym_idx = cur; + cur += n; + n_chunks++; + C += (uint32_t)n; + remaining -= n; + } + } + } else { + /* OPTIMIZED (default): original bit-decomposition of c_L. */ + for (int L = 1; L <= max_len; L++) { + int c = table->sym_count[L]; + int cur = per_len_start[L]; + /* Iterate set bits high-to-low so larger chunks come first + within the length (matters only for top-freq-first symbol + assignment within the length). */ + for (int bit = PIVCO_MAX_CODE_LEN; bit >= 0; bit--) { + if (c & (1 << bit)) { + int n = 1 << bit; + int depth; + if (bit >= 2) depth = L - bit; + else if (bit == 1) depth = L - 1; + else depth = L; + chunks[n_chunks].L = (uint16_t)L; + chunks[n_chunks].bit = (uint16_t)bit; + chunks[n_chunks].depth = (uint16_t)depth; + chunks[n_chunks].n_syms = (uint16_t)n; + chunks[n_chunks].sym_idx = cur; + cur += n; + n_chunks++; + } + } + } + } + + + /* For CANONICAL_FLAT, chunks already carry canonical root_codes; assign + symbol codes directly from them and skip the depth-sort + sequential + reassign step (sequential reassign would clobber the canonical + prefixes when chunks span multiple depths). All other modes use + the standard pipeline. */ + if (tree_mode == PIVCO_TREE_MODE_CANONICAL_FLAT) { + for (int ci = 0; ci < n_chunks; ci++) { + int bit = chunks[ci].bit; + int n = chunks[ci].n_syms; + uint16_t root = chunks[ci].root_code; + for (int i = 0; i < n; i++) { + uint8_t sym = flat_items[chunks[ci].sym_idx + i].sym; + table->code[sym] = (uint16_t)(((uint32_t)root << bit) | (uint32_t)i); + } + } + } else { + /* Sort chunks by depth asc (stable; ties keep their natural order + which is L asc by length, larger-bit-first within length). */ + for (int i = 1; i < n_chunks; i++) { + chunk_t cur = chunks[i]; + int j = i - 1; + while (j >= 0 && chunks[j].depth > cur.depth) { + chunks[j + 1] = chunks[j]; + j--; + } + chunks[j + 1] = cur; + } + + + /* Canonical-assign codes to chunks (chunk-level Kraft sum = 1). + Each chunk gets a code prefix of length `chunk.depth`. Within + the chunk, symbol i takes suffix i for i in [0, 2^bit). */ + { + uint32_t code = 0; + int prev_depth = 0; + for (int ci = 0; ci < n_chunks; ci++) { + int d = chunks[ci].depth; + if (d > prev_depth) code <<= (d - prev_depth); + chunks[ci].root_code = (uint16_t)code; + int bit = chunks[ci].bit; + int n = chunks[ci].n_syms; + for (int i = 0; i < n; i++) { + uint8_t sym = flat_items[chunks[ci].sym_idx + i].sym; + table->code[sym] = (uint16_t)((code << bit) | (uint32_t)i); + } + code += 1; + prev_depth = d; + } + } + + } + + /* Populate sorted_symbols / first_sym_idx / first_code from the + new code assignment. These fields are not used by runtime + decoders (only by the tree-walk pass below), but we keep them + in length-asc order for compatibility with anyone inspecting + the table. */ + int sorted_idx = per_len_start[max_len + 1]; + for (int i = 0; i < sorted_idx; i++) + table->sorted_symbols[i] = flat_items[i].sym; + for (int len = 1; len <= max_len; len++) { + table->first_sym_idx[len] = (uint16_t)per_len_start[len]; + uint16_t min_code = 0xFFFF; + for (int i = per_len_start[len]; i < per_len_start[len + 1]; i++) { + uint16_t c = table->code[flat_items[i].sym]; + if (c < min_code) min_code = c; + } + table->first_code[len] = (min_code == 0xFFFF) ? 0 : min_code; + } + + /* The 2^MAX_CODE_LEN flat decode table (decode_sym/decode_len) is used + * ONLY by the traditional flat-table decoder (trad_huffman_decode*), never + * by the production tree-walk path. It is built on demand via + * pivco_build_traditional_table() so the normal build -- and the + * decode-side rebuild from code lengths -- don't pay for the 2 KB fill. */ + + + /* Build the PIVCO tree-walk tree, one node-creating walk per chunk. + A flat subtree (D>=2) stops at its root: the decoder reaches its 2^D + symbols via flat_code_to_sym (filled here), so we never materialize + the 2^D leaves nor the internal nodes below the root -- a large node + saving on full alphabets, which also shrinks the classify and + max_leaf_depth passes. Singletons (D=0) and sibling pairs (D=1) + build their leaves. */ + { + int16_t nc = 0; /* node count */ + table->tree[0].symbol = -1; + table->tree[0].left = -1; + table->tree[0].right = -1; + nc++; + table->tree_root = 0; + uint16_t pool = 0; + + for (int ci = 0; ci < n_chunks; ci++) { + int D = chunks[ci].bit; + int d = chunks[ci].depth; + uint16_t rc = chunks[ci].root_code; + int base = chunks[ci].sym_idx; + + /* Walk rc's d bits MSB-first, creating spine nodes as needed. */ + int16_t cur = 0; + for (int b = d - 1; b >= 0; b--) { + int16_t *child = ((rc >> b) & 1) ? &table->tree[cur].right + : &table->tree[cur].left; + if (*child < 0) { + *child = nc; + table->tree[nc].symbol = -1; + table->tree[nc].left = -1; + table->tree[nc].right = -1; + nc++; + } + cur = *child; + } + + if (D >= 2) { + /* Flat root: mark + fill code_to_sym; no children built. + Leaf i of the chunk has in-subtree code i (low D bits of + its canonical code), so flat_code_to_sym[base+i] is its + i-th symbol. */ + PIVCO_CHECK(table->tree[cur].left == -1 && + table->tree[cur].right == -1); + table->flat_depth[cur] = (uint8_t)D; + table->flat_offset[cur] = pool; + int n = 1 << D; + for (int i = 0; i < n; i++) + table->flat_code_to_sym[pool + i] = flat_items[base + i].sym; + pool = (uint16_t)(pool + n); + } else if (D == 1) { + /* Sibling pair: two leaf children (suffix 0 -> left). */ + table->tree[cur].left = nc; + table->tree[nc].symbol = (int16_t)flat_items[base].sym; + table->tree[nc].left = -1; table->tree[nc].right = -1; nc++; + table->tree[cur].right = nc; + table->tree[nc].symbol = (int16_t)flat_items[base + 1].sym; + table->tree[nc].left = -1; table->tree[nc].right = -1; nc++; + } else { + /* Singleton: cur is the leaf at depth d. */ + table->tree[cur].symbol = (int16_t)flat_items[base].sym; + } + } + table->tree_node_count = nc; + } + + + /* Classify each node for decode-dispatch, by children's leafness: + * FLAT (subtree, D>=2) > BOTH_LEAVES > LEAF_LEFT > FULL. + * Canonical code assignment always puts a lone leaf child on the + * 0/left side (shorter code = smaller left-aligned value), so a + * right-leaf-only node cannot occur — asserted. */ + for (int16_t i = 0; i < table->tree_node_count; i++) { + const pivco_tree_node_t *node = &table->tree[i]; + + if (node->symbol >= 0) { + table->node_type[i] = (uint8_t)PIVCO_NODE_LEAF; + continue; + } + + /* Internal node */ + if (table->flat_depth[i] >= 2) { + table->node_type[i] = (uint8_t)PIVCO_NODE_INTERNAL_FLAT; + continue; + } + + int left_leaf = (table->tree[node->left].symbol >= 0); + int right_leaf = (table->tree[node->right].symbol >= 0); + + if (left_leaf && right_leaf) { + table->node_type[i] = (uint8_t)PIVCO_NODE_BOTH_LEAVES; + } else if (left_leaf) { + table->node_type[i] = (uint8_t)PIVCO_NODE_LEAF_LEFT; + } else { + PIVCO_CHECK(!right_leaf); + table->node_type[i] = (uint8_t)PIVCO_NODE_INTERNAL_FULL; + } + } + + + /* Populate max_leaf_depth[node] for every internal node. Used by + * the encoder to detect when a subtree's remaining bits fit in a + * byte and can be processed with uint8-wide partitions. Iterative + * post-order via tree_node_count traversal: tree nodes are + * allocated in order of construction (children before parents in + * our build), so a single pass from node 0 to tree_node_count + * fills max_leaf_depth bottom-up. + * + * BUT: that ordering is not guaranteed in general. Use recursion + * for correctness; the depth is small (<= PIVCO_MAX_CODE_LEN). */ + { + /* Iterative DFS via an explicit small stack. Simpler than + * thinking about node-allocation order, and recursion-free. */ + int stack[2 * PIVCO_MAX_TREE_NODES]; + int top = 0; + stack[top++] = table->tree_root; + /* First pass: count children visited per node, leaf := 0. */ + memset(table->max_leaf_depth, 0, sizeof(table->max_leaf_depth)); + int order[PIVCO_MAX_TREE_NODES]; + int order_n = 0; + while (top > 0) { + int16_t id = (int16_t)stack[--top]; + order[order_n++] = id; + const pivco_tree_node_t *n = &table->tree[id]; + /* Flat roots have no materialized children -- treat as terminal. */ + if (n->symbol < 0 && table->flat_depth[id] < 2) { + stack[top++] = n->left; + stack[top++] = n->right; + } + } + /* Process in reverse (children before parents). */ + for (int oi = order_n - 1; oi >= 0; oi--) { + int16_t id = (int16_t)order[oi]; + const pivco_tree_node_t *n = &table->tree[id]; + if (n->symbol >= 0) { + table->max_leaf_depth[id] = 0; + } else if (table->flat_depth[id] >= 2) { + /* All 2^D leaves sit D levels below this flat root. */ + table->max_leaf_depth[id] = table->flat_depth[id]; + } else { + uint8_t l = table->max_leaf_depth[n->left]; + uint8_t r = table->max_leaf_depth[n->right]; + table->max_leaf_depth[id] = (uint8_t)(1 + (l > r ? l : r)); + } + } + } + + /* partbyrank: one in-order pass assigns every leaf its rank and every + * internal node its split_rank / flat_base_rank (see assign_inorder_ranks). */ + assign_inorder_ranks(table, table->tree_root, 0); + + fill_enc_init_aux(table); /* x86 2tab/4tab gather tables (or NULL elsewhere) */ + + return PIVCO_OK; +} + +/* Public API: build a table from code lengths alone. The tree is fully + * determined by the lengths (within-tier order is symbol-value), so encoder + * and decoder reconstruct identical tables with no extra wire info. Goes + * straight to build_table_finish -- no synthetic frequencies, no Huffman + * heap (the lengths are already final). */ +int pivco_build_table_from_code_lens( + const pivco_cfg_t *cfg, + const uint8_t code_lens[PIVCO_MAX_SYMBOLS], + pivco_table_t *table) +{ + if (!code_lens || !table) return PIVCO_ERR_NULL; + if (!cfg) cfg = &pivco_cfg_default; + if ((unsigned)cfg->flat_layout > PIVCO_FLAT_VERTICAL_128) + return PIVCO_ERR_BAD_CFG; + /* Clear everything except the 4 KB decode_sym/decode_len pair: those are + * filled independently by pivco_build_traditional_table() and are + * never read by the bulk decoder, so zeroing them here is wasted work. */ + { + size_t skip_end = offsetof(pivco_table_t, decode_len) + + sizeof(table->decode_len); + memset(table, 0, offsetof(pivco_table_t, decode_sym)); + memset((char *)table + skip_end, 0, sizeof(*table) - skip_end); + } + + int n_used = 0, last = 0; + for (int i = 0; i < PIVCO_MAX_SYMBOLS; i++) + if (code_lens[i] > 0) { n_used++; last = i; } + if (n_used == 0) return PIVCO_ERR_EMPTY; + table->num_symbols = (uint16_t)n_used; + + if (n_used == 1) { + build_single_symbol_table(last, table); + table->fse_enabled = (uint8_t)(cfg->fse_enabled ? 1 : 0); + table->flat_layout = (uint8_t)cfg->flat_layout; + return PIVCO_OK; + } + return build_table_finish(code_lens, table, cfg); +} + +/* Fill the 2^MAX_CODE_LEN flat decode table (decode_sym/decode_len) read by + * the traditional flat-table decoder (trad_huffman_decode*). Call once after + * the table is built; the production tree-walk decoder does not need it, so + * pivco_build_table no longer fills it automatically. */ +void pivco_build_traditional_table(pivco_table_t *table) +{ + if (!table) return; + /* Defensive base fill covers any gap for incomplete codes (single sym); + * sorted_symbols[0] = shortest-code (most frequent) symbol. */ + memset(table->decode_sym, table->sorted_symbols[0], sizeof(table->decode_sym)); + memset(table->decode_len, 1, sizeof(table->decode_len)); + for (int s = 0; s < PIVCO_MAX_SYMBOLS; s++) { + int len = table->code_len[s]; + if (len <= 0) continue; + int shift = PIVCO_MAX_CODE_LEN - len; + uint32_t base = (uint32_t)table->code[s] << shift; + uint32_t count = (uint32_t)1 << shift; + memset(&table->decode_sym[base], s, count); + memset(&table->decode_len[base], len, count); + } +} diff --git a/third_party/pivco/src/joint_lengths.c b/third_party/pivco/src/joint_lengths.c new file mode 100644 index 0000000..9fa4dcc --- /dev/null +++ b/third_party/pivco/src/joint_lengths.c @@ -0,0 +1,1348 @@ +/* ---------- Joint code-length / flat-shape optimization ---------- + * + * pivco_build_table derives code lengths that minimize + * compressed bits. This pass additionally bends them -- at an + * explicitly priced, guard-bounded cost in bits -- so the per-length + * class counts land on round binary numbers and the OPTIMIZED chunk + * decomposition in build_table_finish yields fewer, larger flat + * subtrees and fewer merge passes. Encoder side only: the wire + * carries plain code lengths, so ANY decoder reads the output and + * both sides rebuild identical tables. + * + * Chunk model: choosing lengths IS choosing at most one chunk per + * (level L <= PIVCO_MAX_CODE_LEN, flat depth b <= min(8, L)) -- a + * chunk holds 2^b symbols at length L inside a depth-b flat, so each + * of its symbols' occurrences costs L bits and L - b merge passes. + * Objective: + * J = sum_s n_s * (L_s + lambda*(L_s - b_s + kappa[b_s])) + * + lambda * gamma * blocks * records + * subject to chunk-root Kraft equality. For a fixed chunk multiset + * the optimal symbol assignment deals freq-sorted symbols into + * cost-sorted chunks (rearrangement inequality), which turns the + * solve into a DP over (symbols placed, open slots); lambda = 0 + * degenerates to the Huffman baseline, so the result can only improve + * in-model, and a kind-aware time model guards against out-of-model + * regressions. When the slot DP's validity condition fails (lambda > + * 1/7 under zero kappa), the baseline is kept -- the same contract as + * a guard reject. + * + * The effort modes (pivco_effort_t) pick the solve tier: BALANCED + * runs a coarse grouped solve, FASTER_DECOMPRESS the auto tier, + * FASTEST_DECOMPRESS the exact DP. Grouping by g = 2^G solves the + * identical problem G levels shallower (a group of g freq-sorted + * symbols at real level L is a depth-G flat) at 4^G fewer states; + * near-optimal solutions are dense enough that g = 2 loses ~0.13% of + * J on average, g = 4 ~0.25% (measured on LZ-literal data), and the + * guard still rejects any bad case. + */ + +#include "pivco_huffman.h" +#include +#include +#include + +/* The guard's class-count bins index `length & 15`. */ +_Static_assert(PIVCO_MAX_CODE_LEN <= 15, + "joint pass assumes code lengths fit 4 bits"); + +/* Flat-depth cap: a depth-b flat holds 2^b symbols and the per-table + * flat_code_to_sym pool is PIVCO_MAX_SYMBOLS entries, so b <= 8. */ +#define JL_MAX_FLAT 8 + +/* Upper bound on the chunks of one Kraft-complete decomposition (per + * length, one chunk per set bit of the class count). The proven max + * at max_len 11 is 33; 64 covers any PIVCO_MAX_CODE_LEN up to 15. */ +#define JL_MAX_CHUNKS 64 + +/* Max compact DP row width: j <= 128 at sigma = 256, padded to x4. */ +#define JL_WMAX 132 + +/* ---------- SIMD selection ---------- + * + * The DP row sweeps have NEON and SSE4.1 register-resident fast + * paths; every other build -- including the x86 SSE2 floor -- takes + * the generic sweep's scalar tail, which is the complete algorithm. + * PIVCO_JOINT_SCALAR forces the scalar path for A/B and debugging. + * Candidate costs can differ between the FMA and non-FMA forms in + * the last ulp, which only ever flips ties between equal-cost + * shapes. */ +#if !defined(PIVCO_JOINT_SCALAR) && defined(__ARM_NEON) +#include +#define JL_SIMD_NEON 1 +#define JL_REGROWS 1 +#elif !defined(PIVCO_JOINT_SCALAR) && defined(__SSE4_1__) +#include +#define JL_SIMD_X86 1 +#define JL_REGROWS 1 +#else +#define JL_REGROWS 0 +#endif + +#if defined(__clang__) +#define JL_UNROLL _Pragma("clang loop unroll(full)") +#else +#define JL_UNROLL +#endif + +#if defined(JL_SIMD_X86) +/* f32x4 / u16x4 idioms for the SSE sweeps (u16x4 rides the low half + * of an __m128i; blends see garbage in the high half, stores are + * 8-byte). FMA when the compiler has it (-mfma); mul+add otherwise. */ +#if defined(__FMA__) +#define JL_FMLA(acc, x, y) _mm_fmadd_ps((x), (y), (acc)) +#else +#define JL_FMLA(acc, x, y) _mm_add_ps((acc), _mm_mul_ps((x), (y))) +#endif +/* vextq_f32(a, b, N) / vext_u16(a, b, N) twins */ +#define JL_EXTQ_F32(a, b, N) _mm_castsi128_ps(_mm_alignr_epi8( \ + _mm_castps_si128(b), _mm_castps_si128(a), 4 * (N))) +#define JL_EXT_U16(a, b, N) \ + _mm_srli_si128(_mm_unpacklo_epi64((a), (b)), 2 * (N)) +/* narrow a 4 x u32 compare mask to 4 x u16 (vmovn_u32 twin) */ +#define JL_MOVN32(m) \ + _mm_packs_epi32(_mm_castps_si128(m), _mm_castps_si128(m)) +#endif + +/* ---------- effort knob: pivco_cfg_t.effort, passed down from + * pivco_build_table per call. ---------- */ + +/* ---------- model knobs ---------- + * + * Internal for now; the effort modes only vary `gran`. gamma and + * kappa are decode-cost model constants in merge element-pass units: + * gamma prices one schedule record per PIVCO_BLOCK_SIZE-symbol block + * (dispatch + wire header), kappa[b] prices one symbol of a depth-b + * flat kernel (all-zero models the kernels free), mu_cst prices a + * lone-leaf merge relative to a full partition. The defaults were + * tuned on windowed LZ-literal workloads (Apple M-class); + * re-measuring them against this decoder's kernels is future tuning, + * not correctness -- the adoption guard prices both sides with the + * same model. */ +typedef struct { + double lambda; /* bits one merge pass is worth */ + int gran; /* solve tier: 0 auto (exact DP to 64 symbols, + then grouped), 1 exact DP, 2/4/8 fixed + grouping, -1 coarse auto (one grouping step + chunkier: most of auto's decode win at a + fraction of its solve cost) */ + double guard_bits; /* adopt only if bits <= guard_bits * baseline */ + double guard_time; /* ... and time <= guard_time * baseline */ + double gamma; /* decode cost per schedule record per block */ + double kappa[JL_MAX_FLAT + 1]; /* flat-kernel cost/symbol at depth b */ + double mu_cst; /* lone-leaf merge cost relative to a full merge */ + /* FSE decode tax (active only while the encoder's FSE dispatch is + * enabled): a merge whose bitmap the per-node FSE coder commits + * decodes ~fse_tau full-merge passes SLOWER per element (measured + * ~3-5x a raw merge on Apple M-class). The guard's commit + * predictor mirrors the coder's bytes-shrink rule to first order: + * commit iff the node sees >= fse_wmin elements per block and its + * bitmap skew clears the coder's efficiency fse_eta plus marker + * overhead, 1 - H2(q) > (1 - fse_eta) + 16/W_block. Without this + * term the pass concentrates routing into fewer, more skewed + * bitmaps, FSE commits on them for a real ratio win, and the + * decode tax swamps the merge-pass savings (bell_s10 was shipping + * -24% under PHA without this term). */ + double fse_tau; /* extra full-merge passes per element; 0 off */ + int fse_enabled; /* from build cfg (was the process global) */ + double fse_eta; /* modeled FSE efficiency threshold */ + double fse_wmin; /* min elements/block for a commit attempt */ +} joint_params_t; + +static const joint_params_t joint_defaults = { + 0.1, /* lambda */ + 0, /* gran (set per effort mode) */ + 1.015, /* guard_bits */ + 0.90, /* guard_time */ + 170.0, /* gamma */ + {0}, /* kappa */ + 1.0, /* mu_cst */ + 4.0, /* fse_tau (measured Apple M-class) */ + 1, /* fse_enabled (overwritten per call) */ + 0.85, /* fse_eta */ + 64.0, /* fse_wmin */ +}; + +/* ---------- kind-aware decode-time model (the adoption guard) ---------- + * + * The per-occurrence model above prices every merge alike, but the + * decoder's merges differ by KIND: a merge with a lone-leaf child uses + * the cheap cst kernels, a merge of two internal streams pays the full + * partition. Tree arrangement is deterministic from the chunk + * multiset (build_table_finish depth-sorts chunk roots and assigns + * canonical prefixes), so we simulate the skeleton exactly and price + * each node by kind. Used on BOTH sides of the guard's comparison; + * the DP keeps its separable search cost (the guard is where + * mispricing must not survive). */ + +/* One realized chunk: depth = tree depth of the chunk root, bit = flat + * depth b (2^bit symbols), weight = total occurrence count -- the same + * (depth, bit) naming as build_table_finish's chunk_t. */ +typedef struct { uint8_t depth, bit; double weight; } jl_chunk_t; + +/* Subtree at depth d spanning chunks ch[*i..): consumes them, returns + * the subtree's decode-time units and its weight; *kind reports what + * the parent sees (0 = lone leaf, 1 = internal). + * + * Iterative, explicit frame stack: phase 0 frames are waiting on their + * left child, phase 1 on their right. */ +static double sim_subtree_time(const jl_chunk_t *ch, int n, int *i, int d, + const joint_params_t *jp, const double *kap, + double scale, int fse_on, + int *recs, double *weight_out, int *kind) +{ + struct { + double tl, wl; + int kl; + uint8_t d, phase; + } stk[PIVCO_MAX_CODE_LEN + 2]; + int sp = 0; + double rt, rw; + int rkind; + +enter: + if (d > PIVCO_MAX_CODE_LEN) { /* non-tiling multiset: cut the walk; + * the caller's i != n check reports + * failure. Unreachable from the + * in-file callers (their multisets + * are Kraft-exact by construction) + * -- pure stack-safety. */ + rt = 0.0; rw = 0; rkind = 1; + goto unwind; + } + if (*i < n && ch[*i].depth == d) { + const jl_chunk_t *c = &ch[(*i)++]; + rw = c->weight; + if (c->bit == 0) { rkind = 0; rt = 0.0; goto unwind; } + rkind = 1; + (*recs)++; /* pair/flat record */ + rt = c->weight * kap[c->bit]; + goto unwind; + } + stk[sp].d = (uint8_t)d; + stk[sp].phase = 0; + sp++; + (*recs)++; /* merge record */ + d++; + goto enter; + +unwind: + if (sp == 0) { + *weight_out = rw; + *kind = rkind; + return rt; + } + if (stk[sp - 1].phase == 0) { /* left child done */ + stk[sp - 1].tl = rt; + stk[sp - 1].wl = rw; + stk[sp - 1].kl = rkind; + stk[sp - 1].phase = 1; + d = stk[sp - 1].d + 1; + goto enter; /* right child */ + } + { /* right child done */ + const double wl = stk[sp - 1].wl, wr = rw; + const double tl = stk[sp - 1].tl, tr = rt; + const int kl = stk[sp - 1].kl, kr = rkind; + const double w = wl + wr; + double t; + if (kl == 0 || kr == 0) + t = w * jp->mu_cst; /* one lone leaf: cst merge */ + else + t = w; /* full partition */ + /* FSE decode tax on predicted-committed bitmaps (see the + * fse_tau field doc): commit iff the merge sees enough + * elements per block and its left/right split is skewed + * enough that the modeled FSE saving clears the marker. */ + if (fse_on && w > 0) { + const double wb = w * scale; /* elements per block */ + if (wb >= jp->fse_wmin) { + const double q = wl / w; + if (q > 0 && q < 1) { + const double h2 = -(q * log2(q) + + (1 - q) * log2(1 - q)); + if (1.0 - h2 > (1.0 - jp->fse_eta) + 16.0 / wb) + t += jp->fse_tau * w; + } + } + } + rt = t + tl + tr; + rw = w; + rkind = 1; + sp--; + goto unwind; + } +} + +/* Kind-aware decode time for a chunk list (any order; sorted here into + * the order build_table_finish realizes: depth asc, bit asc. The + * builder stable-sorts its L-ascending generation by depth only, and + * equal-depth chunks from lower classes have smaller bit, so + * depth-then-bit ascending IS that order -- and since a class emits + * each bit at most once, (depth, bit) is unique and the sort is + * total). */ +static double chunk_list_time(jl_chunk_t *ch, int n, + const joint_params_t *jp, const double *kap, + double total_weight) +{ + for (int i = 1; i < n; i++) { + jl_chunk_t c = ch[i]; + int j = i - 1; + while (j >= 0 && (ch[j].depth > c.depth || + (ch[j].depth == c.depth && ch[j].bit > c.bit))) { + ch[j + 1] = ch[j]; + j--; + } + ch[j + 1] = c; + } + /* The FSE tax needs each merge's per-block element count; weights + * scale to one PIVCO_BLOCK_SIZE-symbol block. Resolve the FSE + * toggle once per pricing (both guard sides see the same value). */ + const double scale = total_weight > 0 + ? (double)PIVCO_BLOCK_SIZE / total_weight : 0.0; + const int fse_on = jp->fse_tau > 0 && jp->fse_enabled; + int i = 0, kind, recs = 0; + double w; + double t = sim_subtree_time(ch, n, &i, 0, jp, kap, scale, fse_on, + &recs, &w, &kind); + if (i != n) return -1.0; /* malformed multiset (cannot happen) */ + if (jp->gamma > 0) { /* per-record fixed cost x blocks */ + double blocks = ceil(total_weight / (double)PIVCO_BLOCK_SIZE); + if (blocks < 1) blocks = 1; + t += jp->gamma * (double)recs * blocks; + } + return t; +} + +/* ---------- slot-ledger DP (exact for lambda <= 1/7) ---------- + * + * A state is (k symbols placed, s open slots at the current level); + * Kraft EQUALITY forces s <= sigma - k at every level. Levels are + * processed ascending, chunk types within a level in cost order; that + * equals GLOBAL chunk-cost order -- the sorted-matching exactness + * requirement -- iff dp_take_order's spread bound holds (kappa = 0 + * recovers the classic lambda <= 1/7). Three structural facts keep + * the walk small and L1-resident: + * + * DIAGONALS. A take (k, s) -> (k + 2^b, s - 2^b) preserves t = k + s, + * so within a level the DP decomposes into independent diagonals. + * Stored diagonal-major, all take sweeps of a level run over one short + * row; the plane is traversed once per level (the doubling). + * + * PARITY. Level-entry states have even s (they come from the doubling + * s' = 2s) and takes with b >= 1 preserve s-parity, so the live + * lattice is k == t (mod 2): compact index j = (k - (t&1))/2 halves + * each row. b = 0 -- the only parity flip, always last in the level's + * cost order -- is folded into the doubling (an odd-s cell's unique + * source is its even-lattice predecessor plus one lone leaf) and + * reconstructed from s-parity at backtrack. The deepest level never + * takes b = 0: entry s is even and the terminal needs takes summing to + * s exactly. + * + * CAPACITY BAND. A state at level L can place at most s * 2^h more + * symbols (h = levels below), so sigma - k <= (t - k) << h is + * necessary -- and met by every completing trajectory, making the + * prune exact. Feasibility is preserved cell-to-cell by takes and by + * the doubling, so pruned -- hence stale -- cells are never read. + * + * Terminal: (k = sigma, s = 0) after the deepest level. Per-level u16 + * pick rows (bits 1..8; bit 0 is implicit in parity) are archived per + * diagonal for backtrack. */ + +/* Largest compact index j on diagonal t whose k = 2j + (t&1) can still + * feed sigma - k leaves through (t - k) slots h levels above the + * bottom; -1 if the whole row is infeasible. */ +static int dp_row_cap(int t, int h, int sigma) +{ + const int p = t & 1; + int kcap; + if (h == 0) { + kcap = t; /* t == sigma: all k feasible */ + } else { + const int num = (t << h) - sigma; + if (num < 0) return -1; + kcap = num / ((1 << h) - 1); + if (kcap > t) kcap = t; + } + if (kcap < p) return -1; + return (kcap - p) >> 1; +} + +/* Within-level sweep/deal order under kernel costs. cost(L, b) = + * L(1+lam) + g(b) with g(b) = lam*(kap[b] - b): the within-level cost + * order is L-independent, so one sorted order serves every level. + * Exactness of the slot DP needs (a) cross-level monotonicity: + * spread(g) <= 1 + lam (kappa = 0 recovers lam <= 1/7), and (b) b = 0 + * dearest within the level (the parity fold runs it last). + * + * Checking the spread over ALL b <= bcap is one notch conservative: a + * chunk with 2^b >= sigma holds the whole alphabet, so chunk-root + * Kraft equality makes it the unique chunk of its solution (root + * depth 0, the full-flat code) -- it co-occurs with nothing and its + * one multiset is priced order-free, so only b with 2^b < sigma need + * the bound. At kappa = 0 that proves lam <= 1/6 for sigma >= 129 + * (1/5 at sigma <= 128, 1/4 at <= 64, ...); relax here -- keeping + * every b takeable, only the spread restricted -- when lambda tuning + * wants the headroom. + * + * Fills border[0..*nb) with b = 1..bcap by ascending g; returns 1 iff + * both conditions hold (on 0 the caller keeps the baseline). */ +static int dp_take_order(double lam, const double *kap, int bcap, + int border[JL_MAX_FLAT], int *nb) +{ + double g[JL_MAX_FLAT + 1]; + double gmin = 0, gmax = 0; + for (int b = 0; b <= bcap; b++) { + g[b] = lam * (kap[b] - (double)b); + if (b == 0 || g[b] < gmin) gmin = g[b]; + if (b == 0 || g[b] > gmax) gmax = g[b]; + } + if (gmax - gmin > (1.0 + lam) * (1.0 - 1e-9)) return 0; + int n = 0; + for (int b = 1; b <= bcap; b++) { + if (g[b] > g[0] + 1e-12) return 0; /* b0 must stay dearest */ + int i = n++; + while (i > 0 && (g[border[i - 1]] > g[b] + || (g[border[i - 1]] == g[b] && border[i - 1] < b))) { + border[i] = border[i - 1]; + i--; + } + border[i] = b; /* ties: larger b first */ + } + *nb = n; + return 1; +} + +/* lmax/bcap parameterize the level range and flat-depth cap so the + * same solver runs the exact problem (PIVCO_MAX_CODE_LEN, 8) and the + * 2^G-grouped coarse problem (PIVCO_MAX_CODE_LEN - G, 8 - G): a group + * of 2^G sorted symbols at real level L is a depth-G flat, so the + * coarse problem is this problem shifted by G with an identical cost + * form. tc0/tc1: per-take J constants (lambda * gamma * blocks * + * records added) for b = 0 and b >= 1 takes. On success fills + * out_BL[L] with the level's takes as a b-bitmask and returns the + * optimal J; returns -1.0 on an order-condition failure or OOM. */ +static double solve_slot_dp(const double *P, int sigma, double lam, + int lmax, int bcap, const double *kap, + double tc0, double tc1, + uint16_t out_BL[PIVCO_MAX_CODE_LEN + 1]) +{ + int border[JL_MAX_FLAT], nb; + if (!dp_take_order(lam, kap, bcap, border, &nb)) + return -1.0; + /* sigma <= 32 rows fit five q-registers at a fixed W = 20, and + * sigma <= 64 rows nine at W = 36: the take sweeps then run + * register-resident per diagonal (loads/stores once per row + * instead of per item), which is where the grouped tiers' time + * lives. The register sweeps store their full fixed width, so W + * must equal it exactly on the small-sigma tiers. */ + const int sm32 = sigma <= 32; + const int sm64 = !sm32 && sigma <= 64; + const int W = sm32 ? 20 : sm64 ? 36 : (((sigma >> 1) + 2) + 3) & ~3; + const size_t plane = (size_t)(sigma + 1) * (size_t)W; + /* One f32 cost plane + lmax u16 backtrack planes (~880 KB for the + * exact solve at sigma = 256; the grouped tiers use a fraction). */ + uint8_t *buf = (uint8_t *)malloc(plane * (4 + 2 * (size_t)lmax)); + if (!buf) return -1.0; + float *cost = (float *)buf; + uint16_t *arch = (uint16_t *)(cost + plane); + float dPt[2][JL_MAX_FLAT + 1][JL_WMAX]; /* [t&1][b][j]: P[k+2^b]-P[k] */ + float dP0[PIVCO_MAX_SYMBOLS + 1]; /* P[k] - P[k-1] */ + + for (int p = 0; p < 2; p++) + for (int b = 1; b <= bcap; b++) { + const int cnk = 1 << b; + for (int j = 0; j < W; j++) { + const int k = 2 * j + p; + dPt[p][b][j] = k + cnk <= sigma + ? (float)(P[k + cnk] - P[k]) : 0.0f; + } + } + dP0[0] = 0.0f; + for (int k = 1; k <= sigma; k++) dP0[k] = (float)(P[k] - P[k - 1]); + + /* Per-level diagonal band: t <= min(2^L, sigma) states exist, and + * the capacity band needs t >= ceil(sigma / 2^h). */ + int tlo[PIVCO_MAX_CODE_LEN + 1], thi[PIVCO_MAX_CODE_LEN + 1]; + for (int L = 1; L <= lmax; L++) { + const int h = lmax - L; + thi[L] = (1 << L) > sigma ? sigma : (1 << L); + tlo[L] = (sigma + (1 << h) - 1) >> h; + if (tlo[L] < 1) tlo[L] = 1; + } + + /* Lazy init: the doubling that produces a level writes each row up + * to its cap, so only the level-1 band rows need priming. Lanes + * beyond a row's cap stay indeterminate and ARE loaded by the + * fixed-width register sweeps; dest > source keeps that junk out + * of every in-band cell (see the sweep comments) -- memory-safe + * and result-clean, but MSan users should define + * PIVCO_JOINT_SCALAR or pre-zero the buffer. */ + for (int t = tlo[1]; t <= thi[1]; t++) + for (int j = 0; j < W; j++) cost[(size_t)t * W + j] = INFINITY; + cost[2 * W + 0] = 0.0f; /* level-1 entry: k = 0, s = 2, t = 2 */ + + for (int L = 1; L <= lmax; L++) { + const int h = lmax - L; + const int bmax = L < bcap ? L : bcap; + uint16_t *archL = arch + (size_t)(L - 1) * plane; + for (int t = tlo[L]; t <= thi[L]; t++) { + const int p = t & 1; + const int jcap = dp_row_cap(t, h, sigma); + if (jcap < 0) continue; + float *row = cost + (size_t)t * W; + uint16_t *prow = archL + (size_t)t * W; /* picks, archived + * in place */ +#if JL_REGROWS + if (jcap >= 20 && jcap < 36) { + /* Nine-group register-resident sweep for rows whose + * dests all fit lanes 0..35: every wide sm64 row (the + * grouped tiers' bulk) and the mid-band rows of + * full-width exact solves. Same junk-propagation + * safety as below: dest > source always, so beyond-cap + * lanes never contaminate the band. Candidates are + * computed on the fly per dest group, descending, so + * sources are pre-item. Narrower rows fall through to + * the five-group body -- it only touches lanes 0..19, + * which cover every dest, and processing 9 groups for + * a 2-group band costs more than it saves. */ +#if defined(JL_SIMD_NEON) + float32x4_t r[9]; + uint16x4_t pk[9]; + const float32x4_t vinf = vdupq_n_f32(INFINITY); + const uint16x4_t z16 = vdup_n_u16(0); + JL_UNROLL + for (int g = 0; g < 9; g++) { + r[g] = vld1q_f32(row + 4 * g); + pk[g] = vdup_n_u16(0); + } + for (int oi = 0; oi < nb; oi++) { + const int b = border[oi]; + if (b > bmax) continue; + const int jstep = 1 << (b - 1); + if (jcap - jstep < 0) continue; + const float a = (float)((double)L + + lam * ((double)(L - b) + kap[b])); + const float32x4_t va = vdupq_n_f32(a); + const float32x4_t vtc = vdupq_n_f32((float)tc1); + const float *dpb = dPt[p][b]; + const uint16x4_t vbit = vdup_n_u16((uint16_t)(1u << b)); +#define JL9_CAND(g) \ + vaddq_f32(vfmaq_f32(r[g], vld1q_f32(dpb + 4 * (g)), va), vtc) +#define JL9_TAKE(g, s, kq) do { \ + const uint32x4_t m_ = vcltq_f32((s), r[g]); \ + r[g] = vbslq_f32(m_, (s), r[g]); \ + pk[g] = vbsl_u16(vmovn_u32(m_), (kq), pk[g]); } while (0) +#define JL9_SHIFTK(K) do { \ + JL_UNROLL \ + for (int g = 8; g >= (K); g--) { \ + const float32x4_t c_ = JL9_CAND(g - (K)); \ + const uint16x4_t kq_ = vorr_u16(pk[g - (K)], vbit); \ + JL9_TAKE(g, c_, kq_); \ + } } while (0) +#define JL9_EXT(N) do { \ + float32x4_t chi_ = JL9_CAND(8); \ + uint16x4_t khi_ = vorr_u16(pk[8], vbit); \ + JL_UNROLL \ + for (int g = 8; g >= 1; g--) { \ + const float32x4_t clo_ = JL9_CAND(g - 1); \ + const uint16x4_t klo_ = vorr_u16(pk[g - 1], vbit); \ + JL9_TAKE(g, vextq_f32(clo_, chi_, N), \ + vext_u16(klo_, khi_, N)); \ + chi_ = clo_; khi_ = klo_; \ + } \ + JL9_TAKE(0, vextq_f32(vinf, chi_, N), \ + vext_u16(z16, khi_, N)); } while (0) + switch (jstep) { + case 1: JL9_EXT(3); break; + case 2: JL9_EXT(2); break; + case 4: JL9_SHIFTK(1); break; + case 8: JL9_SHIFTK(2); break; + case 16: JL9_SHIFTK(4); break; + default: JL9_SHIFTK(8); break; /* 32 */ + } +#undef JL9_CAND +#undef JL9_TAKE +#undef JL9_SHIFTK +#undef JL9_EXT + } + JL_UNROLL + for (int g = 0; g < 9; g++) { + vst1q_f32(row + 4 * g, r[g]); + vst1_u16(prow + 4 * g, pk[g]); + } +#elif defined(JL_SIMD_X86) + __m128 r[9]; + __m128i pk[9]; + const __m128 vinf = _mm_set1_ps(INFINITY); + const __m128i z16 = _mm_setzero_si128(); + JL_UNROLL + for (int g = 0; g < 9; g++) { + r[g] = _mm_loadu_ps(row + 4 * g); + pk[g] = _mm_setzero_si128(); + } + for (int oi = 0; oi < nb; oi++) { + const int b = border[oi]; + if (b > bmax) continue; + const int jstep = 1 << (b - 1); + if (jcap - jstep < 0) continue; + const float a = (float)((double)L + + lam * ((double)(L - b) + kap[b])); + const __m128 va = _mm_set1_ps(a); + const __m128 vtc = _mm_set1_ps((float)tc1); + const float *dpb = dPt[p][b]; + const __m128i vbit = _mm_set1_epi16((short)(1u << b)); +#define JL9_CAND(g) \ + _mm_add_ps(JL_FMLA(r[g], _mm_loadu_ps(dpb + 4 * (g)), va), vtc) +#define JL9_TAKE(g, s, kq) do { \ + const __m128 m_ = _mm_cmplt_ps((s), r[g]); \ + r[g] = _mm_blendv_ps(r[g], (s), m_); \ + pk[g] = _mm_blendv_epi8(pk[g], (kq), JL_MOVN32(m_)); } while (0) +#define JL9_SHIFTK(K) do { \ + JL_UNROLL \ + for (int g = 8; g >= (K); g--) { \ + const __m128 c_ = JL9_CAND(g - (K)); \ + const __m128i kq_ = _mm_or_si128(pk[g - (K)], vbit); \ + JL9_TAKE(g, c_, kq_); \ + } } while (0) +#define JL9_EXT(N) do { \ + __m128 chi_ = JL9_CAND(8); \ + __m128i khi_ = _mm_or_si128(pk[8], vbit); \ + JL_UNROLL \ + for (int g = 8; g >= 1; g--) { \ + const __m128 clo_ = JL9_CAND(g - 1); \ + const __m128i klo_ = _mm_or_si128(pk[g - 1], vbit); \ + JL9_TAKE(g, JL_EXTQ_F32(clo_, chi_, N), \ + JL_EXT_U16(klo_, khi_, N)); \ + chi_ = clo_; khi_ = klo_; \ + } \ + JL9_TAKE(0, JL_EXTQ_F32(vinf, chi_, N), \ + JL_EXT_U16(z16, khi_, N)); } while (0) + switch (jstep) { + case 1: JL9_EXT(3); break; + case 2: JL9_EXT(2); break; + case 4: JL9_SHIFTK(1); break; + case 8: JL9_SHIFTK(2); break; + case 16: JL9_SHIFTK(4); break; + default: JL9_SHIFTK(8); break; /* 32 */ + } +#undef JL9_CAND +#undef JL9_TAKE +#undef JL9_SHIFTK +#undef JL9_EXT + } + JL_UNROLL + for (int g = 0; g < 9; g++) { + _mm_storeu_ps(row + 4 * g, r[g]); + _mm_storel_epi64((__m128i *)(prow + 4 * g), pk[g]); + } +#endif /* backend nine-group sweep */ + continue; + } + if (sm32 || sm64 || jcap < 20) { + /* Whole row in five registers across every item -- used + * whenever every dest fits lanes 0..19: all of + * sigma <= 32, narrow sm64 rows, and the narrow-band + * rows of full-width exact solves (deep levels, band + * edges); wider lanes are simply left untouched. No + * lane masking: a candidate's dest is always above its + * source, so lanes beyond the cap only ever contaminate + * lanes beyond the cap, and nothing in band ever reads + * them (same argument the in-place generic sweep + * relies on). Shift-ins at the low edge are +inf. */ +#if defined(JL_SIMD_NEON) + float32x4_t r0 = vld1q_f32(row), r1 = vld1q_f32(row + 4), + r2 = vld1q_f32(row + 8), r3 = vld1q_f32(row + 12), + r4 = vld1q_f32(row + 16); + uint16x4_t p0 = vdup_n_u16(0), p1 = p0, p2 = p0, p3 = p0, + p4 = p0; + const float32x4_t vinf = vdupq_n_f32(INFINITY); + const uint16x4_t z16 = vdup_n_u16(0); + for (int oi = 0; oi < nb; oi++) { + const int b = border[oi]; + if (b > bmax) continue; + const int jstep = 1 << (b - 1); + if (jcap - jstep < 0) continue; + const float a = (float)((double)L + + lam * ((double)(L - b) + kap[b])); + const float32x4_t va = vdupq_n_f32(a); + const float32x4_t vtc = vdupq_n_f32((float)tc1); + const float *dpb = dPt[p][b]; + float32x4_t c0 = vaddq_f32(vfmaq_f32(r0, vld1q_f32(dpb), va), vtc); + float32x4_t c1 = vaddq_f32(vfmaq_f32(r1, vld1q_f32(dpb + 4), va), vtc); + float32x4_t c2 = vaddq_f32(vfmaq_f32(r2, vld1q_f32(dpb + 8), va), vtc); + float32x4_t c3 = vaddq_f32(vfmaq_f32(r3, vld1q_f32(dpb + 12), va), vtc); + float32x4_t c4 = vaddq_f32(vfmaq_f32(r4, vld1q_f32(dpb + 16), va), vtc); + const uint16x4_t vbit = vdup_n_u16((uint16_t)(1u << b)); + uint16x4_t q0 = vorr_u16(p0, vbit), q1 = vorr_u16(p1, vbit), + q2 = vorr_u16(p2, vbit), q3 = vorr_u16(p3, vbit), + q4 = vorr_u16(p4, vbit); + float32x4_t s0, s1, s2, s3, s4; + uint16x4_t k0, k1, k2, k3, k4; + switch (jstep) { + case 1: + s0 = vextq_f32(vinf, c0, 3); s1 = vextq_f32(c0, c1, 3); + s2 = vextq_f32(c1, c2, 3); s3 = vextq_f32(c2, c3, 3); + s4 = vextq_f32(c3, c4, 3); + k0 = vext_u16(z16, q0, 3); k1 = vext_u16(q0, q1, 3); + k2 = vext_u16(q1, q2, 3); k3 = vext_u16(q2, q3, 3); + k4 = vext_u16(q3, q4, 3); + break; + case 2: + s0 = vextq_f32(vinf, c0, 2); s1 = vextq_f32(c0, c1, 2); + s2 = vextq_f32(c1, c2, 2); s3 = vextq_f32(c2, c3, 2); + s4 = vextq_f32(c3, c4, 2); + k0 = vext_u16(z16, q0, 2); k1 = vext_u16(q0, q1, 2); + k2 = vext_u16(q1, q2, 2); k3 = vext_u16(q2, q3, 2); + k4 = vext_u16(q3, q4, 2); + break; + case 4: + s0 = vinf; s1 = c0; s2 = c1; s3 = c2; s4 = c3; + k0 = z16; k1 = q0; k2 = q1; k3 = q2; k4 = q3; + break; + case 8: + s0 = vinf; s1 = vinf; s2 = c0; s3 = c1; s4 = c2; + k0 = z16; k1 = z16; k2 = q0; k3 = q1; k4 = q2; + break; + default: /* 16 */ + s0 = vinf; s1 = vinf; s2 = vinf; s3 = vinf; s4 = c0; + k0 = z16; k1 = z16; k2 = z16; k3 = z16; k4 = q0; + break; + } + uint32x4_t m; + m = vcltq_f32(s0, r0); r0 = vbslq_f32(m, s0, r0); + p0 = vbsl_u16(vmovn_u32(m), k0, p0); + m = vcltq_f32(s1, r1); r1 = vbslq_f32(m, s1, r1); + p1 = vbsl_u16(vmovn_u32(m), k1, p1); + m = vcltq_f32(s2, r2); r2 = vbslq_f32(m, s2, r2); + p2 = vbsl_u16(vmovn_u32(m), k2, p2); + m = vcltq_f32(s3, r3); r3 = vbslq_f32(m, s3, r3); + p3 = vbsl_u16(vmovn_u32(m), k3, p3); + m = vcltq_f32(s4, r4); r4 = vbslq_f32(m, s4, r4); + p4 = vbsl_u16(vmovn_u32(m), k4, p4); + } + vst1q_f32(row, r0); vst1q_f32(row + 4, r1); + vst1q_f32(row + 8, r2); vst1q_f32(row + 12, r3); + vst1q_f32(row + 16, r4); + vst1_u16(prow, p0); vst1_u16(prow + 4, p1); + vst1_u16(prow + 8, p2); vst1_u16(prow + 12, p3); + vst1_u16(prow + 16, p4); +#elif defined(JL_SIMD_X86) + __m128 r0 = _mm_loadu_ps(row), r1 = _mm_loadu_ps(row + 4), + r2 = _mm_loadu_ps(row + 8), r3 = _mm_loadu_ps(row + 12), + r4 = _mm_loadu_ps(row + 16); + __m128i p0 = _mm_setzero_si128(), p1 = p0, p2 = p0, p3 = p0, + p4 = p0; + const __m128 vinf = _mm_set1_ps(INFINITY); + const __m128i z16 = _mm_setzero_si128(); + for (int oi = 0; oi < nb; oi++) { + const int b = border[oi]; + if (b > bmax) continue; + const int jstep = 1 << (b - 1); + if (jcap - jstep < 0) continue; + const float a = (float)((double)L + + lam * ((double)(L - b) + kap[b])); + const __m128 va = _mm_set1_ps(a); + const __m128 vtc = _mm_set1_ps((float)tc1); + const float *dpb = dPt[p][b]; + __m128 c0 = _mm_add_ps(JL_FMLA(r0, _mm_loadu_ps(dpb), va), vtc); + __m128 c1 = _mm_add_ps(JL_FMLA(r1, _mm_loadu_ps(dpb + 4), va), vtc); + __m128 c2 = _mm_add_ps(JL_FMLA(r2, _mm_loadu_ps(dpb + 8), va), vtc); + __m128 c3 = _mm_add_ps(JL_FMLA(r3, _mm_loadu_ps(dpb + 12), va), vtc); + __m128 c4 = _mm_add_ps(JL_FMLA(r4, _mm_loadu_ps(dpb + 16), va), vtc); + const __m128i vbit = _mm_set1_epi16((short)(1u << b)); + __m128i q0 = _mm_or_si128(p0, vbit), q1 = _mm_or_si128(p1, vbit), + q2 = _mm_or_si128(p2, vbit), q3 = _mm_or_si128(p3, vbit), + q4 = _mm_or_si128(p4, vbit); + __m128 s0, s1, s2, s3, s4; + __m128i k0, k1, k2, k3, k4; + switch (jstep) { + case 1: + s0 = JL_EXTQ_F32(vinf, c0, 3); s1 = JL_EXTQ_F32(c0, c1, 3); + s2 = JL_EXTQ_F32(c1, c2, 3); s3 = JL_EXTQ_F32(c2, c3, 3); + s4 = JL_EXTQ_F32(c3, c4, 3); + k0 = JL_EXT_U16(z16, q0, 3); k1 = JL_EXT_U16(q0, q1, 3); + k2 = JL_EXT_U16(q1, q2, 3); k3 = JL_EXT_U16(q2, q3, 3); + k4 = JL_EXT_U16(q3, q4, 3); + break; + case 2: + s0 = JL_EXTQ_F32(vinf, c0, 2); s1 = JL_EXTQ_F32(c0, c1, 2); + s2 = JL_EXTQ_F32(c1, c2, 2); s3 = JL_EXTQ_F32(c2, c3, 2); + s4 = JL_EXTQ_F32(c3, c4, 2); + k0 = JL_EXT_U16(z16, q0, 2); k1 = JL_EXT_U16(q0, q1, 2); + k2 = JL_EXT_U16(q1, q2, 2); k3 = JL_EXT_U16(q2, q3, 2); + k4 = JL_EXT_U16(q3, q4, 2); + break; + case 4: + s0 = vinf; s1 = c0; s2 = c1; s3 = c2; s4 = c3; + k0 = z16; k1 = q0; k2 = q1; k3 = q2; k4 = q3; + break; + case 8: + s0 = vinf; s1 = vinf; s2 = c0; s3 = c1; s4 = c2; + k0 = z16; k1 = z16; k2 = q0; k3 = q1; k4 = q2; + break; + default: /* 16 */ + s0 = vinf; s1 = vinf; s2 = vinf; s3 = vinf; s4 = c0; + k0 = z16; k1 = z16; k2 = z16; k3 = z16; k4 = q0; + break; + } + __m128 m; + m = _mm_cmplt_ps(s0, r0); r0 = _mm_blendv_ps(r0, s0, m); + p0 = _mm_blendv_epi8(p0, k0, JL_MOVN32(m)); + m = _mm_cmplt_ps(s1, r1); r1 = _mm_blendv_ps(r1, s1, m); + p1 = _mm_blendv_epi8(p1, k1, JL_MOVN32(m)); + m = _mm_cmplt_ps(s2, r2); r2 = _mm_blendv_ps(r2, s2, m); + p2 = _mm_blendv_epi8(p2, k2, JL_MOVN32(m)); + m = _mm_cmplt_ps(s3, r3); r3 = _mm_blendv_ps(r3, s3, m); + p3 = _mm_blendv_epi8(p3, k3, JL_MOVN32(m)); + m = _mm_cmplt_ps(s4, r4); r4 = _mm_blendv_ps(r4, s4, m); + p4 = _mm_blendv_epi8(p4, k4, JL_MOVN32(m)); + } + _mm_storeu_ps(row, r0); _mm_storeu_ps(row + 4, r1); + _mm_storeu_ps(row + 8, r2); _mm_storeu_ps(row + 12, r3); + _mm_storeu_ps(row + 16, r4); + _mm_storel_epi64((__m128i *)prow, p0); + _mm_storel_epi64((__m128i *)(prow + 4), p1); + _mm_storel_epi64((__m128i *)(prow + 8), p2); + _mm_storel_epi64((__m128i *)(prow + 12), p3); + _mm_storel_epi64((__m128i *)(prow + 16), p4); +#endif /* backend five-group sweep */ + continue; + } +#endif /* JL_REGROWS */ + memset(prow, 0, (size_t)(jcap + 1) * sizeof(uint16_t)); + for (int oi = 0; oi < nb; oi++) { + const int b = border[oi]; + if (b > bmax) continue; + const int jstep = 1 << (b - 1); /* = 2^b slots / 2 */ + const int jhi = jcap - jstep; /* dest j <= jcap */ + if (jhi < 0) continue; + const float a = (float)((double)L + + lam * ((double)(L - b) + kap[b])); + const float tc = (float)tc1; + const float *dpb = dPt[p][b]; + int j = jhi; + /* 0/1 in-place: dest j + jstep > src j, so iterate j + * descending -- a written dest is never re-read as a + * source for the same chunk type. Stores are + * unconditional: everything is L1-resident, so blending + * beats the data-dependent branch of an "improved?" + * early-out. */ +#if defined(JL_SIMD_NEON) + const float32x4_t va = vdupq_n_f32(a); + const float32x4_t vtc = vdupq_n_f32(tc); + const uint16x4_t vbit = vdup_n_u16((uint16_t)(1u << b)); + for (; j >= 7; j -= 8) { + const int b1 = j - 3, b2 = j - 7; + float32x4_t s1 = vld1q_f32(row + b1); + float32x4_t s2 = vld1q_f32(row + b2); + float32x4_t c1 = vaddq_f32( + vfmaq_f32(s1, vld1q_f32(dpb + b1), va), vtc); + float32x4_t c2 = vaddq_f32( + vfmaq_f32(s2, vld1q_f32(dpb + b2), va), vtc); + float32x4_t d1 = vld1q_f32(row + b1 + jstep); + float32x4_t d2 = vld1q_f32(row + b2 + jstep); + uint32x4_t m1 = vcltq_f32(c1, d1); + uint32x4_t m2 = vcltq_f32(c2, d2); + vst1q_f32(row + b1 + jstep, vbslq_f32(m1, c1, d1)); + vst1q_f32(row + b2 + jstep, vbslq_f32(m2, c2, d2)); + uint16x4_t pv1 = vorr_u16(vld1_u16(prow + b1), vbit); + uint16x4_t pv2 = vorr_u16(vld1_u16(prow + b2), vbit); + uint16x4_t qv1 = vld1_u16(prow + b1 + jstep); + uint16x4_t qv2 = vld1_u16(prow + b2 + jstep); + vst1_u16(prow + b1 + jstep, + vbsl_u16(vmovn_u32(m1), pv1, qv1)); + vst1_u16(prow + b2 + jstep, + vbsl_u16(vmovn_u32(m2), pv2, qv2)); + } + for (; j >= 3; j -= 4) { + const int base = j - 3; + float32x4_t src = vld1q_f32(row + base); + float32x4_t cand = vaddq_f32( + vfmaq_f32(src, vld1q_f32(dpb + base), va), vtc); + float32x4_t dst = vld1q_f32(row + base + jstep); + uint32x4_t m = vcltq_f32(cand, dst); + vst1q_f32(row + base + jstep, vbslq_f32(m, cand, dst)); + uint16x4_t pm = vmovn_u32(m); + uint16x4_t pv = vorr_u16(vld1_u16(prow + base), vbit); + uint16x4_t qv = vld1_u16(prow + base + jstep); + vst1_u16(prow + base + jstep, vbsl_u16(pm, pv, qv)); + } +#elif defined(JL_SIMD_X86) + const __m128 va = _mm_set1_ps(a); + const __m128 vtc = _mm_set1_ps(tc); + const __m128i vbit = _mm_set1_epi16((short)(1u << b)); + for (; j >= 7; j -= 8) { + const int b1 = j - 3, b2 = j - 7; + __m128 s1 = _mm_loadu_ps(row + b1); + __m128 s2 = _mm_loadu_ps(row + b2); + __m128 c1 = _mm_add_ps(JL_FMLA(s1, _mm_loadu_ps(dpb + b1), va), vtc); + __m128 c2 = _mm_add_ps(JL_FMLA(s2, _mm_loadu_ps(dpb + b2), va), vtc); + __m128 d1 = _mm_loadu_ps(row + b1 + jstep); + __m128 d2 = _mm_loadu_ps(row + b2 + jstep); + __m128 m1 = _mm_cmplt_ps(c1, d1); + __m128 m2 = _mm_cmplt_ps(c2, d2); + _mm_storeu_ps(row + b1 + jstep, _mm_blendv_ps(d1, c1, m1)); + _mm_storeu_ps(row + b2 + jstep, _mm_blendv_ps(d2, c2, m2)); + __m128i pv1 = _mm_or_si128( + _mm_loadl_epi64((const __m128i *)(prow + b1)), vbit); + __m128i pv2 = _mm_or_si128( + _mm_loadl_epi64((const __m128i *)(prow + b2)), vbit); + __m128i qv1 = _mm_loadl_epi64((const __m128i *)(prow + b1 + jstep)); + __m128i qv2 = _mm_loadl_epi64((const __m128i *)(prow + b2 + jstep)); + _mm_storel_epi64((__m128i *)(prow + b1 + jstep), + _mm_blendv_epi8(qv1, pv1, JL_MOVN32(m1))); + _mm_storel_epi64((__m128i *)(prow + b2 + jstep), + _mm_blendv_epi8(qv2, pv2, JL_MOVN32(m2))); + } + for (; j >= 3; j -= 4) { + const int base = j - 3; + __m128 src = _mm_loadu_ps(row + base); + __m128 cand = _mm_add_ps( + JL_FMLA(src, _mm_loadu_ps(dpb + base), va), vtc); + __m128 dst = _mm_loadu_ps(row + base + jstep); + __m128 m = _mm_cmplt_ps(cand, dst); + _mm_storeu_ps(row + base + jstep, _mm_blendv_ps(dst, cand, m)); + __m128i pv = _mm_or_si128( + _mm_loadl_epi64((const __m128i *)(prow + base)), vbit); + __m128i qv = _mm_loadl_epi64((const __m128i *)(prow + base + jstep)); + _mm_storel_epi64((__m128i *)(prow + base + jstep), + _mm_blendv_epi8(qv, pv, JL_MOVN32(m))); + } +#endif /* backend in-place sweep */ + for (; j >= 0; j--) { + const float v = row[j]; + if (!(v < INFINITY)) continue; + const float cand = v + a * dpb[j] + tc; + if (cand < row[j + jstep]) { + row[j + jstep] = cand; + prow[j + jstep] = (uint16_t)(prow[j] | (1u << b)); + } + } + } + } + if (L == lmax) break; + /* Doubling s' = 2s with the b = 0 take folded in. Dest cell + * (t', k) has the unique source (t = (t'+k)/2, k): even-lattice + * there if k == t (mod 2), else the odd-s product of a lone + * leaf taken at level L from (t, k-1). In place, t' and j' + * descending: sources live on rows <= t', and the single + * same-row read (t = t', only at k = t') happens before its + * cell is overwritten. + * + * Branchless: on dest row t' the source diagonal is t = t0 + j' + * (t0 = (t'+p')/2), so the level-L band check hoists to a + * j'-range, and the source parity d = (t^k)&1 alternates with + * j' -- two constant-stride subloops with the unified source + * index (k - d - (t&1))/2. The subloop containing the top cell + * runs first (it holds the only same-row read). */ + const float a0 = (float)((double)L * (1.0 + lam) + lam * kap[0]); + const float tcz = (float)tc0; + for (int tp = thi[L + 1]; tp >= tlo[L + 1]; tp--) { + const int pp = tp & 1; + const int jcap2 = dp_row_cap(tp, h - 1, sigma); + if (jcap2 < 0) continue; + float *nrow = cost + (size_t)tp * W; + const int t0 = (tp + pp) >> 1; + int jlo = tlo[L] - t0; if (jlo < 0) jlo = 0; + int jhi2 = thi[L] - t0; if (jhi2 > jcap2) jhi2 = jcap2; + for (int jp2 = jcap2; jp2 > jhi2; jp2--) nrow[jp2] = INFINITY; + for (int jp2 = jlo - 1; jp2 >= 0; jp2--) nrow[jp2] = INFINITY; + for (int half = 0; half < 2; half++) { + int jp2 = jhi2 - half; + if (jp2 < jlo) continue; + const int k1 = 2 * jp2 + pp; + const int t1 = t0 + jp2; + const int d = (t1 ^ k1) & 1; + /* Signed source index into cost[]: at the odd branch's + * degenerate k = 0 cell the offset (k1 - d - (t1 & 1)) + * would go negative, and casting it to size_t would + * wrap the pointer backwards (UB); keeping the whole + * index signed and indexing cost[]/dP0[] avoids forming + * any out-of-array pointer (a negative sentinel index + * is just an integer). */ + ptrdiff_t si = (ptrdiff_t)t1 * W + ((k1 - d - (t1 & 1)) >> 1); + if (d == 0) { + for (; jp2 >= jlo; jp2 -= 2, si -= 2 * W + 2) + nrow[jp2] = cost[si]; + } else { + /* k = 0 has no lone-leaf predecessor: if this + * chain reaches cell (jp2 = 0, k = 0), stop above + * it and mark it unreachable. */ + int floor2 = jlo, patch0 = 0; + if (pp == 0 && (jp2 & 1) == 0 && jlo == 0) { + floor2 = 2; + patch0 = 1; + } + ptrdiff_t di = k1; + for (; jp2 >= floor2; jp2 -= 2, si -= 2 * W + 2, di -= 4) + nrow[jp2] = cost[si] + a0 * dP0[di] + tcz; + if (patch0) + nrow[0] = INFINITY; + } + } + } + } + + double J = cost[(size_t)sigma * W + (size_t)((sigma - (sigma & 1)) >> 1)]; + if (J < INFINITY) { + /* Backtrack: invert each level's transition; odd end-of-level + * s means the folded b0 was taken there -- recover its bits + * from the even-lattice predecessor and set bit 0. */ + int k = sigma, s = 0; + for (int L = lmax; L >= 1; L--) { + const int t = k + s; + const int p = t & 1; + const uint16_t *arow = arch + (size_t)(L - 1) * plane + + (size_t)t * W; + uint16_t BL; + if ((k ^ t) & 1) + BL = (uint16_t)(arow[(k - 1 - p) >> 1] | 1u); + else + BL = arow[(k - p) >> 1]; + out_BL[L] = BL; + int cL = 0; + for (int b = 0; b <= JL_MAX_FLAT; b++) + if (BL & (1 << b)) cL += 1 << b; + k -= cL; + s += cL; /* slots at level L entry (even) */ + if (L > 1) s >>= 1; /* pre-doubling slots left */ + } + } else { + J = -1.0; + } + free(buf); + return J; +} + +/* Realized chunk lists (depth, bit, weight) for BOTH lens images -- + * the incoming baseline lb and the deal's candidate lc -- priced as + * the tables build_table_finish will build: within a class the builder + * takes symbols in ascending symbol order and splits the count + * largest-set-bit first, so chunk membership -- and with it each + * chunk's weight -- is a function of the lengths alone, NOT of the + * deal that chose them. Pricing the guard on realized weights (both + * sides) keeps it honest: the DP's sorted matching of heavy symbols to + * cheap chunks is a search relaxation the canonical rebuild does not + * reproduce. + * + * cnt* are per-class symbol counts, supplied by the caller; bins 0 and + * PIVCO_MAX_CODE_LEN+1..15 are trash (absent / garbage lengths -- + * internal lengths never exceed the cap). One fused ascending + * 256-symbol sweep deals every class's chunk cursor on both sides + * simultaneously -- ascending symbol order IS the builder's membership + * order -- with absent symbols draining into a zero-weight dummy chunk + * via the &15 trash bins, branchlessly (their weight contribution is + * 0). Frequencies narrow to u32 as in the caller's leaf collection; + * u64 accumulators keep the sums exact with integer adds. */ +static void realized_chunk_weights(const uint8_t *lb, const uint8_t *lc, + const int cntb[16], const int cntc[16], + const uint64_t freq[PIVCO_MAX_SYMBOLS], + jl_chunk_t chb[JL_MAX_CHUNKS], int *nb_out, + jl_chunk_t chc[JL_MAX_CHUNKS], int *nc_out) +{ + int curb[16], curc[16], leftb[16], leftc[16], endb[16], endc[16]; + int nb = 0, nc = 0; + for (int L = 1; L <= PIVCO_MAX_CODE_LEN; L++) { + curb[L] = nb; + curc[L] = nc; + for (int b = JL_MAX_FLAT; b >= 0; b--) { + if (cntb[L] & (1 << b)) { + chb[nb].depth = (uint8_t)(b ? L - b : L); + chb[nb].bit = (uint8_t)b; + nb++; + } + if (cntc[L] & (1 << b)) { + chc[nc].depth = (uint8_t)(b ? L - b : L); + chc[nc].bit = (uint8_t)b; + nc++; + } + } + endb[L] = nb; + endc[L] = nc; + leftb[L] = curb[L] < nb ? 1 << chb[curb[L]].bit : 1; + leftc[L] = curc[L] < nc ? 1 << chc[curc[L]].bit : 1; + } + for (int t = 0; t < 16; t++) + if (t == 0 || t > PIVCO_MAX_CODE_LEN) { + curb[t] = nb; endb[t] = nb; leftb[t] = 0x7fffffff; + curc[t] = nc; endc[t] = nc; leftc[t] = 0x7fffffff; + } + uint64_t wb[JL_MAX_CHUNKS + 1] = {0}, wc[JL_MAX_CHUNKS + 1] = {0}; + for (int s = 0; s < PIVCO_MAX_SYMBOLS; s++) { + const uint32_t f = freq[s] > UINT32_MAX + ? UINT32_MAX : (uint32_t)freq[s]; + const int Lb = lb[s] & 15, Lc = lc[s] & 15; + wb[curb[Lb]] += f; + if (--leftb[Lb] == 0 && ++curb[Lb] < endb[Lb]) + leftb[Lb] = 1 << chb[curb[Lb]].bit; + wc[curc[Lc]] += f; + if (--leftc[Lc] == 0 && ++curc[Lc] < endc[Lc]) + leftc[Lc] = 1 << chc[curc[Lc]].bit; + } + for (int i = 0; i < nb; i++) chb[i].weight = (double)wb[i]; + for (int i = 0; i < nc; i++) chc[i].weight = (double)wc[i]; + *nb_out = nb; + *nc_out = nc; +} + +/* ---------- driver ---------- */ + +typedef struct { uint32_t freq; uint16_t sym; } jl_leaf_t; + +/* Stable ascending (freq, sym) sort. Leaves arrive in symbol order + * (the stable seed), so a stable freq sort IS the (freq, sym) order. + * Small alphabets insertion-sort; larger ones take an LSD radix over + * only the frequency bytes that VARY across the set (vary = OR ^ AND + * of all freqs, a free by-product of the caller's scan) -- a constant + * byte is an identity pass, so it is skipped outright. */ +static void sort_leaves(jl_leaf_t *leaf, int n, uint32_t vary) +{ + int i, j; + if (n <= 40) { + for (i = 1; i < n; i++) { + jl_leaf_t cur = leaf[i]; + for (j = i - 1; j >= 0 && leaf[j].freq > cur.freq; j--) + leaf[j + 1] = leaf[j]; + leaf[j + 1] = cur; + } + return; + } + int shift[4], npass = 0; + for (int b = 0; b < 32; b += 8) + if ((vary >> b) & 0xFF) shift[npass++] = b; + if (npass == 0) return; /* all frequencies equal */ + /* u8 bins cannot go wrong at n <= 256: a bin could only reach 256 + * if every leaf shared that byte, but such a plane does not vary + * and is skipped, so varying-plane bins are <= 255. A prefix that + * wraps to 0 is only stored for an empty bin (never indexed), and + * the final in-scatter increment that wraps is never read again. */ + uint8_t cnt[4][256]; + memset(cnt, 0, (size_t)npass * sizeof(cnt[0])); + for (i = 0; i < n; i++) /* all planes in one pass */ + for (int p = 0; p < npass; p++) + cnt[p][(leaf[i].freq >> shift[p]) & 0xFF]++; + jl_leaf_t tmp[PIVCO_MAX_SYMBOLS], *src = leaf, *dst = tmp; + for (int p = 0; p < npass; p++) { + unsigned sum = 0; + for (int k = 0; k < 256; k++) { + unsigned c = cnt[p][k]; + cnt[p][k] = (uint8_t)sum; + sum += c; + } + for (i = 0; i < n; i++) + dst[cnt[p][(src[i].freq >> shift[p]) & 0xFF]++] = src[i]; + jl_leaf_t *t = src; src = dst; dst = t; + } + if (src != leaf) memcpy(leaf, src, (size_t)n * sizeof(*leaf)); +} + +/* Core over the ascending-sorted leaf array (reversed in place here; + * ghost-padding may append). Overwrites lengths[] on adoption; any + * reject leaves them untouched. */ +static int joint_core(jl_leaf_t *sf, int sigma, + const uint64_t freq[PIVCO_MAX_SYMBOLS], + uint8_t lengths[PIVCO_MAX_SYMBOLS], + const joint_params_t *jp) +{ + const double lam = jp->lambda; + const double *kap = jp->kappa; + if (sigma < 2) return -1; + + for (int i = 0; i < sigma / 2; i++) { /* ascending -> descending */ + jl_leaf_t tmp = sf[i]; + sf[i] = sf[sigma - 1 - i]; + sf[sigma - 1 - i] = tmp; + } + double P[PIVCO_MAX_SYMBOLS + 1]; + P[0] = 0.0; + for (int i = 0; i < sigma; i++) P[i + 1] = P[i] + (double)sf[i].freq; + + /* Baseline class counts (the guard itself is priced after the + * deal, one fused pass covering both sides; internal lengths are + * <= PIVCO_MAX_CODE_LEN so the &15 bins are exact, with 0 + * collecting absent symbols). */ + int cntb[16] = {0}; + for (int i = 0; i < sigma; i++) + cntb[lengths[sf[i].sym] & 15]++; + + /* Per-take fixed-cost constants: lambda * gamma * blocks, one + * record for b = 0 takes (the skeleton merge above the leaf), two + * for deeper chunks (the flat record + its stitch merge). */ + double blocks = ceil(P[sigma] / (double)PIVCO_BLOCK_SIZE); + if (blocks < 1) blocks = 1; + const double tc0 = lam * jp->gamma * blocks; + const double tc1 = 2.0 * tc0; + + /* Tier resolve. Granularity g = 2^G groups the freq-sorted + * symbols by g and solves the identical problem G levels shallower + * (see the header comment); sigma is ghost-padded to a multiple of + * g with zero-frequency unused byte values -- real leaves the + * encoder never emits; there are always enough since + * sigma % g != 0 implies sigma < 256. */ + int gran = jp->gran; + if (gran != -1 && gran != 1 && gran != 2 && gran != 4 && gran != 8) + gran = 0; + if (gran == 0) /* auto: keep the solve cheap at every sigma */ + gran = sigma <= 64 ? 1 : sigma <= 128 ? 2 : 4; + else if (gran == -1) /* coarse auto: one granularity step chunkier */ + gran = sigma <= 64 ? 2 : sigma <= 128 ? 4 : 8; + int obuf[JL_MAX_FLAT], on; + if (gran > 1 && (sigma < 8 * gran + || !dp_take_order(lam, + kap + (gran == 8 ? 3 : gran == 4 ? 2 : 1), + JL_MAX_FLAT - (gran == 8 ? 3 : gran == 4 ? 2 : 1), + obuf, &on))) + gran = 1; + const int glog = gran == 8 ? 3 : gran == 4 ? 2 : gran == 2 ? 1 : 0; + int sigma_pad = sigma; + if (glog) { + const int pad = (gran - (sigma % gran)) % gran; + int added = 0; + for (int s = 0; s < PIVCO_MAX_SYMBOLS && added < pad; s++) + if (!freq[s]) { + sf[sigma_pad].freq = 0; + sf[sigma_pad].sym = (uint16_t)s; + P[sigma_pad + 1] = P[sigma]; + sigma_pad++; + added++; + } + if (added < pad) return -1; /* unreachable: pad <= 256-sigma */ + } + + uint16_t BL[PIVCO_MAX_CODE_LEN + 1] = {0}; + if (glog) { + double Pg[PIVCO_MAX_SYMBOLS / 2 + 2]; + const int sp = sigma_pad / gran; + for (int i = 0; i <= sp; i++) Pg[i] = P[i * gran]; + uint16_t BLc[PIVCO_MAX_CODE_LEN + 1] = {0}; + /* kap + glog: local b' prices the real depth b' + glog; a + * grouped b' = 0 take is a real 2^glog flat, hence tc1 twice */ + if (solve_slot_dp(Pg, sp, lam, PIVCO_MAX_CODE_LEN - glog, + JL_MAX_FLAT - glog, kap + glog, tc1, tc1, BLc) < 0) + return -1; + for (int L = 1; L <= PIVCO_MAX_CODE_LEN - glog; L++) + BL[L + glog] = (uint16_t)(BLc[L] << glog); + } else if (solve_slot_dp(P, sigma, lam, PIVCO_MAX_CODE_LEN, JL_MAX_FLAT, + kap, tc0, tc1, BL) < 0) { + return -1; /* order condition failed (lambda > 1/7) or OOM */ + } + + /* Collect the chosen chunks in GLOBAL per-occurrence cost order -- + * under kappa the plain "L ascending, b descending" deal is no + * longer the cost order, and the sorted matching the solver + * assumes must be the assignment we actually realize. */ + struct { double cost; uint8_t L, b; uint16_t size; } chunks[JL_MAX_CHUNKS]; + int nchunks = 0; + for (int L = 1; L <= PIVCO_MAX_CODE_LEN; L++) + for (int b = 0; b <= JL_MAX_FLAT; b++) + if (BL[L] & (1 << b)) { + double c = (double)L + lam * ((double)(L - b) + kap[b]); + int i = nchunks++; + while (i > 0 && (chunks[i - 1].cost > c + || (chunks[i - 1].cost == c + && (chunks[i - 1].L > L + || (chunks[i - 1].L == L + && chunks[i - 1].b < b))))) { + chunks[i] = chunks[i - 1]; + i--; + } + chunks[i].cost = c; + chunks[i].L = (uint8_t)L; + chunks[i].b = (uint8_t)b; + chunks[i].size = (uint16_t)(1 << b); + } + + /* Deal freq-sorted symbols to the chunks in that same order, into + * a CANDIDATE lens image (the caller's lengths hold the baseline + * until the guard passes). Ghosts (sorted last) take the dearest + * chunks: unused byte values receive real codes the encoder never + * emits. dp_bits is exact off the deal -- bits depend only on + * per-symbol length, which the rebuild preserves. */ + uint8_t cand[PIVCO_MAX_SYMBOLS]; + double dp_bits = 0, dp_time; + memcpy(cand, lengths, PIVCO_MAX_SYMBOLS); + { + int cur = 0; + for (int i = 0; i < nchunks; i++) { + dp_bits += (P[cur + chunks[i].size] - P[cur]) * chunks[i].L; + for (int j = 0; j < chunks[i].size; j++) + cand[sf[cur++].sym] = chunks[i].L; + } + if (cur != sigma_pad) return -1; + } + /* Apply the adoption guard on the tables the decoder will actually + * build: the deal's heavy-to-cheap matching is not realizable (the + * rebuild redistributes a class's symbols over its chunks in + * symbol order), so both sides price the realized weights. Ghost + * chunks carry zero weight, so real symbols are scored exactly. */ + double base_bits = 0, base_time; + { + jl_chunk_t chb[JL_MAX_CHUNKS], chc[JL_MAX_CHUNKS]; + int cntc[16] = {0}, nb, nc; + for (int i = 0; i < nchunks; i++) + cntc[chunks[i].L] += chunks[i].size; + realized_chunk_weights(lengths, cand, cntb, cntc, freq, + chb, &nb, chc, &nc); + for (int i = 0; i < nb; i++) + base_bits += chb[i].weight * (double)(chb[i].depth + chb[i].bit); + base_time = chunk_list_time(chb, nb, jp, kap, P[sigma]); + dp_time = chunk_list_time(chc, nc, jp, kap, P[sigma]); + if (base_time < 0 || dp_time < 0) return -1; + } + if (!(dp_time <= jp->guard_time * base_time + && dp_bits <= jp->guard_bits * base_bits)) + return -1; + memcpy(lengths, cand, PIVCO_MAX_SYMBOLS); + return 0; +} + +int pivco_joint_optimize_lengths(const uint64_t freq[PIVCO_MAX_SYMBOLS], + uint8_t lengths[PIVCO_MAX_SYMBOLS], + const pivco_cfg_t *cfg) +{ + if (!cfg) cfg = &pivco_cfg_default; + const pivco_effort_t effort = cfg->effort; + if (!freq || !lengths) return -1; + if (effort == PIVCO_EFFORT_PLAIN) return -1; + /* The chunk model prices the OPTIMIZED decomposition; under the + * other (ablation) tree modes the plain Huffman lengths are kept. */ + if (cfg->tree_mode != PIVCO_TREE_MODE_OPTIMIZED) return -1; + + joint_params_t jp = joint_defaults; + jp.fse_enabled = cfg->fse_enabled; + jp.gran = effort == PIVCO_EFFORT_FASTER_DECOMPRESS ? 0 + : effort == PIVCO_EFFORT_FASTEST_DECOMPRESS ? 1 + : -1; /* BALANCED -- and FASTEST_COMPRESS reaching a bare + * build_table, where no input size is available to + * resolve it (the pivcohuf file codec resolves it + * by size before building) */ + + /* Frequencies SATURATE to u32: a symbol with >= 2^32 occurrences + * keeps maximal weight instead of wrapping toward zero (a wrap + * once demoted a dominant symbol from length 1 to 6 and adopted a + * 6x-bigger shape). Above the clamp only relative order among + * >= 4 GiB symbols is lost, which cannot change any sensible + * shape. Correctness is unaffected either way: every index below + * is bounded structurally, never by frequency values. */ + jl_leaf_t leaf[PIVCO_MAX_SYMBOLS]; + uint32_t orv = 0, andv = ~(uint32_t)0; + int n = 0; + for (int i = 0; i < PIVCO_MAX_SYMBOLS; i++) + if (freq[i]) { + uint32_t f = freq[i] > UINT32_MAX ? UINT32_MAX + : (uint32_t)freq[i]; + leaf[n].freq = f; + leaf[n].sym = (uint16_t)i; + n++; + orv |= f; + andv &= f; + } else { + /* Harden the documented precondition: freq-0 symbols carry + * no code. A stale nonzero length here once survived into + * an adopted set and broke Kraft (sum 2050/2048). */ + lengths[i] = 0; + } + if (n < 2) return -1; + sort_leaves(leaf, n, orv ^ andv); + return joint_core(leaf, n, freq, lengths, &jp); +} diff --git a/third_party/pivco/src/pivco_check.h b/third_party/pivco/src/pivco_check.h new file mode 100644 index 0000000..7a5dc04 --- /dev/null +++ b/third_party/pivco/src/pivco_check.h @@ -0,0 +1,32 @@ +#ifndef PIVCO_CHECK_H +#define PIVCO_CHECK_H + +/* ---------- Internal invariant checks ---------- + * + * Policy: a violated internal invariant CRASHES, in every build type. + * It suggests corruption that might not be recoverable, and continuing + * risks silently wrong output. Never use assert() directly. + * + * PIVCO_CHECK(cond) always on, Release included. The failure + * path is one out-of-line noreturn call, so a + * check costs a single predictable test+branch. + * PIVCO_CHECK_DEBUG(cond) compiled out under NDEBUG -- reserve for + * checks too hot to keep in Release. + */ + +__attribute__((noreturn)) +void pivco_check_fail(const char *expr, const char *file, int line); + +#define PIVCO_CHECK(cond) \ + do { \ + if (__builtin_expect(!(cond), 0)) \ + pivco_check_fail(#cond, __FILE__, __LINE__); \ + } while (0) + +#ifdef NDEBUG +#define PIVCO_CHECK_DEBUG(cond) ((void)0) +#else +#define PIVCO_CHECK_DEBUG(cond) PIVCO_CHECK(cond) +#endif + +#endif /* PIVCO_CHECK_H */ diff --git a/third_party/pivco/src/pivco_huffman.c b/third_party/pivco/src/pivco_huffman.c new file mode 100644 index 0000000..e2e1e7a --- /dev/null +++ b/third_party/pivco/src/pivco_huffman.c @@ -0,0 +1,223 @@ +#include "pivco_huffman.h" + +#include + +/* PIVCO_CHECK failure path (see pivco_check.h): print and crash -- + * internal invariants must fail loudly in every build type. */ +#include "pivco_check.h" +#include +#include +void pivco_check_fail(const char *expr, const char *file, int line) +{ + fprintf(stderr, "PIVCO_CHECK failed: %s (%s:%d)\n", expr, file, line); + fflush(NULL); + abort(); +} + +/* ---------- FSE per-table-id stats storage ---------- + * + * Backend-neutral home for the FSE-encode instrumentation counters. + * Defined here so codec.c (compiled per-backend) and any legacy + * backend-specific .c files all link against the same storage; before + * this lived in pivco_huffman_neon.c as static, which broke + * pivco_bench_fse_table_use on x86 hosts where neon.c isn't compiled. + * + * Slot 0 of `commit` counts "FSE attempted but rejected" (codeword-cost + * gate refused or the FSE library returned fallback). Slots 1..25 of + * commit/bytes_in/bytes_out are per-table-id committed FSE encodes. + * attempt[t_id] counts every call to pivco_fse_compress for table t_id + * whether or not it committed. Not thread-safe -- debug instrumentation + * only; the codec mutates these inline during encode. */ +uint64_t g_pivco_fse_commit [PIVCO_FSE_STATS_SLOTS]; +uint64_t g_pivco_fse_attempt [PIVCO_FSE_STATS_SLOTS]; +uint64_t g_pivco_fse_bytes_in [PIVCO_FSE_STATS_SLOTS]; +uint64_t g_pivco_fse_bytes_out[PIVCO_FSE_STATS_SLOTS]; + +#define PIVCO_FSE_ROOT_LOG_MAX 65536 +pivco_fse_root_event_t g_pivco_fse_root_log[PIVCO_FSE_ROOT_LOG_MAX]; +int g_pivco_fse_root_n; + +void pivco_fse_stats_reset(void) +{ + memset(g_pivco_fse_commit, 0, sizeof(g_pivco_fse_commit)); + memset(g_pivco_fse_attempt, 0, sizeof(g_pivco_fse_attempt)); + memset(g_pivco_fse_bytes_in, 0, sizeof(g_pivco_fse_bytes_in)); + memset(g_pivco_fse_bytes_out, 0, sizeof(g_pivco_fse_bytes_out)); + g_pivco_fse_root_n = 0; +} + +void pivco_fse_stats_get(uint64_t commit[PIVCO_FSE_STATS_SLOTS], + uint64_t attempt[PIVCO_FSE_STATS_SLOTS], + uint64_t bytes_in[PIVCO_FSE_STATS_SLOTS], + uint64_t bytes_out[PIVCO_FSE_STATS_SLOTS]) +{ + memcpy(commit, g_pivco_fse_commit, sizeof(g_pivco_fse_commit)); + memcpy(attempt, g_pivco_fse_attempt, sizeof(g_pivco_fse_attempt)); + memcpy(bytes_in, g_pivco_fse_bytes_in, sizeof(g_pivco_fse_bytes_in)); + memcpy(bytes_out, g_pivco_fse_bytes_out, sizeof(g_pivco_fse_bytes_out)); +} + +int pivco_fse_root_count(void) +{ + return g_pivco_fse_root_n; +} + +void pivco_fse_root_get(int idx, pivco_fse_root_event_t *out) +{ + if (idx < 0 || idx >= g_pivco_fse_root_n) { + memset(out, 0, sizeof(*out)); + return; + } + *out = g_pivco_fse_root_log[idx]; +} + +/* Compile-time backend choice for the dispatched entries: the build + * enables exactly one SIMD tier (or none), so there is nothing to + * select at runtime. */ +int pivco_encode(pivco_encoder_t *enc, const pivco_table_t *table, + const uint8_t *symbols, size_t n, + uint8_t *out, size_t *out_len) +{ + int rc; +#ifdef PIVCO_HAS_AVX512 + rc = pivco_encode_avx512(enc, table, symbols, n, out, out_len); +#elif defined(PIVCO_HAS_NEON) + rc = pivco_encode_neon(enc, table, symbols, n, out, out_len); +#elif defined(PIVCO_HAS_SSE4) + rc = pivco_encode_x86(enc, table, symbols, n, out, out_len); +#else + rc = pivco_encode_scalar(enc, table, symbols, n, out, out_len); +#endif + if (rc == PIVCO_OK) { + enc->stats.blocks++; + enc->stats.bytes_in += n; + enc->stats.bytes_out += *out_len; + } + return rc; +} + +int pivco_decode(pivco_decoder_t *dec, const pivco_table_t *table, + const uint8_t *in, size_t in_len, + uint8_t *symbols, size_t *consumed) +{ + int rc; +#ifdef PIVCO_HAS_AVX512 + rc = pivco_decode_bu_avx512(dec, table, in, in_len, symbols, consumed); +#elif defined(PIVCO_HAS_NEON) + rc = pivco_decode_bu_neon(dec, table, in, in_len, symbols, consumed); +#elif defined(PIVCO_HAS_SSE4) + rc = pivco_decode_bu_x86(dec, table, in, in_len, symbols, consumed); +#else + rc = pivco_decode_scalar(dec, table, in, in_len, symbols, consumed); +#endif + if (rc == PIVCO_OK) { + dec->stats.blocks++; + dec->stats.bytes_in += in_len; + dec->stats.bytes_out += *consumed ? *consumed : 0; + } + return rc; +} + +/* ---------------- contexts ---------------- */ + +const pivco_cfg_t pivco_cfg_default = { + .tree_mode = PIVCO_TREE_MODE_OPTIMIZED, + .effort = PIVCO_EFFORT_PLAIN, + .fse_enabled = 1, + .flat_layout = PIVCO_FLAT_VERTICAL, +}; + +#include "pivco_huffman_common.h" +#include "pivco_huffman_primitives.h" /* arch-selected; prim_histogram_chunk */ + +static pivco_scratch_t *scratch_create(void) +{ + pivco_scratch_t *sc = (pivco_scratch_t *)calloc(1, sizeof(*sc)); + if (!sc) return NULL; + sc->enc_cap = PIVCO_ENC_SCRATCH_BYTES(PIVCO_WIRE_MAX_N); + sc->dec_cap = PIVCO_DEC_SCRATCH_BYTES(PIVCO_WIRE_MAX_N) + + DECODE_SCRATCH_ALIGN + DECODE_SCRATCH_SHIFT; + /* PH_CTX_ALLOC=1 placement A/B probe: start from a deliberately + * small 256 KiB cap instead of the full preallocation, mimicking + * the retired TLS arenas' grow-on-demand footprint (the ensure + * helpers realloc up on first oversized block). */ +#define PH_CTX_ALLOC_PROBE_CAP ((size_t)256 * 1024) + const char *am = getenv("PH_CTX_ALLOC"); + if (am && am[0] == '1') { + sc->enc_cap = PH_CTX_ALLOC_PROBE_CAP; + sc->dec_cap = PH_CTX_ALLOC_PROBE_CAP + + DECODE_SCRATCH_ALIGN + DECODE_SCRATCH_SHIFT; + sc->enc = (uint8_t *)malloc(sc->enc_cap); + sc->dec = (uint8_t *)malloc(sc->dec_cap); + } else { + sc->enc = (uint8_t *)malloc(sc->enc_cap); + sc->dec = (uint8_t *)malloc(sc->dec_cap); + } + if (!sc->enc || !sc->dec) { + free(sc->enc); free(sc->dec); free(sc); + return NULL; + } + return sc; +} + +static void scratch_free(pivco_scratch_t *sc) +{ + if (!sc) return; + free(sc->enc); free(sc->dec); free(sc); +} + +pivco_encoder_t *pivco_encoder_create(void) +{ + pivco_encoder_t *e = (pivco_encoder_t *)calloc(1, sizeof(*e)); + if (!e) return NULL; + e->internal = scratch_create(); + if (!e->internal) { free(e); return NULL; } + prim_codec_init(); + return e; +} + +pivco_decoder_t *pivco_decoder_create(void) +{ + pivco_decoder_t *d = (pivco_decoder_t *)calloc(1, sizeof(*d)); + if (!d) return NULL; + d->internal = scratch_create(); + if (!d->internal) { free(d); return NULL; } + prim_codec_init(); + return d; +} + +void pivco_encoder_free(pivco_encoder_t *e) +{ + if (!e) return; + scratch_free((pivco_scratch_t *)e->internal); + free(e); +} + +void pivco_decoder_free(pivco_decoder_t *d) +{ + if (!d) return; + scratch_free((pivco_scratch_t *)d->internal); + free(d); +} + +/* ---------------- histogram ---------------- */ + +int pivco_histogram(pivco_encoder_t *enc, const uint8_t *in, size_t n, + uint64_t freq[PIVCO_MAX_SYMBOLS]) +{ + if (!enc || (!in && n) || !freq) return PIVCO_ERR_NULL; + pivco_scratch_t *sc = (pivco_scratch_t *)enc->internal; + /* hist bins live in the tail of the encode arena (never concurrent + * with an encode call: contexts are single-threaded) */ + uint8_t *scratch = sc->enc + sc->enc_cap - PIVCO_PRIM_HIST_SCRATCH_MAX; + size_t off = 0; + while (off < n) { + size_t len = n - off; + if (len > PIVCO_PRIM_HIST_CHUNK) len = PIVCO_PRIM_HIST_CHUNK; + uint32_t h32[256] = {0}; + prim_histogram_chunk(in + off, len, h32, scratch); + for (int sym = 0; sym < 256; sym++) freq[sym] += h32[sym]; + off += len; + } + return PIVCO_OK; +} diff --git a/third_party/pivco/src/pivco_huffman_avx2_pack.h b/third_party/pivco/src/pivco_huffman_avx2_pack.h new file mode 100644 index 0000000..ea64a4e --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_avx2_pack.h @@ -0,0 +1,95 @@ +/* pivco_huffman_avx2_pack.h — flat-subtree D-bit pack (AVX2 port of + * ryg's multiply-as-shift pack). Mirrors pivco_huffman_avx512_pack.h + * for the x86 backend hosts that have AVX2 but not AVX-512 VBMI2. + * + * 32 codes per ymm iter via byte-laid intermediate: + * - vpmaddubsw c0 word[i] = code[2i] + code[2i+1] * 2^D (2D bits) + * - vpmaddwd c1 dword[i] = word[2i] + word[2i+1] * 2^(2D) (4D bits) + * - vpsrlq + (a&c)|(b&~c) via vpand/vpandn/vpor (8D bits) + * - vpshufb compact per-128-bit lane (2D bytes) + * - 2 x vmovdqu storeu (low + high 128-bit halves) + * + * The trailing junk in each 128-bit store (16 - 2D bytes) gets + * overwritten by the next iter's low store. Caller's output buffer + * needs at least 16 bytes of slack past the last valid byte of the + * packed stream so the LAST iter's trailing junk lands somewhere safe; + * PIVCO_MAX_ENCODED_SIZE = 2 * block_size gives plenty. + * + * Internal header. Not part of the public API. */ + +#ifndef PIVCO_HUFFMAN_AVX2_PACK_H +#define PIVCO_HUFFMAN_AVX2_PACK_H + +#if !defined(__AVX2__) || !defined(__SSE4_1__) +#error "pivco_huffman_avx2_pack.h requires AVX2 + SSE4.1" +#endif + +#include +#include +#include + +/* Load 32 ranks, subtract base (1 rank/byte) — the byte-laid intermediate the + * multiply-as-shift pack expects, with no u16 narrow. The local code is already + * in [0,2^D) (rank - flat_base_rank over a depth-D flat subtree), so no mask to D + * bits is needed. */ +static inline __m256i pivco_pack_load_byte_avx2(const uint8_t *ranks, uint8_t base) +{ + return _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)ranks), + _mm256_set1_epi8((char)base)); +} + +/* Per-D compact-shuf tables: pattern is bytes [0..D-1] from positions + * 0..D-1, bytes [D..2D-1] from positions 8..8+D-1, junk at 2D..15. + * Replicated identically in both 128-bit halves of the ymm shuf. */ +#define PIVCO_PACK_AVX2_COMPACT_D2 _mm256_setr_epi8( \ + 0, 1, 8, 9, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, \ + 0, 1, 8, 9, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1) +#define PIVCO_PACK_AVX2_COMPACT_D3 _mm256_setr_epi8( \ + 0, 1, 2, 8, 9,10, -1,-1, -1,-1,-1,-1, -1,-1,-1,-1, \ + 0, 1, 2, 8, 9,10, -1,-1, -1,-1,-1,-1, -1,-1,-1,-1) +#define PIVCO_PACK_AVX2_COMPACT_D5 _mm256_setr_epi8( \ + 0, 1, 2, 3, 4, 8, 9,10, 11,12, -1,-1, -1,-1,-1,-1, \ + 0, 1, 2, 3, 4, 8, 9,10, 11,12, -1,-1, -1,-1,-1,-1) +#define PIVCO_PACK_AVX2_COMPACT_D6 _mm256_setr_epi8( \ + 0, 1, 2, 3, 4, 5, 8, 9, 10,11,12,13, -1,-1,-1,-1, \ + 0, 1, 2, 3, 4, 5, 8, 9, 10,11,12,13, -1,-1,-1,-1) +#define PIVCO_PACK_AVX2_COMPACT_D7 _mm256_setr_epi8( \ + 0, 1, 2, 3, 4, 5, 6, 8, 9,10,11,12,13,14, -1,-1, \ + 0, 1, 2, 3, 4, 5, 6, 8, 9,10,11,12,13,14, -1,-1) + +#define PIVCO_PACK_AVX2_DN(NAME, D_VAL, COMPACT_SHUF) \ +static inline int NAME(uint8_t *out, const uint8_t *ranks, \ + int n, uint8_t base) \ +{ \ + const __m256i c0 = _mm256_set1_epi16( \ + (int16_t)(((1 << (D_VAL)) << 8) | 1)); \ + const __m256i c1 = _mm256_set1_epi32( \ + (int32_t)(((int32_t)1 << (2*(D_VAL))) << 16) | 1); \ + const __m256i c3 = _mm256_set1_epi64x( \ + (int64_t)(((int64_t)1 << (4*(D_VAL))) - 1)); \ + const __m256i compact = COMPACT_SHUF; \ + int i = 0; \ + for (; i + 32 <= n; i += 32) { \ + __m256i cb = pivco_pack_load_byte_avx2(ranks + i, base); \ + __m256i x = _mm256_maddubs_epi16(c0, cb); \ + x = _mm256_madd_epi16(x, c1); \ + __m256i xs = _mm256_srli_epi64(x, 32 - 4*(D_VAL)); \ + x = _mm256_or_si256(_mm256_and_si256(x, c3), \ + _mm256_andnot_si256(c3, xs)); \ + __m256i out_y = _mm256_shuffle_epi8(x, compact); \ + int outpos = (i * (D_VAL)) >> 3; \ + _mm_storeu_si128((__m128i *)(out + outpos), \ + _mm256_castsi256_si128(out_y)); \ + _mm_storeu_si128((__m128i *)(out + outpos + 2*(D_VAL)), \ + _mm256_extracti128_si256(out_y, 1)); \ + } \ + return i; \ +} +PIVCO_PACK_AVX2_DN(pack_d2_avx2_x86, 2, PIVCO_PACK_AVX2_COMPACT_D2) +PIVCO_PACK_AVX2_DN(pack_d3_avx2_x86, 3, PIVCO_PACK_AVX2_COMPACT_D3) +PIVCO_PACK_AVX2_DN(pack_d5_avx2_x86, 5, PIVCO_PACK_AVX2_COMPACT_D5) +PIVCO_PACK_AVX2_DN(pack_d6_avx2_x86, 6, PIVCO_PACK_AVX2_COMPACT_D6) +PIVCO_PACK_AVX2_DN(pack_d7_avx2_x86, 7, PIVCO_PACK_AVX2_COMPACT_D7) +#undef PIVCO_PACK_AVX2_DN + +#endif /* PIVCO_HUFFMAN_AVX2_PACK_H */ diff --git a/third_party/pivco/src/pivco_huffman_avx512_flat.h b/third_party/pivco/src/pivco_huffman_avx512_flat.h new file mode 100644 index 0000000..a3cef03 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_avx512_flat.h @@ -0,0 +1,361 @@ +/* pivco_huffman_avx512_flat.h — flat-subtree D-bit code unpackers (AVX-512 VBMI2). + * + * Internal header. Mirrors src/pivco_huffman_neon_flat.h: each + * `flat_dN_unpack_avx512()` (and the fast/safe pair for D ∈ {3,5,6}) + * reads N D-bit codes from a packed bitstream and returns them in a + * 128-bit vector lane (one byte per code, value < 2^D). Used by the + * production decoder (pivco_huffman_avx512.c) and the per-D microbench + * (bench/bench_micro.c). + * + * Tables are folded into the helpers as `_mm_setr_epi8` constants so + * each helper is fully self-contained. All helpers are `static + * inline` — values fold into the inlined function and no extern + * symbols are emitted. + * + * The "fast" variants for D=3, D=5, D=6 use power-of-2 byte loads + * (8 / 16 bytes) which overread the valid bm region by a few bytes + * but compile to a single load instruction. The "safe" variants use + * the exact byte count — caller picks the safe form for the final + * chunk. See the AVX-512 revisit ship note in IDEAS.md for context. + * + * Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_AVX512_FLAT_H +#define PIVCO_HUFFMAN_AVX512_FLAT_H + +#if !defined(__AVX512BW__) || !defined(__AVX512VBMI__) || !defined(__AVX512VBMI2__) +#error "pivco_huffman_avx512_flat.h requires AVX-512 BW + VBMI + VBMI2" +#endif + +#include +#include +#include + +/* D=2: 16 codes from 4 bytes of bm. Replicate 4 bytes to 16 bytes, then + * multishift with offsets {0,2,..,14, 16,18,..,30} across 2 uint64 lanes. */ +static inline __m128i flat_d2_unpack_avx512(const uint8_t *bm_ptr) +{ + uint32_t packed; + memcpy(&packed, bm_ptr, 4); + __m128i data = _mm_set1_epi32((int32_t)packed); + const __m128i ctrl = _mm_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, + 16, 18, 20, 22, 24, 26, 28, 30); + __m128i raw = _mm_multishift_epi64_epi8(ctrl, data); + return _mm_and_si128(raw, _mm_set1_epi8(0x03)); +} + +/* D=3 fast: loads 8 bytes (2 past the end of the 6-valid-byte region). + * Caller must guarantee buffer slack. */ +static inline __m128i flat_d3_unpack_avx512_fast(const uint8_t *bm_ptr) +{ + uint64_t packed; + memcpy(&packed, bm_ptr, 8); + __m128i data = _mm_set1_epi64x((int64_t)packed); + const __m128i ctrl = _mm_setr_epi8( + 0, 3, 6, 9, 12, 15, 18, 21, + 24, 27, 30, 33, 36, 39, 42, 45); + __m128i raw = _mm_multishift_epi64_epi8(ctrl, data); + return _mm_and_si128(raw, _mm_set1_epi8(0x07)); +} + +/* D=3 safe: 6-byte memcpy for the last chunk. */ +static inline __m128i flat_d3_unpack_avx512_safe(const uint8_t *bm_ptr) +{ + uint64_t packed = 0; + memcpy(&packed, bm_ptr, 6); + __m128i data = _mm_set1_epi64x((int64_t)packed); + const __m128i ctrl = _mm_setr_epi8( + 0, 3, 6, 9, 12, 15, 18, 21, + 24, 27, 30, 33, 36, 39, 42, 45); + __m128i raw = _mm_multishift_epi64_epi8(ctrl, data); + return _mm_and_si128(raw, _mm_set1_epi8(0x07)); +} + +/* D=4: 16 codes from 8 bytes of bm. 2 codes per byte, no cross-byte + * carries. */ +static inline __m128i flat_d4_unpack_avx512(const uint8_t *bm_ptr) +{ + uint64_t packed; + memcpy(&packed, bm_ptr, 8); + __m128i data = _mm_set1_epi64x((int64_t)packed); + const __m128i ctrl = _mm_setr_epi8( + 0, 4, 8, 12, 16, 20, 24, 28, + 32, 36, 40, 44, 48, 52, 56, 60); + __m128i raw = _mm_multishift_epi64_epi8(ctrl, data); + return _mm_and_si128(raw, _mm_set1_epi8(0x0F)); +} + +/* D=5 fast: 16 codes from 10 valid bytes, with a 16-byte load. */ +static inline __m128i flat_d5_unpack_avx512_fast(const uint8_t *bm_ptr) +{ + __m128i raw = _mm_loadu_si128((const __m128i *)bm_ptr); + const __m128i shuf = _mm_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, + 2, 3, 4, 5, 6, 7, 8, 9); + __m128i data = _mm_shuffle_epi8(raw, shuf); + const __m128i ctrl = _mm_setr_epi8( + 0, 5, 10, 15, 20, 25, 30, 35, + 24, 29, 34, 39, 44, 49, 54, 59); + __m128i ms = _mm_multishift_epi64_epi8(ctrl, data); + return _mm_and_si128(ms, _mm_set1_epi8(0x1F)); +} + +/* D=5 safe: 10-byte memcpy for the last chunk. */ +static inline __m128i flat_d5_unpack_avx512_safe(const uint8_t *bm_ptr) +{ + uint8_t buf[16] = {0}; + memcpy(buf, bm_ptr, 10); + return flat_d5_unpack_avx512_fast(buf); +} + +/* ---- 64-at-a-time unpacks (codes/call = 64, one zmm output) ---------- + * + * All variants follow the same shape: load enough bytes for lane 7 to be + * fully sourced (`max_byte = D*7 + 7`), `vpermb` to gather the 8 per-lane + * windows, `vpmultishiftqb` with per-lane ctrl {0, D, 2D, …, 7D}, then + * AND with the D-bit mask. The load width is 32 B (ymm zext) when + * `max_byte ≤ 31`, 64 B (zmm) otherwise. Strict end-of-stream slack is + * `ceil(load_bytes * 8 / D)` codes — the caller leaves that much room at + * the end and runs the existing 16-wide tail there. */ + +/* D=2 x4: 64 codes from 16 valid bm bytes via a 32-byte ymm load (lane 7 + * needs byte 21). */ +static inline __m512i flat_d2_unpack64_avx512_fast(const uint8_t *bm_ptr) +{ + __m256i raw256 = _mm256_loadu_si256((const __m256i *)bm_ptr); + __m512i raw = _mm512_zextsi256_si512(raw256); + const __m512i shuf = _mm512_set_epi8( + 21, 20, 19, 18, 17, 16, 15, 14, /* lane 7 */ + 19, 18, 17, 16, 15, 14, 13, 12, /* lane 6 */ + 17, 16, 15, 14, 13, 12, 11, 10, /* lane 5 */ + 15, 14, 13, 12, 11, 10, 9, 8, /* lane 4 */ + 13, 12, 11, 10, 9, 8, 7, 6, /* lane 3 */ + 11, 10, 9, 8, 7, 6, 5, 4, /* lane 2 */ + 9, 8, 7, 6, 5, 4, 3, 2, /* lane 1 */ + 7, 6, 5, 4, 3, 2, 1, 0); /* lane 0 */ + __m512i data = _mm512_permutexvar_epi8(shuf, raw); + /* {0,2,4,6,8,10,12,14} packed LE: 0x0E 0C 0A 08 06 04 02 00 */ + const __m512i ctrl = _mm512_set1_epi64( + (int64_t)0x0E0C0A0806040200LL); + __m512i ms = _mm512_multishift_epi64_epi8(ctrl, data); + return _mm512_and_si512(ms, _mm512_set1_epi8(0x03)); +} + +/* D=2 safe x4: 16-byte memcpy. */ +static inline __m512i flat_d2_unpack64_avx512_safe(const uint8_t *bm_ptr) +{ + uint8_t buf[32] = {0}; + memcpy(buf, bm_ptr, 16); + return flat_d2_unpack64_avx512_fast(buf); +} + +/* D=3 x4: 64 codes from 24 valid bm bytes via a 32-byte ymm load (lane 7 + * needs byte 28). */ +static inline __m512i flat_d3_unpack64_avx512_fast(const uint8_t *bm_ptr) +{ + __m256i raw256 = _mm256_loadu_si256((const __m256i *)bm_ptr); + __m512i raw = _mm512_zextsi256_si512(raw256); + const __m512i shuf = _mm512_set_epi8( + 28, 27, 26, 25, 24, 23, 22, 21, /* lane 7 */ + 25, 24, 23, 22, 21, 20, 19, 18, /* lane 6 */ + 22, 21, 20, 19, 18, 17, 16, 15, /* lane 5 */ + 19, 18, 17, 16, 15, 14, 13, 12, /* lane 4 */ + 16, 15, 14, 13, 12, 11, 10, 9, /* lane 3 */ + 13, 12, 11, 10, 9, 8, 7, 6, /* lane 2 */ + 10, 9, 8, 7, 6, 5, 4, 3, /* lane 1 */ + 7, 6, 5, 4, 3, 2, 1, 0); /* lane 0 */ + __m512i data = _mm512_permutexvar_epi8(shuf, raw); + const __m512i ctrl = _mm512_set1_epi64( + (int64_t)0x15120F0C09060300LL); /* {0,3,6,9,12,15,18,21} */ + __m512i ms = _mm512_multishift_epi64_epi8(ctrl, data); + return _mm512_and_si512(ms, _mm512_set1_epi8(0x07)); +} + +/* D=3 safe x4: 24-byte memcpy. */ +static inline __m512i flat_d3_unpack64_avx512_safe(const uint8_t *bm_ptr) +{ + uint8_t buf[32] = {0}; + memcpy(buf, bm_ptr, 24); + return flat_d3_unpack64_avx512_fast(buf); +} + +/* D=4 x4: 64 codes from 32 valid bm bytes via a 32-byte ymm load. Lane 7 + * reads bytes 28..35; bytes 32..35 are the (zero) high half of the zext + * but the output mask 0x0F drops their contribution. */ +static inline __m512i flat_d4_unpack64_avx512_fast(const uint8_t *bm_ptr) +{ + __m256i raw256 = _mm256_loadu_si256((const __m256i *)bm_ptr); + __m512i raw = _mm512_zextsi256_si512(raw256); + const __m512i shuf = _mm512_set_epi8( + 35, 34, 33, 32, 31, 30, 29, 28, /* lane 7 (top 4 bytes zero) */ + 31, 30, 29, 28, 27, 26, 25, 24, /* lane 6 */ + 27, 26, 25, 24, 23, 22, 21, 20, /* lane 5 */ + 23, 22, 21, 20, 19, 18, 17, 16, /* lane 4 */ + 19, 18, 17, 16, 15, 14, 13, 12, /* lane 3 */ + 15, 14, 13, 12, 11, 10, 9, 8, /* lane 2 */ + 11, 10, 9, 8, 7, 6, 5, 4, /* lane 1 */ + 7, 6, 5, 4, 3, 2, 1, 0); /* lane 0 */ + __m512i data = _mm512_permutexvar_epi8(shuf, raw); + /* {0,4,8,12,16,20,24,28} packed LE: 0x1C 18 14 10 0C 08 04 00 */ + const __m512i ctrl = _mm512_set1_epi64( + (int64_t)0x1C1814100C080400LL); + __m512i ms = _mm512_multishift_epi64_epi8(ctrl, data); + return _mm512_and_si512(ms, _mm512_set1_epi8(0x0F)); +} + +/* D=4 safe x4: 32-byte memcpy. */ +static inline __m512i flat_d4_unpack64_avx512_safe(const uint8_t *bm_ptr) +{ + uint8_t buf[32] = {0}; + memcpy(buf, bm_ptr, 32); + return flat_d4_unpack64_avx512_fast(buf); +} + +/* D=5 fast x4: 64 codes from 40 valid bm bytes in a single zmm chain. + * Lane k of the multishift input gets bytes [5k .. 5k+7] of bm (lane stride + * 5 — code 8k starts at bit 40k = byte 5k bit 0), so each 64-bit lane + * holds 8 codes at bit offsets {0,5,10,15,20,25,30,35}. Highest source + * byte is 42 → caller must guarantee ≥43 valid bm bytes (64-byte load + * over-reads by 21). */ +static inline __m512i flat_d5_unpack64_avx512_fast(const uint8_t *bm_ptr) +{ + __m512i raw = _mm512_loadu_si512((const __m512i *)bm_ptr); + /* _mm512_set_epi8 takes bytes top-first; rows below are written + * lane-7 ... lane-0 so the in-memory layout is + * lane 0: bm[0..7], lane 1: bm[5..12], ..., lane 7: bm[35..42] */ + const __m512i shuf = _mm512_set_epi8( + 42, 41, 40, 39, 38, 37, 36, 35, /* lane 7 */ + 37, 36, 35, 34, 33, 32, 31, 30, /* lane 6 */ + 32, 31, 30, 29, 28, 27, 26, 25, /* lane 5 */ + 27, 26, 25, 24, 23, 22, 21, 20, /* lane 4 */ + 22, 21, 20, 19, 18, 17, 16, 15, /* lane 3 */ + 17, 16, 15, 14, 13, 12, 11, 10, /* lane 2 */ + 12, 11, 10, 9, 8, 7, 6, 5, /* lane 1 */ + 7, 6, 5, 4, 3, 2, 1, 0); /* lane 0 */ + __m512i data = _mm512_permutexvar_epi8(shuf, raw); + /* bit-offsets {0,5,10,15,20,25,30,35} packed LE into a u64. */ + const __m512i ctrl = _mm512_set1_epi64( + (int64_t)0x231E19140F0A0500LL); + __m512i ms = _mm512_multishift_epi64_epi8(ctrl, data); + return _mm512_and_si512(ms, _mm512_set1_epi8(0x1F)); +} + +/* D=5 safe x4: 40-byte memcpy for the last chunk. */ +static inline __m512i flat_d5_unpack64_avx512_safe(const uint8_t *bm_ptr) +{ + uint8_t buf[64] = {0}; + memcpy(buf, bm_ptr, 40); + return flat_d5_unpack64_avx512_fast(buf); +} + +/* D=6 x4: 64 codes from 48 valid bm bytes via a 64-byte zmm load (lane 7 + * needs byte 49). */ +static inline __m512i flat_d6_unpack64_avx512_fast(const uint8_t *bm_ptr) +{ + __m512i raw = _mm512_loadu_si512((const __m512i *)bm_ptr); + const __m512i shuf = _mm512_set_epi8( + 49, 48, 47, 46, 45, 44, 43, 42, /* lane 7 */ + 43, 42, 41, 40, 39, 38, 37, 36, /* lane 6 */ + 37, 36, 35, 34, 33, 32, 31, 30, /* lane 5 */ + 31, 30, 29, 28, 27, 26, 25, 24, /* lane 4 */ + 25, 24, 23, 22, 21, 20, 19, 18, /* lane 3 */ + 19, 18, 17, 16, 15, 14, 13, 12, /* lane 2 */ + 13, 12, 11, 10, 9, 8, 7, 6, /* lane 1 */ + 7, 6, 5, 4, 3, 2, 1, 0); /* lane 0 */ + __m512i data = _mm512_permutexvar_epi8(shuf, raw); + /* {0,6,12,18,24,30,36,42} packed LE: 0x2A 24 1E 18 12 0C 06 00 */ + const __m512i ctrl = _mm512_set1_epi64( + (int64_t)0x2A241E18120C0600LL); + __m512i ms = _mm512_multishift_epi64_epi8(ctrl, data); + return _mm512_and_si512(ms, _mm512_set1_epi8(0x3F)); +} + +/* D=6 safe x4: 48-byte memcpy. */ +static inline __m512i flat_d6_unpack64_avx512_safe(const uint8_t *bm_ptr) +{ + uint8_t buf[64] = {0}; + memcpy(buf, bm_ptr, 48); + return flat_d6_unpack64_avx512_fast(buf); +} + +/* D=7 x4: 64 codes from 56 valid bm bytes via a 64-byte zmm load (lane 7 + * needs byte 56). */ +static inline __m512i flat_d7_unpack64_avx512_fast(const uint8_t *bm_ptr) +{ + __m512i raw = _mm512_loadu_si512((const __m512i *)bm_ptr); + const __m512i shuf = _mm512_set_epi8( + 56, 55, 54, 53, 52, 51, 50, 49, /* lane 7 */ + 49, 48, 47, 46, 45, 44, 43, 42, /* lane 6 */ + 42, 41, 40, 39, 38, 37, 36, 35, /* lane 5 */ + 35, 34, 33, 32, 31, 30, 29, 28, /* lane 4 */ + 28, 27, 26, 25, 24, 23, 22, 21, /* lane 3 */ + 21, 20, 19, 18, 17, 16, 15, 14, /* lane 2 */ + 14, 13, 12, 11, 10, 9, 8, 7, /* lane 1 */ + 7, 6, 5, 4, 3, 2, 1, 0); /* lane 0 */ + __m512i data = _mm512_permutexvar_epi8(shuf, raw); + /* {0,7,14,21,28,35,42,49} packed LE: 0x31 2A 23 1C 15 0E 07 00 */ + const __m512i ctrl = _mm512_set1_epi64( + (int64_t)0x312A231C150E0700LL); + __m512i ms = _mm512_multishift_epi64_epi8(ctrl, data); + return _mm512_and_si512(ms, _mm512_set1_epi8(0x7F)); +} + +/* D=7 safe x4: 56-byte memcpy. */ +static inline __m512i flat_d7_unpack64_avx512_safe(const uint8_t *bm_ptr) +{ + uint8_t buf[64] = {0}; + memcpy(buf, bm_ptr, 56); + return flat_d7_unpack64_avx512_fast(buf); +} + +/* D=6 fast: 16 codes from 12 valid bytes, with a 16-byte load. */ +static inline __m128i flat_d6_unpack_avx512_fast(const uint8_t *bm_ptr) +{ + __m128i raw = _mm_loadu_si128((const __m128i *)bm_ptr); + const __m128i shuf = _mm_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, + 4, 5, 6, 7, 8, 9, 10, 11); + __m128i data = _mm_shuffle_epi8(raw, shuf); + const __m128i ctrl = _mm_setr_epi8( + 0, 6, 12, 18, 24, 30, 36, 42, + 16, 22, 28, 34, 40, 46, 52, 58); + __m128i ms = _mm_multishift_epi64_epi8(ctrl, data); + return _mm_and_si128(ms, _mm_set1_epi8(0x3F)); +} + +/* D=6 safe: 12-byte memcpy for the last chunk. */ +static inline __m128i flat_d6_unpack_avx512_safe(const uint8_t *bm_ptr) +{ + uint8_t buf[16] = {0}; + memcpy(buf, bm_ptr, 12); + return flat_d6_unpack_avx512_fast(buf); +} + +/* D=7: 16 codes = 112 bits = 14 bytes. Code i is at bit 7i. Two 64-bit + * windows (input bytes 0..7 and 7..14) each hold 8 codes at offsets + * {0,7,14,21,28,35,42,49}; vpmultishift extracts them. Mask to 7 bits. */ +static inline __m128i flat_d7_unpack_avx512_fast(const uint8_t *bm_ptr) +{ + __m128i raw = _mm_loadu_si128((const __m128i *)bm_ptr); + const __m128i shuf = _mm_setr_epi8( + 0, 1, 2, 3, 4, 5, 6, 7, + 7, 8, 9, 10, 11, 12, 13, 14); + __m128i data = _mm_shuffle_epi8(raw, shuf); + const __m128i ctrl = _mm_setr_epi8( + 0, 7, 14, 21, 28, 35, 42, 49, + 0, 7, 14, 21, 28, 35, 42, 49); + __m128i ms = _mm_multishift_epi64_epi8(ctrl, data); + return _mm_and_si128(ms, _mm_set1_epi8(0x7F)); +} + +/* D=7 safe: 14-byte memcpy for the last chunk (avoid the 16-byte over-read). */ +static inline __m128i flat_d7_unpack_avx512_safe(const uint8_t *bm_ptr) +{ + uint8_t buf[16] = {0}; + memcpy(buf, bm_ptr, 14); + return flat_d7_unpack_avx512_fast(buf); +} + +#endif /* PIVCO_HUFFMAN_AVX512_FLAT_H */ diff --git a/third_party/pivco/src/pivco_huffman_avx512_pack.h b/third_party/pivco/src/pivco_huffman_avx512_pack.h new file mode 100644 index 0000000..b32302a --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_avx512_pack.h @@ -0,0 +1,244 @@ +/* pivco_huffman_avx512_pack.h — flat-subtree D-bit pack (AVX-512 VBMI2). + * + * 64 codes per zmm iter via byte-laid intermediate + vpmultishiftqb. + * + * The codes_la input lane has the D-bit code at [right_shift, right_shift+D) + * (left-aligned encoder format). load_codes_byte right-shifts, narrows + * u16 -> u8 via vpmovwb, and assembles 64 codes into one zmm (one byte + * per code, low D bits valid). + * + * Pack strategy per D: + * - D=2, D=4: codes don't cross byte boundaries, so 4 (D=2) or 2 (D=4) + * vpermb gathers at code-stride 4 / 2, plus a fixed left-shift per + * group, OR'd together. Simpler than multishift. + * - D=3, D=5, D=6, D=7: codes cross byte boundaries. Split codes into + * G groups (codes mod G, G = ceil(8/D)+1 for D=3, 3 for D=5, 2 for + * D=6/7) such that within each group no output byte gets contribution + * from two same-group codes. For each group: mask the byte-laid + * input + vpmultishiftqb with broadcast ctrl that pulls each code's + * bits to its absolute bit position in the packed stream. OR the G + * group results; vpermb compacts the 8 lanes' 0..D-1 bytes into a + * contiguous 8*D-byte stream; masked store writes the valid prefix. + * + * Op count per 64 codes (D=5 example): + * load_codes_byte (~5 ops) + 3*(mask + multishift) + 2 OR + vpermb + * + masked store ~= 12 ops, vs the prior 8-codes-per-iter sllv + + * reduce_add path's ~6 ops/chunk * 8 chunks = ~48 ops. + * + * Same-session microbench (ns/code on c8i / c8a) vs v1 vector and BMI2: + * + * v1-vec bmi2 this + * D=2 0.204/.088 0.068/.071 0.046/0.021 + * D=3 0.206/.106 0.102/.115 0.050/0.022 + * D=4 0.206/.114 0.089/.074 0.038/0.014 + * D=5 0.207/.106 0.104/.099 0.046/0.019 + * D=6 0.209/.108 0.104/.114 0.043/0.016 + * D=7 0.211/.125 0.117/.116 0.043/0.017 + * + * Internal header. Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_AVX512_PACK_H +#define PIVCO_HUFFMAN_AVX512_PACK_H + +#if !defined(__AVX512BW__) || !defined(__AVX512VBMI__) || !defined(__AVX512VBMI2__) +#error "pivco_huffman_avx512_pack.h requires AVX-512 BW + VBMI + VBMI2" +#endif + +#include +#include +#include + +/* Compact-shuf tables for D=3,5,6,7: gather lane k bytes [0..D-1] into + * output bytes [k*D .. k*D+D-1]. Bytes beyond 8*D are 0 (masked store). */ +static const uint8_t pivco_pack_compact_d3[64] __attribute__((aligned(64))) = { + 0, 1, 2, 8, 9,10, 16,17,18, 24,25,26, 32,33,34, 40,41,42, 48,49,50, 56,57,58, + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0 +}; +static const uint8_t pivco_pack_compact_d5[64] __attribute__((aligned(64))) = { + 0, 1, 2, 3, 4, 8, 9,10,11,12, 16,17,18,19,20, 24,25,26,27,28, + 32,33,34,35,36, 40,41,42,43,44, 48,49,50,51,52, 56,57,58,59,60, + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0 +}; +static const uint8_t pivco_pack_compact_d6[64] __attribute__((aligned(64))) = { + 0, 1, 2, 3, 4, 5, 8, 9,10,11,12,13, 16,17,18,19,20,21, 24,25,26,27,28,29, + 32,33,34,35,36,37, 40,41,42,43,44,45, 48,49,50,51,52,53, 56,57,58,59,60,61, + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0 +}; +static const uint8_t pivco_pack_compact_d7[64] __attribute__((aligned(64))) = { + 0, 1, 2, 3, 4, 5, 6, 8, 9,10,11,12,13,14, 16,17,18,19,20,21,22, 24,25,26,27,28,29,30, + 32,33,34,35,36,37,38, 40,41,42,43,44,45,46, 48,49,50,51,52,53,54, 56,57,58,59,60,61,62, + 0,0,0,0,0,0,0,0 +}; + + +/* ---- rank-based (partbyrank) variants ------------------------------------- + * The flat local code is (rank - base), already a D-bit value in each byte — + * so the byte-laid `cb` comes straight from a u8 load + subtract (no u16 load + * + cvtepi16_epi8 narrow). The pack BACKEND is byte-for-byte the same. */ +static inline __m512i pivco_pack_load_byte(const uint8_t *ranks, uint8_t base) +{ + return _mm512_sub_epi8(_mm512_loadu_si512((const __m512i *)ranks), + _mm512_set1_epi8((char)base)); +} + +static inline int pack_d2_avx512(uint8_t *out, const uint8_t *ranks, + int n, uint8_t base) +{ + /* Group g (g in 0..3) gathers ranks (g, g+4, g+8, ..., g+60) into the + * low 16 output bytes; group g's bits land at position 2g within each + * output byte. */ + const __m512i shuf0 = _mm512_set_epi8( + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 60,56,52,48,44,40,36,32, 28,24,20,16,12,8,4,0); + const __m512i shuf1 = _mm512_set_epi8( + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 61,57,53,49,45,41,37,33, 29,25,21,17,13,9,5,1); + const __m512i shuf2 = _mm512_set_epi8( + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 62,58,54,50,46,42,38,34, 30,26,22,18,14,10,6,2); + const __m512i shuf3 = _mm512_set_epi8( + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 63,59,55,51,47,43,39,35, 31,27,23,19,15,11,7,3); + int i = 0; + for (; i + 64 <= n; i += 64) { + /* Local code is in [0,2^D) (rank - flat_base_rank over the flat subtree), + * so the slli-then-OR below can't leak high bits across byte boundaries + * within a u32 lane -- no mask needed. */ + __m512i cb = pivco_pack_load_byte(ranks + i, base); + __m512i g0 = _mm512_permutexvar_epi8(shuf0, cb); + __m512i g1 = _mm512_permutexvar_epi8(shuf1, cb); + __m512i g2 = _mm512_permutexvar_epi8(shuf2, cb); + __m512i g3 = _mm512_permutexvar_epi8(shuf3, cb); + __m512i packed = _mm512_or_si512( + _mm512_or_si512(g0, _mm512_slli_epi32(g1, 2)), + _mm512_or_si512(_mm512_slli_epi32(g2, 4), _mm512_slli_epi32(g3, 6))); + _mm512_mask_storeu_epi8(out + ((i * 2) >> 3), + (__mmask64)0xFFFFULL, packed); + } + return i; +} + +/* D=3: 4 groups (ranks mod 4). Each chunk of 8 ranks -> 3 output bytes. */ + +static inline int pack_d3_avx512(uint8_t *out, const uint8_t *ranks, + int n, uint8_t base) +{ + const __m512i mA = _mm512_set1_epi64((int64_t)0x0000000700000007ULL); /* bytes 0,4 */ + const __m512i mB = _mm512_set1_epi64((int64_t)0x0000070000000700ULL); /* bytes 1,5 */ + const __m512i mC = _mm512_set1_epi64((int64_t)0x0007000000070000ULL); /* bytes 2,6 */ + const __m512i mD = _mm512_set1_epi64((int64_t)0x0700000007000000ULL); /* bytes 3,7 */ + /* Per-byte multishift ctrls (lo->hi byte order). Byte 2 of cA reads + * a zero region of lane_A (Group A doesn't contribute to output byte + * 2; pulling from a masked-zero byte avoids leaking rank 0 in). */ + const __m512i cA = _mm512_set1_epi64((int64_t)0x0000000000081C00ULL); /* {0,28,8,...} */ + const __m512i cB = _mm512_set1_epi64((int64_t)0x0000000000292105ULL); /* {5,33,41,...} */ + const __m512i cC = _mm512_set1_epi64((int64_t)0x00000000002E120AULL); /* {10,18,46,...} */ + const __m512i cD = _mm512_set1_epi64((int64_t)0x0000000000331700ULL); /* {0,23,51,...} */ + int i = 0; + for (; i + 64 <= n; i += 64) { + __m512i cb = pivco_pack_load_byte(ranks + i, base); + __m512i a = _mm512_multishift_epi64_epi8(cA, _mm512_and_si512(cb, mA)); + __m512i b = _mm512_multishift_epi64_epi8(cB, _mm512_and_si512(cb, mB)); + __m512i c = _mm512_multishift_epi64_epi8(cC, _mm512_and_si512(cb, mC)); + __m512i d = _mm512_multishift_epi64_epi8(cD, _mm512_and_si512(cb, mD)); + __m512i packed = _mm512_or_si512(_mm512_or_si512(a, b), + _mm512_or_si512(c, d)); + __m512i compact = _mm512_permutexvar_epi8( + _mm512_load_si512((const __m512i *)pivco_pack_compact_d3), packed); + _mm512_mask_storeu_epi8(out + ((i * 3) >> 3), + (__mmask64)0xFFFFFFULL, compact); + } + return i; +} + +/* D=4 (2 ranks per byte): 2 groups (even/odd), gather + shift + OR. */ + +static inline int pack_d4_avx512(uint8_t *out, const uint8_t *ranks, + int n, uint8_t base) +{ + const __m512i shuf0 = _mm512_set_epi8( + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 62,60,58,56,54,52,50,48, 46,44,42,40,38,36,34,32, + 30,28,26,24,22,20,18,16, 14,12,10, 8, 6, 4, 2, 0); + const __m512i shuf1 = _mm512_set_epi8( + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, + 63,61,59,57,55,53,51,49, 47,45,43,41,39,37,35,33, + 31,29,27,25,23,21,19,17, 15,13,11, 9, 7, 5, 3, 1); + int i = 0; + for (; i + 64 <= n; i += 64) { + /* Local code in [0,2^D); no mask needed (see D=2). */ + __m512i cb = pivco_pack_load_byte(ranks + i, base); + __m512i g0 = _mm512_permutexvar_epi8(shuf0, cb); + __m512i g1 = _mm512_permutexvar_epi8(shuf1, cb); + __m512i packed = _mm512_or_si512(g0, _mm512_slli_epi32(g1, 4)); + _mm512_mask_storeu_epi8(out + ((i * 4) >> 3), + (__mmask64)0xFFFFFFFFULL, packed); + } + return i; +} + +/* D=5/6/7 pack via ryg multiply-as-shift (port of AVX2 a1aa6b9): + * - mask byte-laid ranks to D bits + * - vpmaddubsw c0 word[i] = cb[2i] + cb[2i+1] * 2^D (2D bits) + * - vpmaddwd c1 dword[i] = word[2i] + word[2i+1] * 2^(2D) (4D bits) + * - vpsrlq + vpternlogq 0xE4 to merge dword[2i+1] into dword[2i]'s + * u64 lane: bits [0..4D-1] = dword[2i], bits [4D..8D-1] = dword[2i+1] + * - vpermb compact + masked store + * + * Beats the per-group multishift path on Intel (Granite Rapids -16 to + * -23% cyc/elem, Sapphire Rapids -10 to -18%); ties or marginally loses + * on AMD (Zen 4 D=5 -24% else tied, Zen 5 D=5 -10% / D=6,7 +5%). See + * scratch bench results in the commit message. + * + * For D=5 only, the two-byte mults of vpmaddubsw can produce u16 lanes + * up to 31 + 31*32 = 1023 (fits u16), then vpmaddwd up to 1023 + 1023 * + * 1024 ≈ 1.05M (fits u32) -- 4D = 20 bits is the maximum used. Same + * envelope analysis for D=6 (24 bits) and D=7 (28 bits). */ + +/* D=8: byte-aligned; 64 ranks -> 64 bytes (sub base + store). */ +static inline int pack_d8_avx512(uint8_t *out, const uint8_t *ranks, int n, uint8_t base) +{ + int i = 0; + for (; i + 64 <= n; i += 64) + _mm512_storeu_si512((__m512i *)(out + i), pivco_pack_load_byte(ranks + i, base)); + return i; +} + +#define PIVCO_PACK_AVX512_RYG_DN(NAME, D_VAL, COMPACT_TAB, STORE_MASK) \ +static inline int NAME(uint8_t *out, const uint8_t *ranks, \ + int n, uint8_t base) \ +{ \ + const __m512i c0 = _mm512_set1_epi16( \ + (int16_t)(((1 << (D_VAL)) << 8) | 1)); \ + const __m512i c1 = _mm512_set1_epi32( \ + (int32_t)(((int32_t)1 << (2*(D_VAL))) << 16) | 1); \ + const __m512i c3 = _mm512_set1_epi64( \ + (int64_t)(((int64_t)1 << (4*(D_VAL))) - 1)); \ + int i = 0; \ + for (; i + 64 <= n; i += 64) { \ + /* local code already in [0,2^D) -- no per-byte mask needed */ \ + __m512i cb = pivco_pack_load_byte(ranks + i, base); \ + __m512i x = _mm512_maddubs_epi16(c0, cb); \ + x = _mm512_madd_epi16(x, c1); \ + __m512i xs = _mm512_srli_epi64(x, 32 - 4*(D_VAL)); \ + /* (x & c3) | (xs & ~c3) via vpternlogq 0xE4 */ \ + x = _mm512_ternarylogic_epi64(x, xs, c3, 0xE4); \ + __m512i compact = _mm512_permutexvar_epi8( \ + _mm512_load_si512((const __m512i *)COMPACT_TAB), x); \ + _mm512_mask_storeu_epi8(out + ((i * (D_VAL)) >> 3), \ + (__mmask64)(STORE_MASK), compact); \ + } \ + return i; \ +} +PIVCO_PACK_AVX512_RYG_DN(pack_d5_avx512, 5, pivco_pack_compact_d5, 0xFFFFFFFFFFULL) +PIVCO_PACK_AVX512_RYG_DN(pack_d6_avx512, 6, pivco_pack_compact_d6, 0xFFFFFFFFFFFFULL) +PIVCO_PACK_AVX512_RYG_DN(pack_d7_avx512, 7, pivco_pack_compact_d7, 0x00FFFFFFFFFFFFFFULL) +#undef PIVCO_PACK_AVX512_RYG_DN + +#endif /* PIVCO_HUFFMAN_AVX512_PACK_H */ diff --git a/third_party/pivco/src/pivco_huffman_codec.c b/third_party/pivco/src/pivco_huffman_codec.c new file mode 100644 index 0000000..5c8f63e --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_codec.c @@ -0,0 +1,621 @@ +/* pivco_huffman_codec.c — unified encode + bottom-up decode. + * + * One source file, compiled once per backend tier (CMake passes + * -DPIVCO_BACKEND_{SCALAR,NEON,X86,AVX512}). The tree walk + dispatch + * + wire format are identical across backends; the per-node SIMD work + * lives in pivco_huffman_primitives_.h, selected via the + * router pivco_huffman_primitives.h. + * + * Two responsibilities only: + * + * 1. Walk the Huffman tree (encode recursion + BU decode recursion). + * 2. Read/write per-node wire records via pivco_huffman_wire.h. + * + * Everything backend-shaped (bitmap build, partition, flat-decode, + * merge, etc.) is a `prim_*` call. No vector types here. + * + * The bottom-up decoder is the production path (top-down has been + * parked). Encode is shared. + */ + +#include "pivco_huffman.h" +#include "pivco_huffman_common.h" +#include "pivco_huffman_wire.h" +#include "pivco_huffman_primitives.h" +#include "pivco_prof.h" +#ifdef PIVCO_HAS_FSE +#include "pivco_fse.h" +#endif + +#include +#include +#include "pivco_check.h" + +/* Decode scratch arena, owned by the caller's pivco_decoder_t (the + * pivco_scratch_t behind its `internal` pointer). Preallocated at + * context create for PIVCO_WIRE_MAX_N and reused across blocks (the + * ensure below still grows it if ever needed; never shrinks) -- block + * size stays a pure runtime parameter (the wire N header carries the + * per-block count). */ + + +/* The returned base is aligned to a 16 KiB boundary plus 64*39 bytes — + * see DECODE_SCRATCH_ALIGN / DECODE_SCRATCH_SHIFT in + * pivco_huffman_common.h for the page-split rationale. */ +static uint8_t *decode_scratch_ensure(pivco_scratch_t *sc, size_t need) +{ + need += DECODE_SCRATCH_ALIGN + DECODE_SCRATCH_SHIFT; + if (need > sc->dec_cap) { + uint8_t *p = (uint8_t *)realloc(sc->dec, need); + if (!p) return NULL; + sc->dec = p; + sc->dec_cap = need; + } + uintptr_t p = (uintptr_t)sc->dec; + p = ((p + DECODE_SCRATCH_ALIGN - 1) & ~(DECODE_SCRATCH_ALIGN - 1)) + + DECODE_SCRATCH_SHIFT; + return (uint8_t *)p; +} + +/* MERGE_OVERREAD: the SIMD merges load their source buffers in + * full-vector chunks, so they may read (never write) up to this many + * bytes past a source's end. Every buffer the decode walk hands to a + * merge therefore needs this much trailing slack inside the arena; the + * caller's `symbols` buffer, which guarantees none, is only ever a + * merge destination (writes are exact). */ +#define MERGE_OVERREAD ((size_t)PIVCO_PRIM_MERGE_OVERREAD) + +/* Growable encode scratch arena, owned by the caller's pivco_encoder_t + * (the pivco_scratch_t behind its `internal` pointer): holds the + * per-block ranks buffer + the tree-walk's right-half recursion + * scratch, preallocated at context create and reused across blocks + * (never shrinks), so a block-loop encode doesn't malloc/free per + * block. The former thread_local stopgap (and its thread-death leak) + * is gone -- ownership and lifetime are the context's. */ + + +static uint8_t *encode_scratch_ensure(pivco_scratch_t *sc, size_t need) +{ + if (need > sc->enc_cap) { + uint8_t *p = (uint8_t *)realloc(sc->enc, need); + if (!p) return NULL; + sc->enc = p; + sc->enc_cap = need; + } + return sc->enc; +} + +/* ---------- FSE dispatch parameters ---------- + * + * The thresholds match the NEON encoder's settings so the wire format + * is byte-identical across backends: same skew threshold, same per- + * codeword cost gate, same minimum bitmap size. See docs/FSE-V0.md for the + * derivation of each value. Overridable at build time via -D... . */ +#ifndef PIVCO_FSE_MIN_THRESHOLD +#define PIVCO_FSE_MIN_THRESHOLD 0.625 +#endif +#ifndef PIVCO_FSE_MIN_RATIO +#define PIVCO_FSE_MIN_RATIO 0.95 +#endif +#ifndef PIVCO_FSE_MIN_BITMAP_BYTES +#define PIVCO_FSE_MIN_BITMAP_BYTES 32 +#endif + +/* FSE per-table-id stats live in src/pivco_huffman.c (backend-neutral + * TU) so the symbols resolve regardless of backend. codec.c writes + * the counters every time it commits or rejects an FSE attempt. */ +extern uint64_t g_pivco_fse_commit [PIVCO_FSE_STATS_SLOTS]; +extern uint64_t g_pivco_fse_attempt [PIVCO_FSE_STATS_SLOTS]; +extern uint64_t g_pivco_fse_bytes_in [PIVCO_FSE_STATS_SLOTS]; +extern uint64_t g_pivco_fse_bytes_out[PIVCO_FSE_STATS_SLOTS]; + +/* ---------- Backend → entry-point name ---------- */ + +#if defined(PIVCO_BACKEND_SCALAR) +# define CODEC_ENCODE_ENTRY pivco_encode_scalar +# define CODEC_DECODE_ENTRY pivco_decode_scalar +#elif defined(PIVCO_BACKEND_NEON) +# define CODEC_ENCODE_ENTRY pivco_encode_neon +# define CODEC_DECODE_ENTRY pivco_decode_bu_neon +#elif defined(PIVCO_BACKEND_X86) +# define CODEC_ENCODE_ENTRY pivco_encode_x86 +# define CODEC_DECODE_ENTRY pivco_decode_bu_x86 +#elif defined(PIVCO_BACKEND_AVX512) +# define CODEC_ENCODE_ENTRY pivco_encode_avx512 +# define CODEC_DECODE_ENTRY pivco_decode_bu_avx512 +#else +# error "pivco_huffman_codec.c needs PIVCO_BACKEND_{SCALAR,NEON,X86,AVX512}" +#endif + +/* ---------- Encode tree walk ---------- * + * + * DFS, emitting records in decompression order (an Euler walk): the + * partition runs at node entry (it routes the ranks the recursion + * needs) and the K_right header is written there too, but the node's + * marker+bitmap record is emitted after the children's regions — + * exactly where the decoder's merge consumes it, so the decoder reads + * the stream strictly forward. At each non-flat internal node, + * `ranks[0..n)` holds the surviving leaves' in-order ranks; partition + * routes each by `rank > split_rank[node]`, leaving the left half in + * place in `ranks[0..n_left)` and compacting the right half into + * `tmp[0..n_right)`. The recursion descends left on `ranks`, right on + * `tmp`. The bitmap is staged in a stack buffer across the recursion + * (its final stream position depends on the children's — FSE-variable — + * encoded sizes, so it can't be written in place up front); + * ≤ bitmap_bytes(N)+64 per level, tree height ≤ PIVCO_MAX_CODE_LEN + * levels. */ + +/* Arch-agnostic FSE attempt on a freshly-built raw bitmap. + * + * Inputs: + * marker_slot — points at the 1-byte FSE marker (currently 0 = raw) + * bm — points at the ceil(n/8)-byte raw bitmap region + * immediately after the marker + * nbytes — bitmap_bytes(n) + * n / n_left / n_right — partition counts (for the skew test) + * depth — for the codeword-cost gate + * out_ptr — cursor; advanced past the FSE payload on commit + * + * On commit: rewrites *marker_slot, replaces bm with [fse_len:u16 + * LE][fse_payload], advances *out_ptr to one past the payload. Stats + * (g_pivco_fse_*) are bumped. + * + * On no-commit / no-attempt: stream and stats untouched. + * + * No-op when PIVCO_HAS_FSE is not defined. */ +static inline void codec_maybe_fse_attempt(int fse_on, uint8_t *marker_slot, + uint8_t *bm, int nbytes, + int n, int n_left, int n_right, + int depth, uint8_t **out_ptr) +{ +#ifdef PIVCO_HAS_FSE + if (!fse_on) return; + if (nbytes < PIVCO_FSE_MIN_BITMAP_BYTES) return; + + int n_major = (n_left >= n_right) ? n_left : n_right; + double p_major = (n > 0) ? (double)n_major / (double)n : 0.0; + if (p_major < PIVCO_FSE_MIN_THRESHOLD) return; + + int t_id = pivco_fse_select_table(p_major); + if (t_id < 1) return; + PIVCO_CHECK(t_id < PIVCO_FSE_STATS_SLOTS); /* guards the g_pivco_fse_* indexing */ + + int xor_flag = (n_right > n_left); + uint8_t scratch[(size_t)nbytes + 16]; + if (xor_flag) { + for (int i = 0; i < nbytes; i++) scratch[i] = (uint8_t)~bm[i]; + } else { + memcpy(scratch, bm, (size_t)nbytes); + } + + uint8_t fse_out[(size_t)nbytes + 64]; + size_t fse_len = 0; + pivco_fse_status_t rc = pivco_fse_compress(t_id, scratch, (size_t)nbytes, + fse_out, sizeof(fse_out), + &fse_len); + g_pivco_fse_attempt[t_id]++; + if (rc != PIVCO_FSE_OK) { + g_pivco_fse_commit[0]++; /* slot 0 = attempted, rejected */ + return; + } + /* Per-codeword commit gate (see docs/FSE-V0.md): + * raw: every codeword through this node costs (depth + 1) bits + * fse: (depth + (fse_len + 2 wire-prefix) * 8 / n) bits + * Commit iff (depth + fse_frac) <= MIN_RATIO * (depth + 1). */ + double fse_frac = (double)(fse_len + 2) * 8.0 / (double)n; + double codeword_ratio = ((double)depth + fse_frac) / + ((double)depth + 1.0); + if (codeword_ratio > (double)PIVCO_FSE_MIN_RATIO) { + g_pivco_fse_commit[0]++; + return; + } + + /* Commit: rewrite marker + bitmap region with [fse_len][payload], + * adjust the wire cursor to one past the payload. */ + *marker_slot = (uint8_t)((xor_flag ? 0x80 : 0) | t_id); + uint8_t *p = bm; + *p++ = (uint8_t)( fse_len & 0xFF); + *p++ = (uint8_t)((fse_len >> 8) & 0xFF); + memcpy(p, fse_out, fse_len); + *out_ptr = p + fse_len; + + g_pivco_fse_commit [t_id]++; + g_pivco_fse_bytes_in [t_id] += (uint64_t)nbytes; + g_pivco_fse_bytes_out[t_id] += (uint64_t)(fse_len + 3); +#else + (void)marker_slot; (void)bm; (void)nbytes; + (void)n; (void)n_left; (void)n_right; (void)depth; (void)out_ptr; +#endif +} + +static void codec_encode_node(const pivco_table_t *table, + int16_t node_id, + uint8_t *ranks, int n, + int depth, + uint8_t **out_ptr, + uint8_t *tmp) +{ + if (n == 0) return; + PROF_COUNT_ONLY(PROF_ENC_NODE_VISIT, n); + + const pivco_tree_node_t *node = &table->tree[node_id]; + if (node->symbol >= 0) return; /* leaf — nothing to emit */ + + /* Flat-subtree fast path: pack n*D bits, no marker, no K_right. */ + if (table->flat_depth[node_id] >= 2) { + int D = table->flat_depth[node_id]; + int total_bytes = (n * D + 7) >> 3; + PROF_TIC(); + prim_enc_pack_dN(ranks, n, D, table->flat_base_rank[node_id], *out_ptr, + table->flat_layout); + PROF_TOC(PROF_ENC_FLAT, n); + *out_ptr += total_bytes; + return; + } + + /* Non-flat internal node. codec.c owns: K_right header, FSE marker + * byte, optional FSE-attempt on the raw bitmap. The arch-specific + * primitive does only the SIMD-bound work: build the raw bitmap and + * partition the ranks. + * + * The bitmap is built into a stack staging buffer (+64 slack + * absorbs the SIMD partitions' over-wide tail stores) and copied + * into the stream after the children's regions. */ + int nbytes = bitmap_bytes(n); + uint8_t bm_stage[(size_t)nbytes + 64]; + + /* Pick the partition variant by node_type, mirroring the decode-side + * dispatch. The bitmap (and thus the wire bytes) is identical across + * variants; only the encode-internal scatter work differs — a leaf child + * never reads its scattered side, so that side's scatter is skipped: + * BOTH_LEAVES stores nothing, LEAF_LEFT only the right (compacted into + * tmp), FULL both. */ + uint8_t thr = table->split_rank[node_id]; + int n_right; + PROF_TIC(); + switch ((pivco_node_type_t)table->node_type[node_id]) { + case PIVCO_NODE_BOTH_LEAVES: + n_right = prim_enc_partition_none(ranks, n, thr, bm_stage); break; + case PIVCO_NODE_LEAF_LEFT: + n_right = prim_enc_partition_right(ranks, n, thr, bm_stage, tmp); break; + default: + n_right = prim_enc_partition_full(ranks, n, thr, bm_stage, tmp); break; + } + PROF_TOC(PROF_ENC_NODE_FULL, n); + int n_left = n - n_right; + + /* One K_right header per recursion site, consumed by the decoder + * at node entry so it can size both children before their regions + * arrive. */ + wire_write_kr_header(table, node_id, out_ptr, n_right); + + /* Emit the larger-K child's region first: the decoder can then + * decode it into scratch that the smaller, not-yet-decoded + * sibling's buffer overlaps (hole-reuse), shrinking the arena + * high-water. The two rank buffers (ranks=left, tmp=right) and + * the shared deeper scratch tmp+n_right are mutually disjoint, so + * the call order is free. A leaf child emits nothing, so this + * only changes the stream at INTERNAL_FULL nodes — exactly where + * the decoder reorders. */ + if (n_right > n_left) { + codec_encode_node(table, node->right, tmp, n_right, depth + 1, + out_ptr, tmp + n_right); + codec_encode_node(table, node->left, ranks, n_left, depth + 1, + out_ptr, tmp + n_right); + } else { + codec_encode_node(table, node->left, ranks, n_left, depth + 1, + out_ptr, tmp + n_right); + codec_encode_node(table, node->right, tmp, n_right, depth + 1, + out_ptr, tmp + n_right); + } + + /* Emit this node's record: marker + staged bitmap. The FSE attempt + * may rewrite marker+bm in place with [fse_len][payload] and pull + * *out_ptr back to the payload end. No-op otherwise. */ + uint8_t *marker_slot = *out_ptr; + *marker_slot = 0; + *out_ptr += 1; + uint8_t *bm = *out_ptr; + memcpy(bm, bm_stage, (size_t)nbytes); + *out_ptr += nbytes; + codec_maybe_fse_attempt(table->fse_enabled, marker_slot, bm, nbytes, + n, n_left, n_right, depth, out_ptr); +} + +int CODEC_ENCODE_ENTRY(pivco_encoder_t *enc_ctx, const pivco_table_t *table, const uint8_t *symbols, size_t n, uint8_t *out, size_t *out_len) +{ + if (!symbols || !table || !out || !out_len) return PIVCO_ERR_NULL; + if (n == 0 || n > PIVCO_WIRE_MAX_N) return PIVCO_ERR_OVERFLOW; + prim_codec_init(); + + const int N = (int)n; + + /* Block header: write N as the first 2 bytes so the decoder can + * recover it without an out-of-band channel. */ + uint8_t *ptr = out; + wire_write_block_n(ptr, N); + ptr += PIVCO_BLOCK_N_BYTES; + + /* One heap block: the per-block ranks buffer + the recursion's right-half + * scratch (see the tree-walk note above). +64 slack on ranks absorbs the + * SIMD partition's over-wide (16/64-byte) tail store at end-of-buffer; the + * scratch holds one right-half per recursion level, hence (MAX_CODE_LEN+2)*N. */ + const size_t ranks_capacity = (size_t)N + 64; + const size_t tmp_capacity = (size_t)N * (PIVCO_MAX_CODE_LEN + 2); + uint8_t *ranks = encode_scratch_ensure((pivco_scratch_t *)enc_ctx->internal, + ranks_capacity + tmp_capacity); + if (!ranks) return PIVCO_ERR_NULL; + uint8_t *tmp = ranks + ranks_capacity; + + /* ranks[i] = in-order rank of symbols[i] (gather table->sym_to_rank). */ + PROF_COUNT_ONLY(PROF_ENC_ENTRY, N); + PROF_TIC(); + prim_enc_init(ranks, N, symbols, table->sym_to_rank, &table->enc_init_aux); + PROF_TOC(PROF_ENC_INIT, N); + + codec_encode_node(table, table->tree_root, ranks, N, 0, &ptr, tmp); + + *out_len = (size_t)(ptr - out); + return PIVCO_OK; +} + +/* ---------- Bottom-up decode tree walk (ping-pong scratch) ---------- * + * + * Each call decodes a subtree's K symbols into out[0,K). Internal + * nodes recurse into their children, then merge per the node's bitmap. + * The flat-subtree fast path bypasses recursion entirely. + * + * The wire is in decompression order — larger-K child first — so the + * input cursor is consumed strictly forward and each record is loaded + * exactly where it is used: the K_right header at node entry, the + * children's regions during their recursion, the node's bitmap right + * before its merge. + * + * Scratch placement is a two-buffer ping-pong (out, tmp): + * + * - the larger child decodes in place into out's tail + * out[K_small, K): safe under the merge, whose write cursor can + * never overtake its tail-side read cursor (by the time it writes + * out[i] it has consumed at least i - K_small tail bytes); + * - the smaller child decodes into tmp[0, K_small), and its own + * recursion uses out's still-empty prefix out[0, K_small) as its + * partner — the pair (tmp, out-prefix) ping-pongs down the + * smaller-child spine. + * + * The caller guarantees tmp capacity floor(K/2): a node's smaller child + * is at most floor(K/2), and everything a smaller child's subtree puts + * in its partner stays inside out[0, K_small). The walk's footprint is + * therefore out[0,K), plus at most floor(K/2) bytes past tmp (the + * largest smaller-child on the larger-child spine), plus MERGE_OVERREAD + * read slack past whichever region ends last. + * + * Dispatch on node_type, computed at build-table time (by children's + * leafness — a leaf child's symbol goes straight into the parent's + * merge, so the walk never recurses into a leaf): + * + * INTERNAL_FLAT — packed-bits flat decode into out + * BOTH_LEAVES — both children leaves, merge_cst_cst directly + * LEAF_LEFT — left child leaf, recurse right (in place, into + * out's tail), merge_cst_vec + * INTERNAL_FULL — both children internal: larger child in place, + * smaller via the ping-pong partner, merge_vec_vec */ + +static void codec_decode_subtree(const pivco_table_t *table, + int16_t node_id, int K, + uint8_t *out, uint8_t *tmp, + const uint8_t **in_ptr) +{ + if (K == 0) return; + + const pivco_tree_node_t *node = &table->tree[node_id]; + + switch ((pivco_node_type_t)table->node_type[node_id]) { + + case PIVCO_NODE_LEAF: + /* Unreachable: every parent consumes a leaf child via its + * cst_* merge instead of recursing into it. */ + pivco_check_fail("codec_decode_subtree dispatched on a leaf", + __FILE__, __LINE__); + + case PIVCO_NODE_INTERNAL_FLAT: { + int D = table->flat_depth[node_id]; + int total_bytes = (K * D + 7) >> 3; + const uint8_t *bm = *in_ptr; + *in_ptr += total_bytes; + const uint8_t *c2s = + &table->flat_code_to_sym[table->flat_offset[node_id]]; + prim_merge_flat(out, K, bm, D, c2s, table->flat_layout); + return; + } + + case PIVCO_NODE_BOTH_LEAVES: { + /* No K_right header (kr_header_needed returns false). */ + uint8_t bm_scratch[(size_t)bitmap_bytes(K) + 16]; + const uint8_t *bm = wire_read_bitmap(in_ptr, K, bm_scratch); + prim_merge_cst_cst(bm, K, + (uint8_t)table->tree[node->left].symbol, + (uint8_t)table->tree[node->right].symbol, + out); + return; + } + + case PIVCO_NODE_LEAF_LEFT: { + /* One internal child (right); the leaf contributes the K_left + * symbols the merge fills into out's prefix. The right child + * decodes in place into out's tail, whatever its share of K. */ + int K_right = wire_read_kr_header(table, node_id, in_ptr); + uint8_t *right_buf = out + (K - K_right); + codec_decode_subtree(table, node->right, K_right, + right_buf, tmp, in_ptr); + + uint8_t bm_scratch[(size_t)bitmap_bytes(K) + 16]; + const uint8_t *bm = wire_read_bitmap(in_ptr, K, bm_scratch); + prim_merge_cst_vec(bm, K, + (uint8_t)table->tree[node->left].symbol, + right_buf, out); + return; + } + + case PIVCO_NODE_INTERNAL_FULL: + default: { + /* Both children internal. Stream order == decode order == + * larger first (strict >, ties left-first — must match the + * encoder). */ + int K_right = wire_read_kr_header(table, node_id, in_ptr); + int K_left = K - K_right; + uint8_t *left_buf, *right_buf; + if (K_right > K_left) { + right_buf = out + K_left; /* larger, in place */ + left_buf = tmp; /* smaller, ping-pong */ + codec_decode_subtree(table, node->right, K_right, + right_buf, tmp, in_ptr); + codec_decode_subtree(table, node->left, K_left, + left_buf, out, in_ptr); + } else { + left_buf = out + K_right; /* larger, in place */ + right_buf = tmp; /* smaller, ping-pong */ + codec_decode_subtree(table, node->left, K_left, + left_buf, tmp, in_ptr); + codec_decode_subtree(table, node->right, K_right, + right_buf, out, in_ptr); + } + + uint8_t bm_scratch[(size_t)bitmap_bytes(K) + 16]; + const uint8_t *bm = wire_read_bitmap(in_ptr, K, bm_scratch); + prim_merge_vec_vec(bm, K, left_buf, right_buf, out); + return; + } + } +} + +int CODEC_DECODE_ENTRY(pivco_decoder_t *dec_ctx, const pivco_table_t *table, const uint8_t *in, size_t in_len, uint8_t *symbols, size_t *consumed) +{ + if (!in || !table || !symbols || !consumed) return PIVCO_ERR_NULL; + (void)in_len; + prim_codec_init(); + + /* Block header: first 2 bytes are N (symbol count for this block). */ + const uint8_t *ptr = in; + const int N = wire_read_block_n(&ptr); + if (N <= 0 || N > PIVCO_WIRE_MAX_N) return PIVCO_ERR_CORRUPT; + const pivco_tree_node_t *root = &table->tree[table->tree_root]; + + /* Root-is-leaf: fill everything with the single symbol. */ + if (root->symbol >= 0) { + memset(symbols, (uint8_t)root->symbol, (size_t)N); + *consumed = 0; + return PIVCO_OK; + } + + /* Fast path: BOTH_LEAVES at root — a 2-symbol (or single-symbol) + * tree, where the whole block collapses to "read the K-bit + * partition, blend two symbols". Skips the recursive + * codec_decode_subtree machinery (switch dispatch + bm_scratch + * stack frame + scratch TLS reference / arena ensure). Worth −26% + * on two_sym decode on older narrow x86 (IvyBridge), noise on + * modern hosts. TODO: consider removing this extreme-case + * optimization. */ + if ((pivco_node_type_t)table->node_type[table->tree_root] + == PIVCO_NODE_BOTH_LEAVES) { + uint8_t bm_scratch[(size_t)bitmap_bytes(N) + 16]; + const uint8_t *bm = wire_read_bitmap(&ptr, N, bm_scratch); + const pivco_tree_node_t *left_child = &table->tree[root->left]; + const pivco_tree_node_t *right_child = &table->tree[root->right]; + prim_merge_cst_cst(bm, N, + (uint8_t)left_child->symbol, + (uint8_t)right_child->symbol, + symbols); + *consumed = (size_t)(ptr - in); + return PIVCO_OK; + } + + /* Flat root: the whole tree is one packed-bits region, decoded + * straight into symbols (exact writes, no merge, no scratch). */ + if ((pivco_node_type_t)table->node_type[table->tree_root] + == PIVCO_NODE_INTERNAL_FLAT) { + int D = table->flat_depth[table->tree_root]; + int total_bytes = (N * D + 7) >> 3; + const uint8_t *bm = ptr; + ptr += total_bytes; + prim_merge_flat(symbols, N, bm, D, + &table->flat_code_to_sym[table->flat_offset[table->tree_root]], + table->flat_layout); + *consumed = (size_t)(ptr - in); + return PIVCO_OK; + } + + /* Remaining root shapes recurse. The ping-pong walk parks children + * in its out buffer's tail and prefix and the merges read their + * sources with up-to-MERGE_OVERREAD slack past the end — guarantees + * the caller's `symbols` doesn't offer. So the root's children + * decode into the context's arena (preallocated, reused + * across blocks) and only the root's own merge, whose writes are + * exact, targets `symbols`. + * + * Arena bound: (MAX_CODE_LEN+2)·N, loose. The ping-pong walk's + * true high-water is under 1.5·N — but only on valid streams, so + * that figure is not a safe allocation target. Against invalid + * data (a bitmap popcount disagreeing with the declared child + * counts) the floor is 2·N plus kernel-overtouch pad: with splits + * bounded (0 <= KR <= K — not currently checked), every recursive + * (out, tmp) placement satisfies out + 2K <= 2·N and tmp + K <= + * 2·N as arena offsets, which caps hostile cursor excursions too. */ + size_t need = (size_t)N * (PIVCO_MAX_CODE_LEN + 2) + MERGE_OVERREAD; + + if ((pivco_node_type_t)table->node_type[table->tree_root] + == PIVCO_NODE_LEAF_LEFT) { + /* One internal child: it decodes at the arena base with the + * space after it as ping-pong partner; the cst_vec merge fills + * symbols. */ + int K_right = wire_read_kr_header(table, table->tree_root, &ptr); + uint8_t *scratch = decode_scratch_ensure((pivco_scratch_t *)dec_ctx->internal, need); + if (!scratch) return PIVCO_ERR_NULL; + codec_decode_subtree(table, root->right, K_right, + scratch, scratch + K_right, &ptr); + + uint8_t bm_scratch[(size_t)bitmap_bytes(N) + 16]; + const uint8_t *bm = wire_read_bitmap(&ptr, N, bm_scratch); + prim_merge_cst_vec(bm, N, + (uint8_t)table->tree[root->left].symbol, + scratch, symbols); + *consumed = (size_t)(ptr - in); + return PIVCO_OK; + } + + /* INTERNAL_FULL root — hybrid hole-reuse placement. Both children + * decode into the arena's first N bytes, [larger | smaller], and + * the final merge writes symbols. The larger child (first on the + * wire) uses the smaller sibling's still-empty slot as its + * ping-pong partner — hole-reuse; when a smaller-child on its spine + * outgrows that slot, the partner writes spill past N into fresh + * arena. The smaller root child then decodes into its slot with a + * fresh partner beyond N. */ + int K_right = wire_read_kr_header(table, table->tree_root, &ptr); + int K_left = N - K_right; + uint8_t *scratch = decode_scratch_ensure((pivco_scratch_t *)dec_ctx->internal, need); + if (!scratch) return PIVCO_ERR_NULL; + + uint8_t *buf_left, *buf_right; + if (K_right > K_left) { /* right larger -> first on the wire */ + buf_right = scratch; + buf_left = scratch + K_right; + codec_decode_subtree(table, root->right, K_right, + buf_right, /*tmp=*/buf_left, &ptr); + codec_decode_subtree(table, root->left, K_left, + buf_left, /*tmp=*/scratch + N, &ptr); + } else { + buf_left = scratch; + buf_right = scratch + K_left; + codec_decode_subtree(table, root->left, K_left, + buf_left, /*tmp=*/buf_right, &ptr); + codec_decode_subtree(table, root->right, K_right, + buf_right, /*tmp=*/scratch + N, &ptr); + } + + uint8_t bm_scratch[(size_t)bitmap_bytes(N) + 16]; + const uint8_t *bm = wire_read_bitmap(&ptr, N, bm_scratch); + prim_merge_vec_vec(bm, N, buf_left, buf_right, symbols); + + *consumed = (size_t)(ptr - in); + return PIVCO_OK; +} diff --git a/third_party/pivco/src/pivco_huffman_common.h b/third_party/pivco/src/pivco_huffman_common.h new file mode 100644 index 0000000..7eb1eda --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_common.h @@ -0,0 +1,138 @@ +#ifndef PIVCO_HUFFMAN_COMMON_H +#define PIVCO_HUFFMAN_COMMON_H + +#include "pivco_huffman.h" +#include "pivco_huffman_vertical.h" +#include + +/* ---------- Bitmap utilities ---------- */ + +/* Popcount of a bitmap stored as bytes. n_bytes = ceil(count / 8). */ +static inline int fast_popcount(const uint8_t *bitmap, int n_bytes) +{ + int count = 0; + int i = 0; + /* Process 8 bytes at a time */ + for (; i + 8 <= n_bytes; i += 8) { + uint64_t w; + memcpy(&w, bitmap + i, 8); + count += __builtin_popcountll(w); + } + /* Remaining bytes */ + for (; i < n_bytes; i++) { + count += __builtin_popcount(bitmap[i]); + } + return count; +} + +/* Extract indices of set bits in bitmap into active[]. + Returns number of set bits. */ +static inline int bitmap_extract(const uint8_t *bitmap, int n_bytes, + const uint16_t *old_active, + uint16_t *new_active) +{ + int out = 0; + int bit_idx = 0; + for (int i = 0; i < n_bytes; i++) { + uint8_t byte = bitmap[i]; + while (byte) { + int bit = __builtin_ctz(byte); + new_active[out++] = old_active[bit_idx + bit]; + byte &= byte - 1; /* clear lowest set bit */ + } + bit_idx += 8; + } + return out; +} + +/* Same but active[j] == j (identity mapping) */ +static inline int bitmap_extract_identity(const uint8_t *bitmap, int n_bytes, + uint16_t *new_active) +{ + int out = 0; + int bit_idx = 0; + for (int i = 0; i < n_bytes; i++) { + uint8_t byte = bitmap[i]; + while (byte) { + int bit = __builtin_ctz(byte); + new_active[out++] = (uint16_t)(bit_idx + bit); + byte &= byte - 1; + } + bit_idx += 8; + } + return out; +} + +/* Set bit j in bitmap */ +static inline void bitmap_set(uint8_t *bitmap, int j) +{ + bitmap[j >> 3] |= (1u << (j & 7)); +} + +/* Get bit j from bitmap */ +static inline int bitmap_get(const uint8_t *bitmap, int j) +{ + return (bitmap[j >> 3] >> (j & 7)) & 1; +} + +/* Bytes needed for a bitmap of n bits */ +static inline int bitmap_bytes(int n) +{ + return (n + 7) >> 3; +} + +/* K_right wire-format header decision (2026-05-12). + * + * Wire format: at each non-flat internal node that recurses into at + * least one non-leaf child, the encoder writes a 2-byte little-endian + * uint16 K_right header at node entry. The BU decoder reads it there + * instead of running popcount, and uses it to size both children + * before their regions arrive. + * + * Condition: node has at least one child that's NOT a leaf. Encodes the + * exact set of popcount call sites in the BU decoder. Both-leaf cases + * and HALF_*-with-leaf cases get no header (decoder uses merge_cst_cst + * directly). + * + * The "needs header" decision is a pure function of the tree topology and + * matches across encoder and decoder via this shared helper. */ +static inline int kr_header_needed(const pivco_table_t *table, + int16_t node_id) +{ + const pivco_tree_node_t *n = &table->tree[node_id]; + if (n->symbol >= 0) return 0; /* leaf */ + if (table->flat_depth[node_id] >= 2) return 0; /* flat path */ + return (table->tree[n->left].symbol < 0) + || (table->tree[n->right].symbol < 0); +} + +#define KR_HEADER_BYTES 2 /* uint16 little-endian */ + +/* Context-owned scratch (the former per-thread arenas). Lives behind + * the public contexts' `internal` pointer; grown by the codec's ensure + * helpers, preallocated for PIVCO_WIRE_MAX_N at context create. */ +typedef struct { + uint8_t *enc; size_t enc_cap; /* encode ranks + recursion tmp */ + uint8_t *dec; size_t dec_cap; /* decode ping-pong arena */ +} pivco_scratch_t; + +#define PIVCO_ENC_SCRATCH_BYTES(N) \ + ((size_t)(N) + 64 + (size_t)(N) * (PIVCO_MAX_CODE_LEN + 2) \ + + PIVCO_PRIM_HIST_SCRATCH_MAX) +#define PIVCO_PRIM_HIST_SCRATCH_MAX (4 * 16 * 1024 + 64) +#define PIVCO_DEC_SCRATCH_BYTES(N) \ + ((size_t)(N) * (PIVCO_MAX_CODE_LEN + 2) + 64) + +/* The decode arena's usable base is aligned to a 16 KiB boundary plus + * 64*39 bytes. The in-place walk parks every nested child chain so it + * ends exactly at base + N; when that address sits within [-16, +256) + * of a 16 KiB page boundary, the merge tails' 16 B over-reads become + * page-split loads, ~25 cycles each on Apple M4 (page size == 16 KiB == + * the block size, so a page-aligned allocation hits the band + * deterministically — dna_fasta PH -11% E2E). The odd 64-aligned + * offset pins the chain end mid-page on every host. Allocations must + * carry this much extra headroom (see decode_scratch_ensure). */ +#define DECODE_SCRATCH_ALIGN ((uintptr_t)16384) /* Apple page size */ +#define DECODE_SCRATCH_SHIFT (64 * 39) + +#endif /* PIVCO_HUFFMAN_COMMON_H */ diff --git a/third_party/pivco/src/pivco_huffman_hist_scalar.h b/third_party/pivco/src/pivco_huffman_hist_scalar.h new file mode 100644 index 0000000..44fb9ab --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_hist_scalar.h @@ -0,0 +1,42 @@ +/* pivco_hist_scalar.h — the shared scalar histogram core. + * + * Included by every backend primitives header; backends without their + * own implementation alias prim_histogram_chunk to this (currently + * scalar, NEON and SSE4.1 — an 8-cursor u64-load variant measured on + * par on M4/Graviton4/Zen3, so one portable core suffices; AVX-512 has + * its own, see primitives_avx512.h). Not a backend header itself: no + * prim_* aliases here, specialized name only. + * + * Contract: see prim_histogram_chunk in pivco_primitives.h. + */ +#ifndef PIVCO_HUFFMAN_HIST_SCALAR_H +#define PIVCO_HUFFMAN_HIST_SCALAR_H + +#include +#include + +#define PIVCO_PRIM_HIST_CHUNK ((size_t)1 << 30) +#define PIVCO_PRIM_HIST_SCRATCH (4 * 16 * 1024 + 64) + +/* 4 interleaved u32 sub-histograms break the same-bucket + * read-modify-write dependence that caps the naive loop at L1 + * store-to-load latency; the 4 KB working set stays L1-resident. */ +static inline void histogram_chunk_scalar(const uint8_t *in, size_t n, + uint32_t hist[256], + uint8_t *scratch) +{ + (void)scratch; + uint32_t h0[256] = {0}, h1[256] = {0}, h2[256] = {0}, h3[256] = {0}; + size_t i = 0; + for (; i + 4 <= n; i += 4) { + h0[in[i + 0]]++; + h1[in[i + 1]]++; + h2[in[i + 2]]++; + h3[in[i + 3]]++; + } + for (; i < n; i++) h0[in[i]]++; + for (int s = 0; s < 256; s++) + hist[s] += h0[s] + h1[s] + h2[s] + h3[s]; +} + +#endif /* PIVCO_HUFFMAN_HIST_SCALAR_H */ diff --git a/third_party/pivco/src/pivco_huffman_neon_flat.h b/third_party/pivco/src/pivco_huffman_neon_flat.h new file mode 100644 index 0000000..3fa3600 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_neon_flat.h @@ -0,0 +1,259 @@ +/* pivco_huffman_neon_flat.h — flat-subtree D-bit code unpackers (NEON). + * + * Internal header. Each `flat_dN_unpack()` reads N D-bit codes from a + * packed bitstream and returns them in NEON vector lanes (one byte per + * code, value < 2^D). Used by the production decoder + * (pivco_huffman_neon.c) and the per-D microbench (bench/bench_micro.c). + * + * All helpers + unpack tables live here so the two TUs share a single + * source of truth without giving up inlining. Tables are `static const` + * (per-TU) and helpers are `static inline` — values fold into the + * inlined function and no extern symbols are emitted. + * + * Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_NEON_FLAT_H +#define PIVCO_HUFFMAN_NEON_FLAT_H + +#ifndef __aarch64__ +#error "pivco_huffman_neon_flat.h requires aarch64 NEON" +#endif + +#include +#include +#include + +/* D=2 unpack constants: each byte of input holds 4 codes; replicate each + * input byte to 4 output lanes, then right-shift lane k by 2k to align + * the desired 2-bit code at the low bits. */ +static const uint8_t flat_d2_dup_tab[16] = { + 0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3 +}; +static const int8_t flat_d2_shift_tab[16] = { + 0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6, 0,-2,-4,-6 +}; + +/* Unpack 16 consecutive D=2 codes from 4 bytes of bm into a 16-lane byte + * vector (values 0..3). */ +static inline uint8x16_t flat_d2_unpack(const uint8_t *bm_ptr) +{ + uint32_t packed; + memcpy(&packed, bm_ptr, 4); + uint8x16_t bm_lo = vreinterpretq_u8_u32( + vsetq_lane_u32(packed, vdupq_n_u32(0), 0)); + uint8x16_t dup = vqtbl1q_u8(bm_lo, vld1q_u8(flat_d2_dup_tab)); + uint8x16_t shifted = vshlq_u8(dup, vld1q_s8(flat_d2_shift_tab)); + return vandq_u8(shifted, vdupq_n_u8(0x03)); +} + +/* D=3 unpack: 3 bytes = 24 bits = 8 codes. Two of the 8 codes cross a + * byte boundary, so we work in uint16 lanes (each holding a 16-bit + * window with enough bits to shift out any one 3-bit code). */ +static const uint8_t flat_d3_shuf_tab[16] = { + /* 5 lanes of (b0, b1): for codes 0..4 (shifts 0,3,6,9,12) */ + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + /* 3 lanes of (b1, b2): for codes 5..7 (shifts 7,10,13) */ + 1, 2, 1, 2, 1, 2 +}; +static const int16_t flat_d3_shift_tab[8] = { + 0, -3, -6, -9, -12, -7, -10, -13 +}; + +/* D=3 _safe: byte-wise vsetq_lane_u8 loads, exact 3-byte read. Use for + * the final iteration where the bm region may end within the next + * 16 bytes after bm_ptr. Same shuf+shift as _fast. */ +static inline uint8x8_t flat_d3_unpack_safe(const uint8_t *bm_ptr) +{ + uint8x16_t bm_lo = vdupq_n_u8(0); + bm_lo = vsetq_lane_u8(bm_ptr[0], bm_lo, 0); + bm_lo = vsetq_lane_u8(bm_ptr[1], bm_lo, 1); + bm_lo = vsetq_lane_u8(bm_ptr[2], bm_lo, 2); + uint8x16_t shuffled = vqtbl1q_u8(bm_lo, vld1q_u8(flat_d3_shuf_tab)); + uint16x8_t w = vreinterpretq_u16_u8(shuffled); + uint16x8_t shifted = vshlq_u16(w, vld1q_s16(flat_d3_shift_tab)); + uint16x8_t masked = vandq_u16(shifted, vdupq_n_u16(0x07)); + return vmovn_u16(masked); +} + +/* D=3 _fast: single 16-byte vector load. Reads 13 bytes past the valid + * 3-byte region — caller (merge_flat_dN_neon) bounds the loop so this + * overread stays within the bm scratch region. Same pattern as the + * AVX-512 flat_d3_unpack_avx512_fast and x86 flat_d3_unpack_x86. + * + * The original byte-wise vsetq_lane_u8 chain was the v0.2 workaround + * for a Neoverse-V2 store-forward stall on the previous + * `memcpy(&packed, 5/6) + vsetq_lane_u64` form (which routed via the + * stack). A direct vld1q_u8 doesn't go through the stack at all and + * doesn't reintroduce that pathology. */ +static inline uint8x8_t flat_d3_unpack_fast(const uint8_t *bm_ptr) +{ + uint8x16_t bm_lo = vld1q_u8(bm_ptr); + uint8x16_t shuffled = vqtbl1q_u8(bm_lo, vld1q_u8(flat_d3_shuf_tab)); + uint16x8_t w = vreinterpretq_u16_u8(shuffled); + uint16x8_t shifted = vshlq_u16(w, vld1q_s16(flat_d3_shift_tab)); + uint16x8_t masked = vandq_u16(shifted, vdupq_n_u16(0x07)); + return vmovn_u16(masked); +} + +/* D=4 unpack: 8 bytes hold 16 codes (2 per byte, no byte-crossings). + * Replicate each byte to 2 lanes and shift lane k by (k & 1) * 4. */ +static const uint8_t flat_d4_dup_tab[16] = { + 0,0, 1,1, 2,2, 3,3, 4,4, 5,5, 6,6, 7,7 +}; +static const int8_t flat_d4_shift_tab[16] = { + 0,-4, 0,-4, 0,-4, 0,-4, 0,-4, 0,-4, 0,-4, 0,-4 +}; + +/* Unpack 16 consecutive D=4 codes from 8 bytes of bm. */ +static inline uint8x16_t flat_d4_unpack(const uint8_t *bm_ptr) +{ + uint64_t packed; + memcpy(&packed, bm_ptr, 8); + uint8x16_t bm_lo = vreinterpretq_u8_u64( + vsetq_lane_u64(packed, vdupq_n_u64(0), 0)); + uint8x16_t dup = vqtbl1q_u8(bm_lo, vld1q_u8(flat_d4_dup_tab)); + uint8x16_t shifted = vshlq_u8(dup, vld1q_s8(flat_d4_shift_tab)); + return vandq_u8(shifted, vdupq_n_u8(0x0F)); +} + +/* D=5 unpack: 5 bytes = 40 bits = 8 codes. 5 of 8 codes cross byte + * boundaries, so we work in uint16 lanes. Lane layout: + * lanes 0,1,2: (b0, b1) — codes 0,1,2 (shifts 0, 5, 10) + * lane 3: (b1, b2) — code 3 (shift 7) + * lanes 4,5: (b2, b3) — codes 4,5 (shifts 4, 9) + * lanes 6,7: (b3, b4) — codes 6,7 (shifts 6, 11) */ +static const uint8_t flat_d5_shuf_tab[16] = { + 0,1, 0,1, 0,1, 1,2, 2,3, 2,3, 3,4, 3,4 +}; +static const int16_t flat_d5_shift_tab[8] = { + 0, -5, -10, -7, -4, -9, -6, -11 +}; + +/* Unpack 8 consecutive D=5 codes from 5 bytes starting at bm_ptr. + * + * Bytes are inserted into vector lanes one-by-one via `vsetq_lane_u8`. + * The compiler emits direct byte-into-vector loads (`ldr b` / `ld1 + * {v.b}[k]` / `ins`). An earlier version used + * `memcpy(&packed, bm_ptr, 5) + vsetq_lane_u64(packed, ...)`, which + * the compiler implemented via a stack round-trip + * (int-load -> stack-store -> vector-load-from-stack). On Neoverse-V2 + * (Graviton 4) the int-store -> vector-load forward stalls hard, + * costing ~20x throughput vs the byte-wise pattern. M4 absorbs the + * stall, so the old form looked fine there. */ +static inline uint8x8_t flat_d5_unpack_safe(const uint8_t *bm_ptr) +{ + uint8x16_t bm_lo = vdupq_n_u8(0); + bm_lo = vsetq_lane_u8(bm_ptr[0], bm_lo, 0); + bm_lo = vsetq_lane_u8(bm_ptr[1], bm_lo, 1); + bm_lo = vsetq_lane_u8(bm_ptr[2], bm_lo, 2); + bm_lo = vsetq_lane_u8(bm_ptr[3], bm_lo, 3); + bm_lo = vsetq_lane_u8(bm_ptr[4], bm_lo, 4); + uint8x16_t shuffled = vqtbl1q_u8(bm_lo, vld1q_u8(flat_d5_shuf_tab)); + uint16x8_t w = vreinterpretq_u16_u8(shuffled); + uint16x8_t shifted = vshlq_u16(w, vld1q_s16(flat_d5_shift_tab)); + uint16x8_t masked = vandq_u16(shifted, vdupq_n_u16(0x1F)); + return vmovn_u16(masked); +} + +/* D=5 _fast: single 16-byte vector load (overreads 11 bytes). */ +static inline uint8x8_t flat_d5_unpack_fast(const uint8_t *bm_ptr) +{ + uint8x16_t bm_lo = vld1q_u8(bm_ptr); + uint8x16_t shuffled = vqtbl1q_u8(bm_lo, vld1q_u8(flat_d5_shuf_tab)); + uint16x8_t w = vreinterpretq_u16_u8(shuffled); + uint16x8_t shifted = vshlq_u16(w, vld1q_s16(flat_d5_shift_tab)); + uint16x8_t masked = vandq_u16(shifted, vdupq_n_u16(0x1F)); + return vmovn_u16(masked); +} + +/* D=6 unpack: 3 bytes = 24 bits = 4 codes. 2 of 4 codes cross byte + * boundaries (codes 1 and 2). To produce 8 codes we process 6 bytes. + * Lane layout: + * lanes 0,1: (b0, b1) — codes 0, 1 (shifts 0, 6) + * lanes 2,3: (b1, b2) — codes 2, 3 (shifts 4, 10) + * lanes 4,5: (b3, b4) — codes 4, 5 (shifts 0, 6) + * lanes 6,7: (b4, b5) — codes 6, 7 (shifts 4, 10) */ +static const uint8_t flat_d6_shuf_tab[16] = { + 0,1, 0,1, 1,2, 1,2, 3,4, 3,4, 4,5, 4,5 +}; +static const int16_t flat_d6_shift_tab[8] = { + 0, -6, -4, -10, 0, -6, -4, -10 +}; + +/* Unpack 8 consecutive D=6 codes from 6 bytes starting at bm_ptr. + * Same byte-wise vector-lane load as flat_d5_unpack — see that + * function's comment for the Neoverse-V2 store-forward rationale. */ +static inline uint8x8_t flat_d6_unpack_safe(const uint8_t *bm_ptr) +{ + uint8x16_t bm_lo = vdupq_n_u8(0); + bm_lo = vsetq_lane_u8(bm_ptr[0], bm_lo, 0); + bm_lo = vsetq_lane_u8(bm_ptr[1], bm_lo, 1); + bm_lo = vsetq_lane_u8(bm_ptr[2], bm_lo, 2); + bm_lo = vsetq_lane_u8(bm_ptr[3], bm_lo, 3); + bm_lo = vsetq_lane_u8(bm_ptr[4], bm_lo, 4); + bm_lo = vsetq_lane_u8(bm_ptr[5], bm_lo, 5); + uint8x16_t shuffled = vqtbl1q_u8(bm_lo, vld1q_u8(flat_d6_shuf_tab)); + uint16x8_t w = vreinterpretq_u16_u8(shuffled); + uint16x8_t shifted = vshlq_u16(w, vld1q_s16(flat_d6_shift_tab)); + uint16x8_t masked = vandq_u16(shifted, vdupq_n_u16(0x3F)); + return vmovn_u16(masked); +} + +/* D=6 _fast: single 16-byte vector load (overreads 10 bytes). */ +static inline uint8x8_t flat_d6_unpack_fast(const uint8_t *bm_ptr) +{ + uint8x16_t bm_lo = vld1q_u8(bm_ptr); + uint8x16_t shuffled = vqtbl1q_u8(bm_lo, vld1q_u8(flat_d6_shuf_tab)); + uint16x8_t w = vreinterpretq_u16_u8(shuffled); + uint16x8_t shifted = vshlq_u16(w, vld1q_s16(flat_d6_shift_tab)); + uint16x8_t masked = vandq_u16(shifted, vdupq_n_u16(0x3F)); + return vmovn_u16(masked); +} + +/* D=7 unpack: 8 codes = 56 bits = 7 bytes. Code i starts at bit 7i, i.e. + * byte (7i>>3) with in-byte offset (7i&7). Each 7-bit code fits inside a + * 2-byte (16-bit) window starting at its byte (max in-byte offset is 7, so + * 7+7=14 <= 16), so we use the same uint16-lane shuffle+shift as D5/D6. + * Lane layout (low,high byte / right-shift): + * 0:(b0,b1)/0 1:(b0,b1)/7 2:(b1,b2)/6 3:(b2,b3)/5 + * 4:(b3,b4)/4 5:(b4,b5)/3 6:(b5,b6)/2 7:(b6,b6)/1 (lane7 high byte is + * masked off, so it reuses b6 -- only bytes 0..6 are read). */ +static const uint8_t flat_d7_shuf_tab[16] = { + 0,1, 0,1, 1,2, 2,3, 3,4, 4,5, 5,6, 6,6 +}; +static const int16_t flat_d7_shift_tab[8] = { + 0, -7, -6, -5, -4, -3, -2, -1 +}; + +/* Unpack 8 consecutive D=7 codes from 7 bytes starting at bm_ptr. Byte-wise + * vector-lane load (see flat_d5_unpack's Neoverse-V2 store-forward note). */ +static inline uint8x8_t flat_d7_unpack_safe(const uint8_t *bm_ptr) +{ + uint8x16_t bm_lo = vdupq_n_u8(0); + bm_lo = vsetq_lane_u8(bm_ptr[0], bm_lo, 0); + bm_lo = vsetq_lane_u8(bm_ptr[1], bm_lo, 1); + bm_lo = vsetq_lane_u8(bm_ptr[2], bm_lo, 2); + bm_lo = vsetq_lane_u8(bm_ptr[3], bm_lo, 3); + bm_lo = vsetq_lane_u8(bm_ptr[4], bm_lo, 4); + bm_lo = vsetq_lane_u8(bm_ptr[5], bm_lo, 5); + bm_lo = vsetq_lane_u8(bm_ptr[6], bm_lo, 6); + uint8x16_t shuffled = vqtbl1q_u8(bm_lo, vld1q_u8(flat_d7_shuf_tab)); + uint16x8_t w = vreinterpretq_u16_u8(shuffled); + uint16x8_t shifted = vshlq_u16(w, vld1q_s16(flat_d7_shift_tab)); + uint16x8_t masked = vandq_u16(shifted, vdupq_n_u16(0x7F)); + return vmovn_u16(masked); +} + +/* D=7 _fast: single 16-byte vector load (overreads 9 bytes). */ +static inline uint8x8_t flat_d7_unpack_fast(const uint8_t *bm_ptr) +{ + uint8x16_t bm_lo = vld1q_u8(bm_ptr); + uint8x16_t shuffled = vqtbl1q_u8(bm_lo, vld1q_u8(flat_d7_shuf_tab)); + uint16x8_t w = vreinterpretq_u16_u8(shuffled); + uint16x8_t shifted = vshlq_u16(w, vld1q_s16(flat_d7_shift_tab)); + uint16x8_t masked = vandq_u16(shifted, vdupq_n_u16(0x7F)); + return vmovn_u16(masked); +} + +#endif /* PIVCO_HUFFMAN_NEON_FLAT_H */ diff --git a/third_party/pivco/src/pivco_huffman_neon_pack.h b/third_party/pivco/src/pivco_huffman_neon_pack.h new file mode 100644 index 0000000..53326e9 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_neon_pack.h @@ -0,0 +1,80 @@ +/* pivco_huffman_neon_pack.h — flat-subtree D-bit pack (NEON), D=5/6/7. + * D=2/3/4 stay on their per-D special cases in + * pivco_huffman_primitives_neon.h (byte-aligned paired adds for D=2/4, + * a converging-shift pack for D=3). + * + * Variable-shift "converging pyramid" pack, 16 codes per q-vector iter. + * Each pairing level shifts the two halves of a lane pair towards each + * other with one USHL of {+s,-s} per-lane counts, so the fields meet at + * the lane boundary and each level is a single instruction: + * L1 u8 {8-D, 0}: u16 = pair << (8-D) + * L2 u16 {8-D, -(8-D)}: u32 = quad << (16-2D) + * L3 u32 {16-2D, -(16-2D)}: u64 = octet << (32-4D) + * The compact shuffle absorbs the whole bytes of the final (32-4D) + * re-basing shift (its tables start at byte 1 for D=5/6), leaving a + * residual >>4 for D=5/7 and no final shift for D=6. + * + * Internal header. Not part of the public API. */ + +#ifndef PIVCO_HUFFMAN_NEON_PACK_H +#define PIVCO_HUFFMAN_NEON_PACK_H + +#ifndef __aarch64__ +#error "pivco_huffman_neon_pack.h requires aarch64 NEON" +#endif + +#include +#include + +/* Per-D compact shuffles: gather the 2D valid packed bytes (D from each + * 128-bit lane's low half, after the final re-basing shift is folded in) + * into output lanes [0..2D); 0xff indices zero the trailing lanes. */ +static const uint8_t pivco_pack_compact_d5_neon[16] = { + 1, 2, 3, 4, 5, 9, 10, 11, 12, 13, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +}; +static const uint8_t pivco_pack_compact_d6_neon[16] = { + 1, 2, 3, 4, 5, 6, 9, 10, 11, 12, 13, 14, 0xff, 0xff, 0xff, 0xff +}; +static const uint8_t pivco_pack_compact_d7_neon[16] = { + 0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 0xff, 0xff +}; + +/* D and BITSHR as compile-time constants so the USHL count tables and + * the vshrq_n_u64 fold. Returns the number of codes packed by the SIMD + * loop; the caller's scalar tail packs [i, n). */ +#define PIVCO_PACK_NEON_DN(NAME, D_VAL, BITSHR, COMPACT_TAB) \ +static inline int NAME(uint8_t *out, const uint8_t *ranks, \ + int n, uint8_t base) \ +{ \ + static const int8_t sh1[16] = { 8-(D_VAL),0, 8-(D_VAL),0, 8-(D_VAL),0, \ + 8-(D_VAL),0, 8-(D_VAL),0, 8-(D_VAL),0, \ + 8-(D_VAL),0, 8-(D_VAL),0 }; \ + static const int16_t sh2[8] = { 8-(D_VAL), -(8-(D_VAL)), \ + 8-(D_VAL), -(8-(D_VAL)), \ + 8-(D_VAL), -(8-(D_VAL)), \ + 8-(D_VAL), -(8-(D_VAL)) }; \ + static const int32_t sh3[4] = { 16-2*(D_VAL), -(16-2*(D_VAL)), \ + 16-2*(D_VAL), -(16-2*(D_VAL)) }; \ + const int8x16_t s1 = vld1q_s8(sh1); \ + const int16x8_t s2 = vld1q_s16(sh2); \ + const int32x4_t s3 = vld1q_s32(sh3); \ + const uint8x16_t compact = vld1q_u8(COMPACT_TAB); \ + const int total_bytes = (n * (D_VAL) + 7) >> 3; \ + int i = 0; \ + for (; i + 16 <= n && ((i * (D_VAL)) >> 3) + 16 <= total_bytes; i += 16) { \ + uint8x16_t cb = vsubq_u8(vld1q_u8(ranks + i), vdupq_n_u8(base)); \ + uint16x8_t w16 = vreinterpretq_u16_u8(vshlq_u8(cb, s1)); \ + uint32x4_t w32 = vreinterpretq_u32_u16(vshlq_u16(w16, s2)); \ + uint64x2_t w64 = vreinterpretq_u64_u32(vshlq_u32(w32, s3)); \ + if (BITSHR) w64 = vshrq_n_u64(w64, (BITSHR) ? (BITSHR) : 1); \ + vst1q_u8(out + ((i * (D_VAL)) >> 3), \ + vqtbl1q_u8(vreinterpretq_u8_u64(w64), compact)); \ + } \ + return i; \ +} +PIVCO_PACK_NEON_DN(pack_d5_neon, 5, 4, pivco_pack_compact_d5_neon) +PIVCO_PACK_NEON_DN(pack_d6_neon, 6, 0, pivco_pack_compact_d6_neon) +PIVCO_PACK_NEON_DN(pack_d7_neon, 7, 4, pivco_pack_compact_d7_neon) +#undef PIVCO_PACK_NEON_DN + +#endif /* PIVCO_HUFFMAN_NEON_PACK_H */ diff --git a/third_party/pivco/src/pivco_huffman_neon_tables.c b/third_party/pivco/src/pivco_huffman_neon_tables.c new file mode 100644 index 0000000..bd7ed4b --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_neon_tables.c @@ -0,0 +1,112 @@ +/* pivco_huffman_neon_tables.c — runtime construction of shared NEON tables. + * + * Storage + init functions extracted from the legacy NEON .c files so + * both the legacy code and the new codec.c-compiled-as-NEON object + * library link against one copy. See pivco_huffman_neon_tables.h for + * the contract. */ + +#include "pivco_huffman_neon_tables.h" + +/* ---------- Encoder partition: compress_tab[256][32] + compress_popcnt[256] + * + * For each 8-bit partition mask: + * - bytes 0..15 → vqtbl1q indices packing the bit=1 (right) uint16 + * lanes to the front of the destination register + * - bytes 16..31 → indices for the complement (bit=0 → left) + * Both halves are loaded with a single `ldp q0, q1` (32 bytes, + * contiguous), one cache-line access instead of two scattered lookups. + * Lanes past the popcount are filled with 0xFF so vqtbl1q writes + * arbitrary garbage past `n_{right,left}` — the caller bounds writes + * via compress_popcnt[mask]. + */ +uint8_t compress_tab[256][32] __attribute__((aligned(32))); +uint8_t compress_popcnt[256] __attribute__((aligned(64))); +int compress_table_ready = 0; + +void init_compress_table(void) +{ + if (compress_table_ready) return; + for (int mask = 0; mask < 256; mask++) { + /* Right (bit=1): pack selected to front. */ + int out_r = 0; + for (int i = 0; i < 8; i++) { + if (mask & (1 << i)) { + compress_tab[mask][out_r * 2] = (uint8_t)(i * 2); + compress_tab[mask][out_r * 2 + 1] = (uint8_t)(i * 2 + 1); + out_r++; + } + } + compress_popcnt[mask] = (uint8_t)out_r; + for (int j = out_r * 2; j < 16; j++) + compress_tab[mask][j] = 0xFF; + + /* Left (bit=0): pack complement to front. */ + int out_l = 0; + for (int i = 0; i < 8; i++) { + if (!(mask & (1 << i))) { + compress_tab[mask][16 + out_l * 2] = (uint8_t)(i * 2); + compress_tab[mask][16 + out_l * 2 + 1] = (uint8_t)(i * 2 + 1); + out_l++; + } + } + for (int j = out_l * 2; j < 16; j++) + compress_tab[mask][16 + j] = 0xFF; + } + compress_table_ready = 1; +} + +/* ---------- BU merge V4: expand_tab + expand_tab_pre + expand_popcnt + * + * `expand_tab[m][k]` is the lane index (0..15) for output position k + * of an 8-element merge controlled by mask byte m. Values 0..7 select + * from the left input; 8..15 select from the right input. + * + * `expand_tab_pre[nr0][m1][k]` pre-bakes the iter-1 vqtbl2 indices + * after iter-0 consumed `nr0` right bytes and `(8 - nr0)` left bytes + * from a 16-byte L_full and 16-byte R_full source pair. Without the + * pre-bake, iter-1's shuf depends on iter-0's nr0 via 4 vector ALU + * ops on the critical path; the table folds those into one indexed + * load. Table size: 9 × 256 × 8 = 18 432 bytes; L1d-resident on + * every target. + * + * Layout per (nr0, m1, k): + * L-lane idx = expand_tab[m1][k] + (8 - nr0) ∈ [(8-nr0)..15] + * R-lane idx = expand_tab[m1][k] + 8 + nr0 ∈ [(16+nr0)..(23+nr0)] + * + * `expand_popcnt[m]` = popcount(m); the count of right bytes the + * caller consumes for mask byte m. + */ +uint8_t expand_tab [256][8] __attribute__((aligned(32))); +uint8_t expand_tab_pre[9][256][8] __attribute__((aligned(64))); +uint8_t expand_popcnt [256] __attribute__((aligned(64))); +int expand_table_ready = 0; + +void init_expand_table(void) +{ + if (expand_table_ready) return; + for (int m = 0; m < 256; m++) { + int n_zeros = 0, n_ones = 0; + for (int k = 0; k < 8; k++) { + if (m & (1 << k)) { + expand_tab[m][k] = (uint8_t)(8 + n_ones); + n_ones++; + } else { + expand_tab[m][k] = (uint8_t)n_zeros; + n_zeros++; + } + } + expand_popcnt[m] = (uint8_t)n_ones; + } + /* Pre-adjusted (nr0, m1) shuf table — see header doc. */ + for (int nr0 = 0; nr0 <= 8; nr0++) { + for (int m = 0; m < 256; m++) { + for (int k = 0; k < 8; k++) { + uint8_t raw = expand_tab[m][k]; + expand_tab_pre[nr0][m][k] = + (raw < 8) ? (uint8_t)(raw + (8 - nr0)) /* L-lane */ + : (uint8_t)(raw + 8 + nr0); /* R-lane */ + } + } + } + expand_table_ready = 1; +} diff --git a/third_party/pivco/src/pivco_huffman_neon_tables.h b/third_party/pivco/src/pivco_huffman_neon_tables.h new file mode 100644 index 0000000..2c7a728 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_neon_tables.h @@ -0,0 +1,56 @@ +/* pivco_huffman_neon_tables.h — shared NEON lookup tables. + * + * Two table families live here so both the legacy NEON .c files and + * the codec.c-compiled-as-NEON object library can share one runtime + * copy each (the largest, `expand_tab_pre`, is 18 KB — material to + * avoid duplicating per-TU). + * + * compress_tab + compress_popcnt + init_compress_table + * Encoder partition shuffle table for the 8-element NEON + * `partition_8` primitive. Indexed by an 8-bit partition mask; + * bytes 0..15 give the vqtbl1q indices that pack the right half + * (bit==1) to the front of the destination lane; bytes 16..31 + * give the indices for the complement (bit==0 → left half). + * Both halves loaded with one `ldp q0, q1` (32 bytes contiguous). + * + * expand_tab / expand_tab_pre / expand_popcnt + init_expand_table + * BU `merge` V4 strategy: per (nr0, m1) precomputed shuf + * vectors for the 32-byte (L_full, R_full) source register pair. + * 18 432 bytes; fits L1d on every target. See the long comment + * at the definition for the (nr0, m1) algebra. + * + * Both init functions are idempotent and lazy: the first caller pays + * the construction cost, subsequent callers no-op. Not thread-safe; + * the caller must guarantee a single init thread (in practice the + * encode/decode entry points call init at top before recursing). + * + * Internal header. Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_NEON_TABLES_H +#define PIVCO_HUFFMAN_NEON_TABLES_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Encoder partition (8 elements per call) shuffle / popcount tables. */ +extern uint8_t compress_tab [256][32]; +extern uint8_t compress_popcnt [256]; +extern int compress_table_ready; +void init_compress_table(void); + +/* BU merge V4 expand / popcount tables. */ +extern uint8_t expand_tab [256][8]; +extern uint8_t expand_tab_pre [9][256][8]; +extern uint8_t expand_popcnt [256]; +extern int expand_table_ready; +void init_expand_table(void); + +#ifdef __cplusplus +} +#endif + +#endif /* PIVCO_HUFFMAN_NEON_TABLES_H */ diff --git a/third_party/pivco/src/pivco_huffman_pack_bmi2.h b/third_party/pivco/src/pivco_huffman_pack_bmi2.h new file mode 100644 index 0000000..818e58e --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_pack_bmi2.h @@ -0,0 +1,50 @@ +/* pivco_huffman_pack_bmi2.h — BMI2 pext flat-subtree pack. + * + * The contiguous-wire pack (concatenate n D-bit codes LSB-first into one + * bitstream) maps perfectly onto BMI2 `pext`: each code's D bits live at + * [right_shift, right_shift+D) within its 16-bit codes_la lane, and pext + * gathers the mask-selected bits of a 64-bit window into a contiguous low + * field. One pext packs 4 codes (a 4×u16 window); two cover a group of 8. + * + * Unlike the AVX-512 vector pack (widen u16->u64, sllv, cross-lane reduce), + * this needs NO cross-lane reduction and keeps the contiguous wire format + * (no transpose) — the fast path for short contiguous packs (cf. Lemire's + * LittleIntPacker / TurboPFor, which are scalar BMI2 for short arrays). + * + * Shared by the x86 + AVX-512 primitive backends (both imply BMI2 on the + * hosts we target; NEON has no pext and keeps its uint32-lane pack). + * Not part of the public API. + */ +#ifndef PIVCO_HUFFMAN_PACK_BMI2_H +#define PIVCO_HUFFMAN_PACK_BMI2_H + +#if defined(__BMI2__) +#include +#include +#include /* _pext_u64 */ + +/* Pack n D-bit codes (D in 2..8) from codes_la into out, contiguous LSB-first. + * Returns the number of codes packed (a multiple of 8); the caller's scalar + * tail finishes the residual. Writes exactly ceil(8D/8)=D bytes per group of + * 8, byte-aligned, so consecutive groups tile without overlap or over-read. */ +static inline int pack_dN_bmi2(uint8_t *out, const uint16_t *codes_la, + int n, int D, int right_shift) +{ + /* D bits at [right_shift, right_shift+D) replicated across the four + * 16-bit lanes of a 64-bit window. */ + uint64_t field = (((uint64_t)1 << D) - 1) << right_shift; + uint64_t mask = field | (field << 16) | (field << 32) | (field << 48); + int i = 0; + for (; i + 8 <= n; i += 8) { + uint64_t w0, w1; + memcpy(&w0, codes_la + i, 8); /* codes i .. i+3 */ + memcpy(&w1, codes_la + i + 4, 8); /* codes i+4 .. i+7 */ + uint64_t packed = _pext_u64(w0, mask) + | (_pext_u64(w1, mask) << (4 * D)); /* 8D bits */ + memcpy(out + ((i * D) >> 3), &packed, (size_t)D); /* 8D bits = D bytes */ + } + return i; +} +#endif /* __BMI2__ */ + +#endif /* PIVCO_HUFFMAN_PACK_BMI2_H */ diff --git a/third_party/pivco/src/pivco_huffman_primitives.h b/third_party/pivco/src/pivco_huffman_primitives.h new file mode 100644 index 0000000..c0497c8 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_primitives.h @@ -0,0 +1,241 @@ +/* pivco_huffman_primitives.h — backend-primitive interface (router header). + * + * pivco_huffman_codec.c is compiled once per backend tier; this header + * pulls in the right primitive implementation header based on the + * PIVCO_BACKEND_* macro that CMake passes to that translation unit. + * + * Every backend primitive header MUST provide static-inline + * implementations under specialized names (e.g. `part_core_scalar`, + * `part_core_neon`, etc.) and declare the aliases the codec uses + * (`prim_enc_init`, ...). The aliases forward to the specialized + * name via always-inline static-inline wrappers, so that: + * + * - codec.c reads cleanly (calls `prim_X` consistently) + * - stepping into the alias drops you on the specialized name + * - grep for `part_core_scalar` finds exactly the scalar impl + * + * =========================================================================== + * Primitive contract — every backend must implement these + * =========================================================================== + * + * Boundary convention. Primitives own only the SIMD-bound work: + * building the raw partition bitmap, partitioning the rank array, packing + * N·D-bit flat regions, the BU merge kernels. Everything else -- + * the K_right header, the FSE marker byte, the optional FSE-attempt + * on the raw bitmap, the per-stats bookkeeping -- is arch-agnostic + * glue and lives in codec.c. This split is structural: when a new + * backend is added, it inherits all of the wire-format and FSE logic + * automatically by going through codec.c; there is no per-backend + * feature flag to "remember to wire FSE here too" (the original cause + * of the scalar↔SSE wire-format-drift bug this refactor exists to + * fix). + * + * void prim_histogram_chunk(const uint8_t *in, size_t n, + * uint32_t hist[256], uint8_t *scratch); + * + * Adds the byte counts of in[0..n) into hist[256]. Caller + * guarantees n <= PIVCO_PRIM_HIST_CHUNK (so no u32 counter can + * overflow) and provides PIVCO_PRIM_HIST_SCRATCH bytes of scratch + * (the AVX-512 bin buffers; other backends ignore it). No + * alignment requirements; reads and writes are exact. Backends + * without their own implementation alias the shared scalar core + * (pivco_huffman_hist_scalar.h). + * + * Lifecycle: + * + * void prim_codec_init(void); + * + * Idempotent lazy-init for any backend-specific runtime tables + * (e.g. NEON's compress_tab + expand_tab pre-bakes). codec.c + * calls this once at every encode/decode entry. Scalar's + * implementation is empty; NEON's calls init_compress_table and + * init_expand_table. + * + * --------------------------------------------------------------------------- + * ENCODE PRIMITIVES + * --------------------------------------------------------------------------- + * + * void prim_enc_init(uint8_t ranks[n], int n, + * const uint8_t *symbols, + * const uint8_t sym_to_rank[256], + * const pivco_enc_init_aux_t *aux); + * + * Build the per-block in-order rank array, gathering from `sym_to_rank` + * (= table->sym_to_rank) indexed by each input symbol: + * + * ranks[i] = sym_to_rank[symbols[i]] + * + * `aux` (= &table->enc_init_aux) carries arch-specific precomputed gather + * tables — its fields are non-NULL only on arches that use them (x86 SSE/AVX2 + * uses `s2r_hi` = sym_to_rank<<8 for the 2tab no-OR merge). A backend that + * consumes a field asserts it is non-NULL; backends that don't need it ignore + * `aux`. + * + * Each leaf's rank is its left-to-right position among the tree's leaves + * (partbyrank). A subtree's leaves form a contiguous rank range, so + * the per-node routing test reduces to an 8-bit compare against the node's + * split_rank (below). ranks is built once per block; the partition mutates + * it in place across the recursion (the left half stays, the right half is + * compacted into a scratch buffer for the right child to recurse on). + * + * ENCODE PARTITION FAMILY (prim_enc_partition_{full,left,right,none}) + * + * Every non-flat internal node builds the same n-bit partition bitmap from + * ranks[0..n): bit j = (ranks[j] > split_rank), where split_rank is the max + * rank in the node's left subtree. Because in-order rank order == the old + * left-aligned-code order, this is byte-identical to the former code_la + * bit-test — wire format and decoder are unchanged. The four members + * differ only in how many halves they additionally scatter; the codec picks + * by node_type, mirroring the decode-side prim_merge_* family 1:1: + * + * node_type primitive scatters outputs + * INTERNAL_FULL prim_enc_partition_full both left in place, + * right->right_out + * LEAF_LEFT prim_enc_partition_right right only right->right_out + * BOTH_LEAVES prim_enc_partition_none neither (bitmap only) + * + * SUFFIX CONVENTION: the suffix names the NON-TRIVIAL child subtree — + * the side whose ranks are emitted for further recursion (LEAF_LEFT's + * right child is the subtree, left is a leaf, so + * prim_enc_partition_right emits the right ranks). `_none` = zero + * outputs (both children leaves); it still writes the bitmap, so it + * is exactly the bitmap-build step. + * + * Common contract (all four): + * Writes ceil(n/8) bytes into bm. Bit j (j in [0..n)) is + * (ranks[j] > thr); lands at bit (j & 7) of bm[j >> 3]. + * + * Signatures (thr = table->split_rank[node]): + * int prim_enc_partition_full (uint8_t *ranks, int n, uint8_t thr, + * uint8_t *bm, uint8_t *right_out); + * // left stays in ranks[0..n_left); right->right_out[0..n_right) + * int prim_enc_partition_right(uint8_t *ranks, int n, uint8_t thr, + * uint8_t *bm, uint8_t *right_out); + * // emits right_out[0..n_right); left side not produced + * int prim_enc_partition_none (uint8_t *ranks, int n, uint8_t thr, + * uint8_t *bm); + * // bitmap only (no scatter) + * All three return n_right (caller derives n_left = n - n_right). + * _full keeps the left ranks in place in ranks[0..n_left); _right + * emits to right_out (ranks untouched); _none emits no ranks. + * + * SHARED SCATTER CORE: the compress-table scatter used here is the + * same operation the top-down decoder needs (read bitmap + scatter vs. + * build bitmap + scatter). Keep the scatter core factored so a future + * prim_dec_partition_* family (TD decode, once ph-td is de-forked) can + * reuse it rather than re-implementing — that reuse is the main reason + * to land the half/none split as named members now. + * + * codec.c wraps the chosen member at every non-flat internal node: + * + * marker_slot = *out_ptr; *marker_slot = 0; *out_ptr += 1; + * bm = *out_ptr; *out_ptr += bitmap_bytes(n); + * n_right = prim_enc_partition_(ranks, n, split_rank, bm, ...); + * codec_maybe_fse_attempt(...); // may rewrite marker + bm, + * // adjust *out_ptr on commit + * wire_commit_kr_header(kr_slot, n_right); + * + * IMPLEMENTATION: _right/_none share one parameterized core + * (part_core_, EMIT_RIGHT compile-time flag; the scalar core + * additionally carries EMIT_LEFT because scalar _full rides it too). + * On NEON/x86, _full stays HAND-WRITTEN (part_full_) because + * the generic core's both-sides specialization scheduled ~8% slower on + * the hot common path (measured on M4 for the former code_la + * partition). bench_prim numbers that motivated the split (M4/NEON): + * _none (bitmap only) ~-54% vs _full, fused build+half (_right) ~-26% + * vs _full — the *unfused* "build then partition-half" route is a wash + * (the re-read eats the one-sided-scatter saving), so _right is + * fused build+scatter. End-to-end encode gain lands on skewed dists + * (AVX-512 calgary/proba80 +16-18%, dna +8%; smaller on NEON/SSE); balanced + * inputs (english) are flat. + * + * void prim_enc_pack_dN(const uint8_t *ranks, int n, int D, uint8_t base, + * uint8_t *out_packed, int vertical); + * + * Flat-subtree path. In a flat subtree (all 2^D leaves at the same depth), + * the in-subtree local code is `ranks[i] - base`, already a D-bit value + * (base = table->flat_base_rank[node] = the min rank in the subtree). Pack + * those local codes into out_packed[ceil(n*D/8)] bytes, laid out per the + * pivco_flat_layout_t value in `vertical`: LSB-first natural (0), hybrid + * vertical (1: 512- then 128-lane blocks + natural tail), or 128-only + * vertical (2) — see pivco_huffman_vertical.h. Callers pass + * table->flat_layout. + * + * Because (rank - base) is already an 8-bit value in the low bits, the + * packers read straight from the u8 rank array — no u16 load + shift + + * narrow round-trip that the former code_la pack needed. + * + * --------------------------------------------------------------------------- + * DECODE PRIMITIVES (bottom-up) + * --------------------------------------------------------------------------- + * + * void prim_merge_flat(uint8_t *out, int n, + * const uint8_t *bm, int D, + * const uint8_t *c2s, int vertical); + * + * Unpack n D-bit codes from bm[], look each up in c2s[2^D], write + * the resulting symbols to out[0..n). bm is `ceil(n*D/8)` bytes, + * laid out per `vertical` (must match the encode-side value). + * + * void prim_merge_cst_cst(const uint8_t *bm, int K, + * uint8_t left_sym, uint8_t right_sym, + * uint8_t *out); + * + * Both-leaves merge: for j in [0..K), + * out[j] = (bit_j ? right_sym : left_sym). + * + * void prim_merge_cst_vec(const uint8_t *bm, int K, + * uint8_t left_sym, + * const uint8_t *right_buf, + * uint8_t *out); + * + * Half-leaf merge, constant LEFT: out[j] = (bit_j ? right_buf[r++] + * : left_sym). Used by LEAF_LEFT (the left child is a leaf). + * + * void prim_merge_vec_vec(const uint8_t *bm, int K, + * const uint8_t *left_buf, + * const uint8_t *right_buf, + * uint8_t *out); + * + * Full BU merge: out[j] = (bit_j ? right_buf[r++] : left_buf[l++]). + * + * =========================================================================== + * Internal header. Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_PRIMITIVES_H +#define PIVCO_HUFFMAN_PRIMITIVES_H + +/* Backend selection. The per-backend codec TUs pass an explicit + * PIVCO_BACKEND_* define (CMake) and keep full control. Any OTHER TU + * may simply include this header: with no explicit define, the build + * architecture picks the backend (the same tier CMake's codec dispatch + * prefers for the build), so callers of prim_* never name a backend. */ +#if !defined(PIVCO_BACKEND_SCALAR) && !defined(PIVCO_BACKEND_NEON) \ + && !defined(PIVCO_BACKEND_X86) && !defined(PIVCO_BACKEND_AVX512) +# if defined(__aarch64__) || defined(__ARM_NEON) +# define PIVCO_BACKEND_NEON 1 +# elif defined(__AVX512VBMI2__) +# define PIVCO_BACKEND_AVX512 1 +# elif defined(__SSE4_1__) +# define PIVCO_BACKEND_X86 1 +# else +# define PIVCO_BACKEND_SCALAR 1 +# endif +#endif + +#if defined(PIVCO_BACKEND_SCALAR) +# include "pivco_huffman_primitives_scalar.h" +# define PIVCO_PRIM_BACKEND_NAME "scalar" +#elif defined(PIVCO_BACKEND_NEON) +# include "pivco_huffman_primitives_neon.h" +# define PIVCO_PRIM_BACKEND_NAME "neon" +#elif defined(PIVCO_BACKEND_X86) +# include "pivco_huffman_primitives_x86.h" +# define PIVCO_PRIM_BACKEND_NAME "sse/avx2" +#elif defined(PIVCO_BACKEND_AVX512) +# include "pivco_huffman_primitives_avx512.h" +# define PIVCO_PRIM_BACKEND_NAME "avx512" +#endif + +#endif /* PIVCO_HUFFMAN_PRIMITIVES_H */ diff --git a/third_party/pivco/src/pivco_huffman_primitives_avx512.h b/third_party/pivco/src/pivco_huffman_primitives_avx512.h new file mode 100644 index 0000000..2116d45 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_primitives_avx512.h @@ -0,0 +1,993 @@ +/* pivco_huffman_primitives_avx512.h — AVX-512 VBMI2 primitive + * implementations of the codec primitive interface (see + * pivco_huffman_primitives.h). + * + * Specialized names end in `_avx512`; the codec calls the aliases + * `prim_*` defined at the bottom as always-inline wrappers. + * + * Floor: AVX-512 F + BW + VBMI + VBMI2 + VPOPCNTDQ. Pivco's AVX-512 + * tier is built with all of these (-mavx512f -mavx512bw -mavx512vbmi + * -mavx512vbmi2 -mavx512vpopcntdq via CMakeLists.txt), and the runtime + * dispatcher (pivco_huffman.c::resolve_impl) only routes to this + * backend when /proc/cpuinfo advertises avx512_vbmi2. The header errors + * out if the macro contract isn't met -- catches misconfigured builds + * before they produce silently-incorrect output. + * + * Notable kernels (all kept symmetric with the legacy + * pivco_avx512.c bodies they replace): + * + * - enc_init_avx512: 64-char vpermex2var_epi8 byte-split table lookup + * (chunked over a 256-entry uint16 LUT split into 8 byte-half + * chunks). ~0.19 ops/char vs scalar's ~1.0. See the comment block + * in the function for the lookup geometry. + * + * - build_bitmap_partition_avx512: stride-32 vpcompressw main loop + * (one ZMM = 32 uint16 codes per iter), with an SSE-stride-8 tail + * also using vpcompressw via VL. No shuffle table -- compressw is + * the table-free analog of the SSE pshufb + compress_tab dance. + * + * - BU merge_vec_vec_avx512 family: 64-byte vpexpandb main loop (one + * ZMM load+expand per side, OR'd together), SSE stride-16 tail + * using expand_tab from pivco_huffman_x86_tables (the only x86 + * table this backend depends on; codec_init_avx512 inits it). + * + * - merge_flat_avx512: D=2/3/4/5/6 vector unpacks via the + * shared pivco_huffman_avx512_flat.h helpers (D=5/6 use + * vpmultishiftqb / vpermb -- the AVX-512-only fast paths). + * + * - pack_dN_avx512: D=2..7 via 64-codes-per-zmm byte-laid + + * vpmultishiftqb (D=3,5,6,7) or vpermb-stride + shift (D=2,4). See + * pivco_huffman_avx512_pack.h. D=8 via vpmovwb byte narrow. + * + * Internal header. Included by pivco_huffman_primitives.h when + * PIVCO_BACKEND_AVX512 is defined. Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_PRIMITIVES_AVX512_H +#define PIVCO_HUFFMAN_PRIMITIVES_AVX512_H + +#if !defined(PIVCO_HAS_AVX512) +#error "pivco_huffman_primitives_avx512.h requires PIVCO_HAS_AVX512" +#endif +#if !defined(__AVX512VBMI2__) || !defined(__AVX512VPOPCNTDQ__) +#error "pivco_huffman_primitives_avx512.h requires AVX-512 VBMI2 + VPOPCNTDQ" +#endif + +#include "pivco_huffman.h" +#include "pivco_huffman_common.h" +#include "pivco_huffman_x86_tables.h" /* expand_tab for BU tail */ +#include "pivco_huffman_avx512_flat.h" /* flat_d{2,3,4,5,6}_unpack_avx512* */ +#include "pivco_huffman_x86_vertical.h" +#include "pivco_huffman_avx512_pack.h" /* pack_d{2..7}_avx512 — vpmultishiftqb */ +#include "pivco_prof.h" + +#include +#include +#include + +/* Backend lifecycle. Only the BU merge SSE-stride tail needs a + * runtime table (expand_tab in pivco_huffman_x86_tables.c). AVX-512 + * partition and encode are entirely table-free (vpcompressw is the + * "table" -- it's hardware). */ +static inline void codec_init_avx512(void) +{ + init_expand_table_x86(); +} + +/* ---------- Decode primitives (bottom-up) ---------- */ + +/* popcount_K_right_avx512 — count "1" bits in the first K bits of bm. + * 64-byte main loop with VPOPCNTQ; 1c throughput. No codec.c caller + * (codec uses wire_read_kr_header for the value at read time); kept + * for symmetry with primitives_x86.h + primitives_neon.h. `nbytes` + * is derivable from K. */ +static inline int popcount_K_right_avx512(const uint8_t *bm, + int nbytes, int K) +{ + (void)nbytes; + PROF_TIC(); + int full_bytes = K >> 3; + int partial_bits = K & 7; + int b = 0; + + __m512i acc = _mm512_setzero_si512(); + for (; b + 64 <= full_bytes; b += 64) { + __m512i v = _mm512_loadu_si512((const __m512i *)(bm + b)); + acc = _mm512_add_epi64(acc, _mm512_popcnt_epi64(v)); + } + int K_right = (int)_mm512_reduce_add_epi64(acc); + + for (; b + 8 <= full_bytes; b += 8) { + uint64_t v; + memcpy(&v, bm + b, 8); + K_right += __builtin_popcountll(v); + } + for (; b < full_bytes; b++) { + K_right += __builtin_popcount(bm[b]); + } + if (partial_bits) { + uint8_t valid_mask = (uint8_t)((1u << partial_bits) - 1); + K_right += __builtin_popcount(bm[full_bytes] & valid_mask); + } + PROF_TOC(PROF_BU_POPCOUNT_K, K); + return K_right; +} + +/* merge_vec_vec_avx512 — VBMI2 64-byte main loop via vpexpandb (two + * masked expand-loads OR'd together), SSE stride-16/-8 tails using + * expand_tab from x86_tables. ~0.023 ns/byte on Xeon Ice Lake+. */ +static inline void merge_vec_vec_avx512(const uint8_t *bm, int K, + const uint8_t *left, + const uint8_t *right, + uint8_t *out) +{ + PROF_TIC(); + int lc = 0, rc = 0; + int j = 0; + for (; j + 64 <= K; j += 64) { + uint64_t mask; + memcpy(&mask, bm + (j >> 3), 8); + __mmask64 m = (__mmask64)mask; + __mmask64 nm = ~m; + /* Merge-masked expands, not maskz: Zen 4/5 have a false dependency on + * the output register of zero-masked compress/expand (issue #11, + * aadaa-fgtaa), which serializes iterations at expand latency. The + * asm barrier keeps the compiler from folding the zero back into a + * maskz form. Expanding R into L also replaces the OR. */ + __m512i zero = _mm512_setzero_si512(); asm("":"+v"(zero)); + __m512i L = _mm512_mask_expandloadu_epi8(zero, nm, left + lc); + __m512i o = _mm512_mask_expandloadu_epi8(L, m, right + rc); + _mm512_storeu_si512((__m512i *)(out + j), o); + int nr = __builtin_popcountll(mask); + rc += nr; lc += (64 - nr); + } + /* 2x-unrolled SSE stride-16: see primitives_x86.h. */ + for (; j + 16 <= K; j += 16) { + uint8_t m0 = bm[j >> 3]; + __m128i L0 = _mm_loadl_epi64((const __m128i *)(left + lc)); + __m128i R0 = _mm_loadl_epi64((const __m128i *)(right + rc)); + __m128i both0 = _mm_unpacklo_epi64(L0, R0); + __m128i shuf0 = _mm_loadl_epi64((const __m128i *)expand_tab[m0]); + __m128i o0 = _mm_shuffle_epi8(both0, shuf0); + _mm_storel_epi64((__m128i *)(out + j), o0); + int nr0 = expand_popcnt[m0]; + rc += nr0; lc += (8 - nr0); + + uint8_t m1 = bm[(j >> 3) + 1]; + __m128i L1 = _mm_loadl_epi64((const __m128i *)(left + lc)); + __m128i R1 = _mm_loadl_epi64((const __m128i *)(right + rc)); + __m128i both1 = _mm_unpacklo_epi64(L1, R1); + __m128i shuf1 = _mm_loadl_epi64((const __m128i *)expand_tab[m1]); + __m128i o1 = _mm_shuffle_epi8(both1, shuf1); + _mm_storel_epi64((__m128i *)(out + j + 8), o1); + int nr1 = expand_popcnt[m1]; + rc += nr1; lc += (8 - nr1); + } + for (; j + 8 <= K; j += 8) { + uint8_t m = bm[j >> 3]; + __m128i L = _mm_loadl_epi64((const __m128i *)(left + lc)); + __m128i R = _mm_loadl_epi64((const __m128i *)(right + rc)); + __m128i both = _mm_unpacklo_epi64(L, R); + __m128i shuf = _mm_loadl_epi64((const __m128i *)expand_tab[m]); + __m128i o = _mm_shuffle_epi8(both, shuf); + _mm_storel_epi64((__m128i *)(out + j), o); + int nr = expand_popcnt[m]; + rc += nr; lc += (8 - nr); + } + for (; j < K; j++) { + int mb = (bm[j >> 3] >> (j & 7)) & 1; + out[j] = mb ? right[rc++] : left[lc++]; + } + PROF_TOC(PROF_BU_MERGE_VEC_VEC, K); +} + +/* merge_cst_vec_avx512 — left input is a broadcast constant. */ +static inline void merge_cst_vec_avx512(const uint8_t *bm, int K, + uint8_t left_sym, + const uint8_t *right, + uint8_t *out) +{ + PROF_TIC(); + int rc = 0; + int j = 0; + __m128i Lbcast8 = _mm_set1_epi8((char)left_sym); + __m512i Lbcast64 = _mm512_set1_epi8((char)left_sym); + for (; j + 64 <= K; j += 64) { + uint64_t mask; + memcpy(&mask, bm + (j >> 3), 8); + __mmask64 m = (__mmask64)mask; + /* expand straight into the broadcast (issue #11: avoids the Zen 4/5 + * maskz false dep and drops the blend) */ + __m512i o = _mm512_mask_expandloadu_epi8(Lbcast64, m, right + rc); + _mm512_storeu_si512((__m512i *)(out + j), o); + rc += __builtin_popcountll(mask); + } + for (; j + 16 <= K; j += 16) { + uint8_t m0 = bm[j >> 3]; + __m128i R0 = _mm_loadl_epi64((const __m128i *)(right + rc)); + __m128i both0 = _mm_unpacklo_epi64(Lbcast8, R0); + __m128i shuf0 = _mm_loadl_epi64((const __m128i *)expand_tab[m0]); + __m128i o0 = _mm_shuffle_epi8(both0, shuf0); + _mm_storel_epi64((__m128i *)(out + j), o0); + rc += expand_popcnt[m0]; + + uint8_t m1 = bm[(j >> 3) + 1]; + __m128i R1 = _mm_loadl_epi64((const __m128i *)(right + rc)); + __m128i both1 = _mm_unpacklo_epi64(Lbcast8, R1); + __m128i shuf1 = _mm_loadl_epi64((const __m128i *)expand_tab[m1]); + __m128i o1 = _mm_shuffle_epi8(both1, shuf1); + _mm_storel_epi64((__m128i *)(out + j + 8), o1); + rc += expand_popcnt[m1]; + } + for (; j + 8 <= K; j += 8) { + uint8_t m = bm[j >> 3]; + __m128i R = _mm_loadl_epi64((const __m128i *)(right + rc)); + __m128i both = _mm_unpacklo_epi64(Lbcast8, R); + __m128i shuf = _mm_loadl_epi64((const __m128i *)expand_tab[m]); + __m128i o = _mm_shuffle_epi8(both, shuf); + _mm_storel_epi64((__m128i *)(out + j), o); + rc += expand_popcnt[m]; + } + for (; j < K; j++) { + int mb = (bm[j >> 3] >> (j & 7)) & 1; + out[j] = mb ? right[rc++] : left_sym; + } + PROF_TOC(PROF_BU_MERGE_CST_VEC, K); +} + +/* merge_cst_cst_avx512 — both inputs are constants. Native AVX-512 + * stride-64: read 64 bm bits as a kmask, single mask_blend_epi8 of two + * broadcast registers, one 64-byte store. SSE 16-byte and scalar tails. */ +static inline void merge_cst_cst_avx512(const uint8_t *bm, int K, + uint8_t left_sym, uint8_t right_sym, + uint8_t *out) +{ + PROF_TIC(); + __m512i vL_64 = _mm512_set1_epi8((char)left_sym); + __m512i vR_64 = _mm512_set1_epi8((char)right_sym); + int j = 0; + for (; j + 64 <= K; j += 64) { + uint64_t mask; + memcpy(&mask, bm + (j >> 3), 8); + __m512i o = _mm512_mask_blend_epi8((__mmask64)mask, vL_64, vR_64); + _mm512_storeu_si512((__m512i *)(out + j), o); + } + __m128i vL = _mm_set1_epi8((char)left_sym); + __m128i vR = _mm_set1_epi8((char)right_sym); + __m128i bits = _mm_setr_epi8(1,2,4,8,16,32,64,(char)128, + 1,2,4,8,16,32,64,(char)128); + __m128i shuf = _mm_setr_epi8(0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1); + for (; j + 16 <= K; j += 16) { + __m128i bm_pair = _mm_cvtsi32_si128(*(const uint16_t *)(bm + (j >> 3))); + __m128i bm_dup = _mm_shuffle_epi8(bm_pair, shuf); + __m128i masked = _mm_and_si128(bm_dup, bits); + __m128i mask8 = _mm_cmpeq_epi8(masked, bits); + __m128i o = _mm_blendv_epi8(vL, vR, mask8); + _mm_storeu_si128((__m128i *)(out + j), o); + } + for (; j < K; j++) { + int mb = (bm[j >> 3] >> (j & 7)) & 1; + out[j] = mb ? right_sym : left_sym; + } + PROF_TOC(PROF_BU_MERGE_CST_CST, K); +} + +/* ---------- Flat-subtree decode (contiguous output) ---------- */ + +/* Extract D bits at bit position `bit_pos` from `in`. D <= 16. */ +static inline uint32_t extract_D_bits_avx512(const uint8_t *in, + int bit_pos, int D) +{ + int byte_idx = bit_pos >> 3; + int bit_off = bit_pos & 7; + uint32_t val = (uint32_t)in[byte_idx]; + if (bit_off + D > 8) val |= ((uint32_t)in[byte_idx + 1]) << 8; + if (bit_off + D > 16) val |= ((uint32_t)in[byte_idx + 2]) << 16; + return (val >> bit_off) & ((1u << D) - 1); +} + +/* Per-D direct decode (D-bit packed bm -> symbols). Reads n*D packed + * bits, looks up each D-bit code in c2s, writes the resulting bytes + * to symbols[0..n). merge_flat_avx512 is a switch + * dispatcher to the per-D specialisation; structure mirrors the NEON + * file at pivco_huffman_primitives_neon.h. + * + * Per-D unpack helpers (flat_d{2,3,4,5,6,7}_unpack_avx512*) come from + * pivco_huffman_avx512_flat.h. */ + +/* D=2: c2s = 4 entries; codes < 4 use only low 2 bits so vpermb on a + * 64-byte register whose first 4 bytes are c2s works. Wide 64-at-a-time + * path first, then existing 16-wide tail + scalar nibble unpack. */ +static inline void merge_flat_d2_avx512(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + uint32_t c2s_lo; + memcpy(&c2s_lo, c2s, 4); + __m128i c2s_xmm = _mm_set1_epi32((int32_t)c2s_lo); + __m512i c2s_zmm = _mm512_castsi128_si512(c2s_xmm); + int i = 0; + /* Slack: strict bound is ceil(32*8/2)=128 codes — happens to match the + * "i + 128" symmetry of the other widths exactly. */ + for (; i + 128 <= n; i += 64) { + __m512i codes = flat_d2_unpack64_avx512_fast(bm + (i >> 2)); + __m512i syms = _mm512_permutexvar_epi8(codes, c2s_zmm); + _mm512_storeu_si512((__m512i *)(symbols + i), syms); + } + for (; i + 16 <= n; i += 16) { + __m128i codes = flat_d2_unpack_avx512(bm + (i >> 2)); + __m128i syms = _mm_shuffle_epi8(c2s_xmm, codes); + _mm_storeu_si128((__m128i *)(symbols + i), syms); + } + for (; i + 4 <= n; i += 4) { + uint8_t b = bm[i >> 2]; + symbols[i ] = c2s[(b ) & 3]; + symbols[i + 1] = c2s[(b >> 2) & 3]; + symbols[i + 2] = c2s[(b >> 4) & 3]; + symbols[i + 3] = c2s[(b >> 6) & 3]; + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_avx512(bm, i * 2, 2); + symbols[i] = c2s[code]; + } +} + +/* D=3: c2s = 8 entries fits in low 8 bytes of an xmm; codes < 8 use only + * the low 3 bits so vpermb against a 64-byte register whose first 8 bytes + * are c2s (the rest don't-care) lands on the right entry. Wide 64-at-a + * -time path first (using flat_d3_unpack64), then existing 16-wide tail. */ +static inline void merge_flat_d3_avx512(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + uint64_t c2s_lo; + memcpy(&c2s_lo, c2s, 8); + __m128i c2s_xmm = _mm_cvtsi64_si128((int64_t)c2s_lo); + __m512i c2s_zmm = _mm512_castsi128_si512(c2s_xmm); + int i = 0; + /* Slack: strict bound is ceil(32*8/3)=86 codes; use 128 to drop cleanly + * into the 16-wide tail (same pattern as merge_flat_d5_avx512). */ + for (; i + 128 <= n; i += 64) { + __m512i codes = flat_d3_unpack64_avx512_fast(bm + ((i * 3) >> 3)); + __m512i syms = _mm512_permutexvar_epi8(codes, c2s_zmm); + _mm512_storeu_si512((__m512i *)(symbols + i), syms); + } + int fast_end = n >= 16 ? n - 16 : 0; + for (; i + 16 <= fast_end; i += 16) { + __m128i codes = flat_d3_unpack_avx512_fast(bm + ((i * 3) >> 3)); + __m128i syms = _mm_shuffle_epi8(c2s_xmm, codes); + _mm_storeu_si128((__m128i *)(symbols + i), syms); + } + if (i + 16 <= n) { + __m128i codes = flat_d3_unpack_avx512_safe(bm + ((i * 3) >> 3)); + __m128i syms = _mm_shuffle_epi8(c2s_xmm, codes); + _mm_storeu_si128((__m128i *)(symbols + i), syms); + i += 16; + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_avx512(bm, i * 3, 3); + symbols[i] = c2s[code]; + } +} + +/* D=4: c2s = 16 entries; codes < 16 use only low 4 bits so vpermb on a + * 64-byte register whose first 16 bytes are c2s (rest don't-care) works. + * Wide 64-at-a-time path first, then existing 16-wide tail. No + * over-read concern for the wide path since flat_d4_unpack64's 32-byte + * load only consumes bytes 0..31 = 32 valid bytes, but ymm read of bm + * still over-reads past the bm region. */ +static inline void merge_flat_d4_avx512(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + __m128i c2s_xmm = _mm_loadu_si128((const __m128i *)c2s); + __m512i c2s_zmm = _mm512_castsi128_si512(c2s_xmm); + int i = 0; + /* Slack: strict bound is ceil(32*8/4)=64 codes; use 128 to drop into + * the 16-wide tail and stay symmetric with the other widths. */ + for (; i + 128 <= n; i += 64) { + __m512i codes = flat_d4_unpack64_avx512_fast(bm + ((i * 4) >> 3)); + __m512i syms = _mm512_permutexvar_epi8(codes, c2s_zmm); + _mm512_storeu_si512((__m512i *)(symbols + i), syms); + } + for (; i + 16 <= n; i += 16) { + __m128i codes = flat_d4_unpack_avx512(bm + (i >> 1)); + __m128i syms = _mm_shuffle_epi8(c2s_xmm, codes); + _mm_storeu_si128((__m128i *)(symbols + i), syms); + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_avx512(bm, i * 4, 4); + symbols[i] = c2s[code]; + } +} + +/* D=5: c2s = 32 entries. Fast path uses the 64-at-a-time zmm unpack + * (flat_d5_unpack64_avx512_fast) + vpermb over a zmm broadcast of the + * 32-byte c2s; codes < 32 keep the high half don't-care so the cast is + * safe. Tail of 16..63 codes drops to the 16-at-a-time vpermb-ymm + * form, and the last <16 codes go scalar. */ +static inline void merge_flat_d5_avx512(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + __m256i c2s_ymm = _mm256_loadu_si256((const __m256i *)c2s); + __m512i c2s_zmm = _mm512_castsi256_si512(c2s_ymm); + int i = 0; + /* 64-wide fast loop. Leave a 64-element safety margin so the + * 64-byte zmm load past bm[i*5/8] never reads into uninitialised + * pages — the tail handles the remainder. */ + int fast64_end = n >= 64 ? n - 64 : 0; + for (; i + 64 <= fast64_end; i += 64) { + __m512i codes = flat_d5_unpack64_avx512_fast(bm + ((i * 5) >> 3)); + __m512i syms = _mm512_permutexvar_epi8(codes, c2s_zmm); + _mm512_storeu_si512((__m512i *)(symbols + i), syms); + } + /* 16-wide fast loop drains down to ≤16 remaining. */ + int fast16_end = n >= 16 ? n - 16 : 0; + for (; i + 16 <= fast16_end; i += 16) { + __m128i codes = flat_d5_unpack_avx512_fast(bm + ((i * 5) >> 3)); + __m256i codes_ext = _mm256_zextsi128_si256(codes); + __m256i syms_full = _mm256_permutexvar_epi8(codes_ext, c2s_ymm); + _mm_storeu_si128((__m128i *)(symbols + i), + _mm256_castsi256_si128(syms_full)); + } + if (i + 16 <= n) { + __m128i codes = flat_d5_unpack_avx512_safe(bm + ((i * 5) >> 3)); + __m256i codes_ext = _mm256_zextsi128_si256(codes); + __m256i syms_full = _mm256_permutexvar_epi8(codes_ext, c2s_ymm); + _mm_storeu_si128((__m128i *)(symbols + i), + _mm256_castsi256_si128(syms_full)); + i += 16; + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_avx512(bm, i * 5, 5); + symbols[i] = c2s[code]; + } +} + +/* D=6: c2s = 64 entries fits in a zmm. Wide 64-at-a-time path first + * (flat_d6_unpack64 + vpermb-zmm), then existing 16-wide tail. */ +static inline void merge_flat_d6_avx512(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + __m512i c2s_zmm = _mm512_loadu_si512((const __m512i *)c2s); + int i = 0; + /* Slack: strict bound is ceil(64*8/6)=86 codes; use 128 to match the + * other widths and drop into the 16-wide tail. */ + for (; i + 128 <= n; i += 64) { + __m512i codes = flat_d6_unpack64_avx512_fast(bm + ((i * 6) >> 3)); + __m512i syms = _mm512_permutexvar_epi8(codes, c2s_zmm); + _mm512_storeu_si512((__m512i *)(symbols + i), syms); + } + int fast_end = n >= 16 ? n - 16 : 0; + for (; i + 16 <= fast_end; i += 16) { + __m128i codes = flat_d6_unpack_avx512_fast(bm + ((i * 6) >> 3)); + __m512i codes_ext = _mm512_castsi128_si512(codes); + __m512i syms_full = _mm512_permutexvar_epi8(codes_ext, c2s_zmm); + _mm_storeu_si128((__m128i *)(symbols + i), + _mm512_castsi512_si128(syms_full)); + } + if (i + 16 <= n) { + __m128i codes = flat_d6_unpack_avx512_safe(bm + ((i * 6) >> 3)); + __m512i codes_ext = _mm512_castsi128_si512(codes); + __m512i syms_full = _mm512_permutexvar_epi8(codes_ext, c2s_zmm); + _mm_storeu_si128((__m128i *)(symbols + i), + _mm512_castsi512_si128(syms_full)); + i += 16; + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_avx512(bm, i * 6, 6); + symbols[i] = c2s[code]; + } +} + +/* D=7: c2s = 128 entries spans two zmm tables. One vpermi2b looks up + * the full 128-byte table in a single op. Wide 64-at-a-time path first + * (flat_d7_unpack64 + vpermi2b), then existing 16-wide tail. */ +static inline void merge_flat_d7_avx512(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + __m512i c2s_lo = _mm512_loadu_si512((const __m512i *)c2s); + __m512i c2s_hi = _mm512_loadu_si512((const __m512i *)(c2s + 64)); + int i = 0; + /* Slack: strict bound is ceil(64*8/7)=74 codes; use 128 for symmetry + * with the other widths. */ + for (; i + 128 <= n; i += 64) { + __m512i codes = flat_d7_unpack64_avx512_fast(bm + ((i * 7) >> 3)); + __m512i syms = _mm512_permutex2var_epi8(c2s_lo, codes, c2s_hi); + _mm512_storeu_si512((__m512i *)(symbols + i), syms); + } + int fast_end = n >= 16 ? n - 16 : 0; + for (; i + 16 <= fast_end; i += 16) { + __m128i codes = flat_d7_unpack_avx512_fast(bm + ((i * 7) >> 3)); + __m512i syms = _mm512_permutex2var_epi8(c2s_lo, + _mm512_castsi128_si512(codes), c2s_hi); + _mm_storeu_si128((__m128i *)(symbols + i), + _mm512_castsi512_si128(syms)); + } + if (i + 16 <= n) { + __m128i codes = flat_d7_unpack_avx512_safe(bm + ((i * 7) >> 3)); + __m512i syms = _mm512_permutex2var_epi8(c2s_lo, + _mm512_castsi128_si512(codes), c2s_hi); + _mm_storeu_si128((__m128i *)(symbols + i), + _mm512_castsi512_si128(syms)); + i += 16; + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_avx512(bm, i * 7, 7); + symbols[i] = c2s[code]; + } +} + +/* D=8: a depth-8 flat region is the full 256-symbol alphabet at equal code + * length, whose canonical c2s is the identity permutation -- the byte-aligned + * codes ARE the symbols, so the whole decode is a memcpy. See the derivation + * at merge_flat_d8_neon in pivco_huffman_primitives_neon.h. */ +static inline void merge_flat_d8_avx512(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + (void)c2s; + memcpy(symbols, bm, (size_t)n); +} + +/* merge_flat_avx512 — D-bit flat-subtree decode into a + * contiguous output buffer. Dispatches to the per-D specialisation. */ +static inline void merge_flat_avx512(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s) +{ + PROF_TIC(); + switch (D) { + case 2: merge_flat_d2_avx512(out, n, bm, c2s); break; + case 3: merge_flat_d3_avx512(out, n, bm, c2s); break; + case 4: merge_flat_d4_avx512(out, n, bm, c2s); break; + case 5: merge_flat_d5_avx512(out, n, bm, c2s); break; + case 6: merge_flat_d6_avx512(out, n, bm, c2s); break; + case 7: merge_flat_d7_avx512(out, n, bm, c2s); break; + case 8: merge_flat_d8_avx512(out, n, bm, c2s); break; + default: + for (int i = 0; i < n; i++) { + uint32_t code = extract_D_bits_avx512(bm, i * D, D); + out[i] = c2s[code]; + } + break; + } + PROF_TOC(PROF_BU_MERGE_FLAT, n); +} + +/* ---------- Encode primitives: rank-based encoding (8-bit in-order ranks) ---------- + * Partition 8-bit leaf ranks against split_rank via vpcompressb (64/iter). + * Flat pack subtracts flat_base_rank then reuses pack_dN_avx512. */ +#include + +/* init_avx512 — gather ranks[i] = sym_to_rank[symbols[i]] via two vpermi2b + * over the 256-byte LUT. Simpler than the code_la enc_init (output is a single + * rank byte, no lo/hi code split): each vpermi2b covers 128 entries indexed by + * the symbol's low 7 bits; blend the two halves by bit 7. 64 ranks/iter. + * + * This is the AVX-512 encode bottleneck if left scalar: a scalar 1 MB byte + * gather runs several x longer than the whole vectorized partition tree, so on + * AVX-512 it (not the partition) was what made the rank encode trail code_la. */ +static inline void init_avx512(uint8_t *ranks, int n, + const uint8_t *sym, const uint8_t *s2r) +{ + __m512i t_lo0 = _mm512_loadu_si512((const __m512i *)(s2r + 0)); /* entries [ 0: 64) */ + __m512i t_lo1 = _mm512_loadu_si512((const __m512i *)(s2r + 64)); /* entries [ 64:128) */ + __m512i t_hi0 = _mm512_loadu_si512((const __m512i *)(s2r + 128)); /* entries [128:192) */ + __m512i t_hi1 = _mm512_loadu_si512((const __m512i *)(s2r + 192)); /* entries [192:256) */ + + PROF_TIC(); + int i = 0; + for (; i + 64 <= n; i += 64) { + __m512i c = _mm512_loadu_si512((const __m512i *)(sym + i)); + __mmask64 hib = _mm512_movepi8_mask(c); /* bit 7 of each symbol */ + __m512i lo = _mm512_permutex2var_epi8(t_lo0, c, t_lo1); /* LUT[ c & 127] */ + __m512i hi = _mm512_permutex2var_epi8(t_hi0, c, t_hi1); /* LUT[128 + (c & 127)] */ + _mm512_storeu_si512((__m512i *)(ranks + i), + _mm512_mask_blend_epi8(hib, lo, hi)); + } + for (; i < n; i++) ranks[i] = s2r[sym[i]]; + PROF_TOC(PROF_ENC_INIT, n); +} + +/* full: both sides compacted (right -> tmp, left in place into ranks). */ +static inline int part_full_avx512(uint8_t *ranks, int n, uint8_t thr, + uint8_t *bm, uint8_t *tmp) +{ + int n_left = 0, n_right = 0; + int j = 0; + __m512i vt = _mm512_set1_epi8((char)thr); + for (; j + 64 <= n; j += 64) { + __m512i v = _mm512_loadu_si512((const void *)(ranks + j)); + __mmask64 k = _mm512_cmpgt_epu8_mask(v, vt); + int p = __builtin_popcountll(k); + memcpy(bm + (j >> 3), &k, 8); + /* mask_compress with v as pass-through, not maskz: Zen 4/5 false-dep + * on the maskz destination (issue #11, as in the merges); the lanes + * past popcount are dead either way -- the next store overwrites. */ + _mm512_storeu_si512((void *)(tmp + n_right), _mm512_mask_compress_epi8(v, k, v)); + _mm512_storeu_si512((void *)(ranks + n_left), _mm512_mask_compress_epi8(v, ~k, v)); + n_right += p; + n_left += 64 - p; + } + for (; j < n; j++) { + if ((j & 7) == 0) bm[j >> 3] = 0; + uint8_t r = ranks[j]; + if (r > thr) { bm[j >> 3] |= (uint8_t)(1u << (j & 7)); tmp[n_right++] = r; } + else { ranks[n_left++] = r; } + } + return n_right; +} + +/* right (LEAF_LEFT): compact the right side only, to tmp. */ +static inline int part_right_avx512(uint8_t *ranks, int n, uint8_t thr, + uint8_t *bm, uint8_t *tmp) +{ + int n_right = 0, j = 0; + __m512i vt = _mm512_set1_epi8((char)thr); + for (; j + 64 <= n; j += 64) { + __m512i v = _mm512_loadu_si512((const void *)(ranks + j)); + __mmask64 k = _mm512_cmpgt_epu8_mask(v, vt); + memcpy(bm + (j >> 3), &k, 8); + /* mask_compress into v: see part_full (issue #11 false dep) */ + _mm512_storeu_si512((void *)(tmp + n_right), _mm512_mask_compress_epi8(v, k, v)); + n_right += __builtin_popcountll(k); + } + for (; j < n; j++) { + if ((j & 7) == 0) bm[j >> 3] = 0; + uint8_t r = ranks[j]; + if (r > thr) { bm[j >> 3] |= (uint8_t)(1u << (j & 7)); tmp[n_right++] = r; } + } + return n_right; +} + +/* none (BOTH_LEAVES): bitmap + right count only, no compaction. */ +static inline int part_none_avx512(uint8_t *ranks, int n, uint8_t thr, uint8_t *bm) +{ + int n_right = 0, j = 0; + __m512i vt = _mm512_set1_epi8((char)thr); + for (; j + 64 <= n; j += 64) { + __m512i v = _mm512_loadu_si512((const void *)(ranks + j)); + __mmask64 k = _mm512_cmpgt_epu8_mask(v, vt); + memcpy(bm + (j >> 3), &k, 8); + n_right += __builtin_popcountll(k); + } + for (; j < n; j++) { + if ((j & 7) == 0) bm[j >> 3] = 0; + uint8_t r = ranks[j]; + if (r > thr) { bm[j >> 3] |= (uint8_t)(1u << (j & 7)); n_right++; } + } + return n_right; +} + +/* Flat pack, native u8: the local code (rank - base) is already a D-bit byte, + * so we pack straight from u8 (no u16 widen + cvtepi16 narrow round-trip). + * Per-D kernels in pivco_huffman_avx512_pack.h; scalar tail for the residual. */ +static inline void pack_dN_avx512(uint8_t *out, const uint8_t *ranks, + int n, int D, uint8_t base) +{ + int total_bytes = (n * D + 7) >> 3; + if (total_bytes > 0) out[total_bytes - 1] = 0; + + int i = 0; + switch (D) { + case 2: i = pack_d2_avx512(out, ranks, n, base); break; + case 3: i = pack_d3_avx512(out, ranks, n, base); break; + case 4: i = pack_d4_avx512(out, ranks, n, base); break; + case 5: i = pack_d5_avx512(out, ranks, n, base); break; + case 6: i = pack_d6_avx512(out, ranks, n, base); break; + case 7: i = pack_d7_avx512(out, ranks, n, base); break; + case 8: i = pack_d8_avx512(out, ranks, n, base); break; + default: break; + } + if (i >= n) return; + + int bit_pos = i * D; + int byte_idx = bit_pos >> 3; + int bits_in_buf = bit_pos & 7; + uint64_t buf = bits_in_buf > 0 + ? (uint64_t)out[byte_idx] & ((1u << bits_in_buf) - 1) + : 0; + for (; i < n; i++) { + uint32_t local = (uint32_t)(uint8_t)(ranks[i] - base); /* code in [0,2^D); no mask */ + buf |= (uint64_t)local << bits_in_buf; + bits_in_buf += D; + while (bits_in_buf >= 8) { out[byte_idx++] = (uint8_t)buf; buf >>= 8; bits_in_buf -= 8; } + } + if (bits_in_buf > 0) out[byte_idx] = (uint8_t)(buf & ((1u << bits_in_buf) - 1)); +} + +/* ---------- Aliases consumed by codec.c ---------- */ + +#define PIVCO_PRIM_ALWAYS_INLINE __attribute__((always_inline)) static inline + +/* Widest load a merge kernel issues at a child-buffer cursor (8B expandloadu tail window); + * the cursor can rest AT `size` on the exhausted side, so buffers a + * merge reads need this much trailing slack. Consumed by the decode + * placement logic (scratch_carve / place_tail). */ +#define PIVCO_PRIM_MERGE_OVERREAD 8 + +PIVCO_PRIM_ALWAYS_INLINE void prim_codec_init(void) +{ codec_init_avx512(); } + +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_init(uint8_t *ranks, int n, + const uint8_t *symbols, + const uint8_t *sym_to_rank, + const pivco_enc_init_aux_t *aux) +{ (void)aux; init_avx512(ranks, n, symbols, sym_to_rank); } + +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_full(uint8_t *ranks, + int n, uint8_t thr, + uint8_t *bm, + uint8_t *right_out) +{ return part_full_avx512(ranks, n, thr, bm, right_out); } + +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_right(uint8_t *ranks, + int n, uint8_t thr, + uint8_t *bm, + uint8_t *right_out) +{ return part_right_avx512(ranks, n, thr, bm, right_out); } + +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_none(uint8_t *ranks, + int n, uint8_t thr, + uint8_t *bm) +{ return part_none_avx512(ranks, n, thr, bm); } + +/* Natural-layout kernels exposed for bench_prim's ST_PACK/ST_MERGE_FLAT + * rows (the prim_ entries below produce the layout `vertical` selects: + * the hybrid vertical wire, or natural when 0). */ +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_pack_dN_natural(const uint8_t *ranks, + int n, int D, uint8_t base, uint8_t *out_packed) +{ pack_dN_avx512(out_packed, ranks, n, D, base); } +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_flat_natural(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s) +{ merge_flat_avx512(out, n, bm, D, c2s); } +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_pack_dN(const uint8_t *ranks, + int n, int D, uint8_t base, uint8_t *out_packed, + int vertical) +{ + int n5 = vertical == PIVCO_FLAT_VERTICAL ? pivco_vert_n512(n, D) : 0; + if (n5) vert512_pack_x86_best(out_packed, ranks, n5, D, base); + int r = n - n5, nv = vertical ? pivco_vert_n(r, D) : 0; + uint8_t *o1 = out_packed + (((size_t)n5 * D) >> 3); + if (nv) vert_pack_x86_best(o1, ranks + n5, nv, D, base); + if (r > nv) pack_dN_avx512(o1 + (((size_t)nv * D) >> 3), + ranks + n5 + nv, r - nv, D, base); +} + + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_flat(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s, + int vertical) +{ + int n5 = vertical == PIVCO_FLAT_VERTICAL ? pivco_vert_n512(n, D) : 0; + if (n5) vert512_merge_x86_best(out, n5, bm, D, c2s); + int r = n - n5, nv = vertical ? pivco_vert_n(r, D) : 0; + const uint8_t *bm1 = bm + (((size_t)n5 * D) >> 3); + if (nv) vert_merge_x86_best(out + n5, nv, bm1, D, c2s); + if (r > nv) merge_flat_avx512(out + n5 + nv, r - nv, + bm1 + (((size_t)nv * D) >> 3), D, c2s); +} + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_cst_cst(const uint8_t *bm, int K, + uint8_t left_sym, + uint8_t right_sym, + uint8_t *out) +{ merge_cst_cst_avx512(bm, K, left_sym, right_sym, out); } + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_cst_vec(const uint8_t *bm, int K, + uint8_t left_sym, + const uint8_t *right_buf, + uint8_t *out) +{ merge_cst_vec_avx512(bm, K, left_sym, right_buf, out); } + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_vec_vec(const uint8_t *bm, int K, + const uint8_t *left_buf, + const uint8_t *right_buf, + uint8_t *out) +{ merge_vec_vec_avx512(bm, K, left_buf, right_buf, out); } + + +/* --------------------------------------------------------------------------- + * prim_histogram_chunk — positional (bit-plane) byte histogram. + * + * Port of Harold Aptroot's AVX-512 histogram, used under the MIT + * License (Copyright (c) 2026 Harold Aptroot): + * https://gitlab.com/-/snippets/3745720 (hist.cpp + license.txt) + * Dissected in Jorn Engel's write-up: + * https://github.com/JoernEngel/joernblog/blob/master/histogram.md + * + * Bytes are binned by their top 2 bits into four 16 KB buffers + * (masked vpcompressb), the remaining 6 bits rotated via one GF2P8 + * affine so 1-bit "positional" counters (one bit of each of 512 + * counters per zmm) can absorb them; vpternlog full-adder trees + * promote 1-bit -> 3-bit -> 8-bit -> 16-bit -> 32-bit counters. + * Input-shape independent by construction (~7.5 GB/s GNR, ~12-14 GB/s + * Zen 5 with clang; gcc trails ~10-35%% on the FA chains). + * + * Needs GFNI + BITALG + VBMI2 on top of the backend baseline; without + * the compile flags the shared scalar core is used instead. + * ------------------------------------------------------------------------- */ +#include "pivco_huffman_hist_scalar.h" + +#if defined(__GFNI__) && defined(__AVX512BITALG__) + +#define HIST_FA(hh, ll, a, b, c) do { \ + __m512i _l = _mm512_ternarylogic_epi32((c), (b), (a), 0x96); \ + (hh) = _mm512_ternarylogic_epi32(_l, (b), (a), 0x8E); \ + (ll) = _l; \ +} while (0) + +/* consume one bin: N bytes of 6-bit morsels -> 64 u16 counters */ +static void hist_consume_bin_avx512(uint8_t *data, size_t N, + uint16_t *hist16) +{ + size_t tail = N & 63; + if (tail) { + __m512i *where = (__m512i *)(data + N - tail); + _mm512_store_epi64(where, + _mm512_or_epi64(_mm512_load_epi64(where), + _mm512_movm_epi8(~0ull << tail))); + N = N + 64 - tail; + } + N /= 64; + + __m512i h0 = _mm512_setzero_si512(); + __m512i h1 = _mm512_setzero_si512(); + __m512i w0_0 = _mm512_setzero_si512(); + __m512i w1_0 = _mm512_setzero_si512(); + __m512i w2_0 = _mm512_setzero_si512(); + + static const uint8_t tp_bytes[64] __attribute__((aligned(64))) = { + 0, 8, 16, 24, 32, 40, 48, 56, + 1, 9, 17, 25, 33, 41, 49, 57, + 2, 10, 18, 26, 34, 42, 50, 58, + 3, 11, 19, 27, 35, 43, 51, 59, + 4, 12, 20, 28, 36, 44, 52, 60, + 5, 13, 21, 29, 37, 45, 53, 61, + 6, 14, 22, 30, 38, 46, 54, 62, + 7, 15, 23, 31, 39, 47, 55, 63}; + const __m512i tp = _mm512_load_si512((const void *)tp_bytes); + + do { + size_t M = N > 31 ? 31 : N; + N -= M; + __m512i w = _mm512_setzero_si512(); + do { + __m512i x0 = _mm512_sllv_epi64(_mm512_set1_epi64(1), _mm512_cvtepu8_epi64(_mm_loadu_si64(data + 0))); + __m512i x1 = _mm512_sllv_epi64(_mm512_set1_epi64(1), _mm512_cvtepu8_epi64(_mm_loadu_si64(data + 8))); + __m512i x2 = _mm512_sllv_epi64(_mm512_set1_epi64(1), _mm512_cvtepu8_epi64(_mm_loadu_si64(data + 16))); + __m512i x3 = _mm512_sllv_epi64(_mm512_set1_epi64(1), _mm512_cvtepu8_epi64(_mm_loadu_si64(data + 24))); + __m512i x4 = _mm512_sllv_epi64(_mm512_set1_epi64(1), _mm512_cvtepu8_epi64(_mm_loadu_si64(data + 32))); + __m512i x5 = _mm512_sllv_epi64(_mm512_set1_epi64(1), _mm512_cvtepu8_epi64(_mm_loadu_si64(data + 40))); + __m512i x6 = _mm512_sllv_epi64(_mm512_set1_epi64(1), _mm512_cvtepu8_epi64(_mm_loadu_si64(data + 48))); + __m512i x7 = _mm512_sllv_epi64(_mm512_set1_epi64(1), _mm512_cvtepu8_epi64(_mm_loadu_si64(data + 56))); + data += 64; + + HIST_FA(x1, x2, x0, x1, x2); + HIST_FA(x4, x5, x3, x4, x5); + HIST_FA(x7, w0_0, x6, x7, w0_0); + HIST_FA(x5, w0_0, x2, x5, w0_0); + HIST_FA(x4, x7, x1, x4, x7); + HIST_FA(x5, w1_0, x7, x5, w1_0); + __m512i w3_0; + HIST_FA(w3_0, w2_0, x5, x4, w2_0); + + w3_0 = _mm512_permutexvar_epi8(tp, w3_0); + w3_0 = _mm512_gf2p8affine_epi64_epi8(_mm512_set1_epi64(0x8040201008040201), w3_0, 0); + w3_0 = _mm512_popcnt_epi8(w3_0); + w = _mm512_add_epi8(w, w3_0); + } while (--M); + + h0 = _mm512_add_epi16(h0, _mm512_and_epi64(w, _mm512_set1_epi16(0xFF))); + h1 = _mm512_add_epi16(h1, _mm512_srli_epi16(w, 8)); + } while (N); + + w0_0 = _mm512_permutexvar_epi8(tp, w0_0); + w1_0 = _mm512_permutexvar_epi8(tp, w1_0); + w2_0 = _mm512_permutexvar_epi8(tp, w2_0); + w0_0 = _mm512_gf2p8affine_epi64_epi8(_mm512_set1_epi64(0x8040201008040201), w0_0, 0); + w1_0 = _mm512_gf2p8affine_epi64_epi8(_mm512_set1_epi64(0x8040201008040201), w1_0, 0); + w2_0 = _mm512_gf2p8affine_epi64_epi8(_mm512_set1_epi64(0x8040201008040201), w2_0, 0); + w0_0 = _mm512_popcnt_epi8(w0_0); + w1_0 = _mm512_popcnt_epi8(w1_0); + w2_0 = _mm512_popcnt_epi8(w2_0); + + __m512i w = _mm512_add_epi8(_mm512_add_epi8(w0_0, _mm512_add_epi8(w1_0, w1_0)), + _mm512_slli_epi64(w2_0, 2)); + h0 = _mm512_add_epi16(_mm512_slli_epi16(h0, 3), _mm512_and_epi64(w, _mm512_set1_epi16(0xFF))); + h1 = _mm512_add_epi16(_mm512_slli_epi16(h1, 3), _mm512_srli_epi16(w, 8)); + + _mm512_storeu_epi16(hist16, _mm512_add_epi16(h0, _mm512_loadu_epi16(hist16))); + _mm512_storeu_epi16(hist16 + 32, _mm512_add_epi16(h1, _mm512_loadu_epi16(hist16 + 32))); +} + +static inline void histogram_chunk_avx512(const uint8_t *in, size_t n, + uint32_t hist[256], uint8_t *scratch) +{ + const uint8_t *ptr = in; + size_t N = n; + if (N >= 64) { + const uint8_t *end = ptr + N; + while ((uintptr_t)ptr & 63) hist[*ptr++] += 1; + N = (size_t)(end - ptr); + } + + const size_t bufsize = 16 * 1024; + uint8_t *buffer0 = (uint8_t *)(((uintptr_t)scratch + 63) & ~(uintptr_t)63); + uint8_t *buffer1 = buffer0 + bufsize; + uint8_t *buffer2 = buffer1 + bufsize; + uint8_t *buffer3 = buffer2 + bufsize; + + while (N >= 64) { + uint16_t hist16[256] = {0}; + size_t count0 = 0, count1 = 0, count2 = 0, count3 = 0; + /* consume up to 2^16-64 bytes per round: a u16 counter cannot + * overflow within a round by construction */ + size_t M = N >= 65472 ? 65472 : (N & (size_t)-64); + N -= M; + for (size_t i = 0; i < M; i += 64) { + __m512i data = _mm512_load_si512(ptr + i); + __mmask64 bit7 = _mm512_movepi8_mask(data); + __mmask64 bit6 = _mm512_movepi8_mask(_mm512_add_epi8(data, data)); + __mmask64 b00 = _knot_mask64(_kor_mask64(bit6, bit7)); + __mmask64 b01 = _kandn_mask64(bit7, bit6); + __mmask64 b10 = _kandn_mask64(bit6, bit7); + __mmask64 b11 = _kand_mask64(bit7, bit6); + data = _mm512_gf2p8affine_epi64_epi8(data, _mm512_set1_epi64(0x2010010204080000), 0); + _mm512_storeu_epi8(buffer0 + count0, _mm512_maskz_compress_epi8(b00, data)); + _mm512_storeu_epi8(buffer1 + count1, _mm512_maskz_compress_epi8(b01, data)); + _mm512_storeu_epi8(buffer2 + count2, _mm512_maskz_compress_epi8(b10, data)); + _mm512_storeu_epi8(buffer3 + count3, _mm512_maskz_compress_epi8(b11, data)); + count0 += (size_t)_mm_popcnt_u64(b00); + count1 += (size_t)_mm_popcnt_u64(b01); + count2 += (size_t)_mm_popcnt_u64(b10); + count3 += (size_t)_mm_popcnt_u64(b11); + + if (count0 >= bufsize - 64) { hist_consume_bin_avx512(buffer0, count0, &hist16[0]); count0 = 0; } + if (count1 >= bufsize - 64) { hist_consume_bin_avx512(buffer1, count1, &hist16[64]); count1 = 0; } + if (count2 >= bufsize - 64) { hist_consume_bin_avx512(buffer2, count2, &hist16[128]); count2 = 0; } + if (count3 >= bufsize - 64) { hist_consume_bin_avx512(buffer3, count3, &hist16[192]); count3 = 0; } + } + ptr += M; + + if (count0) hist_consume_bin_avx512(buffer0, count0, &hist16[0]); + if (count1) hist_consume_bin_avx512(buffer1, count1, &hist16[64]); + if (count2) hist_consume_bin_avx512(buffer2, count2, &hist16[128]); + if (count3) hist_consume_bin_avx512(buffer3, count3, &hist16[192]); + + for (size_t i = 0; i < 256; i += 64) { + __m512i h0 = _mm512_loadu_epi16(hist16 + i); + __m512i h1 = _mm512_loadu_epi16(hist16 + i + 32); + __m512i w0 = _mm512_and_epi32(h0, _mm512_set1_epi32(0xFFFF)); + __m512i w1 = _mm512_srli_epi32(h0, 16); + __m512i w2 = _mm512_and_epi32(h1, _mm512_set1_epi32(0xFFFF)); + __m512i w3 = _mm512_srli_epi32(h1, 16); + _mm512_storeu_epi32(hist + i, _mm512_add_epi32(w0, _mm512_loadu_epi32(hist + i))); + _mm512_storeu_epi32(hist + i + 16, _mm512_add_epi32(w1, _mm512_loadu_epi32(hist + i + 16))); + _mm512_storeu_epi32(hist + i + 32, _mm512_add_epi32(w2, _mm512_loadu_epi32(hist + i + 32))); + _mm512_storeu_epi32(hist + i + 48, _mm512_add_epi32(w3, _mm512_loadu_epi32(hist + i + 48))); + } + } + + while (N) hist[ptr[--N]] += 1; +} +PIVCO_PRIM_ALWAYS_INLINE void prim_histogram_chunk(const uint8_t *in, size_t n, + uint32_t hist[256], + uint8_t *scratch) +{ histogram_chunk_avx512(in, n, hist, scratch); } + +#else /* no GFNI/BITALG compile support: fall back to the scalar core */ +PIVCO_PRIM_ALWAYS_INLINE void prim_histogram_chunk(const uint8_t *in, size_t n, + uint32_t hist[256], + uint8_t *scratch) +{ histogram_chunk_scalar(in, n, hist, scratch); } +#endif /* __GFNI__ && __AVX512BITALG__ */ + +#endif /* PIVCO_HUFFMAN_PRIMITIVES_AVX512_H */ diff --git a/third_party/pivco/src/pivco_huffman_primitives_neon.h b/third_party/pivco/src/pivco_huffman_primitives_neon.h new file mode 100644 index 0000000..e94ee46 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_primitives_neon.h @@ -0,0 +1,1772 @@ +/* pivco_huffman_primitives_neon.h — NEON implementations of the codec + * primitive interface (see pivco_huffman_primitives.h). + * + * Specialized names end in `_neon`; the codec calls the aliases + * `prim_*` defined at the bottom as always-inline wrappers. + * + * Internal header. Included by pivco_huffman_primitives.h when + * PIVCO_BACKEND_NEON is defined. Also #included by the legacy + * src/pivco_bu_neon.c during the Phase 3 transition (the + * legacy file calls these primitives directly until step 3.8 retires + * it). Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_PRIMITIVES_NEON_H +#define PIVCO_HUFFMAN_PRIMITIVES_NEON_H + +#if !defined(__aarch64__) +#error "pivco_huffman_primitives_neon.h requires aarch64 NEON" +#endif + +#include "pivco_huffman.h" +#include "pivco_huffman_common.h" +#include "pivco_huffman_neon_tables.h" /* expand_tab*, compress_tab* */ +#include "pivco_huffman_neon_flat.h" /* flat_d{2,3,4,5,6}_unpack */ +#include "pivco_huffman_neon_pack.h" /* pack_d{5,6,7}_neon (variable-shift pack) */ +#include "pivco_prof.h" + +#include +#include +#include + +/* Backend lifecycle. Lazily build the compress_tab pre-bake the NEON + * partition primitives index into, plus the merge shuffle pair. + * Of the expand tables only expand_popcnt (256 B) is still read by the + * shipped kernels (merge-tail cursor advance); expand_tab/expand_tab_pre + * stay built for the bench prim_variants that index them. Idempotent + * and cheap after the first call. */ +static void init_merge_tables(void); /* two-table merge_vec_vec shuffles (below) */ +static inline void codec_init_neon(void) +{ + init_compress_table(); + init_expand_table(); + init_merge_tables(); +} + +/* ---------- Decode primitives (bottom-up) ---------- */ + +/* popcount_K_right_neon — count "1" bits in the first K bits of bm. + * Vectorised: 64-byte main path with 4-wide ILP, then 16-byte mop-up, + * scalar tail for the trailing 0..15 full bytes + the optional partial + * byte (K & 7). `nbytes` is derivable from K; kept for signature + * stability with the BU x86 backend. */ +static inline int popcount_K_right_neon(const uint8_t *bm, int nbytes, int K) +{ + (void)nbytes; + PROF_TIC(); + int full_bytes = K >> 3; + int partial_bits = K & 7; + + uint16x8_t acc_v = vdupq_n_u16(0); + int b = 0; + for (; b + 64 <= full_bytes; b += 64) { + uint8x16_t v0 = vld1q_u8(bm + b); + uint8x16_t v1 = vld1q_u8(bm + b + 16); + uint8x16_t v2 = vld1q_u8(bm + b + 32); + uint8x16_t v3 = vld1q_u8(bm + b + 48); + uint8x16_t c0 = vcntq_u8(v0); + uint8x16_t c1 = vcntq_u8(v1); + uint8x16_t c2 = vcntq_u8(v2); + uint8x16_t c3 = vcntq_u8(v3); + /* 3-level lane-wise add tree, all in u8 (max 32 at root). */ + uint8x16_t s01 = vaddq_u8(c0, c1); + uint8x16_t s23 = vaddq_u8(c2, c3); + uint8x16_t s = vaddq_u8(s01, s23); + acc_v = vaddq_u16(acc_v, vpaddlq_u8(s)); + } + for (; b + 16 <= full_bytes; b += 16) { + uint8x16_t v = vld1q_u8(bm + b); + acc_v = vaddq_u16(acc_v, vpaddlq_u8(vcntq_u8(v))); + } + int K_right = (int)vaddvq_u16(acc_v); + for (; b < full_bytes; b++) K_right += __builtin_popcount(bm[b]); + if (partial_bits) { + uint8_t valid_mask = (uint8_t)((1u << partial_bits) - 1); + K_right += __builtin_popcount(bm[full_bytes] & valid_mask); + } + PROF_TOC(PROF_BU_POPCOUNT_K, K); + return K_right; +} + +/* ---- merge_vec_vec_neon: two-table SABD merge, 64 bytes/iter ---- + * + * One 2-source vqtbl2q over {R16, L16} per 16-byte chunk; the cross-half + * cursor offset is folded into the shuffle index by SABD (|shuf0 - shuf1|), + * so no explicit add. Four chunks per 64-byte iter share one vcnt + 64-bit + * multiply prefix-sum for the per-chunk cursor splits and the L/R advance. The + * two 256x16 index tables (g_merge_shuf0/1, 8 KiB) are built once in + * codec_init_neon. Tail (K mod 64) runs the same SABD merge at 16- and + * 8-wide on the same tables (the 8-wide form stores the low half only), + * so the whole kernel touches only g_merge_shuf0/1 plus the 256-byte + * expand_popcnt (tail cursor advance; aarch64 has no GPR popcount) -- + * the old expand_tab/expand_tab_pre ladder dragged up to 20 KiB of + * cold table lines into L1 for at most two tail iterations per node. */ +static int8_t g_merge_shuf0[256 * 16] __attribute__((aligned(16))); +static int8_t g_merge_shuf1[256 * 16] __attribute__((aligned(16))); +static void init_merge_tables(void) +{ + static int built = 0; + if (built) return; + for (int i = 0; i < 256; i++) { + int8_t pop = 0; + int8_t *o0 = &g_merge_shuf0[i * 16]; + int8_t *o1 = &g_merge_shuf1[i * 16]; + for (int j = 0; j < 8; j++) { + if ((i >> j) & 1) { + o0[j] = pop; o1[j + 8] = (int8_t)(-pop); pop++; + } else { + int8_t v = (int8_t)(-16 - j + pop); + o0[j] = v; o1[j + 8] = (int8_t)(8 - v); + } + } + for (int j = 0; j < 8; j++) { o0[j + 8] = pop; o1[j] = 0; } + } + built = 1; +} +/* one 16-byte merge: 2-source TBL over {R,L}, SABD-fused index. */ +static inline void merge_neon_16B(uint8_t *dest, const uint8_t *l_list, + const uint8_t *r_list, intptr_t mask, + const int8_t *tab0, const int8_t *tab1) +{ + int8x16_t shuf0 = vld1q_s8(&tab0[(mask << 4) & 0xff0]); + int8x16_t shuf1 = vld1q_s8(&tab1[(mask >> 4) & 0xff0]); + uint8x16_t shuf = vreinterpretq_u8_s8(vabdq_s8(shuf0, shuf1)); + uint8x16x2_t src; + src.val[0] = vld1q_u8(r_list); + src.val[1] = vld1q_u8(l_list); + vst1q_u8(dest, vqtbl2q_u8(src, shuf)); +} +/* 8-byte residue on the same tables: the 16-bit-mask path with the high + * mask byte zero (tab1 row 0), storing only the low 8 output lanes. + * Both sides consume <= 8 bytes and every lane index stays in its + * half's low 8 lanes, so 8-byte D-register loads suffice (they + * zero-extend for free) -- no over-read past cursor+8. */ +static inline void merge_neon_8B_lo(uint8_t *dest, const uint8_t *l_list, + const uint8_t *r_list, intptr_t m8, + const int8_t *tab0, const int8_t *tab1) +{ + int8x16_t shuf0 = vld1q_s8(&tab0[(m8 << 4) & 0xff0]); + int8x16_t shuf1 = vld1q_s8(&tab1[0]); + uint8x16_t shuf = vreinterpretq_u8_s8(vabdq_s8(shuf0, shuf1)); + uint8x16x2_t src; + src.val[0] = vcombine_u8(vld1_u8(r_list), vdup_n_u8(0)); + src.val[1] = vcombine_u8(vld1_u8(l_list), vdup_n_u8(0)); + vst1_u8(dest, vget_low_u8(vqtbl2q_u8(src, shuf))); +} +static inline void merge_vec_vec_neon(const uint8_t *bm, int K, + const uint8_t *left, + const uint8_t *right, + uint8_t *out) +{ + PROF_TIC(); + const uint8_t *l_list = left, *r_list = right; + intptr_t i = 0; + /* Software-pipelined one iteration deep: the next iteration's carried + * chain (bitmap load -> vcnt -> 64-bit multiply -> cursor advance, + * ~12cy) is started up front so it resolves under the current four + * merges (~16cy). The popcount loads the bitmap straight into SIMD + * (vld1_u8): a GPR->SIMD fmov costs a load-port uop on Apple and + * would sit mid-chain; mask stays in the GPR for merge_neon_16B's + * SABD index. pfx byte k = sum(bytepopcount[0..k]); bytes 1/3/5/7 + * are the 16-lane chunk boundaries c0, c0+c1, c0+c1+c2, total. */ +#define MERGE_VV_64(msk, pf) do { \ + intptr_t p0 = ((pf) >> 8) & 0xff, p1 = ((pf) >> 24) & 0xff, \ + p2 = ((pf) >> 40) & 0xff, p3 = (pf) >> 56; \ + merge_neon_16B(out + i, l_list, r_list, (msk), g_merge_shuf0, g_merge_shuf1); \ + merge_neon_16B(out + i + 16, l_list + 16 - p0, r_list + p0, (msk) >> 16, g_merge_shuf0, g_merge_shuf1); \ + merge_neon_16B(out + i + 32, l_list + 32 - p1, r_list + p1, (msk) >> 32, g_merge_shuf0, g_merge_shuf1); \ + merge_neon_16B(out + i + 48, l_list + 48 - p2, r_list + p2, (msk) >> 48, g_merge_shuf0, g_merge_shuf1); \ + r_list += p3; l_list += 64 - p3; \ + } while (0) + if (i + 64 <= K) { + uint64_t mask; memcpy(&mask, bm, 8); + uint64_t pfx = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(vld1_u8(bm))), 0) + * 0x0101010101010101ull; + for (; i + 128 <= K; i += 64) { + const uint8_t *nbm = bm + ((i + 64) >> 3); + uint64_t nmask; memcpy(&nmask, nbm, 8); + uint64_t npfx = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(vld1_u8(nbm))), 0) + * 0x0101010101010101ull; + MERGE_VV_64(mask, pfx); + mask = nmask; pfx = npfx; + } + MERGE_VV_64(mask, pfx); + i += 64; + } +#undef MERGE_VV_64 + int j = (int)i; + + /* Residue on the main tables: 16-wide, then 8-wide (low half). */ + for (; j + 16 <= K; j += 16) { + uint16_t m16; memcpy(&m16, bm + (j >> 3), 2); + merge_neon_16B(out + j, l_list, r_list, (intptr_t)m16, + g_merge_shuf0, g_merge_shuf1); + /* expand_popcnt, not __builtin_popcount: aarch64 has no GPR + * popcount (fmov+cnt+addv+fmov, ~7cy) and this sits on the + * serial cursor chain between tail iterations. */ + int pop = expand_popcnt[m16 & 0xff] + expand_popcnt[m16 >> 8]; + r_list += pop; l_list += 16 - pop; + } + if (j + 8 <= K) { + intptr_t m8 = bm[j >> 3]; + merge_neon_8B_lo(out + j, l_list, r_list, m8, + g_merge_shuf0, g_merge_shuf1); + int pop = expand_popcnt[m8]; + r_list += pop; l_list += 8 - pop; + j += 8; + } + /* Scalar tail (1..7 leftover). */ + int lc = (int)(l_list - left), rc = (int)(r_list - right); + for (; j < K; j++) { + int mb = (bm[j >> 3] >> (j & 7)) & 1; + out[j] = mb ? right[rc++] : left[lc++]; + } + PROF_TOC(PROF_BU_MERGE_VEC_VEC, K); +} + +/* merge_cst_vec_neon — left input is a broadcast constant. + * Same V5 strategy as merge_vec_vec_neon; the L lane of every chunk's + * vqtbl2 reads from a duplicated 16-byte register holding left_sym, so + * no L loads are issued in the V5 main loop. */ +/* merge_cst_vec_neon — two-table SABD merge, L = broadcast const (no L load + * or cursor); only the R cursor advances. See merge_vec_vec_neon. */ +static inline void merge_cst_vec_neon(const uint8_t *bm, int K, + uint8_t left_sym, + const uint8_t *right, + uint8_t *out) +{ + PROF_TIC(); + uint8x16_t Lb = vdupq_n_u8(left_sym); + const uint8_t *r_list = right; + intptr_t i = 0; + for (; i + 64 <= K; i += 64) { + uint64_t mask; memcpy(&mask, bm + (i >> 3), 8); + uint8x8_t pop8 = vcnt_u8(vcreate_u8(mask)); + uint64_t pfx = vget_lane_u64(vreinterpret_u64_u8(pop8), 0) * 0x0101010101010101ull; + intptr_t p0 = (pfx >> 8) & 0xff, p1 = (pfx >> 24) & 0xff, p2 = (pfx >> 40) & 0xff, p3 = pfx >> 56; +#define _MCV(off, rd, mk) do { \ + int8x16_t s0 = vld1q_s8(&g_merge_shuf0[(((intptr_t)(mk)) << 4) & 0xff0]); \ + int8x16_t s1 = vld1q_s8(&g_merge_shuf1[(((intptr_t)(mk)) >> 4) & 0xff0]); \ + uint8x16_t sh = vreinterpretq_u8_s8(vabdq_s8(s0, s1)); \ + uint8x16x2_t src; src.val[0] = vld1q_u8(rd); src.val[1] = Lb; \ + vst1q_u8(out + i + (off), vqtbl2q_u8(src, sh)); \ + } while (0) + _MCV(0, r_list, mask); + _MCV(16, r_list + p0, mask >> 16); + _MCV(32, r_list + p1, mask >> 32); + _MCV(48, r_list + p2, mask >> 48); +#undef _MCV + r_list += p3; + } + int j = (int)i; + for (; j + 16 <= K; j += 16) { /* 16-byte ryg tail before the scalar mop-up */ + uint16_t m16; memcpy(&m16, bm + (j >> 3), 2); + int8x16_t s0 = vld1q_s8(&g_merge_shuf0[((intptr_t)m16 << 4) & 0xff0]); + int8x16_t s1 = vld1q_s8(&g_merge_shuf1[((intptr_t)m16 >> 4) & 0xff0]); + uint8x16_t sh = vreinterpretq_u8_s8(vabdq_s8(s0, s1)); + uint8x16x2_t src; src.val[0] = vld1q_u8(r_list); src.val[1] = Lb; + vst1q_u8(out + j, vqtbl2q_u8(src, sh)); + r_list += expand_popcnt[m16 & 0xff] + expand_popcnt[m16 >> 8]; + } + if (j + 8 <= K) { /* 8-wide residue: high mask byte 0, store low half. + * 8B D-load on R -- consumes <= 8, no wider + * over-read than the scalar loop it replaces. */ + intptr_t m8 = bm[j >> 3]; + int8x16_t s0 = vld1q_s8(&g_merge_shuf0[(m8 << 4) & 0xff0]); + int8x16_t s1 = vld1q_s8(&g_merge_shuf1[0]); + uint8x16_t sh = vreinterpretq_u8_s8(vabdq_s8(s0, s1)); + uint8x16x2_t src; + src.val[0] = vcombine_u8(vld1_u8(r_list), vdup_n_u8(0)); + src.val[1] = Lb; + vst1_u8(out + j, vget_low_u8(vqtbl2q_u8(src, sh))); + r_list += expand_popcnt[m8]; + j += 8; + } + int rc = (int)(r_list - right); + for (; j < K; j++) { int mb = (bm[j >> 3] >> (j & 7)) & 1; out[j] = mb ? right[rc++] : left_sym; } + PROF_TOC(PROF_BU_MERGE_CST_VEC, K); +} + +/* merge_cst_cst_neon — both inputs are constants. Treated as a + * D=1 flat decode: a 2-byte (left, right) "c2s" table replicated across + * 16 lanes via vdupq_n_u16, indexed by the bm bit (0 or 1). Bit-spread + * uses the same vqtbl(dup_tab) + vshlq(shift_tab) + vandq pattern as + * merge_flat_d2_neon, scaled down for D=1 (8 codes / bm byte + * instead of 4). Faster than vtst+vand+veor by ~1.6× on M4 NEON and + * ~1.4× on Neoverse V2. */ +static const uint8_t merge_two_dup_tab[16] = {0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1}; +static const int8_t merge_two_shift_tab[16] = {0,-1,-2,-3,-4,-5,-6,-7, + 0,-1,-2,-3,-4,-5,-6,-7}; +static inline void merge_cst_cst_neon(const uint8_t *bm, int K, + uint8_t left_sym, uint8_t right_sym, + uint8_t *out) +{ + PROF_TIC(); + uint16_t lr_word = (uint16_t)left_sym | ((uint16_t)right_sym << 8); + uint8x16_t c2s_vec = vreinterpretq_u8_u16(vdupq_n_u16(lr_word)); + uint8x16_t dup_v = vld1q_u8(merge_two_dup_tab); + int8x16_t shift_v = vld1q_s8(merge_two_shift_tab); + uint8x16_t one_v = vdupq_n_u8(1); + + int j = 0; + for (; j + 16 <= K; j += 16) { + uint16_t bm_word; memcpy(&bm_word, bm + (j >> 3), 2); + uint8x16_t bm_lo = vreinterpretq_u8_u16( + vsetq_lane_u16(bm_word, vdupq_n_u16(0), 0)); + uint8x16_t dup = vqtbl1q_u8(bm_lo, dup_v); + uint8x16_t shifted = vshlq_u8(dup, shift_v); + uint8x16_t idx = vandq_u8(shifted, one_v); + vst1q_u8(out + j, vqtbl1q_u8(c2s_vec, idx)); + } + for (; j + 8 <= K; j += 8) { + uint8x8_t bm_v = vdup_n_u8(bm[j >> 3]); + uint8x8_t dup = vtbl1_u8(bm_v, vget_low_u8(dup_v)); + uint8x8_t shifted = vshl_u8(dup, vget_low_s8(shift_v)); + uint8x8_t idx = vand_u8(shifted, vget_low_u8(one_v)); + vst1_u8(out + j, vtbl1_u8(vget_low_u8(c2s_vec), idx)); + } + for (; j < K; j++) { + int mb = (bm[j >> 3] >> (j & 7)) & 1; + out[j] = mb ? right_sym : left_sym; + } + PROF_TOC(PROF_BU_MERGE_CST_CST, K); +} + +/* ---------- Flat-subtree decode (contiguous output) ---------- + * + * Reads n*D packed bits, looks up each D-bit code in c2s, writes the + * resulting bytes to out[0..n). Output is dense / sequential -- the + * BU codec calls this when it hits a PIVCO_NODE_INTERNAL_FLAT. + * + * One static-inline per supported D (2..8); merge_flat_neon + * is a switch dispatcher. The per-D unpack helpers + * (flat_d{2,3,4,5,6,7}_unpack) come from pivco_huffman_neon_flat.h. + */ + +/* Extract D bits at bit position `bit_pos` from `in`. D <= 16. Used + * by each per-D function's non-aligned scalar tail. */ +static inline uint32_t extract_D_bits_neon(const uint8_t *in, + int bit_pos, int D) +{ + int byte_idx = bit_pos >> 3; + int bit_off = bit_pos & 7; + uint32_t val = (uint32_t)in[byte_idx]; + if (bit_off + D > 8) val |= ((uint32_t)in[byte_idx + 1]) << 8; + if (bit_off + D > 16) val |= ((uint32_t)in[byte_idx + 2]) << 16; + return (val >> bit_off) & ((1u << D) - 1); +} + +/* D=2 (4 codes/byte) */ +static inline void merge_flat_d2_neon(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + int i = 0; + if (n >= 64) { + /* fast path maps each input nibble straight to a symbol pair via two + * prepped tables (TL[n]=c2s[n&3], TH[n]=c2s[(n>>2)&3]) */ + static const uint8_t th_idx[16] = {0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3}; + uint32_t w; memcpy(&w, c2s, 4); + const uint8x16_t TL = vreinterpretq_u8_u32(vdupq_n_u32(w)); /* c2s[n&3] */ + const uint8x16_t TH = vqtbl1q_u8(TL, vld1q_u8(th_idx)); /* c2s[(n>>2)&3] */ + const uint8x16_t m = vdupq_n_u8(0x0F); + for (; i + 64 <= n; i += 64) { + uint8x16_t v = vld1q_u8(bm + (i >> 2)); + uint8x16_t lo = vandq_u8(v, m), hi = vshrq_n_u8(v, 4); + /* four planar 16-symbol vectors, one per 2-bit code position */ + uint8x16x4_t o = {{ vqtbl1q_u8(TL, lo), vqtbl1q_u8(TH, lo), + vqtbl1q_u8(TL, hi), vqtbl1q_u8(TH, hi) }}; + /* store interleaved, restoring the original code order */ + vst4q_u8(symbols + i, o); + } + } + uint8x16_t c2s_vec = vld1q_u8(c2s); + /* smaller inputs/tail => simpler 16-wide path with no extra prep. */ + for (; i + 16 <= n; i += 16) { + uint8x16_t codes = flat_d2_unpack(bm + (i >> 2)); + uint8x16_t syms = vqtbl1q_u8(c2s_vec, codes); + vst1q_u8(symbols + i, syms); + } + for (; i + 4 <= n; i += 4) { + uint8_t b = bm[i >> 2]; + symbols[i ] = c2s[(b ) & 3]; + symbols[i + 1] = c2s[(b >> 2) & 3]; + symbols[i + 2] = c2s[(b >> 4) & 3]; + symbols[i + 3] = c2s[(b >> 6) & 3]; + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_neon(bm, i * 2, 2); + symbols[i] = c2s[code]; + } +} + +/* D=3 (byte-crossing): 32 codes/iter. Use the D=6 6-bit unpack to grab TWO + * D=3 codes per byte (pair6 = c[2k] | c[2k+1]<<3) -- one gather+shift pass does + * 32 codes -- then split lo=&7 (vqtbl1 over c2s16) / hi=>>3 (vqtbl2 over the + * 32-byte repeated table, which ignores the high junk) and interleave with + * vst2q. A single 16-wide pair-gather block mops up the <32 remainder; the + * trailing <=16 codes use the no-overread safe path (bounded by fast_end). */ +static inline void merge_flat_d3_neon(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + const uint8x8_t c2s8 = vld1_u8(c2s); + const uint8x16_t c2s16 = vcombine_u8(c2s8, c2s8); + const uint8x16_t m7 = vdupq_n_u8(7); + int i = 0; + int fast_end = n >= 16 ? n - 16 : 0; + const uint8_t *bp = bm; + if (n >= 48) { + uint8x16x2_t c2s32; c2s32.val[0] = c2s16; c2s32.val[1] = c2s16; + static const uint8_t pair6_shuf_t[16] = { 0,1, 1,2, 3,4, 4,5, 6,7, 7,8, 9,10, 10,11 }; + static const int16_t hshift6_t[8] = { 2,-2, 2,-2, 2,-2, 2,-2 }; + static const int8_t bshr6_t[16] = { -2,0, -2,0, -2,0, -2,0, -2,0, -2,0, -2,0, -2,0 }; + const uint8x16_t pair6_shuf = vld1q_u8(pair6_shuf_t); + const int16x8_t hshift6 = vld1q_s16(hshift6_t); + const int8x16_t bshr6 = vld1q_s8(bshr6_t); + for (; i + 32 <= fast_end; i += 32, bp += 12) { + uint8x16_t packed = vld1q_u8(bp); + uint16x8_t x = vreinterpretq_u16_u8(vqtbl1q_u8(packed, pair6_shuf)); + x = vshlq_u16(x, hshift6); + uint8x16_t pair6 = vshlq_u8(vreinterpretq_u8_u16(x), bshr6); + uint8x16x2_t out; + out.val[0] = vqtbl1q_u8(c2s16, vandq_u8(pair6, m7)); + out.val[1] = vqtbl2q_u8(c2s32, vshrq_n_u8(pair6, 3)); + vst2q_u8(symbols + i, out); + } + } + if (i + 16 <= fast_end) { /* one 16-wide pair-gather block for the <32 remainder */ + static const uint8_t pair_shuf_t[16] = { 0,1, 0,1, 1,2, 2,3, 3,4, 3,4, 4,5, 5,6 }; + static const int16_t hshift_t[8] = { 5,-1, 1, 3, 5,-1, 1, 3 }; + static const int8_t bshr_t[16] = { -5,0, -5,0, -5,0, -5,0, -5,0, -5,0, -5,0, -5,0 }; + uint8x16_t packed = vld1q_u8(bp); + uint16x8_t x = vreinterpretq_u16_u8(vqtbl1q_u8(packed, vld1q_u8(pair_shuf_t))); + x = vshlq_u16(x, vld1q_s16(hshift_t)); + uint8x16_t y = vshlq_u8(vreinterpretq_u8_u16(x), vld1q_s8(bshr_t)); + vst1q_u8(symbols + i, vqtbl1q_u8(c2s16, vandq_u8(y, m7))); + i += 16; bp += 6; + } + for (; i + 8 <= n; i += 8) { + uint8x8_t codes = flat_d3_unpack_safe(bm + ((i * 3) >> 3)); + vst1_u8(symbols + i, vqtbl1_u8(c2s16, codes)); + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_neon(bm, i * 3, 3); + symbols[i] = c2s[code]; + } +} + +/* D=4: codes are nibbles (2/byte), so &0xF / >>4 index the plain c2s directly + * (no dup-shuffle TBL); 32/iter via vzip + plain vst1q. Stock 16-wide tail. */ +static inline void merge_flat_d4_neon(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + uint8x16_t c2s_vec = vld1q_u8(c2s); + const uint8x16_t m = vdupq_n_u8(0x0F); + int i = 0; + for (; i + 32 <= n; i += 32) { + uint8x16_t v = vld1q_u8(bm + (i >> 1)); + uint8x16_t lo = vandq_u8(v, m), hi = vshrq_n_u8(v, 4); + uint8x16_t a = vqtbl1q_u8(c2s_vec, lo), b = vqtbl1q_u8(c2s_vec, hi); + vst1q_u8(symbols + i, vzip1q_u8(a, b)); + vst1q_u8(symbols + i + 16, vzip2q_u8(a, b)); + } + for (; i + 16 <= n; i += 16) { + uint8x16_t codes = flat_d4_unpack(bm + (i >> 1)); + uint8x16_t syms = vqtbl1q_u8(c2s_vec, codes); + vst1q_u8(symbols + i, syms); + } + for (; i + 2 <= n; i += 2) { + uint8_t b = bm[i >> 1]; + symbols[i ] = c2s[b & 0x0F]; + symbols[i + 1] = c2s[b >> 4]; + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_neon(bm, i * 4, 4); + symbols[i] = c2s[code]; + } +} + +/* D=5 (byte-crossing): pair-gather puts two adjacent codes in one u16 lane, + * positioned so a byte reinterpret interleaves even/odd for free (no vtrn1); + * vshr.u8(even lanes) + vand clean to 0..31; vqtbl2 scatter. Setup is gated on the + * block condition; the stock safe path handles the remainder. */ +static inline void merge_flat_d5_neon(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + uint8x16x2_t c2s_vec; + c2s_vec.val[0] = vld1q_u8(c2s); + c2s_vec.val[1] = vld1q_u8(c2s + 16); + int i = 0; + if (n >= 25) { + static const uint8_t pair_shuf_t[16] = { 0,1, 1,2, 2,3, 3,4, 5,6, 6,7, 7,8, 8,9 }; + static const int16_t hshift_t[8] = { 3, 1, -1, -3, 3, 1, -1, -3 }; + static const int8_t bshr_t[16] = { -3,0, -3,0, -3,0, -3,0, -3,0, -3,0, -3,0, -3,0 }; + const uint8x16_t pair_shuf = vld1q_u8(pair_shuf_t); + const int16x8_t hshift = vld1q_s16(hshift_t); + const int8x16_t bshr = vld1q_s8(bshr_t); + const uint8x16_t m31 = vdupq_n_u8(0x1f); + int blocks = (n - 9) >> 4; + for (int b = 0; b < blocks; ++b) { + uint8x16_t packed = vld1q_u8(bm + b * 10); + uint16x8_t x = vreinterpretq_u16_u8(vqtbl1q_u8(packed, pair_shuf)); + x = vshlq_u16(x, hshift); + uint8x16_t y = vshlq_u8(vreinterpretq_u8_u16(x), bshr); + uint8x16_t idx = vandq_u8(y, m31); + vst1q_u8(symbols + (b << 4), vqtbl2q_u8(c2s_vec, idx)); + } + i = blocks << 4; + } + for (; i + 8 <= n; i += 8) { + uint8x8_t codes = flat_d5_unpack_safe(bm + ((i * 5) >> 3)); + uint8x8_t syms = vqtbl2_u8(c2s_vec, codes); + vst1_u8(symbols + i, syms); + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_neon(bm, i * 5, 5); + symbols[i] = c2s[code]; + } +} + +/* D=6: same pair-gather as D=5 (12-bit pairs, even/odd in one u16 lane), but + * the c2s is 64 bytes so the scatter is vqtbl4q. Setup gated; stock safe tail. */ +static inline void merge_flat_d6_neon(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + uint8x16x4_t c2s_vec; + c2s_vec.val[0] = vld1q_u8(c2s); + c2s_vec.val[1] = vld1q_u8(c2s + 16); + c2s_vec.val[2] = vld1q_u8(c2s + 32); + c2s_vec.val[3] = vld1q_u8(c2s + 48); + int i = 0; + if (n >= 24) { + static const uint8_t pair_shuf_t[16] = { 0,1, 1,2, 3,4, 4,5, 6,7, 7,8, 9,10, 10,11 }; + static const int16_t hshift_t[8] = { 2,-2, 2,-2, 2,-2, 2,-2 }; + static const int8_t bshr_t[16] = { -2,0, -2,0, -2,0, -2,0, -2,0, -2,0, -2,0, -2,0 }; + const uint8x16_t pair_shuf = vld1q_u8(pair_shuf_t); + const int16x8_t hshift = vld1q_s16(hshift_t); + const int8x16_t bshr = vld1q_s8(bshr_t); + const uint8x16_t m63 = vdupq_n_u8(0x3f); + int blocks = (n - 8) >> 4; + for (int b = 0; b < blocks; ++b) { + uint8x16_t packed = vld1q_u8(bm + b * 12); + uint16x8_t x = vreinterpretq_u16_u8(vqtbl1q_u8(packed, pair_shuf)); + x = vshlq_u16(x, hshift); + uint8x16_t y = vshlq_u8(vreinterpretq_u8_u16(x), bshr); + uint8x16_t idx = vandq_u8(y, m63); + vst1q_u8(symbols + (b << 4), vqtbl4q_u8(c2s_vec, idx)); + } + i = blocks << 4; + } + for (; i + 8 <= n; i += 8) { + uint8x8_t codes = flat_d6_unpack_safe(bm + ((i * 6) >> 3)); + uint8x8_t syms = vqtbl4_u8(c2s_vec, codes); + vst1_u8(symbols + i, syms); + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_neon(bm, i * 6, 6); + symbols[i] = c2s[code]; + } +} + +/* D=7: 128-entry c2s = 2 * vqtbl4 (= 64). vqtbl4 on the low half + + * vqtbx4 on the high half (with code-64 indexing) — vqtbx keeps the + * first result for out-of-range lanes, so no OR-merge needed. */ +static inline void merge_flat_d7_neon(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + uint8x16x4_t lo, hi; + lo.val[0] = vld1q_u8(c2s); lo.val[1] = vld1q_u8(c2s + 16); + lo.val[2] = vld1q_u8(c2s + 32); lo.val[3] = vld1q_u8(c2s + 48); + hi.val[0] = vld1q_u8(c2s + 64); hi.val[1] = vld1q_u8(c2s + 80); + hi.val[2] = vld1q_u8(c2s + 96); hi.val[3] = vld1q_u8(c2s + 112); + uint8x16_t sub64q = vdupq_n_u8(64); + uint8x8_t sub64 = vdup_n_u8(64); + int i = 0; + int fast_end = n >= 24 ? n - 24 : 0; + /* 16-wide ryg unpack (from the 2026-07-30 csimd study, +14% over the + * two-per-8-helper form on Graviton 4): 16 codes span exactly 14 bytes + * (16*7 bits), so one 16 B window feeds a TBL byte-pair gather; a + * per-lane USHL right-shift ((pos&7) as negative counts) bottoms each + * field, vuzp1 keeps the low bytes, AND 0x7F masks bit 7. Groups are + * byte-aligned every 16 codes (stride 14). Map via independent + * tbl/tbl/orr (no tbx dependency chain). Instruction-for-instruction + * the sequence clang emits for the csimd-ryg-map bench variant. */ + { + static const uint8_t d7_gather_lo_t[16] = + { 0,1, 0,1, 1,2, 2,3, 3,4, 4,5, 5,6, 6,7 }; + static const uint8_t d7_gather_hi_t[16] = + { 7,8, 7,8, 8,9, 9,10, 10,11, 11,12, 12,13, 13,14 }; + static const int16_t d7_shift_t[8] = /* -(pos & 7) */ + { 0, -7, -6, -5, -4, -3, -2, -1 }; + const uint8x16_t gather_lo = vld1q_u8(d7_gather_lo_t); + const uint8x16_t gather_hi = vld1q_u8(d7_gather_hi_t); + const int16x8_t shift = vld1q_s16(d7_shift_t); + const uint8x16_t m7f = vdupq_n_u8(0x7F); + const uint8_t *wp = bm; + for (; i + 16 <= fast_end; i += 16, wp += 14) { + uint8x16_t win = vld1q_u8(wp); + uint16x8_t vl = vshlq_u16(vreinterpretq_u16_u8(vqtbl1q_u8(win, gather_lo)), shift); + uint16x8_t vh = vshlq_u16(vreinterpretq_u16_u8(vqtbl1q_u8(win, gather_hi)), shift); + uint8x16_t codes = vandq_u8(vuzp1q_u8(vreinterpretq_u8_u16(vl), + vreinterpretq_u8_u16(vh)), m7f); + uint8x16_t s = vorrq_u8(vqtbl4q_u8(lo, codes), + vqtbl4q_u8(hi, vsubq_u8(codes, sub64q))); + vst1q_u8(symbols + i, s); + } + } + for (; i + 8 <= fast_end; i += 8) { + uint8x8_t codes = flat_d7_unpack_fast(bm + ((i * 7) >> 3)); + uint8x8_t s = vqtbl4_u8(lo, codes); + s = vqtbx4_u8(s, hi, vsub_u8(codes, sub64)); + vst1_u8(symbols + i, s); + } + for (; i + 8 <= n; i += 8) { + uint8x8_t codes = flat_d7_unpack_safe(bm + ((i * 7) >> 3)); + uint8x8_t s = vqtbl4_u8(lo, codes); + s = vqtbx4_u8(s, hi, vsub_u8(codes, sub64)); + vst1_u8(symbols + i, s); + } + for (; i < n; i++) { + uint32_t code = extract_D_bits_neon(bm, i * 7, 7); + symbols[i] = c2s[code]; + } +} + +/* D=8: a depth-8 flat region has 2^8 = 256 leaves = the WHOLE byte alphabet, + * all at code length 8. A full-alphabet equal-length canonical code is the + * identity permutation (rank == symbol), so c2s[k] == k and the byte-aligned + * 8-bit codes ARE the symbols: out[i] = c2s[bm[i]] = bm[i]. Hence a plain + * memcpy -- no 256-entry vqtbl4/vqtbx4 needed. The caller (a full-alphabet + * flat root) guarantees c2s == identity for D=8; only reachable for + * near-uniform / incompressible blocks (ratio ~1.0). */ +static inline void merge_flat_d8_neon(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + (void)c2s; + memcpy(symbols, bm, (size_t)n); +} + +/* merge_flat_neon -- D-bit flat-subtree decode into a + * contiguous output buffer. Dispatches to the per-D specialisation. */ +static inline void merge_flat_neon(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s) +{ + PROF_TIC(); + switch (D) { + case 2: merge_flat_d2_neon(out, n, bm, c2s); break; + case 3: merge_flat_d3_neon(out, n, bm, c2s); break; + case 4: merge_flat_d4_neon(out, n, bm, c2s); break; + case 5: merge_flat_d5_neon(out, n, bm, c2s); break; + case 6: merge_flat_d6_neon(out, n, bm, c2s); break; + case 7: merge_flat_d7_neon(out, n, bm, c2s); break; + case 8: merge_flat_d8_neon(out, n, bm, c2s); break; + default: { + /* Generic fallback for any unhandled D <= 16. */ + for (int i = 0; i < n; i++) { + uint32_t code = extract_D_bits_neon(bm, i * D, D); + out[i] = c2s[code]; + } + break; + } + } + PROF_TOC(PROF_BU_MERGE_FLAT, n); +} + + +/* ---------- Encode primitives: rank-based encoding (8-bit in-order ranks) ---------- + * Partition 8-bit leaf ranks against a per-node threshold (split_rank). A + * u8 port of the code_la COM64 partition: masks64_neon builds the + * 8 chunk masks (vcgtq > thr replacing the code bit-test), a vcnt + 0x0101.. + * prefix sum precomputes per-chunk cursors, and each 8-rank chunk is compacted + * by a vtbl1_u8 over ctab8 (the 1-byte-per-rank analog of compress_tab). + * Flat pack subtracts flat_base_rank then reuses the production pack_dN. */ +#include + +/* Per-mask LUTs (built once by build_tabs): + * pc8[m] popcount of mask byte m + * ctab8[m][0:8] right source lanes packed at [0,n_right), 0xff fill + * ctab8[m][8:16] left source lanes packed at [0,n_left), 0xff fill + * vtbl1_u8 returns 0 for the 0xff (out-of-range) padding indices. */ +static uint8_t pc8[256]; +static uint8_t ctab8[256][16] __attribute__((aligned(16))); + +/* p16rev partition LUTs (part_full_neon). One combined index per 16-lane group + * packs {left, forward, front} | {right, reversed, back}; left+right tile the + * 16 lanes so the OR of two disjoint-support tables is exact. + * p16rev_tabA[m0] low-byte (positions 0..7): left -> front [0,8-pc0), + * right -> back lanes 15,14,... (reversed) + * p16rev_tabB0[m1] high-byte (positions 8..15): continues both runs after + * the low byte, for pc0=0. The pc0>0 layout is just this + * one shifted left by pc0 lanes, so tabB[pc0][m1] is + * recovered as a byte-offset load `tabB0[m1] + pc0` (no + * separate per-pc0 table). Padded to 32 B/entry so the + * offset-16 load (pc0<=8) stays inside one cache line; + * 8 KB total vs the former 36 KB (fits L1 alongside tabA). + * The right side is recovered with a single loop-invariant full-reverse + * constant in part_full_neon. */ +static uint8_t p16rev_tabA[256][16] __attribute__((aligned(16))); +static uint8_t p16rev_tabB0[256][32] __attribute__((aligned(32))); +static int tabs_ready = 0; + +static void build_tabs(void) +{ + if (tabs_ready) return; + for (int m = 0; m < 256; m++) { + pc8[m] = (uint8_t)__builtin_popcount(m); + memset(ctab8[m], 0xff, 16); + int qr = 0, ql = 0; + for (int k = 0; k < 8; k++) { + if (m & (1 << k)) ctab8[m][qr++] = (uint8_t)k; /* right -> [0:8] */ + else ctab8[m][8 + ql++] = (uint8_t)k; /* left -> [8:16] */ + } + } + for (int m0 = 0; m0 < 256; m0++) { + memset(p16rev_tabA[m0], 0, 16); + int lp = 0, rp = 15; + for (int k = 0; k < 8; k++) { + if ((m0 >> k) & 1) p16rev_tabA[m0][rp--] = (uint8_t)k; + else p16rev_tabA[m0][lp++] = (uint8_t)k; + } + } + for (int m1 = 0; m1 < 256; m1++) { + memset(p16rev_tabB0[m1], 0, 32); + int lp = 8, rp = 15; /* pc0 = 0 layout; pc0 > 0 handled by the load offset */ + for (int k = 0; k < 8; k++) { + if ((m1 >> k) & 1) p16rev_tabB0[m1][rp--] = (uint8_t)(8 + k); + else p16rev_tabB0[m1][lp++] = (uint8_t)(8 + k); + } + } + tabs_ready = 1; +} + +static const uint8_t BW8[8] = {1, 2, 4, 8, 16, 32, 64, 128}; + +/* 8-bit mask of (ids > thr) over the 8 ranks in `ids`. */ +static inline uint8_t nmask8(uint8x8_t ids, uint8x8_t thr) +{ + return vaddv_u8(vand_u8(vcgt_u8(ids, thr), vld1_u8(BW8))); +} + +/* enc_init: ranks[i] = sym_to_rank[sym[i]], a 256-entry byte gather. + * "simd20" version from #5 by dougallj. + * The s2r table lives in 16 NEON regs (4x uint8x16x4_t). + * Each 16-lane input does one vqtbl4 over the [0,63] half + * + three vqtbx4 over the +64/+128/+192 halves (with offset adjusted). + * The tbl/tbx are microcoded and leave scalar load slots idle, so + * 4 extra symbols/iter are done with GPR gathers interleaved between them + * (20 sym/iter total). + */ +static inline void init_neon(uint8_t *ranks, int n, + const uint8_t *sym, const uint8_t *s2r) +{ + int i = 0; + if (n >= 20) { + uint8x16x4_t t0, t1, t2, t3; + t0.val[0]=vld1q_u8(s2r ); t0.val[1]=vld1q_u8(s2r + 16); + t0.val[2]=vld1q_u8(s2r + 32); t0.val[3]=vld1q_u8(s2r + 48); + t1.val[0]=vld1q_u8(s2r + 64); t1.val[1]=vld1q_u8(s2r + 80); + t1.val[2]=vld1q_u8(s2r + 96); t1.val[3]=vld1q_u8(s2r +112); + t2.val[0]=vld1q_u8(s2r +128); t2.val[1]=vld1q_u8(s2r +144); + t2.val[2]=vld1q_u8(s2r +160); t2.val[3]=vld1q_u8(s2r +176); + t3.val[0]=vld1q_u8(s2r +192); t3.val[1]=vld1q_u8(s2r +208); + t3.val[2]=vld1q_u8(s2r +224); t3.val[3]=vld1q_u8(s2r +240); + const uint8x16_t s64 = vdupq_n_u8(64); + const uint8x16_t s128 = vdupq_n_u8(128); + const uint8x16_t s192 = vdupq_n_u8(192); + for (; i + 20 <= n; i += 20) { + uint8x16_t c = vld1q_u8(sym + i); + uint32_t a; memcpy(&a, sym + i + 16, 4); + uint8x16_t r = vqtbl4q_u8(t0, c); + unsigned r0 = s2r[(uint8_t)a]; + r = vqtbx4q_u8(r, t1, vsubq_u8(c, s64)); + unsigned r1 = s2r[(uint8_t)(a >> 8)]; + r = vqtbx4q_u8(r, t2, vsubq_u8(c, s128)); + unsigned r2 = s2r[(uint8_t)(a >> 16)]; + r = vqtbx4q_u8(r, t3, vsubq_u8(c, s192)); + unsigned r3 = s2r[(uint8_t)(a >> 24)]; + vst1q_u8(ranks + i, r); + uint32_t h = r0 | (r1 << 8) | (r2 << 16) | (r3 << 24); + memcpy(ranks + i + 16, &h, 4); + } + } + for (; i < n; i++) ranks[i] = s2r[sym[i]]; +} + +/* Build 8 partition mask bytes for 64 ranks in one vpaddq_u8 reduction tree, + * packed LE into a u64 (byte k = mask of chunk k = ranks[8k .. 8k+7]). The + * rank analog of enc_masks8x8_codes_la_neon: vcgtq replaces the code bit-test, + * and since u8 packs two 8-rank chunks per 128-bit vector, FOUR inputs (not + * eight) feed the pairwise-add tree. Each lane already holds its bit-weight + * (0 or 2^(lane&7)); the 4 vpaddq_u8 collapse all 8 lanes of every chunk into + * one byte, so r's low 8 bytes are mask_0..mask_7 directly. This replaces the + * old 4x mred (12 vpaddq) + 8 vgetq_lane SIMD->GPR extracts with 4 vpaddq + + * one vget_lane_u64 -- the chunk masks now arrive as a single word that also + * feeds a vcnt popcount with no stack round-trip. */ +/* masks64v returns the mask bytes in a D-register so the caller can vcnt + * them before the GPR move, keeping the cursor chain off the GPR->SIMD + * fmov (a load-port uop, ~6cy mid-chain). */ +static inline uint8x8_t masks64v_neon(uint8x16_t v0, uint8x16_t v1, + uint8x16_t v2, uint8x16_t v3, + uint8x16_t vt, uint8x16_t bw) +{ + uint8x16_t w0 = vandq_u8(vcgtq_u8(v0, vt), bw); /* chunks 0,1 */ + uint8x16_t w1 = vandq_u8(vcgtq_u8(v1, vt), bw); /* chunks 2,3 */ + uint8x16_t w2 = vandq_u8(vcgtq_u8(v2, vt), bw); /* chunks 4,5 */ + uint8x16_t w3 = vandq_u8(vcgtq_u8(v3, vt), bw); /* chunks 6,7 */ + uint8x16_t t0 = vpaddq_u8(w0, w1); + uint8x16_t t1 = vpaddq_u8(w2, w3); + uint8x16_t u0 = vpaddq_u8(t0, t1); + return vget_low_u8(vpaddq_u8(u0, u0)); /* low 8 bytes = mask_0..7 */ +} +static inline uint64_t masks64_neon(uint8x16_t v0, uint8x16_t v1, + uint8x16_t v2, uint8x16_t v3, + uint8x16_t vt, uint8x16_t bw) +{ + return vget_lane_u64(vreinterpret_u64_u8( + masks64v_neon(v0, v1, v2, v3, vt, bw)), 0); +} + +/* full: both sides compacted (right -> tmp, left in place into ranks). + * p16rev: per 16-lane group, ONE combined shuffle index packs {left, forward, + * front} | {right, reversed, back}. Left and right exactly tile the 16 lanes, + * so the OR of two disjoint-support tables (p16rev_tabA over the low-byte mask m0, + * p16rev_tabB over [pc0][m1]) is exact. One vqtbl1q over that index yields BOTH + * sides at once: the register IS the left output (store it, advance by the left + * count — the right tail is overwritten by the next group / recursion level); + * the right output is recovered with a second vqtbl1q over the SAME register + * using a single loop-invariant full-reverse constant (full reverse lands the + * top-pc reversed right lanes at output [0,pc); the tail is overwritten). + * vs the prior per-8-chunk ctab8 COM64 path: one table-pair OR + one shuffle + * per 16 lanes instead of two independent 8-lane shuffles — measured 4–22 % + * faster across M4 / Graviton2..4 / Neoverse V3 (see bench_prim `com64` vs + * `p16rev`). The ~40 KB p16rev tables (tabA 4 KB + tabB 36 KB) make it NEON / big- + * L1 only; the 16-byte tail overstore is absorbed by the ranks +64 / tmp +2N + * scratch slack (codec.c). */ +/* Scatter one 64-rank group-set (4 p16rev groups as above). Prefix-summed + * popcounts give each group's store offsets up front, so the cursors + * advance once per 64 (issue #5). Stores run 16 wide, up to +48/+16 past + * the valid counts into the ranks+64 / tmp+2N scratch slack. Returns the + * group-set's right count. */ +__attribute__((always_inline)) static inline +int part64_full_neon(uint8x16_t v0, uint8x16_t v1, uint8x16_t v2, uint8x16_t v3, + uint64_t mask_word, uint64_t pcw, + uint8_t *ldst, uint8_t *rdst) +{ + uint64_t pfx = pcw * 0x0101010101010101ULL; + uint8x16_t vg[4] = { v0, v1, v2, v3 }; + static const uint8_t rev16_a[16] = {15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0}; + uint8x16_t rev16 = vld1q_u8(rev16_a); +#define _PART(g) do { \ + uint8_t m0 = (uint8_t)(mask_word >> (16*(g))); \ + uint8_t m1 = (uint8_t)(mask_word >> (16*(g) + 8)); \ + uint32_t pc0 = (uint32_t)((pcw >> (16*(g))) & 0xFF); \ + uint32_t cr = (g) == 0 ? 0u \ + : (uint32_t)((pfx >> (8*(2*(g) - 1))) & 0xFF); \ + uint8x16_t ri = vorrq_u8(vld1q_u8(p16rev_tabA[m0]), \ + vld1q_u8(&p16rev_tabB0[m1][pc0])); \ + uint8x16_t comb = vqtbl1q_u8(vg[g], ri); \ + vst1q_u8(ldst + (16*(g) - cr), comb); \ + vst1q_u8(rdst + cr, vqtbl1q_u8(comb, rev16)); \ + } while (0) + _PART(0); _PART(1); _PART(2); _PART(3); +#undef _PART + return (int)(pfx >> 56); +} +static inline int part_full_neon(uint8_t *ranks, int n, uint8_t thr, + uint8_t *bm, uint8_t *tmp) +{ + build_tabs(); + int n_left = 0, n_right = 0; + int j = 0; + uint8x16_t vt = vdupq_n_u8(thr); + static const uint8_t bw_a[16] = {1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128}; + uint8x16_t bw = vld1q_u8(bw_a); + /* Software-pipelined one iteration deep (like the decode merges): the + * carried chain (loads -> cgt -> three serial vpaddq -> lane move -> + * multiply -> cursors) is ~2x the loop's port work, so the next + * group-set's mask + popcount start under the current scatter. The + * mask is vcnt'd SIMD-side (masks64v_neon), keeping the GPR->SIMD + * fmov off the chain. */ + if (j + 64 <= n) { + uint8x16_t c0 = vld1q_u8(ranks + j), c1 = vld1q_u8(ranks + j + 16); + uint8x16_t c2 = vld1q_u8(ranks + j + 32), c3 = vld1q_u8(ranks + j + 48); + uint8x8_t mv = masks64v_neon(c0, c1, c2, c3, vt, bw); + uint64_t w = vget_lane_u64(vreinterpret_u64_u8(mv), 0); + uint64_t pcw = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(mv)), 0); + for (; j + 128 <= n; j += 64) { + uint8x16_t n0 = vld1q_u8(ranks + j + 64), n1 = vld1q_u8(ranks + j + 80); + uint8x16_t n2 = vld1q_u8(ranks + j + 96), n3 = vld1q_u8(ranks + j + 112); + uint8x8_t nmv = masks64v_neon(n0, n1, n2, n3, vt, bw); + uint64_t nw = vget_lane_u64(vreinterpret_u64_u8(nmv), 0); + uint64_t npcw = vget_lane_u64(vreinterpret_u64_u8(vcnt_u8(nmv)), 0); + memcpy(bm + (j >> 3), &w, 8); + int tr = part64_full_neon(c0, c1, c2, c3, w, pcw, + ranks + n_left, tmp + n_right); + n_right += tr; + n_left += 64 - tr; + c0 = n0; c1 = n1; c2 = n2; c3 = n3; + w = nw; pcw = npcw; + } + memcpy(bm + (j >> 3), &w, 8); + int tr = part64_full_neon(c0, c1, c2, c3, w, pcw, + ranks + n_left, tmp + n_right); + n_right += tr; + n_left += 64 - tr; + j += 64; + } + for (; j < n; j++) { + if ((j & 7) == 0) bm[j >> 3] = 0; + uint8_t r = ranks[j]; + if (r > thr) { bm[j >> 3] |= (uint8_t)(1u << (j & 7)); tmp[n_right++] = r; } + else { ranks[n_left++] = r; } + } + return n_right; +} + +/* part_core_neon — the one-sided (right/none) rank partition, a + * u8 port of the code_la partition core: same 64/iter COM64 wide path (mask via + * masks64_neon, vcnt + 0x0101.. prefix-sum cursors, per-8-chunk ctab8 + * shuffle), same 8/iter middle loop, same scalar tail. EMIT_RIGHT is + * compile-time, so the none form folds to a pure bitmap build. Right -> + * tmp; the left side is never scattered (a leaf child's ranks are dead). */ +__attribute__((always_inline)) static inline +int part_core_neon(uint8_t *ranks, int n, uint8_t thr, + uint8_t *bm, uint8_t *tmp, int EMIT_RIGHT) +{ + build_tabs(); + int n_right = 0; + int j = 0; + uint8x16_t vt = vdupq_n_u8(thr); + uint8x8_t vt8 = vdup_n_u8(thr); + static const uint8_t bw_a[16] = {1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128}; + uint8x16_t bw = vld1q_u8(bw_a); + for (; j + 64 <= n; j += 64) { + uint8x16_t v0 = vld1q_u8(ranks + j); + uint8x16_t v1 = vld1q_u8(ranks + j + 16); + uint8x16_t v2 = vld1q_u8(ranks + j + 32); + uint8x16_t v3 = vld1q_u8(ranks + j + 48); + uint64_t mask_word = masks64_neon(v0, v1, v2, v3, vt, bw); + memcpy(bm + (j >> 3), &mask_word, 8); + uint8x8_t pc_v = vcnt_u8(vcreate_u8(mask_word)); + uint64_t pc_word = vget_lane_u64(vreinterpret_u64_u8(pc_v), 0); + uint64_t pfx = pc_word * 0x0101010101010101ULL; + uint8x8_t cv[8] = { + vget_low_u8(v0), vget_high_u8(v0), + vget_low_u8(v1), vget_high_u8(v1), + vget_low_u8(v2), vget_high_u8(v2), + vget_low_u8(v3), vget_high_u8(v3), + }; +#define _PART1(K_) do { \ + uint32_t cr = (K_)==0 ? 0u : (uint32_t)((pfx >> (8*((K_)-1))) & 0xFF); \ + if (EMIT_RIGHT) { \ + const uint8_t *tab = ctab8[(uint8_t)(mask_word >> (8*(K_)))]; \ + vst1_u8(tmp + n_right + cr, vtbl1_u8(cv[K_], vld1_u8(tab))); \ + } \ + } while (0) + _PART1(0); _PART1(1); _PART1(2); _PART1(3); + _PART1(4); _PART1(5); _PART1(6); _PART1(7); +#undef _PART1 + n_right += (uint32_t)(pfx >> 56); + } + for (; j + 8 <= n; j += 8) { + uint8x8_t v = vld1_u8(ranks + j); + uint8_t mask = nmask8(v, vt8); + bm[j >> 3] = mask; + if (EMIT_RIGHT) vst1_u8(tmp + n_right, vtbl1_u8(v, vld1_u8(ctab8[mask]))); + n_right += pc8[mask]; + } + for (; j < n; j++) { + if ((j & 7) == 0) bm[j >> 3] = 0; + uint8_t r = ranks[j]; + if (r > thr) { bm[j >> 3] |= (uint8_t)(1u << (j & 7)); + if (EMIT_RIGHT) tmp[n_right] = r; n_right++; } + } + return n_right; +} + +/* Flat pack, native u8: the local code (rank - base) is already a D-bit value + * in the low bits of each byte, so we pack straight from u8 — no u16 widen, no + * round-trip. Per-D byte kernels mirror the code_la packers (D5/6/7 reuse the + * byte-laid backend from pivco_huffman_neon_pack.h via pack_d{5,6,7}). */ + +/* D=2: 16 ranks -> 4 bytes (4 ranks per byte, no byte crossings). */ +static inline int pack_d2_neon(uint8_t *out, const uint8_t *ranks, int n, uint8_t base) +{ + static const int8_t shifts_d2[16] = { 0,2,4,6, 0,2,4,6, 0,2,4,6, 0,2,4,6 }; + const int8x16_t sh = vld1q_s8(shifts_d2); + /* Distribute the base subtract: shift raw ranks, fold four into a + * byte, subtract 85*base once -- + * (r0-b)+4(r1-b)+16(r2-b)+64(r3-b) = r0+4r1+16r2+64r3 - 85b, exact + * mod 256. */ + const uint8x16_t b85 = vdupq_n_u8((uint8_t)(85 * base)); + int i = 0; + for (; i + 64 <= n; i += 64) { + uint8x16_t b0 = vshlq_u8(vld1q_u8(ranks + i), sh); + uint8x16_t b1 = vshlq_u8(vld1q_u8(ranks + i + 16), sh); + uint8x16_t b2 = vshlq_u8(vld1q_u8(ranks + i + 32), sh); + uint8x16_t b3 = vshlq_u8(vld1q_u8(ranks + i + 48), sh); + uint8x16_t r = vpaddq_u8(vpaddq_u8(b0, b1), vpaddq_u8(b2, b3)); + vst1q_u8(out + (i >> 2), vsubq_u8(r, b85)); + } + for (; i + 16 <= n; i += 16) { /* 16-wide cleanup */ + uint8x16_t b = vshlq_u8(vld1q_u8(ranks + i), sh); + uint8x16_t s1 = vpaddq_u8(b, b); + uint8x16_t s2 = vsubq_u8(vpaddq_u8(s1, s1), b85); + uint32_t packed4 = vgetq_lane_u32(vreinterpretq_u32_u8(s2), 0); + memcpy(out + (i >> 2), &packed4, 4); + } + return i; +} + +/* D=3: pair adjacent codes into 6-bit values the D=4 way (per-lane {0,3} + * shifts + one vpaddq: pair = c_even + 8 c_odd, one per byte; base + * subtract distributed as - 9*base, exact since the true pair < 64), + * then run the D=6 variable-shift pyramid on the pairs -- a 3-bit + * LSB-first stream is exactly the 6-bit LSB-first stream of its pairs. + * 32 codes/iter plus a 16-code self-paired cleanup, store-bounded like + * pack_d{5,6,7} with the scalar tail packing the rest. */ +static inline int pack_d3_neon(uint8_t *out, const uint8_t *ranks, int n, uint8_t base) +{ + static const int8_t shifts_p[16] = { 0,3, 0,3, 0,3, 0,3, 0,3, 0,3, 0,3, 0,3 }; + static const int8_t shifts_1[16] = { 2,0, 2,0, 2,0, 2,0, 2,0, 2,0, 2,0, 2,0 }; + static const int16_t shifts_2[8] = { 2,-2, 2,-2, 2,-2, 2,-2 }; + static const int32_t shifts_4[4] = { 4,-4, 4,-4 }; + const int8x16_t shp = vld1q_s8(shifts_p); + const uint8x16_t b9 = vdupq_n_u8((uint8_t)(9 * base)); + const int8x16_t s1 = vld1q_s8(shifts_1); + const int16x8_t s2 = vld1q_s16(shifts_2); + const int32x4_t s3 = vld1q_s32(shifts_4); + const uint8x16_t compact = vld1q_u8(pivco_pack_compact_d6_neon); + const int total_bytes = (n * 3 + 7) >> 3; + int i = 0; + for (; i + 32 <= n && ((i * 3) >> 3) + 16 <= total_bytes; i += 32) { + uint8x16_t b0 = vshlq_u8(vld1q_u8(ranks + i), shp); + uint8x16_t b1 = vshlq_u8(vld1q_u8(ranks + i + 16), shp); + uint8x16_t pair = vsubq_u8(vpaddq_u8(b0, b1), b9); + uint16x8_t w16 = vreinterpretq_u16_u8(vshlq_u8(pair, s1)); + uint32x4_t w32 = vreinterpretq_u32_u16(vshlq_u16(w16, s2)); + uint64x2_t w64 = vreinterpretq_u64_u32(vshlq_u32(w32, s3)); + vst1q_u8(out + ((i * 3) >> 3), + vqtbl1q_u8(vreinterpretq_u8_u64(w64), compact)); + } + for (; i + 16 <= n && ((i * 3) >> 3) + 16 <= total_bytes; i += 16) { + uint8x16_t b = vshlq_u8(vld1q_u8(ranks + i), shp); + uint8x16_t pair = vsubq_u8(vpaddq_u8(b, b), b9); + uint16x8_t w16 = vreinterpretq_u16_u8(vshlq_u8(pair, s1)); + uint32x4_t w32 = vreinterpretq_u32_u16(vshlq_u16(w16, s2)); + uint64x2_t w64 = vreinterpretq_u64_u32(vshlq_u32(w32, s3)); + vst1q_u8(out + ((i * 3) >> 3), + vqtbl1q_u8(vreinterpretq_u8_u64(w64), compact)); + } + return i; +} + +/* D=4: 16 ranks -> 8 bytes. Pair (r[2k], r[2k+1]) into one byte each. */ +static inline int pack_d4_neon(uint8_t *out, const uint8_t *ranks, int n, uint8_t base) +{ + static const int8_t shifts_d4[16] = { 0,4, 0,4, 0,4, 0,4, 0,4, 0,4, 0,4, 0,4 }; + const int8x16_t sh = vld1q_s8(shifts_d4); + /* Distributed base subtract, unrolled once so the vpaddq_u8 pairs two + * full input vectors into one 16-byte store: + * (r0-b)+16(r1-b) = r0+16r1 - 17b, exact mod 256. */ + const uint8x16_t b17 = vdupq_n_u8((uint8_t)(17 * base)); + int i = 0; + for (; i + 32 <= n; i += 32) { + uint8x16_t b0 = vshlq_u8(vld1q_u8(ranks + i), sh); + uint8x16_t b1 = vshlq_u8(vld1q_u8(ranks + i + 16), sh); + vst1q_u8(out + (i >> 1), vsubq_u8(vpaddq_u8(b0, b1), b17)); + } + for (; i + 16 <= n; i += 16) { /* 16-wide cleanup */ + uint8x16_t b = vshlq_u8(vld1q_u8(ranks + i), sh); + vst1_u8(out + (i >> 1), vget_low_u8(vsubq_u8(vpaddq_u8(b, b), b17))); + } + return i; +} + +/* D=8: 16 ranks -> 16 bytes. Byte-aligned; one shift+AND pass. */ +static inline int pack_d8_neon(uint8_t *out, const uint8_t *ranks, int n, uint8_t base) +{ + uint8x16_t vb = vdupq_n_u8(base); + int i = 0; + for (; i + 16 <= n; i += 16) { + vst1q_u8(out + i, vsubq_u8(vld1q_u8(ranks + i), vb)); + } + return i; +} + +/* Dispatcher: SIMD per-D path + scalar tail (packs (rank - base) LSB-first). */ +static inline void pack_dN_neon(uint8_t *out, const uint8_t *ranks, + int n, int D, uint8_t base) +{ + int total_bytes = (n * D + 7) >> 3; + if (total_bytes > 0) out[total_bytes - 1] = 0; + + int i = 0; + switch (D) { + case 2: i = pack_d2_neon(out, ranks, n, base); break; + case 3: i = pack_d3_neon(out, ranks, n, base); break; + case 4: i = pack_d4_neon(out, ranks, n, base); break; + case 5: i = pack_d5_neon(out, ranks, n, base); break; + case 6: i = pack_d6_neon(out, ranks, n, base); break; + case 7: i = pack_d7_neon(out, ranks, n, base); break; + case 8: i = pack_d8_neon(out, ranks, n, base); break; + default: break; + } + if (i >= n) return; + + int bit_pos = i * D; + int byte_idx = bit_pos >> 3; + int bits_in_buf = bit_pos & 7; + uint64_t buf = bits_in_buf > 0 + ? (uint64_t)out[byte_idx] & ((1u << bits_in_buf) - 1) + : 0; + for (; i < n; i++) { + uint32_t local = (uint32_t)(uint8_t)(ranks[i] - base); /* code in [0,2^D); no mask */ + buf |= (uint64_t)local << bits_in_buf; + bits_in_buf += D; + while (bits_in_buf >= 8) { + out[byte_idx++] = (uint8_t)(buf & 0xff); + buf >>= 8; + bits_in_buf -= 8; + } + } + if (bits_in_buf > 0) out[byte_idx] = (uint8_t)(buf & ((1u << bits_in_buf) - 1)); +} + +/* ---------- Aliases consumed by codec.c ---------- */ + +#define PIVCO_PRIM_ALWAYS_INLINE __attribute__((always_inline)) static inline + +#include "pivco_huffman_hist_scalar.h" + +/* NEON has no histogram win over the shared scalar core (measured); + * alias it explicitly. */ +PIVCO_PRIM_ALWAYS_INLINE void prim_histogram_chunk(const uint8_t *in, size_t n, + uint32_t hist[256], + uint8_t *scratch) +{ histogram_chunk_scalar(in, n, hist, scratch); } + + +/* Widest load a merge kernel issues at a child-buffer cursor (16B vld1q at child cursors); + * the cursor can rest AT `size` on the exhausted side, so buffers a + * merge reads need this much trailing slack. Consumed by the decode + * placement logic (scratch_carve / place_tail). */ +#define PIVCO_PRIM_MERGE_OVERREAD 16 + +PIVCO_PRIM_ALWAYS_INLINE void prim_codec_init(void) +{ codec_init_neon(); } + + +/* rank-based encode aliases (consumed by codec.c) */ +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_init(uint8_t *ranks, int n, + const uint8_t *symbols, const uint8_t *sym_to_rank, + const pivco_enc_init_aux_t *aux) +{ (void)aux; init_neon(ranks, n, symbols, sym_to_rank); } +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_full(uint8_t *ranks, int n, + uint8_t thr, uint8_t *bm, uint8_t *right_out) +{ return part_full_neon(ranks, n, thr, bm, right_out); } +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_right(uint8_t *ranks, int n, + uint8_t thr, uint8_t *bm, uint8_t *right_out) +{ return part_core_neon(ranks, n, thr, bm, right_out, 1); } +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_none(uint8_t *ranks, int n, + uint8_t thr, uint8_t *bm) +{ return part_core_neon(ranks, n, thr, bm, NULL, 0); } + +/* ---------- vertical-128 flat kernels (see pivco_huffman_vertical.h) ---------- + * + * Fused decode of the vertical prefix: per block, load the D byte-columns + * (16 B each), then for each of the 8 steps extract one D-bit field from + * every lane with uniform shifts (compile-time constants after per-D + * instantiation), map through c2s, store 16 consecutive symbols. */ +#define PIVCO_VERT_MERGE_NEON_BODY(DV, MAPEXPR, BLK, O, CS, OS) \ + do { \ + uint8x16_t cols[8]; \ + for (int j = 0; j < DV; j++) cols[j] = vld1q_u8((BLK) + (CS) * j); \ + for (int s = 0; s < 8; s++) { \ + int bit = s * DV, j = bit >> 3, off = bit & 7; \ + uint8x16_t w = vshlq_u8(cols[j], vdupq_n_s8((int8_t)-off)); \ + if (off + DV > 8) \ + w = vorrq_u8(w, vshlq_u8(cols[j + 1], \ + vdupq_n_s8((int8_t)(8 - off)))); \ + uint8x16_t codes = vandq_u8(w, maskv); \ + vst1q_u8((O) + (OS) * s, MAPEXPR); \ + } \ + } while (0) +#define PIVCO_VERT_MERGE_NEON(DV, SETUP, MAPEXPR) \ +static void vert_merge_neon_d##DV(uint8_t *out, int n_v, const uint8_t *bm, \ + const uint8_t *c2s) \ +{ \ + SETUP \ + const uint8x16_t maskv = vdupq_n_u8((uint8_t)((1u << DV) - 1)); \ + for (int b = 0; b < n_v >> 7; b++) \ + PIVCO_VERT_MERGE_NEON_BODY(DV, MAPEXPR, \ + bm + (size_t)b * 16 * DV, \ + out + ((size_t)b << 7), 16, 16); \ +} \ +static void vert512_merge_neon_d##DV(uint8_t *out, int n_v, const uint8_t *bm, \ + const uint8_t *c2s) \ +{ \ + SETUP \ + const uint8x16_t maskv = vdupq_n_u8((uint8_t)((1u << DV) - 1)); \ + for (int b = 0; b < n_v >> 9; b++) \ + for (int qt = 0; qt < 4; qt++) \ + PIVCO_VERT_MERGE_NEON_BODY(DV, MAPEXPR, \ + bm + (size_t)b * 64 * DV + 16 * qt, \ + out + ((size_t)b << 9) + 16 * qt, \ + 64, 64); \ +} +PIVCO_VERT_MERGE_NEON(2, const uint8x16_t t = vld1q_u8(c2s);, vqtbl1q_u8(t, codes)) +PIVCO_VERT_MERGE_NEON(3, const uint8x16_t t = vld1q_u8(c2s);, vqtbl1q_u8(t, codes)) +PIVCO_VERT_MERGE_NEON(4, const uint8x16_t t = vld1q_u8(c2s);, vqtbl1q_u8(t, codes)) +PIVCO_VERT_MERGE_NEON(5, + uint8x16x2_t t; t.val[0] = vld1q_u8(c2s); t.val[1] = vld1q_u8(c2s + 16);, + vqtbl2q_u8(t, codes)) +PIVCO_VERT_MERGE_NEON(6, + uint8x16x4_t t; t.val[0] = vld1q_u8(c2s); t.val[1] = vld1q_u8(c2s + 16); + t.val[2] = vld1q_u8(c2s + 32); t.val[3] = vld1q_u8(c2s + 48);, + vqtbl4q_u8(t, codes)) +PIVCO_VERT_MERGE_NEON(7, + uint8x16x4_t lo; lo.val[0] = vld1q_u8(c2s); lo.val[1] = vld1q_u8(c2s + 16); + lo.val[2] = vld1q_u8(c2s + 32); lo.val[3] = vld1q_u8(c2s + 48); + uint8x16x4_t hi; hi.val[0] = vld1q_u8(c2s + 64); hi.val[1] = vld1q_u8(c2s + 80); + hi.val[2] = vld1q_u8(c2s + 96); hi.val[3] = vld1q_u8(c2s + 112); + const uint8x16_t s64 = vdupq_n_u8(64);, + vorrq_u8(vqtbl4q_u8(lo, codes), vqtbl4q_u8(hi, vsubq_u8(codes, s64)))) +#undef PIVCO_VERT_MERGE_NEON +#undef PIVCO_VERT_MERGE_NEON_BODY + +static inline void vert_merge_neon(uint8_t *out, int n_v, const uint8_t *bm, + int D, const uint8_t *c2s) +{ + switch (D) { + case 2: vert_merge_neon_d2(out, n_v, bm, c2s); break; + case 3: vert_merge_neon_d3(out, n_v, bm, c2s); break; + case 4: vert_merge_neon_d4(out, n_v, bm, c2s); break; + case 5: vert_merge_neon_d5(out, n_v, bm, c2s); break; + case 6: vert_merge_neon_d6(out, n_v, bm, c2s); break; + default: vert_merge_neon_d7(out, n_v, bm, c2s); break; + } +} + +static inline void vert512_merge_neon(uint8_t *out, int n_v, const uint8_t *bm, + int D, const uint8_t *c2s) +{ + switch (D) { + case 2: vert512_merge_neon_d2(out, n_v, bm, c2s); break; + case 3: vert512_merge_neon_d3(out, n_v, bm, c2s); break; + case 4: vert512_merge_neon_d4(out, n_v, bm, c2s); break; + case 5: vert512_merge_neon_d5(out, n_v, bm, c2s); break; + case 6: vert512_merge_neon_d6(out, n_v, bm, c2s); break; + default: vert512_merge_neon_d7(out, n_v, bm, c2s); break; + } +} + +/* Encoder mirror, fully unrolled per D: VSLI/VSRI fuse the shift+OR + * accumulate into one op, and each column's first touch is a plain + * move/shift (no zero-init pass). Chronological step order makes the + * insert semantics safe: within a column, offsets only grow, so VSLI's + * preserved-low-bits / VSRI's preserved-high-bits never clobber prior + * contributions. Relies on ranks - base < 2^D (no masking), like the + * natural packs. The group core packs one 16-lane sub-block at runtime + * column/output strides: (16,16) walks 128-value blocks, (64,64) walks + * the four interleaved quarters of a 512-value block. */ +static void vert_pack_neon_d2(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 7; b++) { + uint8_t *blk = out + (size_t)b * 16 * 2; + const uint8_t *r = ranks + ((size_t)b << 7); + uint8x16_t c0, c1; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 16), basev); + c0 = vsliq_n_u8(c0, v1, 2); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 32), basev); + c0 = vsliq_n_u8(c0, v2, 4); + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 48), basev); + c0 = vsliq_n_u8(c0, v3, 6); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 64), basev); + c1 = v4; + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 80), basev); + c1 = vsliq_n_u8(c1, v5, 2); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 96), basev); + c1 = vsliq_n_u8(c1, v6, 4); + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 112), basev); + c1 = vsliq_n_u8(c1, v7, 6); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 16, c1); + } +} +static void vert512_pack_neon_d2(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 9; b++) { + for (int qt = 0; qt < 4; qt++) { + uint8_t *blk = out + (size_t)b * 64 * 2 + 16 * qt; + const uint8_t *r = ranks + ((size_t)b << 9) + 16 * qt; + uint8x16_t c0, c1; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 64), basev); + c0 = vsliq_n_u8(c0, v1, 2); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 128), basev); + c0 = vsliq_n_u8(c0, v2, 4); + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 192), basev); + c0 = vsliq_n_u8(c0, v3, 6); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 256), basev); + c1 = v4; + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 320), basev); + c1 = vsliq_n_u8(c1, v5, 2); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 384), basev); + c1 = vsliq_n_u8(c1, v6, 4); + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 448), basev); + c1 = vsliq_n_u8(c1, v7, 6); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 64, c1); + } + } +} +static void vert_pack_neon_d3(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 7; b++) { + uint8_t *blk = out + (size_t)b * 16 * 3; + const uint8_t *r = ranks + ((size_t)b << 7); + uint8x16_t c0, c1, c2; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 16), basev); + c0 = vsliq_n_u8(c0, v1, 3); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 32), basev); + c0 = vsliq_n_u8(c0, v2, 6); + c1 = vshrq_n_u8(v2, 2); + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 48), basev); + c1 = vsliq_n_u8(c1, v3, 1); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 64), basev); + c1 = vsliq_n_u8(c1, v4, 4); + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 80), basev); + c1 = vsliq_n_u8(c1, v5, 7); + c2 = vshrq_n_u8(v5, 1); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 96), basev); + c2 = vsliq_n_u8(c2, v6, 2); + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 112), basev); + c2 = vsliq_n_u8(c2, v7, 5); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 16, c1); + vst1q_u8(blk + 32, c2); + } +} +static void vert512_pack_neon_d3(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 9; b++) { + for (int qt = 0; qt < 4; qt++) { + uint8_t *blk = out + (size_t)b * 64 * 3 + 16 * qt; + const uint8_t *r = ranks + ((size_t)b << 9) + 16 * qt; + uint8x16_t c0, c1, c2; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 64), basev); + c0 = vsliq_n_u8(c0, v1, 3); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 128), basev); + c0 = vsliq_n_u8(c0, v2, 6); + c1 = vshrq_n_u8(v2, 2); + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 192), basev); + c1 = vsliq_n_u8(c1, v3, 1); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 256), basev); + c1 = vsliq_n_u8(c1, v4, 4); + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 320), basev); + c1 = vsliq_n_u8(c1, v5, 7); + c2 = vshrq_n_u8(v5, 1); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 384), basev); + c2 = vsliq_n_u8(c2, v6, 2); + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 448), basev); + c2 = vsliq_n_u8(c2, v7, 5); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 64, c1); + vst1q_u8(blk + 128, c2); + } + } +} +static void vert_pack_neon_d4(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 7; b++) { + uint8_t *blk = out + (size_t)b * 16 * 4; + const uint8_t *r = ranks + ((size_t)b << 7); + uint8x16_t c0, c1, c2, c3; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 16), basev); + c0 = vsliq_n_u8(c0, v1, 4); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 32), basev); + c1 = v2; + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 48), basev); + c1 = vsliq_n_u8(c1, v3, 4); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 64), basev); + c2 = v4; + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 80), basev); + c2 = vsliq_n_u8(c2, v5, 4); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 96), basev); + c3 = v6; + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 112), basev); + c3 = vsliq_n_u8(c3, v7, 4); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 16, c1); + vst1q_u8(blk + 32, c2); + vst1q_u8(blk + 48, c3); + } +} +static void vert512_pack_neon_d4(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 9; b++) { + for (int qt = 0; qt < 4; qt++) { + uint8_t *blk = out + (size_t)b * 64 * 4 + 16 * qt; + const uint8_t *r = ranks + ((size_t)b << 9) + 16 * qt; + uint8x16_t c0, c1, c2, c3; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 64), basev); + c0 = vsliq_n_u8(c0, v1, 4); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 128), basev); + c1 = v2; + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 192), basev); + c1 = vsliq_n_u8(c1, v3, 4); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 256), basev); + c2 = v4; + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 320), basev); + c2 = vsliq_n_u8(c2, v5, 4); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 384), basev); + c3 = v6; + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 448), basev); + c3 = vsliq_n_u8(c3, v7, 4); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 64, c1); + vst1q_u8(blk + 128, c2); + vst1q_u8(blk + 192, c3); + } + } +} +static void vert_pack_neon_d5(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 7; b++) { + uint8_t *blk = out + (size_t)b * 16 * 5; + const uint8_t *r = ranks + ((size_t)b << 7); + uint8x16_t c0, c1, c2, c3, c4; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 16), basev); + c0 = vsliq_n_u8(c0, v1, 5); + c1 = vshrq_n_u8(v1, 3); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 32), basev); + c1 = vsliq_n_u8(c1, v2, 2); + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 48), basev); + c1 = vsliq_n_u8(c1, v3, 7); + c2 = vshrq_n_u8(v3, 1); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 64), basev); + c2 = vsliq_n_u8(c2, v4, 4); + c3 = vshrq_n_u8(v4, 4); + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 80), basev); + c3 = vsliq_n_u8(c3, v5, 1); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 96), basev); + c3 = vsliq_n_u8(c3, v6, 6); + c4 = vshrq_n_u8(v6, 2); + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 112), basev); + c4 = vsliq_n_u8(c4, v7, 3); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 16, c1); + vst1q_u8(blk + 32, c2); + vst1q_u8(blk + 48, c3); + vst1q_u8(blk + 64, c4); + } +} +static void vert512_pack_neon_d5(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 9; b++) { + for (int qt = 0; qt < 4; qt++) { + uint8_t *blk = out + (size_t)b * 64 * 5 + 16 * qt; + const uint8_t *r = ranks + ((size_t)b << 9) + 16 * qt; + uint8x16_t c0, c1, c2, c3, c4; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 64), basev); + c0 = vsliq_n_u8(c0, v1, 5); + c1 = vshrq_n_u8(v1, 3); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 128), basev); + c1 = vsliq_n_u8(c1, v2, 2); + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 192), basev); + c1 = vsliq_n_u8(c1, v3, 7); + c2 = vshrq_n_u8(v3, 1); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 256), basev); + c2 = vsliq_n_u8(c2, v4, 4); + c3 = vshrq_n_u8(v4, 4); + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 320), basev); + c3 = vsliq_n_u8(c3, v5, 1); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 384), basev); + c3 = vsliq_n_u8(c3, v6, 6); + c4 = vshrq_n_u8(v6, 2); + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 448), basev); + c4 = vsliq_n_u8(c4, v7, 3); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 64, c1); + vst1q_u8(blk + 128, c2); + vst1q_u8(blk + 192, c3); + vst1q_u8(blk + 256, c4); + } + } +} +static void vert_pack_neon_d6(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 7; b++) { + uint8_t *blk = out + (size_t)b * 16 * 6; + const uint8_t *r = ranks + ((size_t)b << 7); + uint8x16_t c0, c1, c2, c3, c4, c5; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 16), basev); + c0 = vsliq_n_u8(c0, v1, 6); + c1 = vshrq_n_u8(v1, 2); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 32), basev); + c1 = vsliq_n_u8(c1, v2, 4); + c2 = vshrq_n_u8(v2, 4); + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 48), basev); + c2 = vsliq_n_u8(c2, v3, 2); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 64), basev); + c3 = v4; + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 80), basev); + c3 = vsliq_n_u8(c3, v5, 6); + c4 = vshrq_n_u8(v5, 2); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 96), basev); + c4 = vsliq_n_u8(c4, v6, 4); + c5 = vshrq_n_u8(v6, 4); + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 112), basev); + c5 = vsliq_n_u8(c5, v7, 2); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 16, c1); + vst1q_u8(blk + 32, c2); + vst1q_u8(blk + 48, c3); + vst1q_u8(blk + 64, c4); + vst1q_u8(blk + 80, c5); + } +} +static void vert512_pack_neon_d6(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 9; b++) { + for (int qt = 0; qt < 4; qt++) { + uint8_t *blk = out + (size_t)b * 64 * 6 + 16 * qt; + const uint8_t *r = ranks + ((size_t)b << 9) + 16 * qt; + uint8x16_t c0, c1, c2, c3, c4, c5; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 64), basev); + c0 = vsliq_n_u8(c0, v1, 6); + c1 = vshrq_n_u8(v1, 2); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 128), basev); + c1 = vsliq_n_u8(c1, v2, 4); + c2 = vshrq_n_u8(v2, 4); + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 192), basev); + c2 = vsliq_n_u8(c2, v3, 2); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 256), basev); + c3 = v4; + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 320), basev); + c3 = vsliq_n_u8(c3, v5, 6); + c4 = vshrq_n_u8(v5, 2); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 384), basev); + c4 = vsliq_n_u8(c4, v6, 4); + c5 = vshrq_n_u8(v6, 4); + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 448), basev); + c5 = vsliq_n_u8(c5, v7, 2); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 64, c1); + vst1q_u8(blk + 128, c2); + vst1q_u8(blk + 192, c3); + vst1q_u8(blk + 256, c4); + vst1q_u8(blk + 320, c5); + } + } +} +static void vert_pack_neon_d7(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 7; b++) { + uint8_t *blk = out + (size_t)b * 16 * 7; + const uint8_t *r = ranks + ((size_t)b << 7); + uint8x16_t c0, c1, c2, c3, c4, c5, c6; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 16), basev); + c0 = vsliq_n_u8(c0, v1, 7); + c1 = vshrq_n_u8(v1, 1); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 32), basev); + c1 = vsliq_n_u8(c1, v2, 6); + c2 = vshrq_n_u8(v2, 2); + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 48), basev); + c2 = vsliq_n_u8(c2, v3, 5); + c3 = vshrq_n_u8(v3, 3); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 64), basev); + c3 = vsliq_n_u8(c3, v4, 4); + c4 = vshrq_n_u8(v4, 4); + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 80), basev); + c4 = vsliq_n_u8(c4, v5, 3); + c5 = vshrq_n_u8(v5, 5); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 96), basev); + c5 = vsliq_n_u8(c5, v6, 2); + c6 = vshrq_n_u8(v6, 6); + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 112), basev); + c6 = vsliq_n_u8(c6, v7, 1); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 16, c1); + vst1q_u8(blk + 32, c2); + vst1q_u8(blk + 48, c3); + vst1q_u8(blk + 64, c4); + vst1q_u8(blk + 80, c5); + vst1q_u8(blk + 96, c6); + } +} +static void vert512_pack_neon_d7(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const uint8x16_t basev = vdupq_n_u8(base); + for (int b = 0; b < n_v >> 9; b++) { + for (int qt = 0; qt < 4; qt++) { + uint8_t *blk = out + (size_t)b * 64 * 7 + 16 * qt; + const uint8_t *r = ranks + ((size_t)b << 9) + 16 * qt; + uint8x16_t c0, c1, c2, c3, c4, c5, c6; + uint8x16_t v0 = vsubq_u8(vld1q_u8(r + 0), basev); + c0 = v0; + uint8x16_t v1 = vsubq_u8(vld1q_u8(r + 64), basev); + c0 = vsliq_n_u8(c0, v1, 7); + c1 = vshrq_n_u8(v1, 1); + uint8x16_t v2 = vsubq_u8(vld1q_u8(r + 128), basev); + c1 = vsliq_n_u8(c1, v2, 6); + c2 = vshrq_n_u8(v2, 2); + uint8x16_t v3 = vsubq_u8(vld1q_u8(r + 192), basev); + c2 = vsliq_n_u8(c2, v3, 5); + c3 = vshrq_n_u8(v3, 3); + uint8x16_t v4 = vsubq_u8(vld1q_u8(r + 256), basev); + c3 = vsliq_n_u8(c3, v4, 4); + c4 = vshrq_n_u8(v4, 4); + uint8x16_t v5 = vsubq_u8(vld1q_u8(r + 320), basev); + c4 = vsliq_n_u8(c4, v5, 3); + c5 = vshrq_n_u8(v5, 5); + uint8x16_t v6 = vsubq_u8(vld1q_u8(r + 384), basev); + c5 = vsliq_n_u8(c5, v6, 2); + c6 = vshrq_n_u8(v6, 6); + uint8x16_t v7 = vsubq_u8(vld1q_u8(r + 448), basev); + c6 = vsliq_n_u8(c6, v7, 1); + vst1q_u8(blk + 0, c0); + vst1q_u8(blk + 64, c1); + vst1q_u8(blk + 128, c2); + vst1q_u8(blk + 192, c3); + vst1q_u8(blk + 256, c4); + vst1q_u8(blk + 320, c5); + vst1q_u8(blk + 384, c6); + } + } +} +static inline void vert_pack_neon(uint8_t *out, const uint8_t *ranks, + int n_v, int D, uint8_t base) +{ + switch (D) { + case 2: vert_pack_neon_d2(out, ranks, n_v, base); break; + case 3: vert_pack_neon_d3(out, ranks, n_v, base); break; + case 4: vert_pack_neon_d4(out, ranks, n_v, base); break; + case 5: vert_pack_neon_d5(out, ranks, n_v, base); break; + case 6: vert_pack_neon_d6(out, ranks, n_v, base); break; + default: vert_pack_neon_d7(out, ranks, n_v, base); break; + } +} + +static inline void vert512_pack_neon(uint8_t *out, const uint8_t *ranks, + int n_v, int D, uint8_t base) +{ + switch (D) { + case 2: vert512_pack_neon_d2(out, ranks, n_v, base); break; + case 3: vert512_pack_neon_d3(out, ranks, n_v, base); break; + case 4: vert512_pack_neon_d4(out, ranks, n_v, base); break; + case 5: vert512_pack_neon_d5(out, ranks, n_v, base); break; + case 6: vert512_pack_neon_d6(out, ranks, n_v, base); break; + default: vert512_pack_neon_d7(out, ranks, n_v, base); break; + } +} + +/* Natural-layout kernels exposed for bench_prim's ST_PACK/ST_MERGE_FLAT + * rows (the prim_ entries below produce the layout `vertical` selects: + * the hybrid vertical wire, or natural when 0). */ +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_pack_dN_natural(const uint8_t *ranks, + int n, int D, uint8_t base, uint8_t *out_packed) +{ pack_dN_neon(out_packed, ranks, n, D, base); } +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_flat_natural(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s) +{ merge_flat_neon(out, n, bm, D, c2s); } +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_pack_dN(const uint8_t *ranks, + int n, int D, uint8_t base, uint8_t *out_packed, + int vertical) +{ + int n5 = vertical == PIVCO_FLAT_VERTICAL ? pivco_vert_n512(n, D) : 0; + if (n5) vert512_pack_neon(out_packed, ranks, n5, D, base); + int r = n - n5, nv = vertical ? pivco_vert_n(r, D) : 0; + uint8_t *o1 = out_packed + (((size_t)n5 * D) >> 3); + if (nv) vert_pack_neon(o1, ranks + n5, nv, D, base); + if (r > nv) pack_dN_neon(o1 + (((size_t)nv * D) >> 3), + ranks + n5 + nv, r - nv, D, base); +} + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_flat(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s, + int vertical) +{ + int n5 = vertical == PIVCO_FLAT_VERTICAL ? pivco_vert_n512(n, D) : 0; + if (n5) vert512_merge_neon(out, n5, bm, D, c2s); + int r = n - n5, nv = vertical ? pivco_vert_n(r, D) : 0; + const uint8_t *bm1 = bm + (((size_t)n5 * D) >> 3); + if (nv) vert_merge_neon(out + n5, nv, bm1, D, c2s); + if (r > nv) merge_flat_neon(out + n5 + nv, r - nv, + bm1 + (((size_t)nv * D) >> 3), D, c2s); +} + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_cst_cst(const uint8_t *bm, int K, + uint8_t left_sym, + uint8_t right_sym, + uint8_t *out) +{ merge_cst_cst_neon(bm, K, left_sym, right_sym, out); } + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_cst_vec(const uint8_t *bm, int K, + uint8_t left_sym, + const uint8_t *right_buf, + uint8_t *out) +{ merge_cst_vec_neon(bm, K, left_sym, right_buf, out); } + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_vec_vec(const uint8_t *bm, int K, + const uint8_t *left_buf, + const uint8_t *right_buf, + uint8_t *out) +{ merge_vec_vec_neon(bm, K, left_buf, right_buf, out); } + +#endif /* PIVCO_HUFFMAN_PRIMITIVES_NEON_H */ diff --git a/third_party/pivco/src/pivco_huffman_primitives_scalar.h b/third_party/pivco/src/pivco_huffman_primitives_scalar.h new file mode 100644 index 0000000..443d0df --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_primitives_scalar.h @@ -0,0 +1,226 @@ +/* pivco_primitives_scalar.h — scalar implementations of the + * codec-primitive interface (see pivco_primitives.h). + * + * Specialized names end in `_scalar`; the codec calls the aliases + * `prim_*` defined at the bottom as always-inline wrappers. + * + * Internal header. Included by pivco_primitives.h when + * PIVCO_BACKEND_SCALAR is defined. Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_PRIMITIVES_SCALAR_H +#define PIVCO_HUFFMAN_PRIMITIVES_SCALAR_H + +#include "pivco_huffman.h" +#include "pivco_huffman_common.h" + +#include +#include + +/* Backend lifecycle. Scalar has no runtime tables to lazy-init. */ +static inline void codec_init_scalar(void) { /* no-op */ } + +/* ---------- Encode primitives: rank-based encoding (8-bit in-order ranks) ---------- * + * Partition compares a per-node threshold (split_rank) to the leaf rank, so + * the values are 8-bit and partition routing is byte-identical to the code_la + * bit-test. Flat pack subtracts flat_base_rank to get the local D-bit code. */ +static inline void enc_init_scalar(uint8_t *ranks, int n, + const uint8_t *symbols, const uint8_t *sym_to_rank) +{ for (int i = 0; i < n; i++) ranks[i] = sym_to_rank[symbols[i]]; } + +static inline int part_core_scalar(uint8_t *ranks, int n, uint8_t thr, + uint8_t *bm, uint8_t *right_out, + int EMIT_RIGHT, int EMIT_LEFT) +{ + memset(bm, 0, (size_t)bitmap_bytes(n)); + int n_left = 0, n_right = 0; + for (int j = 0; j < n; j++) { + uint8_t v = ranks[j]; + if (v > thr) { + bm[j >> 3] |= (uint8_t)(1u << (j & 7)); + if (EMIT_RIGHT) right_out[n_right] = v; + n_right++; + } else { + if (EMIT_LEFT) ranks[n_left] = v; + n_left++; + } + } + return n_right; +} + +static inline void pack_dN_scalar(uint8_t *out, const uint8_t *ranks, + int n, int D, uint8_t base) +{ + uint64_t buf = 0; + int bits_in_buf = 0, byte_idx = 0; + for (int i = 0; i < n; i++) { + uint32_t local = (uint32_t)(uint8_t)(ranks[i] - base); /* code in [0,2^D); no mask */ + buf |= (uint64_t)local << bits_in_buf; + bits_in_buf += D; + while (bits_in_buf >= 8) { + out[byte_idx++] = (uint8_t)(buf & 0xFFu); + buf >>= 8; + bits_in_buf -= 8; + } + } + if (bits_in_buf > 0) out[byte_idx] = (uint8_t)(buf & ((1u << bits_in_buf) - 1)); +} + +/* ---------- Decode primitives ---------- */ + +/* Extract D bits at bit position `bit_pos` from a packed-bit region. */ +static inline uint32_t extract_D_bits_scalar(const uint8_t *in, + int bit_pos, int D) +{ + int byte_idx = bit_pos >> 3; + int bit_off = bit_pos & 7; + uint32_t val = (uint32_t)in[byte_idx]; + if (bit_off + D > 8) val |= ((uint32_t)in[byte_idx + 1]) << 8; + if (bit_off + D > 16) val |= ((uint32_t)in[byte_idx + 2]) << 16; + return (val >> bit_off) & ((1u << D) - 1); +} + +/* Unpack n D-bit codes, look up in c2s, write to out[0..n). */ +static inline void merge_flat_scalar(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s) +{ + if (D == 8) { /* full-alphabet flat: c2s is the identity, codes + * ARE the symbols (see merge_flat_d8_neon) */ + memcpy(out, bm, (size_t)n); + return; + } + for (int i = 0; i < n; i++) { + uint32_t code = extract_D_bits_scalar(bm, i * D, D); + out[i] = c2s[code]; + } +} + +/* Both-leaves merge: per bit, pick left_sym or right_sym. */ +static inline void merge_cst_cst_scalar(const uint8_t *bm, int K, + uint8_t left_sym, + uint8_t right_sym, + uint8_t *out) +{ + for (int j = 0; j < K; j++) { + int bit = (bm[j >> 3] >> (j & 7)) & 1; + out[j] = bit ? right_sym : left_sym; + } +} + +/* Half-leaf merge, constant left: out[j] = (bit_j ? right_buf[r++] : left_sym). */ +static inline void merge_cst_vec_scalar(const uint8_t *bm, int K, + uint8_t left_sym, + const uint8_t *right_buf, + uint8_t *out) +{ + int r = 0; + for (int j = 0; j < K; j++) { + int bit = (bm[j >> 3] >> (j & 7)) & 1; + out[j] = bit ? right_buf[r++] : left_sym; + } +} + +/* Full BU merge: out[j] = (bit_j ? right_buf[r++] : left_buf[l++]). */ +static inline void merge_vec_vec_scalar(const uint8_t *bm, int K, + const uint8_t *left_buf, + const uint8_t *right_buf, + uint8_t *out) +{ + int l = 0, r = 0; + for (int j = 0; j < K; j++) { + int bit = (bm[j >> 3] >> (j & 7)) & 1; + out[j] = bit ? right_buf[r++] : left_buf[l++]; + } +} + +/* ---------- Aliases consumed by codec.c ---------- */ + +#define PIVCO_PRIM_ALWAYS_INLINE __attribute__((always_inline)) static inline + +/* Widest load a merge kernel issues at a child-buffer cursor (byte loads); + * the cursor can rest AT `size` on the exhausted side, so buffers a + * merge reads need this much trailing slack. Consumed by the decode + * placement logic (scratch_carve / place_tail). */ +#define PIVCO_PRIM_MERGE_OVERREAD 1 + +#include "pivco_huffman_hist_scalar.h" + +PIVCO_PRIM_ALWAYS_INLINE void prim_histogram_chunk(const uint8_t *in, size_t n, + uint32_t hist[256], + uint8_t *scratch) +{ histogram_chunk_scalar(in, n, hist, scratch); } + +PIVCO_PRIM_ALWAYS_INLINE void prim_codec_init(void) +{ codec_init_scalar(); } + +/* rank-based encode aliases (consumed by codec.c) */ +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_init(uint8_t *ranks, int n, + const uint8_t *symbols, const uint8_t *sym_to_rank, + const pivco_enc_init_aux_t *aux) +{ (void)aux; enc_init_scalar(ranks, n, symbols, sym_to_rank); } +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_full(uint8_t *ranks, int n, + uint8_t thr, uint8_t *bm, uint8_t *right_out) +{ return part_core_scalar(ranks, n, thr, bm, right_out, 1, 1); } +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_right(uint8_t *ranks, int n, + uint8_t thr, uint8_t *bm, uint8_t *right_out) +{ return part_core_scalar(ranks, n, thr, bm, right_out, 1, 0); } +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_none(uint8_t *ranks, int n, + uint8_t thr, uint8_t *bm) +{ return part_core_scalar(ranks, n, thr, bm, NULL, 0, 0); } +/* Natural-layout kernels exposed for bench_prim's ST_PACK/ST_MERGE_FLAT + * rows (the prim_ entries below produce the layout `vertical` selects: + * the hybrid vertical wire, or natural when 0). */ +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_pack_dN_natural(const uint8_t *ranks, + int n, int D, uint8_t base, uint8_t *out_packed) +{ pack_dN_scalar(out_packed, ranks, n, D, base); } +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_flat_natural(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s) +{ merge_flat_scalar(out, n, bm, D, c2s); } +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_pack_dN(const uint8_t *ranks, + int n, int D, uint8_t base, uint8_t *out_packed, + int vertical) +{ + int n5 = vertical == PIVCO_FLAT_VERTICAL ? pivco_vert_n512(n, D) : 0; + if (n5) vert512_pack_scalar(out_packed, ranks, n5, D, base); + int r = n - n5, nv = vertical ? pivco_vert_n(r, D) : 0; + uint8_t *o1 = out_packed + (((size_t)n5 * D) >> 3); + if (nv) vert_pack_scalar(o1, ranks + n5, nv, D, base); + if (r > nv) pack_dN_scalar(o1 + (((size_t)nv * D) >> 3), + ranks + n5 + nv, r - nv, D, base); +} + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_flat(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s, + int vertical) +{ + int n5 = vertical == PIVCO_FLAT_VERTICAL ? pivco_vert_n512(n, D) : 0; + if (n5) vert512_merge_scalar(out, n5, bm, D, c2s); + int r = n - n5, nv = vertical ? pivco_vert_n(r, D) : 0; + const uint8_t *bm1 = bm + (((size_t)n5 * D) >> 3); + if (nv) vert_merge_scalar(out + n5, nv, bm1, D, c2s); + if (r > nv) merge_flat_scalar(out + n5 + nv, r - nv, + bm1 + (((size_t)nv * D) >> 3), D, c2s); +} + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_cst_cst(const uint8_t *bm, int K, + uint8_t left_sym, + uint8_t right_sym, + uint8_t *out) +{ merge_cst_cst_scalar(bm, K, left_sym, right_sym, out); } + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_cst_vec(const uint8_t *bm, int K, + uint8_t left_sym, + const uint8_t *right_buf, + uint8_t *out) +{ merge_cst_vec_scalar(bm, K, left_sym, right_buf, out); } + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_vec_vec(const uint8_t *bm, int K, + const uint8_t *left_buf, + const uint8_t *right_buf, + uint8_t *out) +{ merge_vec_vec_scalar(bm, K, left_buf, right_buf, out); } + +#endif /* PIVCO_HUFFMAN_PRIMITIVES_SCALAR_H */ diff --git a/third_party/pivco/src/pivco_huffman_primitives_x86.h b/third_party/pivco/src/pivco_huffman_primitives_x86.h new file mode 100644 index 0000000..cdb1c07 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_primitives_x86.h @@ -0,0 +1,1388 @@ +/* pivco_huffman_primitives_x86.h — x86 (SSE4.1 + optional AVX2) primitive + * implementations of the codec primitive interface (see + * pivco_huffman_primitives.h). + * + * Specialized names end in `_x86`; the codec calls the aliases `prim_*` + * defined at the bottom as always-inline wrappers. Two implementation + * tiers gated by PIVCO_HAS_AVX2: the AVX2 tier widens pack_dN to 64-bit + * per-lane shifts via _mm256_sllv_epi64 (D=3/5/6/7) and gives flat + * decode a 32-byte D=4 fast path. The SSE4.1 floor handles D=2/4/8 + * with hand-rolled tricks (_mm_maddubs_epi16 weighted pair-add for D=2/4, + * _mm_mullo_epi32 multiply-as-shift for D=3) and falls back to scalar + * for D=5/6/7 (no uint64 per-lane shift in SSE). + * + * AVX-512 VBMI2 fast paths live in primitives_avx512.h (Phase 5 + * landed 2026-05-14). On AVX-512 hosts the runtime dispatcher routes + * to codec_avx512, so this file does NOT need to gate __AVX512* fast + * paths internally. Even when the codec_x86 OBJECT lib is compiled + * on an AVX-512 host (with -mavx512vbmi2 enabled globally), it's + * never reached at runtime there. + * + * Internal header. Included by pivco_huffman_primitives.h when + * PIVCO_BACKEND_X86 is defined. Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_PRIMITIVES_X86_H +#define PIVCO_HUFFMAN_PRIMITIVES_X86_H + +#if !defined(PIVCO_HAS_SSE4) +#error "pivco_huffman_primitives_x86.h requires PIVCO_HAS_SSE4" +#endif + +#include "pivco_huffman.h" +#include "pivco_huffman_common.h" +#include "pivco_huffman_x86_tables.h" /* compress_tab*, expand_tab* */ +#include "pivco_huffman_x86_flat.h" /* flat_d{2,3,4,5,6}_unpack_x86 */ +#include "pivco_huffman_x86_vertical.h" +#ifdef PIVCO_HAS_AVX2 +#include "pivco_huffman_avx2_pack.h" /* pack_d{2,3,5,6,7}_avx2_x86 (ryg pack) */ +#endif +#include "pivco_prof.h" + +#include /* SSE4.1 */ +#include /* AVX/AVX2/AVX-512 umbrella; + * gated paths drop out cleanly + * on SSE-only builds. */ +#include +#include +#include "pivco_check.h" + +/* Backend lifecycle. Lazily build the compress_tab + expand_tab pre- + * bake tables that the x86 partition / merge primitives index + * into. Idempotent and cheap after the first call. */ +static void init_x86_merge_tables(void); /* two-table merge_vec_vec shuffles (below) */ +static inline void codec_init_x86(void) +{ + init_compress_table_x86(); + init_expand_table_x86(); + init_x86_merge_tables(); +} + +/* ---------- Decode primitives (bottom-up) ---------- */ + +/* popcount_K_right_x86 — count "1" bits in the first K bits of bm. + * Scalar 64-bit POPCNT, 4-way unrolled. No codec.c caller (codec uses + * wire_read_kr_header for the value at read time); kept for signature + * stability with the NEON BU backend. `nbytes` is derivable from K. + * VPOPCNTQ fast path lives in primitives_avx512.h. */ +static inline int popcount_K_right_x86(const uint8_t *bm, int nbytes, int K) +{ + (void)nbytes; + PROF_TIC(); + int full_bytes = K >> 3; + int partial_bits = K & 7; + int b = 0; + int K_right = 0; + + uint64_t a0 = 0, a1 = 0, a2 = 0, a3 = 0; + for (; b + 32 <= full_bytes; b += 32) { + uint64_t v0, v1, v2, v3; + memcpy(&v0, bm + b, 8); + memcpy(&v1, bm + b + 8, 8); + memcpy(&v2, bm + b + 16, 8); + memcpy(&v3, bm + b + 24, 8); + a0 += __builtin_popcountll(v0); + a1 += __builtin_popcountll(v1); + a2 += __builtin_popcountll(v2); + a3 += __builtin_popcountll(v3); + } + K_right = (int)(a0 + a1 + a2 + a3); + + for (; b + 8 <= full_bytes; b += 8) { + uint64_t v; + memcpy(&v, bm + b, 8); + K_right += __builtin_popcountll(v); + } + for (; b < full_bytes; b++) { + K_right += __builtin_popcount(bm[b]); + } + if (partial_bits) { + uint8_t valid_mask = (uint8_t)((1u << partial_bits) - 1); + K_right += __builtin_popcount(bm[full_bytes] & valid_mask); + } + PROF_TOC(PROF_BU_POPCOUNT_K, K); + return K_right; +} + +/* ---- merge_vec_vec_x86: two-table merge (PSHUFB-complement + OR) ---- + * + * x86 has no 2-source PSHUFB, but PSHUFB zeroes any lane whose index MSB is + * set: shuffle R with the merged index (R lanes valid, L lanes -> 0 via the + * 255-off indices), shuffle L with the complemented index (L valid, R -> 0), + * then OR. The high half's +pop0 offset is folded in by replicating pop0 + * across shuf0's top 8 bytes and adding shuf1 (L entries stored 247-off). + * Two index tables (g_x86_merge_shuf0[.][16] + g_x86_merge_shuf1[.][8]) are + * built once in codec_init_x86. With AVX2 the main loop runs 32 B/iter as two + * 128-bit lanes (broadcast + vpblendd index assembly, asm-pinned for + * llvm#203132); the SSE 16 B form handles the residual (and all of K on + * SSE4.1-only hosts). AVX-512 VBMI2 uses the vpexpandb path in + * primitives_avx512.h instead. */ +static uint8_t g_x86_merge_shuf0[256][16] __attribute__((aligned(16))); +static uint8_t g_x86_merge_shuf1[256][8]; +static void init_x86_merge_tables(void) +{ + static int built = 0; + if (built) return; + for (int m = 0; m < 256; m++) { + int rset = 0, rclr = 0, pop = __builtin_popcount(m); + for (int i = 0; i < 8; i++) + g_x86_merge_shuf0[m][i] = ((m >> i) & 1) ? (uint8_t)(rset++) + : (uint8_t)(255 - rclr++); + for (int i = 8; i < 16; i++) g_x86_merge_shuf0[m][i] = (uint8_t)pop; + rset = 0; rclr = 0; + for (int t = 0; t < 8; t++) + g_x86_merge_shuf1[m][t] = ((m >> t) & 1) ? (uint8_t)(rset++) + : (uint8_t)(247 - rclr++); + } + built = 1; +} +#if defined(__AVX2__) +static inline __m256i x86_merge_bcastq(const void *src) +{ return _mm256_broadcastq_epi64(_mm_loadl_epi64((const __m128i *)src)); } +static inline __m256i x86_merge_load_halves(const void *s0, const void *s1) +{ + __m256i v = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)s0)); + return _mm256_inserti128_si256(v, _mm_loadu_si128((const __m128i *)s1), 1); +} +#endif +static inline void merge_vec_vec_x86(const uint8_t *bm, int K, + const uint8_t *left, + const uint8_t *right, + uint8_t *out) +{ + PROF_TIC(); + int lc = 0, rc = 0, j = 0; +#if defined(__AVX2__) + { + const __m256i ones = _mm256_set1_epi8(-1), zeros = _mm256_setzero_si256(); + for (; j + 32 <= K; j += 32) { + uint32_t mask; memcpy(&mask, bm + (j >> 3), 4); + unsigned m0 = mask & 0xff, m1 = (mask >> 8) & 0xff, + m2 = (mask >> 16) & 0xff, m3 = (mask >> 24) & 0xff; + __m256i vShuf02 = x86_merge_load_halves(g_x86_merge_shuf0[m0], g_x86_merge_shuf0[m2]); + __m256i vShuf1 = x86_merge_bcastq(g_x86_merge_shuf1[m1]); + __m256i vShuf3 = x86_merge_bcastq(g_x86_merge_shuf1[m3]); + __asm__("" : "+x"(vShuf1)); __asm__("" : "+x"(vShuf3)); /* dodge llvm#203132 */ + __m256i vShuf13 = _mm256_blend_epi32(vShuf1, vShuf3, 0xf0); + __m256i vShuf13M = _mm256_blend_epi32(zeros, vShuf13, 0xcc); + __m256i vShuf = _mm256_add_epi8(vShuf02, vShuf13M); + int lo_pop = _mm_popcnt_u32(mask & 0xffff); + __m256i vR = x86_merge_load_halves(right + rc, right + rc + lo_pop); + __m256i vL = x86_merge_load_halves(left + lc, left + lc + 16 - lo_pop); + __m256i rr = _mm256_shuffle_epi8(vR, vShuf); + __m256i rl = _mm256_shuffle_epi8(vL, _mm256_xor_si256(vShuf, ones)); + _mm256_storeu_si256((__m256i *)(out + j), _mm256_or_si256(rl, rr)); + int pr = _mm_popcnt_u32(mask); + rc += pr; lc += 32 - pr; + } + } +#endif + { + const __m128i ones = _mm_set1_epi8(-1); + for (; j + 16 <= K; j += 16) { + unsigned lo = bm[j >> 3], hi = bm[(j >> 3) + 1]; + __m128i shuf0 = _mm_load_si128((const __m128i *)g_x86_merge_shuf0[lo]); + __m128i shuf1 = _mm_slli_si128(_mm_loadl_epi64((const __m128i *)g_x86_merge_shuf1[hi]), 8); + __m128i merged = _mm_add_epi8(shuf0, shuf1); + __m128i R16 = _mm_loadu_si128((const __m128i *)(right + rc)); + __m128i L16 = _mm_loadu_si128((const __m128i *)(left + lc)); + __m128i rr = _mm_shuffle_epi8(R16, merged); + __m128i rl = _mm_shuffle_epi8(L16, _mm_xor_si128(merged, ones)); + _mm_storeu_si128((__m128i *)(out + j), _mm_or_si128(rr, rl)); + int pr = __builtin_popcount(lo) + __builtin_popcount(hi); + rc += pr; lc += 16 - pr; + } + } + for (; j < K; j++) { + int mb = (bm[j >> 3] >> (j & 7)) & 1; + out[j] = mb ? right[rc++] : left[lc++]; + } + PROF_TOC(PROF_BU_MERGE_VEC_VEC, K); +} + +/* merge_cst_vec_x86 — left input is a broadcast constant. + * Same 2x-unrolled structure; the L lane is a duplicated 16-byte + * register holding left_sym. */ +/* merge_cst_vec_x86 — two-table merge, L = broadcast const (no L load/cursor); + * only R advances. AVX2 32B main + SSE 16B residual. See merge_vec_vec_x86. */ +static inline void merge_cst_vec_x86(const uint8_t *bm, int K, + uint8_t left_sym, + const uint8_t *right, + uint8_t *out) +{ + PROF_TIC(); + int rc = 0, j = 0; +#if defined(__AVX2__) + { + const __m256i ones = _mm256_set1_epi8(-1), zeros = _mm256_setzero_si256(); + const __m256i vLb = _mm256_set1_epi8((char)left_sym); + for (; j + 32 <= K; j += 32) { + uint32_t mask; memcpy(&mask, bm + (j >> 3), 4); + unsigned m0=mask&0xff, m1=(mask>>8)&0xff, m2=(mask>>16)&0xff, m3=(mask>>24)&0xff; + __m256i vShuf02 = x86_merge_load_halves(g_x86_merge_shuf0[m0], g_x86_merge_shuf0[m2]); + __m256i vShuf1 = x86_merge_bcastq(g_x86_merge_shuf1[m1]); + __m256i vShuf3 = x86_merge_bcastq(g_x86_merge_shuf1[m3]); + __asm__("" : "+x"(vShuf1)); __asm__("" : "+x"(vShuf3)); + __m256i vShuf13 = _mm256_blend_epi32(vShuf1, vShuf3, 0xf0); + __m256i vShuf13M = _mm256_blend_epi32(zeros, vShuf13, 0xcc); + __m256i vShuf = _mm256_add_epi8(vShuf02, vShuf13M); + int lo_pop = _mm_popcnt_u32(mask & 0xffff); + __m256i vR = x86_merge_load_halves(right + rc, right + rc + lo_pop); + __m256i rr = _mm256_shuffle_epi8(vR, vShuf); + __m256i rl = _mm256_shuffle_epi8(vLb, _mm256_xor_si256(vShuf, ones)); + _mm256_storeu_si256((__m256i *)(out + j), _mm256_or_si256(rl, rr)); + rc += _mm_popcnt_u32(mask); + } + } +#endif + { + const __m128i ones = _mm_set1_epi8(-1), Lb = _mm_set1_epi8((char)left_sym); + for (; j + 16 <= K; j += 16) { + unsigned lo = bm[j >> 3], hi = bm[(j >> 3) + 1]; + __m128i shuf0 = _mm_load_si128((const __m128i *)g_x86_merge_shuf0[lo]); + __m128i shuf1 = _mm_slli_si128(_mm_loadl_epi64((const __m128i *)g_x86_merge_shuf1[hi]), 8); + __m128i merged = _mm_add_epi8(shuf0, shuf1); + __m128i R16 = _mm_loadu_si128((const __m128i *)(right + rc)); + __m128i rr = _mm_shuffle_epi8(R16, merged); + __m128i rl = _mm_shuffle_epi8(Lb, _mm_xor_si128(merged, ones)); + _mm_storeu_si128((__m128i *)(out + j), _mm_or_si128(rr, rl)); + rc += __builtin_popcount(lo) + __builtin_popcount(hi); + } + } + for (; j < K; j++) { int mb = (bm[j >> 3] >> (j & 7)) & 1; out[j] = mb ? right[rc++] : left_sym; } + PROF_TOC(PROF_BU_MERGE_CST_VEC, K); +} + +/* merge_cst_cst_x86 — both inputs are constants. vpblendvb-style: + * for each bit in mask, output is right_sym or left_sym. AVX2 widens + * to 32 bytes per iter; SSE4.1 floor handles 16. */ +static inline void merge_cst_cst_x86(const uint8_t *bm, int K, + uint8_t left_sym, uint8_t right_sym, + uint8_t *out) +{ + PROF_TIC(); + __m128i vsym0 = _mm_set1_epi8((char)left_sym); + __m128i vsym1 = _mm_set1_epi8((char)right_sym); + __m128i bits = _mm_setr_epi8(1,2,4,8,16,32,64,(char)128, + 1,2,4,8,16,32,64,(char)128); + __m128i shuf = _mm_setr_epi8(0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1); + int j = 0; +#ifdef PIVCO_HAS_AVX2 + __m256i vsym0_256 = _mm256_set1_epi8((char)left_sym); + __m256i vsym1_256 = _mm256_set1_epi8((char)right_sym); + __m256i bits_256 = _mm256_broadcastsi128_si256(bits); + __m256i shuf_256 = _mm256_broadcastsi128_si256(shuf); + for (; j + 32 <= K; j += 32) { + uint32_t four; + memcpy(&four, bm + (j >> 3), 4); + __m256i bm_quad = _mm256_set_epi32(0, 0, 0, (int)(four >> 16), + 0, 0, 0, (int)(four & 0xFFFF)); + __m256i bm_dup = _mm256_shuffle_epi8(bm_quad, shuf_256); + __m256i masked = _mm256_and_si256(bm_dup, bits_256); + __m256i mask8 = _mm256_cmpeq_epi8(masked, bits_256); + __m256i o = _mm256_blendv_epi8(vsym0_256, vsym1_256, mask8); + _mm256_storeu_si256((__m256i *)(out + j), o); + } +#endif + for (; j + 16 <= K; j += 16) { + __m128i bm_pair = _mm_cvtsi32_si128(*(const uint16_t *)(bm + (j >> 3))); + __m128i bm_dup = _mm_shuffle_epi8(bm_pair, shuf); + __m128i masked = _mm_and_si128(bm_dup, bits); + __m128i mask8 = _mm_cmpeq_epi8(masked, bits); + __m128i o = _mm_blendv_epi8(vsym0, vsym1, mask8); + _mm_storeu_si128((__m128i *)(out + j), o); + } + for (; j < K; j++) { + int mb = (bm[j >> 3] >> (j & 7)) & 1; + out[j] = mb ? right_sym : left_sym; + } + PROF_TOC(PROF_BU_MERGE_CST_CST, K); +} + +/* ---------- Flat-subtree decode (contiguous output) ---------- + * + * Reads n*D packed bits, looks up each D-bit code in c2s, writes the + * resulting bytes to out[0..n). D=4 has a SIMD path (AVX2 32-byte or + * SSE 16-byte); all other D values use the per-D scalar unrolled + * switch below. D=2/3/5/6 require either per-byte variable shifts + * (AVX2's _mm_srlv_*) or vpmultishiftqb (AVX-512 VBMI2) to build per- + * byte codes efficiently, and the scalar unrolled forms win without + * those. AVX-512 VBMI2 D=5/6 fast paths live in primitives_avx512.h. */ + +/* Extract D bits at bit position `bit_pos` from `in`. D <= 16. */ +static inline uint32_t extract_D_bits_x86(const uint8_t *in, + int bit_pos, int D) +{ + int byte_idx = bit_pos >> 3; + int bit_off = bit_pos & 7; + uint32_t val = (uint32_t)in[byte_idx]; + if (bit_off + D > 8) val |= ((uint32_t)in[byte_idx + 1]) << 8; + if (bit_off + D > 16) val |= ((uint32_t)in[byte_idx + 2]) << 16; + return (val >> bit_off) & ((1u << D) - 1); +} + +/* Generic scalar mop-up shared by every merge_flat_dN_x86: decode codes + * [i, n) one at a time. D is always a literal at the call sites, so it + * constant-folds. */ +static inline void merge_flat_tail_x86(uint8_t *symbols, int i, int n, + const uint8_t *bm, int D, + const uint8_t *c2s) +{ + for (; i < n; i++) + symbols[i] = c2s[extract_D_bits_x86(bm, i * D, D)]; +} + +/* merge_flat_dN_x86 — one static inline per supported D (mirrors the NEON + * file's structure); merge_flat_x86 below dispatches. Each writes n D-bit + * symbols contiguously to symbols[]. */ + +/* D=2: 16 codes/iter, unpack + 4-entry pshufb scatter. */ +static inline void merge_flat_d2_x86(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + __m128i c2s_vec = _mm_loadl_epi64((const __m128i *)c2s); /* 4 entries */ + int i = 0; +#if defined(PIVCO_HAS_AVX2) + /* AVX2: one vpsrlvd transpose unpack per 16 codes (terrelln PR #1), + * ~1.3-1.9x faster than two ryg calls. Reads exactly 4 bytes/iter, so + * no over-read slop is needed beyond the last group. */ + for (; i + 16 <= n; i += 16) { + __m128i codes = flat_d2_unpack_avx2(bm + ((i * 2) >> 3)); + __m128i syms = _mm_shuffle_epi8(c2s_vec, codes); + _mm_storeu_si128((__m128i *)(symbols + i), syms); + } +#else + /* SSE4.1: TL/TH prepped nibble tables (issue #5, dougallj x86 port): + * TL[nib]=c2s[nib&3], TH[nib]=c2s[(nib>>2)&3] map input nibbles straight + * to symbol pairs -- no unpack pass; 64 codes/iter, the 4-way interleave + * is a 2-level punpck tree. (On AVX2 builds the vpsrlvd unpack above is + * faster on Intel; this form wins on AMD too -- vendor-dispatch note in + * IDEAS "enc_init 4tab / bc2".) */ + if (n >= 64) { + uint32_t w; memcpy(&w, c2s, 4); + const __m128i TL = _mm_set1_epi32((int)w); /* c2s[nib&3] */ + const __m128i TH = _mm_shuffle_epi8(TL, + _mm_setr_epi8(0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3)); /* c2s[(nib>>2)&3] */ + const __m128i m = _mm_set1_epi8(0x0F); + for (; i + 64 <= n; i += 64) { + __m128i v = _mm_loadu_si128((const __m128i *)(bm + (i >> 2))); + __m128i lo = _mm_and_si128(v, m); + __m128i hi = _mm_and_si128(_mm_srli_epi16(v, 4), m); + __m128i a = _mm_shuffle_epi8(TL, lo); /* code0 of each byte */ + __m128i b = _mm_shuffle_epi8(TH, lo); /* code1 */ + __m128i c = _mm_shuffle_epi8(TL, hi); /* code2 */ + __m128i d = _mm_shuffle_epi8(TH, hi); /* code3 */ + __m128i ab_lo = _mm_unpacklo_epi8(a, b), ab_hi = _mm_unpackhi_epi8(a, b); + __m128i cd_lo = _mm_unpacklo_epi8(c, d), cd_hi = _mm_unpackhi_epi8(c, d); + _mm_storeu_si128((__m128i *)(symbols + i), _mm_unpacklo_epi16(ab_lo, cd_lo)); + _mm_storeu_si128((__m128i *)(symbols + i + 16), _mm_unpackhi_epi16(ab_lo, cd_lo)); + _mm_storeu_si128((__m128i *)(symbols + i + 32), _mm_unpacklo_epi16(ab_hi, cd_hi)); + _mm_storeu_si128((__m128i *)(symbols + i + 48), _mm_unpackhi_epi16(ab_hi, cd_hi)); + } + } + /* 2x ryg D=2 unpack remainder. Each reads a 16-byte window (slop), so + * stop the fast loop a few groups early. */ + int fast_end = n >= 16 ? n - 16 : 0; + for (; i + 16 <= fast_end; i += 16) { + __m128i lo_codes = flat_d2_unpack_x86(bm + ((i * 2) >> 3)); + __m128i hi_codes = flat_d2_unpack_x86(bm + (((i + 8) * 2) >> 3)); + __m128i codes = _mm_unpacklo_epi64(lo_codes, hi_codes); + __m128i syms = _mm_shuffle_epi8(c2s_vec, codes); + _mm_storeu_si128((__m128i *)(symbols + i), syms); + } +#endif + merge_flat_tail_x86(symbols, i, n, bm, 2, c2s); +} + +/* D=3: 32 codes/iter 6-bit pair-gather (issue #5, dougallj; x86 port of the + * NEON kernel): one pshufb positions two adjacent 3-bit codes per byte, the + * bidirectional u16 shift is pmullw-as-shift + one uniform psrlw, and the + * 8-entry c2s is duplicated so any 4-bit index works. Falls through to the + * stock 8-wide ryg path + scalar tail for the remainder. */ +static inline void merge_flat_d3_x86(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + int i = 0; + int fast_end = n >= 16 ? n - 16 : 0; + if (n >= 48) { + const uint8_t *bp = bm; + __m128i c2s8 = _mm_loadl_epi64((const __m128i *)c2s); + __m128i c2s16 = _mm_unpacklo_epi64(c2s8, c2s8); /* 8 entries x2 */ + const __m128i pair6_shuf = _mm_setr_epi8(0,1, 1,2, 3,4, 4,5, 6,7, 7,8, 9,10, 10,11); + const __m128i mul6 = _mm_setr_epi16(16,1, 16,1, 16,1, 16,1); /* <<(4-o) */ + const __m128i m3f_even = _mm_set1_epi16(0x003F); + const __m128i m3f_odd = _mm_set1_epi16(0x3F00); + const __m128i m7 = _mm_set1_epi8(7); + for (; i + 32 <= fast_end; i += 32, bp += 12) { + __m128i packed = _mm_loadu_si128((const __m128i *)bp); + __m128i x = _mm_mullo_epi16(_mm_shuffle_epi8(packed, pair6_shuf), mul6); + /* 12-bit group at bits 4..15: even 6-bit half at 4..9, odd at 10..15 */ + __m128i pair6 = _mm_or_si128( + _mm_and_si128(_mm_srli_epi16(x, 4), m3f_even), + _mm_and_si128(_mm_srli_epi16(x, 2), m3f_odd)); + __m128i lo3 = _mm_and_si128(pair6, m7); + __m128i hi3 = _mm_and_si128(_mm_srli_epi16(pair6, 3), m7); + __m128i s_lo = _mm_shuffle_epi8(c2s16, lo3); + __m128i s_hi = _mm_shuffle_epi8(c2s16, hi3); + _mm_storeu_si128((__m128i *)(symbols + i), _mm_unpacklo_epi8(s_lo, s_hi)); + _mm_storeu_si128((__m128i *)(symbols + i + 16), _mm_unpackhi_epi8(s_lo, s_hi)); + } + } + /* stock 8-wide ryg path + scalar tail */ + __m128i c2s_vec = _mm_loadl_epi64((const __m128i *)c2s); /* 8 entries */ + for (; i + 8 <= fast_end; i += 8) { + __m128i codes = flat_d3_unpack_x86(bm + ((i * 3) >> 3)); + __m128i syms = _mm_shuffle_epi8(c2s_vec, codes); + _mm_storel_epi64((__m128i *)(symbols + i), syms); + } + merge_flat_tail_x86(symbols, i, n, bm, 3, c2s); +} + +/* D=4: nibble codes, 32/iter (AVX2) or 16/iter (SSE). */ +static inline void merge_flat_d4_x86(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ +#ifdef PIVCO_HAS_AVX2 + /* AVX2 32-byte fast path: D=4 means c2s has 16 entries → fits + * in a 128-bit lane, broadcast to both 256-bit lanes. */ + __m128i c2s_lo = _mm_loadu_si128((const __m128i *)c2s); + __m256i c2s_v = _mm256_broadcastsi128_si256(c2s_lo); + __m128i lo_mask128 = _mm_set1_epi8(0x0F); + int i = 0; + for (; i + 32 <= n; i += 32) { + __m128i raw = _mm_loadu_si128((const __m128i *)(bm + (i >> 1))); + __m128i lo = _mm_and_si128(raw, lo_mask128); + __m128i hi = _mm_and_si128(_mm_srli_epi16(raw, 4), lo_mask128); + __m128i codes_lo = _mm_unpacklo_epi8(lo, hi); /* codes 0..15 */ + __m128i codes_hi = _mm_unpackhi_epi8(lo, hi); /* codes 16..31 */ + __m256i codes = _mm256_set_m128i(codes_hi, codes_lo); + __m256i syms = _mm256_shuffle_epi8(c2s_v, codes); + _mm256_storeu_si256((__m256i *)(symbols + i), syms); + } + /* 16-byte SSE fallback for the trailing < 32 elements. */ + __m128i c2s_vec = _mm_loadu_si128((const __m128i *)c2s); + for (; i + 16 <= n; i += 16) { + __m128i codes = flat_d4_unpack_x86(bm + (i >> 1)); + __m128i syms = _mm_shuffle_epi8(c2s_vec, codes); + _mm_storeu_si128((__m128i *)(symbols + i), syms); + } +#else + __m128i c2s_vec = _mm_loadu_si128((const __m128i *)c2s); + int i = 0; + for (; i + 16 <= n; i += 16) { + __m128i codes = flat_d4_unpack_x86(bm + (i >> 1)); + __m128i syms = _mm_shuffle_epi8(c2s_vec, codes); + _mm_storeu_si128((__m128i *)(symbols + i), syms); + } +#endif + for (; i + 2 <= n; i += 2) { + uint8_t b = bm[i >> 1]; + symbols[i ] = c2s[b & 0x0F]; + symbols[i + 1] = c2s[b >> 4]; + } + merge_flat_tail_x86(symbols, i, n, bm, 4, c2s); +} + +/* D=5: 16 codes/iter pair-gather (issue #5, dougallj; x86 port of the NEON + * kernel): one pshufb gathers two adjacent 5-bit codes into each u16 lane, + * pmullw-as-shift aligns the 10-bit pair to the lane top, and two shift+mask + * place code0/code1 in the even/odd byte -- the interleave is free. The + * 32-entry scatter is 2 pshufb + blendv, with bit 4 moved to the sign bit by + * one psllw (idx bytes are pre-masked, so the cross-byte spill is clean). + * Falls through to the stock 8-wide ryg path + scalar tail. */ +/* ---- AVX2 ymm widening of the D=5/6/7 flat decoders ---- + * 32 codes/iter: each 128-bit lane runs the SSE kernel's constants on its + * own packed window (lane1 loads at +S bytes, S = 10/12/14); c2s tables + * broadcast per lane; blendv select tree is byte-wise so lanes never + * interact. Returns codes decoded (multiple of 32); the callers shift + * their frame and let the 128-bit body + scalar tail finish. Measured + * 1.5-1.9x over the 128-bit kernels on c4/c5/c5a/c6a (bench_prim). */ +#ifdef PIVCO_HAS_AVX2 +static inline __m256i flat_bc128_x86(const uint8_t *p) +{ + return _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)p)); +} +static inline __m256i flat_load2_x86(const uint8_t *p, int stride) +{ + return _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)p)), + _mm_loadu_si128((const __m128i *)(p + stride)), 1); +} + +static inline int merge_flat_d5_ymm_x86(uint8_t *symbols, int n, + const uint8_t *bm, const uint8_t *c2s) +{ + const __m256i lo = flat_bc128_x86(c2s), hi = flat_bc128_x86(c2s + 16); + const __m256i pair5_shuf = _mm256_broadcastsi128_si256( + _mm_setr_epi8(0,1, 1,2, 2,3, 3,4, 5,6, 6,7, 7,8, 8,9)); + const __m256i mul5 = _mm256_set1_epi64x(0x0001000400100040ll); /* {64,16,4,1} */ + const __m256i m1f_even = _mm256_set1_epi16(0x001F); + const __m256i m1f_odd = _mm256_set1_epi16(0x1F00); + int pb = (5 * n) >> 3; + int blocks = pb >= 26 ? (pb - 26) / 20 + 1 : 0; + if (blocks > (n >> 5)) blocks = n >> 5; + for (int b = 0; b < blocks; ++b) { + __m256i packed = flat_load2_x86(bm + b * 20, 10); + __m256i x = _mm256_mullo_epi16(_mm256_shuffle_epi8(packed, pair5_shuf), mul5); + __m256i idx = _mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(x, 6), m1f_even), + _mm256_and_si256(_mm256_srli_epi16(x, 3), m1f_odd)); + __m256i rlo = _mm256_shuffle_epi8(lo, idx); + __m256i rhi = _mm256_shuffle_epi8(hi, idx); + __m256i sel = _mm256_slli_epi16(idx, 3); /* bit4 -> sign bit */ + _mm256_storeu_si256((__m256i *)(symbols + (b << 5)), + _mm256_blendv_epi8(rlo, rhi, sel)); + } + return blocks << 5; +} + +static inline int merge_flat_d6_ymm_x86(uint8_t *symbols, int n, + const uint8_t *bm, const uint8_t *c2s) +{ + const __m256i t0 = flat_bc128_x86(c2s), t1 = flat_bc128_x86(c2s + 16); + const __m256i t2 = flat_bc128_x86(c2s + 32), t3 = flat_bc128_x86(c2s + 48); + const __m256i pair6_shuf = _mm256_broadcastsi128_si256( + _mm_setr_epi8(0,1, 1,2, 3,4, 4,5, 6,7, 7,8, 9,10, 10,11)); + const __m256i mul6 = _mm256_set1_epi32(0x00010010); /* {16,1} */ + const __m256i m3f_even = _mm256_set1_epi16(0x003F); + const __m256i m3f_odd = _mm256_set1_epi16(0x3F00); + int pb = (6 * n) >> 3; + int blocks = pb >= 28 ? (pb - 28) / 24 + 1 : 0; + if (blocks > (n >> 5)) blocks = n >> 5; + for (int b = 0; b < blocks; ++b) { + __m256i packed = flat_load2_x86(bm + b * 24, 12); + __m256i x = _mm256_mullo_epi16(_mm256_shuffle_epi8(packed, pair6_shuf), mul6); + __m256i idx = _mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(x, 4), m3f_even), + _mm256_and_si256(_mm256_srli_epi16(x, 2), m3f_odd)); + __m256i r0 = _mm256_shuffle_epi8(t0, idx); + __m256i r1 = _mm256_shuffle_epi8(t1, idx); + __m256i r2 = _mm256_shuffle_epi8(t2, idx); + __m256i r3 = _mm256_shuffle_epi8(t3, idx); + __m256i s4 = _mm256_slli_epi16(idx, 3); /* bit4 -> sign bit */ + __m256i s5 = _mm256_slli_epi16(idx, 2); /* bit5 -> sign bit */ + __m256i a = _mm256_blendv_epi8(r0, r1, s4); + __m256i b2 = _mm256_blendv_epi8(r2, r3, s4); + _mm256_storeu_si256((__m256i *)(symbols + (b << 5)), + _mm256_blendv_epi8(a, b2, s5)); + } + return blocks << 5; +} + +static inline int merge_flat_d7_ymm_x86(uint8_t *symbols, int n, + const uint8_t *bm, const uint8_t *c2s) +{ + const __m256i t0 = flat_bc128_x86(c2s), t1 = flat_bc128_x86(c2s + 16); + const __m256i t2 = flat_bc128_x86(c2s + 32), t3 = flat_bc128_x86(c2s + 48); + const __m256i t4 = flat_bc128_x86(c2s + 64), t5 = flat_bc128_x86(c2s + 80); + const __m256i t6 = flat_bc128_x86(c2s + 96), t7 = flat_bc128_x86(c2s + 112); + const __m256i g_lo = _mm256_broadcastsi128_si256( + _mm_setr_epi8(0,1,2,3, 1,2,3,4, 3,4,5,6, 5,6,7,8)); + const __m256i g_hi = _mm256_broadcastsi128_si256( + _mm_setr_epi8(7,8,9,10, 8,9,10,11, 10,11,12,13, 12,13,14,15)); + const __m256i mul7 = _mm256_broadcastsi128_si256( + _mm_setr_epi32(64,1,4,16)); /* <<(6-o), o={0,6,4,2} */ + const __m256i m7f_even = _mm256_set1_epi32(0x0000007F); + const __m256i m7f_odd = _mm256_set1_epi32(0x00007F00); + int pb = (7 * n) >> 3; + int blocks = pb >= 30 ? (pb - 30) / 28 + 1 : 0; + if (blocks > (n >> 5)) blocks = n >> 5; + for (int b = 0; b < blocks; ++b) { + __m256i packed = flat_load2_x86(bm + b * 28, 14); + __m256i xl = _mm256_mullo_epi32(_mm256_shuffle_epi8(packed, g_lo), mul7); + __m256i xh = _mm256_mullo_epi32(_mm256_shuffle_epi8(packed, g_hi), mul7); + __m256i cl = _mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi32(xl, 6), m7f_even), + _mm256_and_si256(_mm256_srli_epi32(xl, 5), m7f_odd)); + __m256i ch = _mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi32(xh, 6), m7f_even), + _mm256_and_si256(_mm256_srli_epi32(xh, 5), m7f_odd)); + __m256i idx = _mm256_packus_epi32(cl, ch); /* per-lane, in order */ + __m256i r0 = _mm256_shuffle_epi8(t0, idx); + __m256i r1 = _mm256_shuffle_epi8(t1, idx); + __m256i r2 = _mm256_shuffle_epi8(t2, idx); + __m256i r3 = _mm256_shuffle_epi8(t3, idx); + __m256i r4 = _mm256_shuffle_epi8(t4, idx); + __m256i r5 = _mm256_shuffle_epi8(t5, idx); + __m256i r6 = _mm256_shuffle_epi8(t6, idx); + __m256i r7 = _mm256_shuffle_epi8(t7, idx); + __m256i s4 = _mm256_slli_epi16(idx, 3); /* bit4 -> sign bit */ + __m256i s5 = _mm256_slli_epi16(idx, 2); /* bit5 -> sign bit */ + __m256i s6 = _mm256_slli_epi16(idx, 1); /* bit6 -> sign bit */ + __m256i a0 = _mm256_blendv_epi8(r0, r1, s4); + __m256i a1 = _mm256_blendv_epi8(r2, r3, s4); + __m256i a2 = _mm256_blendv_epi8(r4, r5, s4); + __m256i a3 = _mm256_blendv_epi8(r6, r7, s4); + __m256i b0 = _mm256_blendv_epi8(a0, a1, s5); + __m256i b1 = _mm256_blendv_epi8(a2, a3, s5); + _mm256_storeu_si256((__m256i *)(symbols + (b << 5)), + _mm256_blendv_epi8(b0, b1, s6)); + } + return blocks << 5; +} +#endif /* PIVCO_HAS_AVX2 */ + +static inline void merge_flat_d5_x86(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ +#ifdef PIVCO_HAS_AVX2 + { /* ymm blocks first; shift the frame so the 128-bit body and the + * scalar tail below run unchanged on the remainder. */ + int done = merge_flat_d5_ymm_x86(symbols, n, bm, c2s); + symbols += done; bm += (done * 5) >> 3; n -= done; + } +#endif + /* pshufb on either table uses code&15; blend by bit 4. */ + __m128i lo = _mm_loadu_si128((const __m128i *)c2s); /* c2s[0..15] */ + __m128i hi = _mm_loadu_si128((const __m128i *)(c2s + 16)); /* c2s[16..31] */ + int i = 0; + if (n >= 25) { + const __m128i pair5_shuf = _mm_setr_epi8(0,1, 1,2, 2,3, 3,4, 5,6, 6,7, 7,8, 8,9); + const __m128i mul5 = _mm_setr_epi16(64,16,4,1, 64,16,4,1); /* <<(6-o) */ + const __m128i m1f_even = _mm_set1_epi16(0x001F); + const __m128i m1f_odd = _mm_set1_epi16(0x1F00); + int blocks = (n - 9) >> 4; + for (int b = 0; b < blocks; ++b) { + __m128i packed = _mm_loadu_si128((const __m128i *)(bm + b * 10)); + __m128i x = _mm_mullo_epi16(_mm_shuffle_epi8(packed, pair5_shuf), mul5); + /* code0 at bits 6..10, code1 at bits 11..15 of each u16 */ + __m128i idx = _mm_or_si128( + _mm_and_si128(_mm_srli_epi16(x, 6), m1f_even), + _mm_and_si128(_mm_srli_epi16(x, 3), m1f_odd)); + __m128i rlo = _mm_shuffle_epi8(lo, idx); + __m128i rhi = _mm_shuffle_epi8(hi, idx); + __m128i sel = _mm_slli_epi16(idx, 3); /* bit4 -> sign bit */ + _mm_storeu_si128((__m128i *)(symbols + (b << 4)), + _mm_blendv_epi8(rlo, rhi, sel)); + } + i = blocks << 4; + } + const __m128i b4 = _mm_set1_epi8(0x10); + int fast_end = n >= 24 ? n - 24 : 0; + for (; i + 8 <= fast_end; i += 8) { + __m128i codes = flat_d5_unpack_x86(bm + ((i * 5) >> 3)); + __m128i rlo = _mm_shuffle_epi8(lo, codes); + __m128i rhi = _mm_shuffle_epi8(hi, codes); + __m128i sel = _mm_cmpeq_epi8(_mm_and_si128(codes, b4), b4); + __m128i syms = _mm_blendv_epi8(rlo, rhi, sel); + _mm_storel_epi64((__m128i *)(symbols + i), syms); + } + merge_flat_tail_x86(symbols, i, n, bm, 5, c2s); +} + +/* D=6: 16 codes/iter pair-gather (issue #5, dougallj; x86 port of the NEON + * kernel): same gather as D=5 but on 12-bit pairs (the shuffle/mul constants + * match the D=3 pair-gather, which works the same 6-bit grid). The 64-entry + * scatter is 4 pshufb + a 2-level blend with bits 4/5 psllw'd to the sign + * bit. Falls through to the stock 8-wide ryg path + scalar tail. */ +static inline void merge_flat_d6_x86(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ +#ifdef PIVCO_HAS_AVX2 + { /* ymm blocks first; shift the frame so the 128-bit body and the + * scalar tail below run unchanged on the remainder. */ + int done = merge_flat_d6_ymm_x86(symbols, n, bm, c2s); + symbols += done; bm += (done * 6) >> 3; n -= done; + } +#endif + /* four pshufb (code&15 into each quarter) then a 2-level blend by + * bits 5,4 selects the right quarter. */ + __m128i t0 = _mm_loadu_si128((const __m128i *)c2s); + __m128i t1 = _mm_loadu_si128((const __m128i *)(c2s + 16)); + __m128i t2 = _mm_loadu_si128((const __m128i *)(c2s + 32)); + __m128i t3 = _mm_loadu_si128((const __m128i *)(c2s + 48)); + int i = 0; + if (n >= 24) { + const __m128i pair6_shuf = _mm_setr_epi8(0,1, 1,2, 3,4, 4,5, 6,7, 7,8, 9,10, 10,11); + const __m128i mul6 = _mm_setr_epi16(16,1, 16,1, 16,1, 16,1); /* <<(4-o) */ + const __m128i m3f_even = _mm_set1_epi16(0x003F); + const __m128i m3f_odd = _mm_set1_epi16(0x3F00); + int blocks = (n - 8) >> 4; + for (int b = 0; b < blocks; ++b) { + __m128i packed = _mm_loadu_si128((const __m128i *)(bm + b * 12)); + __m128i x = _mm_mullo_epi16(_mm_shuffle_epi8(packed, pair6_shuf), mul6); + /* code0 at bits 4..9, code1 at bits 10..15 of each u16 */ + __m128i idx = _mm_or_si128( + _mm_and_si128(_mm_srli_epi16(x, 4), m3f_even), + _mm_and_si128(_mm_srli_epi16(x, 2), m3f_odd)); + __m128i r0 = _mm_shuffle_epi8(t0, idx); + __m128i r1 = _mm_shuffle_epi8(t1, idx); + __m128i r2 = _mm_shuffle_epi8(t2, idx); + __m128i r3 = _mm_shuffle_epi8(t3, idx); + __m128i s4 = _mm_slli_epi16(idx, 3); /* bit4 -> sign bit */ + __m128i s5 = _mm_slli_epi16(idx, 2); /* bit5 -> sign bit */ + __m128i a = _mm_blendv_epi8(r0, r1, s4); + __m128i b2 = _mm_blendv_epi8(r2, r3, s4); + _mm_storeu_si128((__m128i *)(symbols + (b << 4)), + _mm_blendv_epi8(a, b2, s5)); + } + i = blocks << 4; + } + const __m128i b4 = _mm_set1_epi8(0x10); + const __m128i b5 = _mm_set1_epi8(0x20); + int fast_end = n >= 24 ? n - 24 : 0; + for (; i + 8 <= fast_end; i += 8) { + __m128i codes = flat_d6_unpack_x86(bm + ((i * 6) >> 3)); + __m128i r0 = _mm_shuffle_epi8(t0, codes); + __m128i r1 = _mm_shuffle_epi8(t1, codes); + __m128i r2 = _mm_shuffle_epi8(t2, codes); + __m128i r3 = _mm_shuffle_epi8(t3, codes); + __m128i s4 = _mm_cmpeq_epi8(_mm_and_si128(codes, b4), b4); + __m128i s5 = _mm_cmpeq_epi8(_mm_and_si128(codes, b5), b5); + __m128i a = _mm_blendv_epi8(r0, r1, s4); /* bit5=0: t0/t1 */ + __m128i b = _mm_blendv_epi8(r2, r3, s4); /* bit5=1: t2/t3 */ + __m128i syms = _mm_blendv_epi8(a, b, s5); + _mm_storel_epi64((__m128i *)(symbols + i), syms); + } + merge_flat_tail_x86(symbols, i, n, bm, 6, c2s); +} + +/* D=7: 16 codes/iter u32-lane pair-gather. 14-bit pairs at offsets {0,6,4,2} + * don't fit the u16 windows the D<=6 kernels use (up to 20 bits), so each + * pair gets a 4-byte window in a u32 lane (2 pshufb gathers x 4 pairs), + * pmulld by 2^(6-o) normalizes to bits 6..19, shift+mask place code0/code1 + * in the lane's low bytes, and packus_epi32 compacts to 16 in-order codes. + * The 128-entry scatter is 8 pshufb quarters + a 3-level blend on bits 4/5/6 + * (psllw into blendv's sign bit). + * + * Kept as its own function: the first cut, written directly inside + * merge_flat_d7_x86, cost c3 (IvyBridge) ~12% E2E on every flat-using dist + * via codegen/layout; as a named function the codegen is fine whether or + * not the compiler inlines it (inline policy revisits with dynamic + * dispatch). Decodes the largest 16-multiple prefix that gating allows; + * returns the count of codes written. */ +static inline int merge_flat_d7_pair_x86(uint8_t *symbols, + int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + __m128i t0 = _mm_loadu_si128((const __m128i *)c2s); + __m128i t1 = _mm_loadu_si128((const __m128i *)(c2s + 16)); + __m128i t2 = _mm_loadu_si128((const __m128i *)(c2s + 32)); + __m128i t3 = _mm_loadu_si128((const __m128i *)(c2s + 48)); + __m128i t4 = _mm_loadu_si128((const __m128i *)(c2s + 64)); + __m128i t5 = _mm_loadu_si128((const __m128i *)(c2s + 80)); + __m128i t6 = _mm_loadu_si128((const __m128i *)(c2s + 96)); + __m128i t7 = _mm_loadu_si128((const __m128i *)(c2s + 112)); + const __m128i g_lo = _mm_setr_epi8(0,1,2,3, 1,2,3,4, 3,4,5,6, 5,6,7,8); + const __m128i g_hi = _mm_setr_epi8(7,8,9,10, 8,9,10,11, 10,11,12,13, 12,13,14,15); + const __m128i mul7 = _mm_setr_epi32(64,1,4,16); /* <<(6-o), o={0,6,4,2} */ + const __m128i m7f_even = _mm_set1_epi32(0x0000007F); + const __m128i m7f_odd = _mm_set1_epi32(0x00007F00); + int blocks = (n - 3) >> 4; + for (int b = 0; b < blocks; ++b) { + __m128i packed = _mm_loadu_si128((const __m128i *)(bm + b * 14)); + __m128i xl = _mm_mullo_epi32(_mm_shuffle_epi8(packed, g_lo), mul7); + __m128i xh = _mm_mullo_epi32(_mm_shuffle_epi8(packed, g_hi), mul7); + /* code0 at bits 6..12, code1 at bits 13..19 of each u32 */ + __m128i cl = _mm_or_si128( + _mm_and_si128(_mm_srli_epi32(xl, 6), m7f_even), + _mm_and_si128(_mm_srli_epi32(xl, 5), m7f_odd)); + __m128i ch = _mm_or_si128( + _mm_and_si128(_mm_srli_epi32(xh, 6), m7f_even), + _mm_and_si128(_mm_srli_epi32(xh, 5), m7f_odd)); + __m128i idx = _mm_packus_epi32(cl, ch); /* 16 codes, in order */ + __m128i r0 = _mm_shuffle_epi8(t0, idx); + __m128i r1 = _mm_shuffle_epi8(t1, idx); + __m128i r2 = _mm_shuffle_epi8(t2, idx); + __m128i r3 = _mm_shuffle_epi8(t3, idx); + __m128i r4 = _mm_shuffle_epi8(t4, idx); + __m128i r5 = _mm_shuffle_epi8(t5, idx); + __m128i r6 = _mm_shuffle_epi8(t6, idx); + __m128i r7 = _mm_shuffle_epi8(t7, idx); + __m128i s4 = _mm_slli_epi16(idx, 3); /* bit4 -> sign bit */ + __m128i s5 = _mm_slli_epi16(idx, 2); /* bit5 -> sign bit */ + __m128i s6 = _mm_slli_epi16(idx, 1); /* bit6 -> sign bit */ + __m128i a0 = _mm_blendv_epi8(r0, r1, s4); + __m128i a1 = _mm_blendv_epi8(r2, r3, s4); + __m128i a2 = _mm_blendv_epi8(r4, r5, s4); + __m128i a3 = _mm_blendv_epi8(r6, r7, s4); + __m128i b0 = _mm_blendv_epi8(a0, a1, s5); + __m128i b1 = _mm_blendv_epi8(a2, a3, s5); + _mm_storeu_si128((__m128i *)(symbols + (b << 4)), + _mm_blendv_epi8(b0, b1, s6)); + } + return blocks << 4; +} + +static inline void merge_flat_d7_x86(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ +#ifdef PIVCO_HAS_AVX2 + { /* ymm blocks first; shift the frame so the 128-bit body and the + * scalar tail below run unchanged on the remainder. */ + int done = merge_flat_d7_ymm_x86(symbols, n, bm, c2s); + symbols += done; bm += (done * 7) >> 3; n -= done; + } +#endif + int i = 0; + if (n >= 19) + i = merge_flat_d7_pair_x86(symbols, n, bm, c2s); + for (; i + 8 <= n; i += 8) { + const uint8_t *p = bm + ((i * 7) >> 3); + uint64_t w = (uint64_t)p[0] | ((uint64_t)p[1] << 8) + | ((uint64_t)p[2] << 16) | ((uint64_t)p[3] << 24) + | ((uint64_t)p[4] << 32) | ((uint64_t)p[5] << 40) + | ((uint64_t)p[6] << 48); + symbols[i ] = c2s[(w ) & 0x7F]; + symbols[i + 1] = c2s[(w >> 7) & 0x7F]; + symbols[i + 2] = c2s[(w >> 14) & 0x7F]; + symbols[i + 3] = c2s[(w >> 21) & 0x7F]; + symbols[i + 4] = c2s[(w >> 28) & 0x7F]; + symbols[i + 5] = c2s[(w >> 35) & 0x7F]; + symbols[i + 6] = c2s[(w >> 42) & 0x7F]; + symbols[i + 7] = c2s[(w >> 49) & 0x7F]; + } + merge_flat_tail_x86(symbols, i, n, bm, 7, c2s); +} + +/* D=8: a depth-8 flat region is the full 256-symbol alphabet at equal code + * length, whose canonical c2s is the identity permutation -- the byte-aligned + * codes ARE the symbols, so the whole decode is a memcpy. See the derivation + * at merge_flat_d8_neon in pivco_huffman_primitives_neon.h. */ +static inline void merge_flat_d8_x86(uint8_t *symbols, int n, + const uint8_t *bm, + const uint8_t *c2s) +{ + (void)c2s; + memcpy(symbols, bm, (size_t)n); +} + +/* merge_flat_x86 — D-bit flat-subtree decode into a contiguous output + * buffer. Per-D SIMD paths above; scalar unrolled for the rest. AVX-512 + * D=5/D=6 fast paths live in primitives_avx512.h. */ +static inline void merge_flat_x86(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s) +{ + PROF_TIC(); + switch (D) { + case 2: merge_flat_d2_x86(out, n, bm, c2s); break; + case 3: merge_flat_d3_x86(out, n, bm, c2s); break; + case 4: merge_flat_d4_x86(out, n, bm, c2s); break; + case 5: merge_flat_d5_x86(out, n, bm, c2s); break; + case 6: merge_flat_d6_x86(out, n, bm, c2s); break; + case 7: merge_flat_d7_x86(out, n, bm, c2s); break; + case 8: merge_flat_d8_x86(out, n, bm, c2s); break; + default: + pivco_check_fail("merge_flat_x86: D out of range (flat_depth is 2..8)", + __FILE__, __LINE__); + break; + } + PROF_TOC(PROF_BU_MERGE_FLAT, n); +} + +/* ---------- Encode primitives: rank-based encoding (8-bit in-order ranks) ---------- + * Partition 8-bit ranks against split_rank, a u8 port of the code_la partition + * (part_core_x86): per-8-rank chunk, movemask routing mask, compress_tab pshufb, + * 8-byte storel compaction. No unsigned byte-compare on SSE, so the routing + * mask uses the MIN trick: rank > thr <=> min(rank, thr+1) == thr+1. */ +static uint8_t x86_pc8[256]; +static uint8_t x86_ctab_r[256][16], x86_ctab_l[256][16]; +static uint8_t x86_pre_r[9][256][16], x86_pre_l[9][256][16]; + +/* p16rev partition LUTs (part_full_x86). One combined index per 16-lane group + * packs {left, forward, front} | {right, reversed, back}; left+right tile the + * 16 lanes so the OR of two disjoint-support tables is exact. A fraction of + * the LUT footprint of the ctab_r/l + pre_r/l pair above (12 KB vs 80 KB) — + * the win on x86's 32-48 KB L1. + * x86_p16rev_tabA[m0] low-byte (positions 0..7) + * x86_p16rev_tabB0[m1] high-byte (positions 8..15), pc0=0 layout only: the + * pc0>0 layout is this one shifted left by pc0 lanes, so + * tabB[pc0][m1] is recovered as a byte-offset load + * `tabB0[m1] + pc0` (8 KB vs the former 36 KB). 32 B + * rows keep the offset-16 load inside one cache line. + * The right side is recovered with a single loop-invariant full-reverse + * constant in part_full_x86. */ +static uint8_t x86_p16rev_tabA[256][16] __attribute__((aligned(16))); +static uint8_t x86_p16rev_tabB0[256][32] __attribute__((aligned(32))); +static int x86_tabs_ready = 0; +static void x86_build_tabs(void) +{ + if (x86_tabs_ready) return; + for (int m = 0; m < 256; m++) { + x86_pc8[m] = (uint8_t)__builtin_popcount(m); + memset(x86_ctab_r[m], 0x80, 16); + memset(x86_ctab_l[m], 0x80, 16); + int pr = 0, pl = 0; + for (int k = 0; k < 8; k++) { + if (m & (1 << k)) x86_ctab_r[m][pr++] = (uint8_t)k; /* right -> [0:n_right) */ + else x86_ctab_l[m][pl++] = (uint8_t)k; /* left -> [0:n_left) */ + } + } + /* High-half (lanes 8..15) source positions pre-shifted to output offset + * nlo, for the dense 16-wide compaction (min-merge with the low-half ctab). */ + for (int nlo = 0; nlo <= 8; nlo++) { + for (int m = 0; m < 256; m++) { + memset(x86_pre_r[nlo][m], 0x80, 16); + memset(x86_pre_l[nlo][m], 0x80, 16); + int pr = nlo, pl = nlo; + for (int k = 0; k < 8; k++) { + if (m & (1 << k)) x86_pre_r[nlo][m][pr++] = (uint8_t)(8 + k); + else x86_pre_l[nlo][m][pl++] = (uint8_t)(8 + k); + } + } + } + /* p16rev combined-index tables (low byte over m0, high byte over [pc0][m1]), + * 0-fill on the non-owned lanes so the OR is exact (every lane owned once). */ + for (int m0 = 0; m0 < 256; m0++) { + memset(x86_p16rev_tabA[m0], 0, 16); + int lp = 0, rp = 15; + for (int k = 0; k < 8; k++) { + if ((m0 >> k) & 1) x86_p16rev_tabA[m0][rp--] = (uint8_t)k; + else x86_p16rev_tabA[m0][lp++] = (uint8_t)k; + } + } + for (int m1 = 0; m1 < 256; m1++) { + memset(x86_p16rev_tabB0[m1], 0, 32); + int lp = 8, rp = 15; /* pc0 = 0 layout; pc0 > 0 handled by the load offset */ + for (int k = 0; k < 8; k++) { + if ((m1 >> k) & 1) x86_p16rev_tabB0[m1][rp--] = (uint8_t)(8 + k); + else x86_p16rev_tabB0[m1][lp++] = (uint8_t)(8 + k); + } + } + x86_tabs_ready = 1; +} + +/* 8-bit mask of (rank > thr) for the 8 ranks in the low 8 lanes of `ids8`. */ +static inline uint8_t x86_mask8(__m128i ids8, __m128i thr1) +{ + __m128i ge = _mm_cmpeq_epi8(_mm_min_epu8(ids8, thr1), thr1); + return (uint8_t)_mm_movemask_epi8(ge); +} + +/* Compact the 8 ranks in the low 8 lanes of `v` (chunk mask `m`): right ranks + * to tmp[ro), left ranks in place to ranks[lo). pshufb gathers each side + * contiguously; storel writes exactly 8 bytes (the chunk), the (8 - popcount) + * trailing zeros get overwritten by the next chunk's compaction. The 8-byte + * width (vs the code_la 16-byte store) keeps the in-place left write from + * clobbering the next iter's not-yet-loaded ranks. */ +/* Dense 16-wide compaction: one pshufb + one 16-byte store per side over all 16 + * ranks in `v`. The low-half ctab (lanes 0..7 of chunk mlo) is min-merged with + * the high-half pre table (lanes 8..15 of chunk mhi, pre-shifted to output + * offset rlo): both use 0x80 fill, so min picks the real index at each output + * lane. Half the pshufb + store traffic of the per-8 form — the binding + * resource on a port-bound SSE loop (SSE has no native byte-compress). The + * in-place left 16-byte store is safe: n_left <= j so n_left+16 <= j+16 = the next + * iter's load, no clobber. */ +#define X86_COMPACT16(v, mlo, mhi, rlo, ldst, rdst) \ + do { \ + __m128i ridx_ = _mm_min_epu8( \ + _mm_load_si128((const __m128i *)x86_ctab_r[mlo]), \ + _mm_load_si128((const __m128i *)x86_pre_r[rlo][mhi])); \ + _mm_storeu_si128((__m128i *)(tmp + (rdst)), _mm_shuffle_epi8((v), ridx_)); \ + int llo_ = 8 - (rlo); \ + __m128i lidx_ = _mm_min_epu8( \ + _mm_load_si128((const __m128i *)x86_ctab_l[mlo]), \ + _mm_load_si128((const __m128i *)x86_pre_l[llo_][mhi])); \ + _mm_storeu_si128((__m128i *)(ranks + (ldst)), _mm_shuffle_epi8((v), lidx_)); \ + } while (0) + +/* full: p16rev — per 16-lane group, ONE combined index (OR of the two disjoint + * x86_p16rev_tabA/tabB0) feeds one pshufb that yields {left fwd | right reversed} + * in a single register; that register IS the left output (store it), and the + * right output is recovered with a second pshufb over the SAME register using + * the loop-invariant full-reverse constant. vs the prior dense X86_COMPACT16 + * (two independent min-merged indices): one OR + one table-pair load instead of + * two pminub + four loads per 16 lanes, and the LUTs shrink (12 KB vs 80 KB) — + * the win on x86's 32-48 KB L1. 32 ranks per iter; the 16-byte tail overstore + * is absorbed by the ranks +64 / tmp +2N scratch slack reserved in codec.c. */ +static inline int part_full_x86(uint8_t *ranks, int n, uint8_t thr, + uint8_t *bm, uint8_t *tmp) +{ + x86_build_tabs(); + int n_left = 0, n_right = 0; + int j = 0; + __m128i thr1 = _mm_set1_epi8((char)(thr + 1)); + /* Right recovery via one loop-invariant full-reverse constant: reversing the + * whole comb register lands the top-pc reversed right lanes at output [0,pc) + * (the tail is left-reversed garbage the next group overwrites). */ + static const uint8_t rev16_a[16] = {15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0}; + const __m128i rev16 = _mm_loadu_si128((const __m128i *)rev16_a); + /* cl_/cr_ = lefts/rights already emitted by earlier groups THIS iter, so + * both groups' store addresses are known up front (no serial cursor chain + * between them); n_left/n_right advance once per iter. (issue #5) */ +#define _P16REV(v, mlo_, mhi_, cl_, cr_) do { \ + uint32_t pc0_ = (uint32_t)__builtin_popcount((unsigned)(mlo_)); \ + __m128i cidx_ = _mm_or_si128( \ + _mm_load_si128((const __m128i *)x86_p16rev_tabA[(mlo_)]), \ + _mm_loadu_si128((const __m128i *)&x86_p16rev_tabB0[(mhi_)][pc0_])); \ + __m128i comb_ = _mm_shuffle_epi8((v), cidx_); \ + _mm_storeu_si128((__m128i *)(ranks + n_left + (cl_)), comb_); \ + _mm_storeu_si128((__m128i *)(tmp + n_right + (cr_)), \ + _mm_shuffle_epi8(comb_, rev16)); \ + } while (0) + /* 32 ranks/iter: two SSE movemasks OR'd into a 32-bit routing mask (one + * 4-byte bitmap write), two combined-shuffle compactions. Both 16-byte + * halves are loaded before any in-place left store, so the dense left write + * can't clobber an un-loaded rank. */ + for (; j + 32 <= n; j += 32) { + __m128i v0 = _mm_loadu_si128((const __m128i *)(ranks + j)); + __m128i v1 = _mm_loadu_si128((const __m128i *)(ranks + j + 16)); + uint32_t mlo = (uint16_t)_mm_movemask_epi8( + _mm_cmpeq_epi8(_mm_min_epu8(v0, thr1), thr1)); + uint32_t mhi = (uint16_t)_mm_movemask_epi8( + _mm_cmpeq_epi8(_mm_min_epu8(v1, thr1), thr1)); + uint32_t mm = mlo | (mhi << 16); + memcpy(bm + (j >> 3), &mm, 4); + uint32_t cr1 = (uint32_t)__builtin_popcount(mlo); /* rights in group 0 */ + uint32_t total = (uint32_t)__builtin_popcount(mm); + _P16REV(v0, (uint8_t)mlo, (uint8_t)(mlo >> 8), 0, 0); + _P16REV(v1, (uint8_t)mhi, (uint8_t)(mhi >> 8), 16 - cr1, cr1); + n_right += (int)total; n_left += 32 - (int)total; + } + /* 16-rank tail of the [32k, 32k+31] remainder: one movemask, one compaction. */ + for (; j + 16 <= n; j += 16) { + __m128i v = _mm_loadu_si128((const __m128i *)(ranks + j)); + uint16_t mm = (uint16_t)_mm_movemask_epi8( + _mm_cmpeq_epi8(_mm_min_epu8(v, thr1), thr1)); + memcpy(bm + (j >> 3), &mm, 2); + uint32_t pc16 = (uint32_t)__builtin_popcount((unsigned)mm); + _P16REV(v, (uint8_t)mm, (uint8_t)(mm >> 8), 0, 0); + n_right += (int)pc16; n_left += 16 - (int)pc16; + } +#undef _P16REV + for (; j < n; j++) { + if ((j & 7) == 0) bm[j >> 3] = 0; + uint8_t r = ranks[j]; + if (r > thr) { bm[j >> 3] |= (uint8_t)(1u << (j & 7)); tmp[n_right++] = r; } + else { ranks[n_left++] = r; } + } + return n_right; +} + +/* right/none: 16-wide one-sided compaction, 32 ranks/iter. Per 16-lane + * group, the emitted side's index is the matching half of X86_COMPACT16 (RIGHT: + * min(ctab_r[m0], pre_r[pc0][m1]); LEFT: min(ctab_l[m0], pre_l[8-pc0][m1])), one + * pshufb + one 16-byte store — vs the prior stride-8 form's pshufb + 8-byte store + * every 8 lanes. Reuses the production ctab/pre tables. EMIT_RIGHT is + * compile-time, so the none form folds to a pure bitmap build; the left + * side is never scattered (a leaf child's ranks are dead). */ +__attribute__((always_inline)) static inline +int part_core_x86(uint8_t *ranks, int n, uint8_t thr, + uint8_t *bm, uint8_t *tmp, int EMIT_RIGHT) +{ + x86_build_tabs(); + int n_right = 0; + int j = 0; + __m128i thr1 = _mm_set1_epi8((char)(thr + 1)); +#define _PC16(v, mlo_, mhi_) do { \ + uint32_t pc0_ = x86_pc8[(mlo_)]; \ + uint32_t pc_ = pc0_ + x86_pc8[(mhi_)]; \ + if (EMIT_RIGHT) _mm_storeu_si128((__m128i *)(tmp + n_right), \ + _mm_shuffle_epi8((v), _mm_min_epu8( \ + _mm_load_si128((const __m128i *)x86_ctab_r[(mlo_)]), \ + _mm_load_si128((const __m128i *)x86_pre_r[pc0_][(mhi_)])))); \ + n_right += pc_; \ + } while (0) + for (; j + 32 <= n; j += 32) { + __m128i v0 = _mm_loadu_si128((const __m128i *)(ranks + j)); + __m128i v1 = _mm_loadu_si128((const __m128i *)(ranks + j + 16)); + uint32_t mlo = (uint16_t)_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_min_epu8(v0, thr1), thr1)); + uint32_t mhi = (uint16_t)_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_min_epu8(v1, thr1), thr1)); + uint32_t mm = mlo | (mhi << 16); + memcpy(bm + (j >> 3), &mm, 4); + _PC16(v0, (uint8_t)mlo, (uint8_t)(mlo >> 8)); + _PC16(v1, (uint8_t)mhi, (uint8_t)(mhi >> 8)); + } + for (; j + 16 <= n; j += 16) { + __m128i v = _mm_loadu_si128((const __m128i *)(ranks + j)); + uint16_t mm = (uint16_t)_mm_movemask_epi8(_mm_cmpeq_epi8(_mm_min_epu8(v, thr1), thr1)); + memcpy(bm + (j >> 3), &mm, 2); + _PC16(v, (uint8_t)mm, (uint8_t)(mm >> 8)); + } +#undef _PC16 + for (; j < n; j++) { + if ((j & 7) == 0) bm[j >> 3] = 0; + uint8_t r = ranks[j]; + if (r > thr) { bm[j >> 3] |= (uint8_t)(1u << (j & 7)); + if (EMIT_RIGHT) tmp[n_right] = r; n_right++; } + } + return n_right; +} + +/* Native u8 rank packers (SSE4.1). The flat local code is (rank - base), + * already a D-bit byte, so the byte-laid intermediate comes from a u8 load + + * sub + mask — no u16 srli + saturating narrow. The bit-stitch backend mirrors + * the code_la pack_d{2,3,4,8}_sse_x86 helpers above. */ + +/* SSE4.1 D=2: 16 ranks -> 4 bytes. _mm_maddubs_epi16 weighted pair-add + * with weights {1, 4, 16, 64} (int8 max 127, so 64 fits). */ +static inline int pack_d2_sse_x86(uint8_t *out, const uint8_t *ranks, int n, uint8_t base) +{ + const __m128i weights = _mm_setr_epi8(1, 4, 16, 64, 1, 4, 16, 64, + 1, 4, 16, 64, 1, 4, 16, 64); + const __m128i vb = _mm_set1_epi8((char)base); + int i = 0; + for (; i + 16 <= n; i += 16) { + __m128i bytes = _mm_sub_epi8(_mm_loadu_si128((const __m128i *)(ranks + i)), vb); + __m128i step1 = _mm_maddubs_epi16(bytes, weights); /* local code in [0,2^D); no mask */ + __m128i step2 = _mm_hadd_epi16(step1, _mm_setzero_si128()); + __m128i out_bytes = _mm_packus_epi16(step2, _mm_setzero_si128()); + uint32_t packed4 = (uint32_t)_mm_cvtsi128_si32(out_bytes); + memcpy(out + (i * 2 / 8), &packed4, 4); + } + return i; +} + +/* SSE4.1 D=4: 16 ranks -> 8 bytes. _mm_maddubs_epi16 with weights {1, 16}. */ +static inline int pack_d4_sse_x86(uint8_t *out, const uint8_t *ranks, int n, uint8_t base) +{ + const __m128i weights = _mm_setr_epi8(1, 16, 1, 16, 1, 16, 1, 16, + 1, 16, 1, 16, 1, 16, 1, 16); + const __m128i vb = _mm_set1_epi8((char)base); + int i = 0; + for (; i + 16 <= n; i += 16) { + __m128i bytes = _mm_sub_epi8(_mm_loadu_si128((const __m128i *)(ranks + i)), vb); + __m128i step1 = _mm_maddubs_epi16(bytes, weights); /* local code in [0,2^D); no mask */ + __m128i out_bytes = _mm_packus_epi16(step1, _mm_setzero_si128()); + _mm_storel_epi64((__m128i *)(out + (i * 4 / 8)), out_bytes); + } + return i; +} + +/* SSE4.1 D=8: 16 ranks -> 16 bytes, byte-aligned. */ +static inline int pack_d8_sse_x86(uint8_t *out, const uint8_t *ranks, int n, uint8_t base) +{ + const __m128i vb = _mm_set1_epi8((char)base); + int i = 0; + for (; i + 16 <= n; i += 16) { + _mm_storeu_si128((__m128i *)(out + i), + _mm_sub_epi8(_mm_loadu_si128((const __m128i *)(ranks + i)), vb)); + } + return i; +} + +/* SSE4.1 D=3/5/6/7: 128-bit twin of the AVX2 ryg/pyramid pack (see + * pivco_huffman_avx2_pack.h for the op-by-op story). 16 codes per xmm + * iter: maddubs (2D-bit byte pairs) -> madd (4D-bit dword pairs) -> + * qword fuse (srlq + and/andn/or) -> pshufb compact -> one 16-byte + * store. The store's trailing junk (16 - 2D bytes) is overwritten by + * the next iter; the LAST iter's junk needs slack past the packed + * stream, which PIVCO_MAX_ENCODED_SIZE provides (same contract as the + * AVX2 kernels). Replaces the mullo+hadd 8-codes/iter D=3 form and + * the D=5/6/7 scalar fallback on non-AVX2 hosts: c3/IvyBridge measures + * 5.1x (d3) and 19-22x (d5-d7); the old d3 is asof-b8bf472 in + * bench/prim_variants. D=2/4 keep the simpler maddubs forms above + * (faster than a full pyramid at those widths on every host tested). */ +#define PIVCO_PACK_SSE_DN(NAME, D_VAL, C0,C1,C2, C3,C4,C5, C6,C7,C8, C9,C10,C11, C12,C13) \ +static inline int NAME(uint8_t *out, const uint8_t *ranks, int n, uint8_t base) \ +{ \ + const __m128i c0 = _mm_set1_epi16((int16_t)(((1 << (D_VAL)) << 8) | 1)); \ + const __m128i c1 = _mm_set1_epi32((int32_t)(((int32_t)1 << (2*(D_VAL))) << 16) | 1); \ + const __m128i c3m = _mm_set1_epi64x((int64_t)(((int64_t)1 << (4*(D_VAL))) - 1)); \ + const __m128i compact = _mm_setr_epi8(C0,C1,C2, C3,C4,C5, C6,C7,C8, C9,C10,C11, \ + C12,C13, -1,-1); \ + const __m128i vb = _mm_set1_epi8((char)base); \ + int i = 0; \ + for (; i + 16 <= n; i += 16) { \ + __m128i cb = _mm_sub_epi8(_mm_loadu_si128((const __m128i *)(ranks + i)), vb); \ + __m128i x = _mm_maddubs_epi16(c0, cb); \ + x = _mm_madd_epi16(x, c1); \ + __m128i xs = _mm_srli_epi64(x, 32 - 4*(D_VAL)); \ + x = _mm_or_si128(_mm_and_si128(x, c3m), _mm_andnot_si128(c3m, xs)); \ + _mm_storeu_si128((__m128i *)(out + ((i * (D_VAL)) >> 3)), \ + _mm_shuffle_epi8(x, compact)); \ + } \ + return i; \ +} +/* compact patterns: bytes [0..D-1] from qword0, [D..2D-1] from qword1 (pos 8+). */ +PIVCO_PACK_SSE_DN(pack_d3_sse_x86, 3, 0,1,2, 8,9,10, -1,-1,-1, -1,-1,-1, -1,-1) +PIVCO_PACK_SSE_DN(pack_d5_sse_x86, 5, 0,1,2, 3,4,8, 9,10,11, 12,-1,-1, -1,-1) +PIVCO_PACK_SSE_DN(pack_d6_sse_x86, 6, 0,1,2, 3,4,5, 8,9,10, 11,12,13, -1,-1) +PIVCO_PACK_SSE_DN(pack_d7_sse_x86, 7, 0,1,2, 3,4,5, 6,8,9, 10,11,12, 13,14) +#undef PIVCO_PACK_SSE_DN + +/* Dispatcher: native SIMD per-D path (mirrors pack_dN_x86) + scalar tail. */ +static inline void pack_dN_x86(uint8_t *out, const uint8_t *ranks, + int n, int D, uint8_t base) +{ + int total_bytes = (n * D + 7) >> 3; + if (total_bytes > 0) out[total_bytes - 1] = 0; + + int i = 0; + switch (D) { + case 4: i = pack_d4_sse_x86(out, ranks, n, base); break; + case 8: i = pack_d8_sse_x86(out, ranks, n, base); break; +#ifdef PIVCO_HAS_AVX2 + case 2: i = pack_d2_avx2_x86(out, ranks, n, base); break; + case 3: i = pack_d3_avx2_x86(out, ranks, n, base); break; + case 5: i = pack_d5_avx2_x86(out, ranks, n, base); break; + case 6: i = pack_d6_avx2_x86(out, ranks, n, base); break; + case 7: i = pack_d7_avx2_x86(out, ranks, n, base); break; +#else + case 2: i = pack_d2_sse_x86(out, ranks, n, base); break; + case 3: i = pack_d3_sse_x86(out, ranks, n, base); break; + case 5: i = pack_d5_sse_x86(out, ranks, n, base); break; + case 6: i = pack_d6_sse_x86(out, ranks, n, base); break; + case 7: i = pack_d7_sse_x86(out, ranks, n, base); break; +#endif + default: break; + } + + if (i >= n) return; + + /* Scalar tail. */ + int bit_pos = i * D; + int byte_idx = bit_pos >> 3; + int bits_in_buf = bit_pos & 7; + uint64_t buf = bits_in_buf > 0 + ? (uint64_t)out[byte_idx] & ((1u << bits_in_buf) - 1) + : 0; + for (; i < n; i++) { + uint32_t local = (uint32_t)(uint8_t)(ranks[i] - base); /* code in [0,2^D); no mask */ + buf |= (uint64_t)local << bits_in_buf; + bits_in_buf += D; + while (bits_in_buf >= 8) { + out[byte_idx++] = (uint8_t)(buf & 0xff); + buf >>= 8; + bits_in_buf -= 8; + } + } + if (bits_in_buf > 0) out[byte_idx] = (uint8_t)(buf & ((1u << bits_in_buf) - 1)); +} + +/* ---------- Aliases consumed by codec.c ---------- */ + +#define PIVCO_PRIM_ALWAYS_INLINE __attribute__((always_inline)) static inline + +#include "pivco_huffman_hist_scalar.h" + +/* SSE4.1/AVX2 has no histogram win over the shared scalar core (measured); + * alias it explicitly. */ +PIVCO_PRIM_ALWAYS_INLINE void prim_histogram_chunk(const uint8_t *in, size_t n, + uint32_t hist[256], + uint8_t *scratch) +{ histogram_chunk_scalar(in, n, hist, scratch); } + + +/* Widest load a merge kernel issues at a child-buffer cursor (16B loadu at child cursors); + * the cursor can rest AT `size` on the exhausted side, so buffers a + * merge reads need this much trailing slack. Consumed by the decode + * placement logic (scratch_carve / place_tail). */ +#define PIVCO_PRIM_MERGE_OVERREAD 16 + +PIVCO_PRIM_ALWAYS_INLINE void prim_codec_init(void) +{ codec_init_x86(); } + +/* enc_init 2tab no-OR gather: read 16 input symbols as 2x u64 (frees the load + * ports for the dependent table loads) and merge each rank pair as + * (u16)sym_to_rank[s0] + hi[s1], where hi[s] = sym_to_rank[s]<<8 (aux->s2r_hi, + * built once in the table). Disjoint byte lanes -> + is a single add, no shift + * and the hi load folds in as a memory operand -- the shift+or that x86 can't + * fuse is gone. ~1.6x the naive byte loop across the SSE/AVX2 tier, and the + * only variant with no pathological host (4tab regresses on Skylake, bc2 on all + * Intel). x86-only: on AArch64 the shift folds into orr, so NEON keeps its SIMD + * gather. See IDEAS.md ("enc_init 4tab / bc2") for the A/B/C that chose 2tab. */ +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_init(uint8_t *restrict ranks, int n, + const uint8_t *restrict symbols, + const uint8_t *sym_to_rank, + const pivco_enc_init_aux_t *aux) +{ + PIVCO_CHECK(aux && aux->s2r_hi); + const uint16_t *restrict hi = aux->s2r_hi; +#define PIVCO_LO(x) ((uint16_t)sym_to_rank[(uint8_t)(x)]) +#define PIVCO_HI(x) hi[(uint8_t)(x)] + int i = 0; + for (; i + 16 <= n; i += 16) { + uint64_t a, b; + memcpy(&a, symbols + i, 8); + memcpy(&b, symbols + i + 8, 8); + uint16_t h0 = PIVCO_LO(a) + PIVCO_HI(a >> 8); + uint16_t h1 = PIVCO_LO(a >> 16) + PIVCO_HI(a >> 24); + uint16_t h2 = PIVCO_LO(a >> 32) + PIVCO_HI(a >> 40); + uint16_t h3 = PIVCO_LO(a >> 48) + PIVCO_HI(a >> 56); + uint16_t h4 = PIVCO_LO(b) + PIVCO_HI(b >> 8); + uint16_t h5 = PIVCO_LO(b >> 16) + PIVCO_HI(b >> 24); + uint16_t h6 = PIVCO_LO(b >> 32) + PIVCO_HI(b >> 40); + uint16_t h7 = PIVCO_LO(b >> 48) + PIVCO_HI(b >> 56); + memcpy(ranks + i, &h0, 2); memcpy(ranks + i + 2, &h1, 2); + memcpy(ranks + i + 4, &h2, 2); memcpy(ranks + i + 6, &h3, 2); + memcpy(ranks + i + 8, &h4, 2); memcpy(ranks + i + 10, &h5, 2); + memcpy(ranks + i + 12, &h6, 2); memcpy(ranks + i + 14, &h7, 2); + } + for (; i < n; i++) ranks[i] = sym_to_rank[symbols[i]]; +#undef PIVCO_LO +#undef PIVCO_HI +} + +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_full(uint8_t *ranks, + int n, uint8_t thr, + uint8_t *bm, + uint8_t *right_out) +{ return part_full_x86(ranks, n, thr, bm, right_out); } + +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_right(uint8_t *ranks, + int n, uint8_t thr, + uint8_t *bm, + uint8_t *right_out) +{ return part_core_x86(ranks, n, thr, bm, right_out, 1); } + +PIVCO_PRIM_ALWAYS_INLINE int prim_enc_partition_none(uint8_t *ranks, + int n, uint8_t thr, + uint8_t *bm) +{ return part_core_x86(ranks, n, thr, bm, NULL, 0); } + +/* Natural-layout kernels exposed for bench_prim's ST_PACK/ST_MERGE_FLAT + * rows (the prim_ entries below produce the layout `vertical` selects: + * the hybrid vertical wire, or natural when 0). */ +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_pack_dN_natural(const uint8_t *ranks, + int n, int D, uint8_t base, uint8_t *out_packed) +{ pack_dN_x86(out_packed, ranks, n, D, base); } +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_flat_natural(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s) +{ merge_flat_x86(out, n, bm, D, c2s); } +PIVCO_PRIM_ALWAYS_INLINE void prim_enc_pack_dN(const uint8_t *ranks, + int n, int D, uint8_t base, uint8_t *out_packed, + int vertical) +{ + int n5 = vertical == PIVCO_FLAT_VERTICAL ? pivco_vert_n512(n, D) : 0; + if (n5) vert512_pack_x86_best(out_packed, ranks, n5, D, base); + int r = n - n5, nv = vertical ? pivco_vert_n(r, D) : 0; + uint8_t *o1 = out_packed + (((size_t)n5 * D) >> 3); + if (nv) vert_pack_x86_best(o1, ranks + n5, nv, D, base); + if (r > nv) pack_dN_x86(o1 + (((size_t)nv * D) >> 3), + ranks + n5 + nv, r - nv, D, base); +} + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_flat(uint8_t *out, int n, + const uint8_t *bm, int D, + const uint8_t *c2s, + int vertical) +{ + int n5 = vertical == PIVCO_FLAT_VERTICAL ? pivco_vert_n512(n, D) : 0; + if (n5) vert512_merge_x86_best(out, n5, bm, D, c2s); + int r = n - n5, nv = vertical ? pivco_vert_n(r, D) : 0; + const uint8_t *bm1 = bm + (((size_t)n5 * D) >> 3); + if (nv) vert_merge_x86_best(out + n5, nv, bm1, D, c2s); + if (r > nv) merge_flat_x86(out + n5 + nv, r - nv, + bm1 + (((size_t)nv * D) >> 3), D, c2s); +} + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_cst_cst(const uint8_t *bm, int K, + uint8_t left_sym, + uint8_t right_sym, + uint8_t *out) +{ merge_cst_cst_x86(bm, K, left_sym, right_sym, out); } + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_cst_vec(const uint8_t *bm, int K, + uint8_t left_sym, + const uint8_t *right_buf, + uint8_t *out) +{ merge_cst_vec_x86(bm, K, left_sym, right_buf, out); } + +PIVCO_PRIM_ALWAYS_INLINE void prim_merge_vec_vec(const uint8_t *bm, int K, + const uint8_t *left_buf, + const uint8_t *right_buf, + uint8_t *out) +{ merge_vec_vec_x86(bm, K, left_buf, right_buf, out); } + +#endif /* PIVCO_HUFFMAN_PRIMITIVES_X86_H */ diff --git a/third_party/pivco/src/pivco_huffman_vertical.h b/third_party/pivco/src/pivco_huffman_vertical.h new file mode 100644 index 0000000..c649ffb --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_vertical.h @@ -0,0 +1,117 @@ +/* pivco_huffman_vertical.h — hybrid vertical flat-region bit-packing: + * shared layout definition + scalar reference kernels. + * + * WIRE FORMAT (the PIVCO_FLAT_VERTICAL layout, the default; replaces + * the natural row-major layout inside flat regions only — region byte + * size is unchanged, so no other wire structure moves): + * + * A flat region of n D-bit codes with D in {2..7} stores its first + * n512 = pivco_vert_n512(n) codes in vertical blocks of 512 (64 lanes), + * the next n128 = pivco_vert_n(n - n512) codes in vertical blocks of + * 128 (16 lanes), and the remainder in the legacy natural layout. + * D = 8 and n < 128 stay fully natural (D = 8 is the identity memcpy). + * + * The 512 block is 4 interleaved 16-lane sub-blocks: narrow (128-bit) + * engines process quarter q with the 16-lane kernels at column stride + * 64 and output stride 64; 512-bit engines get one uniform-shift row + * per step. + * + * Block layout (16*D bytes, byte-column major): + * code v of a block (v = 0..127) lives in lane L = v & 15 at + * step s = v >> 4. Lane L's sub-stream is D bytes, little-endian, + * holding its 8 codes at bit offsets {0, D, ..., 7D}. Byte j of + * lane L is stored at block_base + 16*j + L — i.e. byte-column j + * is 16 contiguous bytes, one per lane. + * + * Decode property: extracting bit-field s from every lane of a block + * yields codes 16s .. 16s+15 — sixteen consecutive outputs — from + * byte-column loads + uniform shifts, for ANY D (no per-code byte + * misalignment, the FastLanes idea at 128-value granularity). + * + * The layout itself is selected at table build (pivco_cfg_t.flat_layout, + * baked into table->flat_layout; the pivcohuf container records it in + * its FLAGS byte): PIVCO_FLAT_VERTICAL is the hybrid above, and + * PIVCO_FLAT_VERTICAL_128 skips the 512 span (128-value blocks + + * natural tail only — the fastest decode+encode on ARM servers). + * Within a layout, both sides gate with the pure functions below, so + * nothing per-region goes on the wire. + * + * SIMD implementations live in the backend primitive headers (NEON) and + * pivco_huffman_x86_vertical.h (SSE/AVX2/AVX-512); the scalar kernels + * below are the reference + scalar-backend forms. + */ +#ifndef PIVCO_HUFFMAN_VERTICAL_H +#define PIVCO_HUFFMAN_VERTICAL_H + +#include +#include + +/* Hybrid gates. A flat region with D in {2..7} stores, in order: + * [n512 codes in 512-value/64-lane blocks] + * [n128 codes in 128-value/16-lane blocks] (n128 = gate on the remainder) + * [natural tail] + * Both sides evaluate the same pure functions, so no wire flag is needed. */ +static inline int pivco_vert_n512(int n, int D) +{ + return (D >= 2 && D <= 7 && n >= 512) ? (n & ~511) : 0; +} + +/* Length of the 128-value vertical span (applied to the post-512 remainder; + * 0 = fully natural). */ +static inline int pivco_vert_n(int n, int D) +{ + return (D >= 2 && D <= 7 && n >= 128) ? (n & ~127) : 0; +} + +/* Scalar reference: pack n_v ranks vertically at lane count LN (n_v a + * multiple of 8*LN). */ +static inline void vert_pack_scalar_w(uint8_t *out, const uint8_t *ranks, + int n_v, int D, uint8_t base, int LN) +{ + uint32_t mask = (1u << D) - 1u; + int BV = LN * 8; + for (int b = 0; b < n_v / BV; b++) { + uint8_t *blk = out + (size_t)b * LN * D; + memset(blk, 0, (size_t)LN * D); + for (int v = 0; v < BV; v++) { + uint32_t val = (uint32_t)(uint8_t)(ranks[b * BV + v] - base) & mask; + int L = v % LN, bit = (v / LN) * D, j = bit >> 3, off = bit & 7; + blk[LN * j + L] |= (uint8_t)(val << off); + if (off + D > 8) + blk[LN * (j + 1) + L] |= (uint8_t)(val >> (8 - off)); + } + } +} + +/* Scalar reference: fused decode of a vertical span at lane count LN. */ +static inline void vert_merge_scalar_w(uint8_t *out, int n_v, const uint8_t *bm, + int D, const uint8_t *c2s, int LN) +{ + uint32_t mask = (1u << D) - 1u; + int BV = LN * 8; + for (int b = 0; b < n_v / BV; b++) { + const uint8_t *blk = bm + (size_t)b * LN * D; + for (int v = 0; v < BV; v++) { + int L = v % LN, bit = (v / LN) * D, j = bit >> 3, off = bit & 7; + uint32_t w = blk[LN * j + L]; + if (off + D > 8) + w |= (uint32_t)blk[LN * (j + 1) + L] << 8; + out[b * BV + v] = c2s[(w >> off) & mask]; + } + } +} + +static inline void vert_pack_scalar(uint8_t *out, const uint8_t *ranks, + int n_v, int D, uint8_t base) +{ vert_pack_scalar_w(out, ranks, n_v, D, base, 16); } +static inline void vert_merge_scalar(uint8_t *out, int n_v, const uint8_t *bm, + int D, const uint8_t *c2s) +{ vert_merge_scalar_w(out, n_v, bm, D, c2s, 16); } +static inline void vert512_pack_scalar(uint8_t *out, const uint8_t *ranks, + int n_v, int D, uint8_t base) +{ vert_pack_scalar_w(out, ranks, n_v, D, base, 64); } +static inline void vert512_merge_scalar(uint8_t *out, int n_v, const uint8_t *bm, + int D, const uint8_t *c2s) +{ vert_merge_scalar_w(out, n_v, bm, D, c2s, 64); } + +#endif /* PIVCO_HUFFMAN_VERTICAL_H */ diff --git a/third_party/pivco/src/pivco_huffman_wire.h b/third_party/pivco/src/pivco_huffman_wire.h new file mode 100644 index 0000000..393f2c2 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_wire.h @@ -0,0 +1,201 @@ +/* pivco_huffman_wire.h — single source of truth for the per-node wire format. + * + * All backends MUST consume/produce the per-non-flat-internal-node wire + * record through these helpers. Previously each backend hand-rolled + * the read/write of K_right header + FSE marker byte + bitmap bytes, + * which led to silent drift (scalar+NEON added the FSE marker byte in + * 2026-05-13, x86+AVX-512 didn't — broke scalar↔SSE cross-decoding). + * + * Wire format (v0.8): file data in decompression order, larger-K + * child first. The layout is an Euler walk of the tree — each node's + * K_right split header lands at its pre-order position (on the way + * down), its marker+bitmap record at its post-order position (on the + * way up, after its children's regions): + * + * Per-block header (once, at the very start of each encoded block): + * [block_N: uint16 LE, 2 bytes] symbol count N for this + * block; the decoder reads + * it before starting the + * tree walk. Lets the + * codec encode any N up to + * 65535 — no longer pinned + * to PIVCO_BLOCK_SIZE. + * + * Per non-flat internal node: + * [optional K_right: uint16 LE, 2 bytes] if kr_header_needed(); + * at node entry + * [larger-K child region][smaller child region] recursively, same layout; + * larger first (strict >, + * ties left-first), keyed + * on the K_right header — + * no extra bits + * [FSE marker byte: uint8, 1 byte] always + * [bitmap body] marker == 0: raw n-bit + * bitmap, ceil(n/8) bytes + * marker != 0: 2-byte LE + * fse_len + fse_len bytes + * of FSE-compressed bytes + * + * This is exactly the order the BU decoder consumes bytes. It needs + * both child counts up front to size the children's buffers — and + * forward parsing of variable-size regions requires the sizing + * information in prefix position anyway — but it consumes a node's + * bitmap only at merge time, after both children are decoded. So the + * stream is read strictly forward, each byte touched once, and the + * decoder's L1 working set is a moving window. + * + * The larger-K child goes first so the decoder meets each node's + * dominant half while the smaller sibling's buffer is still empty — + * the decoder can overlap the larger child's working scratch with + * that hole, which is what lets the scratch arena stay near N (see + * the decode tree walk in pivco_huffman_codec.c). Both sides key the + * order on the K_right header already on the wire, so it costs no + * bits. + * + * The K_right header occupies one slot per recursion site into a + * non-leaf child (kr_header_needed()): leaf-only nodes (BOTH_LEAVES, + * LEAF_LEFT's leaf side) carry no count, and empty subtrees (n == 0) + * emit nothing at all. + * + * Flat-subtree nodes do NOT use the per-node record — they emit n·D + * packed bits directly (they have no children, so pre- and post-order + * coincide). The bit layout inside the region follows + * table->flat_layout — natural, hybrid vertical, or 128-only vertical + * (see pivco_huffman_vertical.h); the region byte size is the same in + * every layout. See pivco_huffman.h:flat_depth. + * + * Internal header, not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_WIRE_H +#define PIVCO_HUFFMAN_WIRE_H + +#include "pivco_huffman.h" +#include "pivco_huffman_common.h" +#include "pivco_check.h" +#include "pivco_prof.h" +#ifdef PIVCO_HAS_FSE +#include "pivco_fse.h" +#endif + +#include +#include + +#define PIVCO_BLOCK_N_BYTES 2 /* per-block N header: uint16 little-endian */ + +/* ---------- Per-block N header ---------- */ + +/* Encode: write the block's symbol count N as the first 2 bytes of the + * encoded stream. N <= 65535 (the existing PIVCO_BLOCK_SIZE of 8192/4096 + * leaves plenty of headroom; uint16 caps any future variable-block work + * at the same 65535 limit). */ +static inline void wire_write_block_n(uint8_t *out_ptr, int n) +{ + out_ptr[0] = (uint8_t)(n & 0xFF); + out_ptr[1] = (uint8_t)((n >> 8) & 0xFF); +} + +/* Decode: read the block's symbol count N from the first 2 bytes and + * advance *in_ptr. */ +static inline int wire_read_block_n(const uint8_t **in_ptr) +{ + uint16_t v; + memcpy(&v, *in_ptr, 2); + *in_ptr += PIVCO_BLOCK_N_BYTES; + return (int)v; +} + +/* ---------- Encode side: K_right header ---------- + * + * The encoder's partition runs before anything is emitted for the + * node, so the header value is known and written directly at node + * entry. No-op when the node carries no header (kr_header_needed()). */ +static inline void wire_write_kr_header(const pivco_table_t *table, + int16_t node_id, + uint8_t **out_ptr, int n_right) +{ + if (!kr_header_needed(table, node_id)) return; + (*out_ptr)[0] = (uint8_t)(n_right & 0xFF); + (*out_ptr)[1] = (uint8_t)((n_right >> 8) & 0xFF); + *out_ptr += KR_HEADER_BYTES; +} + +/* Note: the FSE marker byte + bitmap (or FSE payload) is emitted by + * the backend's `prim_encode_node` primitive, not by a helper here. + * Backends that attempt FSE-coding of the bitmap need to make that + * decision after building the raw bitmap, which is intrinsically + * backend-specific; threading a wire-helper through that flow would + * be more complexity than win. The wire FORMAT — 1 byte marker + * followed by raw bitmap (marker == 0) or [fse_len:u16][fse_payload] + * (marker != 0) — is still authoritative here in the header doc, and + * `wire_read_bitmap` below is the corresponding decoder. */ + +/* ---------- Decode side ---------- */ + +/* Read the K_right header at node entry. Every decode-side call site + * dispatches on node_type first (LEAF_LEFT / INTERNAL_FULL), where the + * header is present by construction, so kr_header_needed() -- three + * dependent tree loads re-deriving a statically known truth -- is only + * consulted in debug builds. (The encoder's write side still uses it: + * its walk visits header-less nodes too.) */ +static inline int wire_read_kr_header(const pivco_table_t *table, + int16_t node_id, + const uint8_t **in_ptr) +{ + PIVCO_CHECK_DEBUG(kr_header_needed(table, node_id)); + (void)table; (void)node_id; + PROF_TIC(); + uint16_t v; + memcpy(&v, *in_ptr, 2); + *in_ptr += KR_HEADER_BYTES; + PROF_TOC(PROF_WIRE_KR, 1); + return (int)v; +} + +/* Read the per-node bitmap body (marker + payload). Returns a pointer + * to the usable n-bit bitmap (either pointing into the input stream + * for marker==0, or into the caller-provided `scratch` for the FSE + * path). Advances *in_ptr past the whole record. + * + * scratch must hold at least bitmap_bytes(n) bytes and stay live for + * the entire span where the returned pointer is dereferenced. */ +static inline const uint8_t *wire_read_bitmap(const uint8_t **in_ptr, + int n, + uint8_t *scratch) +{ + PROF_TIC(); + int nbytes = bitmap_bytes(n); + uint8_t marker = **in_ptr; + *in_ptr += 1; + if (marker == 0) { + const uint8_t *bm = *in_ptr; + *in_ptr += nbytes; + PROF_TOC(PROF_WIRE_BITMAP_RAW, n); + return bm; + } +#ifdef PIVCO_HAS_FSE + int t_id = marker & 0x7F; + int xor_flag = (marker >> 7) & 1; + uint16_t fse_len; + memcpy(&fse_len, *in_ptr, 2); + *in_ptr += 2; + size_t out_len = 0; + (void)pivco_fse_decompress(t_id, *in_ptr, fse_len, + scratch, (size_t)nbytes, + (size_t)nbytes, &out_len); + *in_ptr += fse_len; + if (xor_flag) pivco_fse_flip_bits(scratch, (size_t)nbytes); + PROF_TOC(PROF_WIRE_BITMAP_FSE, n); + return scratch; +#else + /* FSE not built but stream uses it — best-effort fallback. The + * caller will produce wrong output; the file codec will catch the + * mismatch. We don't fault, just advance and return zeros. */ + (void)scratch; + *in_ptr += nbytes; + PROF_TOC(PROF_WIRE_BITMAP_FSE, n); + return *in_ptr - nbytes; +#endif +} + +#endif /* PIVCO_HUFFMAN_WIRE_H */ diff --git a/third_party/pivco/src/pivco_huffman_x86_flat.h b/third_party/pivco/src/pivco_huffman_x86_flat.h new file mode 100644 index 0000000..136a5f6 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_x86_flat.h @@ -0,0 +1,145 @@ +/* pivco_huffman_x86_flat.h — flat-subtree D-bit code unpacker (SSE4.1). + * + * One unpack helper per D in {2,3,4,5,6}. All use ryg's PSHUFB+PMULLO + * "multiply-as-shift" trick: gather two adjacent bytes per uint16 lane, + * multiply by a per-lane constant `1 << (16 - D - (pos & 7))` so the + * field lands at the MSB of the lane, then PSRLI by (16 - D) and AND + * the mask to LSB-align. Works on SSE4.1 — no AVX2 vpsrlv, no VBMI2 + * vpmultishiftqb needed. + * + * D=4 has a SSE2 3-op specialisation (psrlw + punpcklbw + and) that + * extracts all 16 codes in one shot from 8 bytes. + * + * Replaces the earlier vpsrlvd-based AVX2 paths: ryg's pattern was + * measured uniformly faster on c3 (Ivy Bridge SSE), c4 (Haswell), + * c5 (Cascade Lake), c5a (Zen 2, -43% to -53%), c6a (Zen 3, -25%). + * + * Internal header. Used by the production decoder + * (pivco_huffman_primitives_x86.h::merge_flat_x86_impl) and the + * per-D microbench (bench/bench_micro.c). + * + * Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_X86_FLAT_H +#define PIVCO_HUFFMAN_X86_FLAT_H + +#if !defined(__SSE4_1__) +#error "pivco_huffman_x86_flat.h requires SSE4.1" +#endif + +#include +#include +#include + +/* D=4 SSE2 unpack: 16 codes from 8 bytes via 3 ops. + * raw = 8-byte load (8 bytes, 16 nibbles) + * top_nib = srli_epi16(raw, 4) — top nibble of each byte moves down + * merged = unpacklo_epi8(raw, top_nib) — interleave: [b0_lo, b0_hi, b1_lo, b1_hi, ...] + * final = and(merged, 0xF) + * Returns __m128i with codes in all 16 lanes (low byte = code value). */ +static inline __m128i flat_d4_unpack_x86(const uint8_t *bm_ptr) +{ + __m128i raw = _mm_loadl_epi64((const __m128i *)bm_ptr); + __m128i top_nib = _mm_srli_epi16(raw, 4); + __m128i merged = _mm_unpacklo_epi8(raw, top_nib); + return _mm_and_si128(merged, _mm_set1_epi8(0xF)); +} + +/* D=2 SSE4.1 unpack: 8 codes from 2 bytes via ryg multiply-as-shift. + * Reads up to 16 bytes (loadu); caller must ensure tail safety. + * Returns __m128i with 8 codes in low 8 bytes. */ +static inline __m128i flat_d2_unpack_x86(const uint8_t *bm_ptr) +{ + __m128i raw = _mm_loadu_si128((const __m128i *)bm_ptr); + /* per-lane shuf: bytes (pos>>3, pos>>3 + 1) for pos = 0,2,4,...,14 */ + const __m128i shuf = _mm_setr_epi8( + 0,1, 0,1, 0,1, 0,1, + 1,2, 1,2, 1,2, 1,2); + __m128i gathered = _mm_shuffle_epi8(raw, shuf); + /* mult = 1 << (16 - D - (pos & 7)), pos&7 = 0,2,4,6 */ + const __m128i mult = _mm_setr_epi16( + 1<<14, 1<<12, 1<<10, 1<<8, + 1<<14, 1<<12, 1<<10, 1<<8); + __m128i mh = _mm_mullo_epi16(gathered, mult); + __m128i lsb = _mm_srli_epi16(mh, 14); /* 16 - D = 14 */ + /* PSRLI by 14 already leaves only 2 bits, no AND needed. Pack u16 -> u8. */ + return _mm_packus_epi16(lsb, _mm_setzero_si128()); +} + +/* D=3 SSE4.1 unpack: 8 codes from 3 bytes (+1 slop) via ryg multiply-as-shift. + * Returns __m128i with 8 codes in low 8 bytes. */ +static inline __m128i flat_d3_unpack_x86(const uint8_t *bm_ptr) +{ + __m128i raw = _mm_loadu_si128((const __m128i *)bm_ptr); + /* pos = 0,3,6,9,12,15,18,21. pos>>3 = 0,0,0,1,1,1,2,2 */ + const __m128i shuf = _mm_setr_epi8( + 0,1, 0,1, 0,1, 1,2, 1,2, 1,2, 2,3, 2,3); + __m128i gathered = _mm_shuffle_epi8(raw, shuf); + /* (pos & 7) = 0,3,6,1,4,7,2,5. mult = 1 << (16 - 3 - (pos&7)) */ + const __m128i mult = _mm_setr_epi16( + 1<<13, 1<<10, 1<<7, 1<<12, 1<<9, 1<<6, 1<<11, 1<<8); + __m128i mh = _mm_mullo_epi16(gathered, mult); + __m128i lsb = _mm_srli_epi16(mh, 13); + return _mm_packus_epi16(lsb, _mm_setzero_si128()); +} + +/* D=5 SSE4.1 unpack: 8 codes from 5 bytes via ryg multiply-as-shift. + * 16-byte loadu over-reads up to 11 bytes — caller bounds the loop. */ +static inline __m128i flat_d5_unpack_x86(const uint8_t *bm_ptr) +{ + __m128i raw = _mm_loadu_si128((const __m128i *)bm_ptr); + /* pos = 0,5,10,15,20,25,30,35. pos>>3 = 0,0,1,1,2,3,3,4 */ + const __m128i shuf = _mm_setr_epi8( + 0,1, 0,1, 1,2, 1,2, 2,3, 3,4, 3,4, 4,5); + __m128i gathered = _mm_shuffle_epi8(raw, shuf); + /* (pos & 7) = 0,5,2,7,4,1,6,3. mult = 1 << (16 - 5 - (pos&7)) */ + const __m128i mult = _mm_setr_epi16( + 1<<11, 1<<6, 1<<9, 1<<4, 1<<7, 1<<10, 1<<5, 1<<8); + __m128i mh = _mm_mullo_epi16(gathered, mult); + __m128i lsb = _mm_srli_epi16(mh, 11); + return _mm_packus_epi16(lsb, _mm_setzero_si128()); +} + +/* D=6 SSE4.1 unpack: 8 codes from 6 bytes via ryg multiply-as-shift. + * 16-byte loadu over-reads up to 10 bytes — caller bounds the loop. */ +static inline __m128i flat_d6_unpack_x86(const uint8_t *bm_ptr) +{ + __m128i raw = _mm_loadu_si128((const __m128i *)bm_ptr); + /* pos = 0,6,12,18,24,30,36,42. pos>>3 = 0,0,1,2,3,3,4,5 */ + const __m128i shuf = _mm_setr_epi8( + 0,1, 0,1, 1,2, 2,3, 3,4, 3,4, 4,5, 5,6); + __m128i gathered = _mm_shuffle_epi8(raw, shuf); + /* (pos & 7) = 0,6,4,2,0,6,4,2. mult = 1 << (16 - 6 - (pos&7)) */ + const __m128i mult = _mm_setr_epi16( + 1<<10, 1<<4, 1<<6, 1<<8, 1<<10, 1<<4, 1<<6, 1<<8); + __m128i mh = _mm_mullo_epi16(gathered, mult); + __m128i lsb = _mm_srli_epi16(mh, 10); + return _mm_packus_epi16(lsb, _mm_setzero_si128()); +} + +#if defined(PIVCO_HAS_AVX2) +#include +/* D=2 AVX2 unpack: 16 codes from 4 bytes. Broadcast the 4-byte window to four + * 32-bit lanes and vpsrlvd lane j by 2*j ({0,2,4,6}), so lane j byte b holds + * code (4*b + j) in its low bits (D=2 packs exactly 4 codes per byte, which is + * what makes this clean — no other D aligns this way). A single pshufb + * transposes that 4x4 byte matrix and the 0x3 mask clears the upper 6 bits, + * leaving code i in byte i. ~1.3-1.9x faster than two ryg flat_d2_unpack_x86 + * calls; from terrelln's PR #1. Reads exactly 4 bytes (memcpy, no over-read). */ +static inline __m128i flat_d2_unpack_avx2(const uint8_t *bm_ptr) +{ + const __m128i s = _mm_setr_epi32(0, 2, 4, 6); + const __m128i m = _mm_set1_epi8(0x3); + const __m128i shuf = _mm_setr_epi8( + 0, 4, 8, 12, + 1, 5, 9, 13, + 2, 6, 10, 14, + 3, 7, 11, 15); + uint32_t packed; memcpy(&packed, bm_ptr, 4); + __m128i v = _mm_srlv_epi32(_mm_set1_epi32((int)packed), s); + return _mm_and_si128(_mm_shuffle_epi8(v, shuf), m); +} +#endif /* PIVCO_HAS_AVX2 */ + +#endif /* PIVCO_HUFFMAN_X86_FLAT_H */ diff --git a/third_party/pivco/src/pivco_huffman_x86_tables.c b/third_party/pivco/src/pivco_huffman_x86_tables.c new file mode 100644 index 0000000..b71890a --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_x86_tables.c @@ -0,0 +1,84 @@ +/* pivco_huffman_x86_tables.c — runtime construction of shared x86 tables. + * + * Storage + init functions extracted from the legacy x86 .c files so + * both the legacy code (until step 4 cutover) and the codec.c-compiled- + * as-x86 object library link against one copy. See header for the + * contract. */ + +#include "pivco_huffman_x86_tables.h" + +/* ---------- Encoder partition: compress_tab[256][32] + compress_popcnt[256] + * + * For each 8-bit partition mask: + * - bytes 0..15 → pshufb indices packing the bit=1 (right) uint16 + * lanes to the front of the destination register + * - bytes 16..31 → indices for the complement (bit=0 → left) + * Both halves loaded as two aligned 16-byte _mm_load_si128 from + * contiguous memory. Lanes past the popcount are set to 0x80 (pshufb + * zero-fill sentinel) so they write zeros instead of arbitrary data. + */ +uint8_t compress_tab[256][32] __attribute__((aligned(32))); +uint8_t compress_popcnt[256] __attribute__((aligned(64))); +int compress_table_ready = 0; + +void init_compress_table_x86(void) +{ + if (compress_table_ready) return; + for (int mask = 0; mask < 256; mask++) { + /* Right (bit=1): pack selected to front. */ + int out_r = 0; + for (int i = 0; i < 8; i++) { + if (mask & (1 << i)) { + compress_tab[mask][out_r * 2] = (uint8_t)(i * 2); + compress_tab[mask][out_r * 2 + 1] = (uint8_t)(i * 2 + 1); + out_r++; + } + } + compress_popcnt[mask] = (uint8_t)out_r; + for (int j = out_r * 2; j < 16; j++) + compress_tab[mask][j] = 0x80; + + /* Left (bit=0): pack complement to front. */ + int out_l = 0; + for (int i = 0; i < 8; i++) { + if (!(mask & (1 << i))) { + compress_tab[mask][16 + out_l * 2] = (uint8_t)(i * 2); + compress_tab[mask][16 + out_l * 2 + 1] = (uint8_t)(i * 2 + 1); + out_l++; + } + } + for (int j = out_l * 2; j < 16; j++) + compress_tab[mask][16 + j] = 0x80; + } + compress_table_ready = 1; +} + +/* ---------- BU merge: expand_tab[256][8] + expand_popcnt[256] + * + * expand_tab[m][k] = lane index (0..15) for output position k of an + * 8-element merge controlled by mask byte m. Values 0..7 select from + * L, 8..15 select from R. Used as pshufb indices over + * _mm_unpacklo_epi64(L8_lo, R8_lo) for the 8-byte merge body. + */ +uint8_t expand_tab[256][8] __attribute__((aligned(32))); +uint8_t expand_popcnt[256] __attribute__((aligned(64))); +int expand_table_ready = 0; + +void init_expand_table_x86(void) +{ + if (expand_table_ready) return; + for (int m = 0; m < 256; m++) { + int n_zeros = 0, n_ones = 0; + for (int k = 0; k < 8; k++) { + if (m & (1 << k)) { + expand_tab[m][k] = (uint8_t)(8 + n_ones); + n_ones++; + } else { + expand_tab[m][k] = (uint8_t)n_zeros; + n_zeros++; + } + } + expand_popcnt[m] = (uint8_t)n_ones; + } + expand_table_ready = 1; +} diff --git a/third_party/pivco/src/pivco_huffman_x86_tables.h b/third_party/pivco/src/pivco_huffman_x86_tables.h new file mode 100644 index 0000000..52274f5 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_x86_tables.h @@ -0,0 +1,51 @@ +/* pivco_huffman_x86_tables.h — shared x86 (SSE4.1 / AVX2) lookup tables. + * + * Two table families: + * + * compress_tab + compress_popcnt + init_compress_table_x86 + * Encoder partition shuffle table for the 8-element SSE pshufb- + * based `partition_8_sse` primitive. Bytes 0..15 of compress_tab + * [mask] are the pshufb indices that pack right-going (bit==1) + * uint16 lanes to the front of the destination; bytes 16..31 are + * the complementary indices for the left half. Out-of-range + * slots are set to 0x80 (pshufb zero-fill sentinel). + * + * expand_tab + expand_popcnt + init_expand_table_x86 + * BU merge per-mask-byte shuffle pattern for the 8-element + * pshufb merge over `_mm_unpacklo_epi64(L8, R8)`. expand_tab[m][k] + * gives the lane index (0..15) for output position k controlled + * by mask byte m -- values 0..7 select from L, 8..15 from R. + * + * Both init functions are idempotent. Not thread-safe; callers are + * expected to invoke prim_codec_init from a single thread before + * recursing. + * + * Internal header. Not part of the public API. + */ + +#ifndef PIVCO_HUFFMAN_X86_TABLES_H +#define PIVCO_HUFFMAN_X86_TABLES_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Encoder partition (8 elements per call) shuffle / popcount tables. */ +extern uint8_t compress_tab [256][32]; +extern uint8_t compress_popcnt [256]; +extern int compress_table_ready; +void init_compress_table_x86(void); + +/* BU merge expand / popcount tables. */ +extern uint8_t expand_tab [256][8]; +extern uint8_t expand_popcnt [256]; +extern int expand_table_ready; +void init_expand_table_x86(void); + +#ifdef __cplusplus +} +#endif + +#endif /* PIVCO_HUFFMAN_X86_TABLES_H */ diff --git a/third_party/pivco/src/pivco_huffman_x86_vertical.h b/third_party/pivco/src/pivco_huffman_x86_vertical.h new file mode 100644 index 0000000..0a625a6 --- /dev/null +++ b/third_party/pivco/src/pivco_huffman_x86_vertical.h @@ -0,0 +1,1499 @@ +/* pivco_huffman_x86_vertical.h — vertical-128 flat kernels, SSE4.1. + * + * Shared by the x86 (SSE4.1/AVX2) and AVX-512 backends: the vertical + * block layout is 16-lane-native (see pivco_huffman_vertical.h), so a + * 128-bit kernel is the natural shape; the AVX-512 backend reuses it + * as-is (wider forms — two blocks per ymm/zmm, vpsrlvw — are possible + * later if profiles ask). + * + * SSE has no per-byte shifts; each step's field extraction synthesizes + * them from u16-lane shifts plus byte masks. With DV and the step + * index compile-time constants (per-D macro instantiation + unrolled + * s-loop), the three cases (off == 0 / field within byte / field + * spanning two columns) fold to straight-line code. + * + * Maps are the same 1/2/4/8-table pshufb(+pblendvb select tree) forms + * used elsewhere in the x86 flat path. + */ +#ifndef PIVCO_HUFFMAN_X86_VERTICAL_H +#define PIVCO_HUFFMAN_X86_VERTICAL_H + +#include +#include +#include "pivco_huffman_vertical.h" + +#define PIVCO_VERT_MERGE_X86_BODY(DV, MAPEXPR, BLK, O, CS, OS) \ + do { \ + __m128i cols[8]; \ + for (int j = 0; j < DV; j++) \ + cols[j] = _mm_loadu_si128((const __m128i *)((BLK) + (CS) * j)); \ + for (int s = 0; s < 8; s++) { \ + const int bit = s * DV, j = bit >> 3, off = bit & 7; \ + __m128i w; \ + if (off == 0) { \ + w = cols[j]; \ + } else if (off + DV <= 8) { \ + w = _mm_srli_epi16(cols[j], off); \ + } else { \ + __m128i lo = _mm_and_si128(_mm_srli_epi16(cols[j], off), \ + _mm_set1_epi8((char)(0xFFu >> off))); \ + __m128i hi = _mm_and_si128(_mm_slli_epi16(cols[j + 1], 8 - off),\ + _mm_set1_epi8((char)(0xFFu << (8 - off)))); \ + w = _mm_or_si128(lo, hi); \ + } \ + __m128i codes = _mm_and_si128(w, maskv); \ + _mm_storeu_si128((__m128i *)((O) + (OS) * s), MAPEXPR); \ + } \ + } while (0) +#define PIVCO_VERT_MERGE_X86(DV, SETUP, MAPEXPR) \ +static void vert_merge_x86_d##DV(uint8_t *out, int n_v, const uint8_t *bm, \ + const uint8_t *c2s) \ +{ \ + SETUP \ + const __m128i maskv = _mm_set1_epi8((char)((1u << DV) - 1)); \ + for (int b = 0; b < n_v >> 7; b++) \ + PIVCO_VERT_MERGE_X86_BODY(DV, MAPEXPR, \ + bm + (size_t)b * 16 * DV, \ + out + ((size_t)b << 7), 16, 16); \ +} \ +static void vert512_merge_x86_d##DV(uint8_t *out, int n_v, const uint8_t *bm, \ + const uint8_t *c2s) \ +{ \ + SETUP \ + const __m128i maskv = _mm_set1_epi8((char)((1u << DV) - 1)); \ + for (int b = 0; b < n_v >> 9; b++) \ + for (int qt = 0; qt < 4; qt++) \ + PIVCO_VERT_MERGE_X86_BODY(DV, MAPEXPR, \ + bm + (size_t)b * 64 * DV + 16 * qt, \ + out + ((size_t)b << 9) + 16 * qt, \ + 64, 64); \ +} + +PIVCO_VERT_MERGE_X86(2, + const __m128i t = _mm_loadu_si128((const __m128i *)c2s);, + _mm_shuffle_epi8(t, codes)) +PIVCO_VERT_MERGE_X86(3, + const __m128i t = _mm_loadu_si128((const __m128i *)c2s);, + _mm_shuffle_epi8(t, codes)) +PIVCO_VERT_MERGE_X86(4, + const __m128i t = _mm_loadu_si128((const __m128i *)c2s);, + _mm_shuffle_epi8(t, codes)) +PIVCO_VERT_MERGE_X86(5, + const __m128i t0 = _mm_loadu_si128((const __m128i *)c2s); + const __m128i t1 = _mm_loadu_si128((const __m128i *)(c2s + 16));, + _mm_blendv_epi8(_mm_shuffle_epi8(t0, codes), + _mm_shuffle_epi8(t1, codes), _mm_slli_epi16(codes, 3))) +PIVCO_VERT_MERGE_X86(6, + const __m128i t0 = _mm_loadu_si128((const __m128i *)c2s); + const __m128i t1 = _mm_loadu_si128((const __m128i *)(c2s + 16)); + const __m128i t2 = _mm_loadu_si128((const __m128i *)(c2s + 32)); + const __m128i t3 = _mm_loadu_si128((const __m128i *)(c2s + 48));, + _mm_blendv_epi8( + _mm_blendv_epi8(_mm_shuffle_epi8(t0, codes), + _mm_shuffle_epi8(t1, codes), _mm_slli_epi16(codes, 3)), + _mm_blendv_epi8(_mm_shuffle_epi8(t2, codes), + _mm_shuffle_epi8(t3, codes), _mm_slli_epi16(codes, 3)), + _mm_slli_epi16(codes, 2))) +PIVCO_VERT_MERGE_X86(7, + __m128i t[8]; + for (int q = 0; q < 8; q++) + t[q] = _mm_loadu_si128((const __m128i *)(c2s + 16 * q));, + _mm_blendv_epi8( + _mm_blendv_epi8( + _mm_blendv_epi8(_mm_shuffle_epi8(t[0], codes), + _mm_shuffle_epi8(t[1], codes), _mm_slli_epi16(codes, 3)), + _mm_blendv_epi8(_mm_shuffle_epi8(t[2], codes), + _mm_shuffle_epi8(t[3], codes), _mm_slli_epi16(codes, 3)), + _mm_slli_epi16(codes, 2)), + _mm_blendv_epi8( + _mm_blendv_epi8(_mm_shuffle_epi8(t[4], codes), + _mm_shuffle_epi8(t[5], codes), _mm_slli_epi16(codes, 3)), + _mm_blendv_epi8(_mm_shuffle_epi8(t[6], codes), + _mm_shuffle_epi8(t[7], codes), _mm_slli_epi16(codes, 3)), + _mm_slli_epi16(codes, 2)), + _mm_slli_epi16(codes, 1))) +#undef PIVCO_VERT_MERGE_X86 +#undef PIVCO_VERT_MERGE_X86_BODY + +static inline void vert_merge_x86v(uint8_t *out, int n_v, const uint8_t *bm, + int D, const uint8_t *c2s) +{ + switch (D) { + case 2: vert_merge_x86_d2(out, n_v, bm, c2s); break; + case 3: vert_merge_x86_d3(out, n_v, bm, c2s); break; + case 4: vert_merge_x86_d4(out, n_v, bm, c2s); break; + case 5: vert_merge_x86_d5(out, n_v, bm, c2s); break; + case 6: vert_merge_x86_d6(out, n_v, bm, c2s); break; + default: vert_merge_x86_d7(out, n_v, bm, c2s); break; + } +} + +static inline void vert512_merge_x86v(uint8_t *out, int n_v, const uint8_t *bm, + int D, const uint8_t *c2s) +{ + switch (D) { + case 2: vert512_merge_x86_d2(out, n_v, bm, c2s); break; + case 3: vert512_merge_x86_d3(out, n_v, bm, c2s); break; + case 4: vert512_merge_x86_d4(out, n_v, bm, c2s); break; + case 5: vert512_merge_x86_d5(out, n_v, bm, c2s); break; + case 6: vert512_merge_x86_d6(out, n_v, bm, c2s); break; + default: vert512_merge_x86_d7(out, n_v, bm, c2s); break; + } +} + +/* Encoder mirror: accumulate byte-columns with the same synthesized + * per-byte shifts. Per-D instantiation for constant folding. */ +#define PIVCO_VERT_PACK_X86_BODY(DV, BLK, R, CS, OS) \ + do { \ + __m128i cols[8]; \ + for (int j = 0; j < DV; j++) cols[j] = _mm_setzero_si128(); \ + for (int s = 0; s < 8; s++) { \ + __m128i val = _mm_sub_epi8( \ + _mm_loadu_si128((const __m128i *)((R) + (OS) * s)), basev); \ + const int bit = s * DV, j = bit >> 3, off = bit & 7; \ + if (off == 0) { \ + cols[j] = _mm_or_si128(cols[j], val); \ + } else { \ + cols[j] = _mm_or_si128(cols[j], \ + _mm_and_si128(_mm_slli_epi16(val, off), \ + _mm_set1_epi8((char)(0xFFu << off)))); \ + if (off + DV > 8) \ + cols[j + 1] = _mm_or_si128(cols[j + 1], \ + _mm_and_si128(_mm_srli_epi16(val, 8 - off), \ + _mm_set1_epi8((char)(0xFFu >> (8 - off)))));\ + } \ + } \ + for (int j = 0; j < DV; j++) \ + _mm_storeu_si128((__m128i *)((BLK) + (CS) * j), cols[j]); \ + } while (0) +#define PIVCO_VERT_PACK_X86(DV) \ +static void vert_pack_x86_d##DV(uint8_t *out, const uint8_t *ranks, \ + int n_v, uint8_t base) \ +{ \ + const __m128i basev = _mm_set1_epi8((char)base); \ + for (int b = 0; b < n_v >> 7; b++) \ + PIVCO_VERT_PACK_X86_BODY(DV, out + (size_t)b * 16 * DV, \ + ranks + ((size_t)b << 7), 16, 16); \ +} \ +static void vert512_pack_x86_d##DV(uint8_t *out, const uint8_t *ranks, \ + int n_v, uint8_t base) \ +{ \ + const __m128i basev = _mm_set1_epi8((char)base); \ + for (int b = 0; b < n_v >> 9; b++) \ + for (int qt = 0; qt < 4; qt++) \ + PIVCO_VERT_PACK_X86_BODY(DV, out + (size_t)b * 64 * DV + 16 * qt, \ + ranks + ((size_t)b << 9) + 16 * qt, \ + 64, 64); \ +} +PIVCO_VERT_PACK_X86(2) +PIVCO_VERT_PACK_X86(3) +PIVCO_VERT_PACK_X86(4) +PIVCO_VERT_PACK_X86(5) +PIVCO_VERT_PACK_X86(6) +PIVCO_VERT_PACK_X86(7) +#undef PIVCO_VERT_PACK_X86 +#undef PIVCO_VERT_PACK_X86_BODY + +static inline void vert_pack_x86v(uint8_t *out, const uint8_t *ranks, + int n_v, int D, uint8_t base) +{ + switch (D) { + case 2: vert_pack_x86_d2(out, ranks, n_v, base); break; + case 3: vert_pack_x86_d3(out, ranks, n_v, base); break; + case 4: vert_pack_x86_d4(out, ranks, n_v, base); break; + case 5: vert_pack_x86_d5(out, ranks, n_v, base); break; + case 6: vert_pack_x86_d6(out, ranks, n_v, base); break; + default: vert_pack_x86_d7(out, ranks, n_v, base); break; + } +} + +static inline void vert512_pack_x86v(uint8_t *out, const uint8_t *ranks, + int n_v, int D, uint8_t base) +{ + switch (D) { + case 2: vert512_pack_x86_d2(out, ranks, n_v, base); break; + case 3: vert512_pack_x86_d3(out, ranks, n_v, base); break; + case 4: vert512_pack_x86_d4(out, ranks, n_v, base); break; + case 5: vert512_pack_x86_d5(out, ranks, n_v, base); break; + case 6: vert512_pack_x86_d6(out, ranks, n_v, base); break; + default: vert512_pack_x86_d7(out, ranks, n_v, base); break; + } +} + + +/* Best-available dispatcher for the vertical prefix. All x86 tiers + * run the shared xmm 16-lane cores here: the wide 128-block forms (ymm + * two-block, zmm srlv/multishift) were removed 2026-08 -- they only + * covered the 128..511 mid-band (<= 4% of elements at 32K blocks, ~0.2% + * E2E) at ~12 KB of text per TU. Resurrect from git @ 87da7ed if that + * band ever matters (see IDEAS.md). */ +static inline void vert_merge_x86_best(uint8_t *out, int n_v, const uint8_t *bm, + int D, const uint8_t *c2s) +{ + vert_merge_x86v(out, n_v, bm, D, c2s); +} + + +/* Best-available vertical pack (xmm cores on all tiers, as above). */ +static inline void vert_pack_x86_best(uint8_t *out, const uint8_t *ranks, + int n_v, int D, uint8_t base) +{ + vert_pack_x86v(out, ranks, n_v, D, base); +} + + +/* ==== 512-value / 64-lane blocks (see pivco_huffman_vertical.h) ==== + * With 64 lanes, one step fills a whole zmm row: extraction is a uniform + * immediate shift + byte masks -- no srlv, no multishift, no reorder, no + * vendor split. AVX2 processes 32-lane half-rows the same way; the SSE + * tier walks the four interleaved 16-lane quarters with the group cores + * above. Pack mirrors merge (contiguous rank loads, monotonic per-row + * offsets let the first touch skip the zero-init). */ +#if defined(__AVX512BW__) && defined(__AVX512VBMI__) && defined(__AVX512VL__) + + +static void vert512_merge_zmm_d2(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m512i tab = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i *)c2s)); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 2; + uint8_t *o = out + ((size_t)b << 9); + __m512i r0 = _mm512_loadu_si512((const void *)(blk + 0)); + __m512i r1 = _mm512_loadu_si512((const void *)(blk + 64)); + __m512i v0 = _mm512_and_si512(r0, _mm512_set1_epi8((char)0x03)); + _mm512_storeu_si512((void *)(o + 0), _mm512_permutexvar_epi8(v0, tab)); + __m512i v1 = _mm512_and_si512(_mm512_srli_epi16(r0, 2), _mm512_set1_epi8((char)0x03)); + _mm512_storeu_si512((void *)(o + 64), _mm512_permutexvar_epi8(v1, tab)); + __m512i v2 = _mm512_and_si512(_mm512_srli_epi16(r0, 4), _mm512_set1_epi8((char)0x03)); + _mm512_storeu_si512((void *)(o + 128), _mm512_permutexvar_epi8(v2, tab)); + __m512i v3 = _mm512_and_si512(_mm512_srli_epi16(r0, 6), _mm512_set1_epi8((char)0x03)); + _mm512_storeu_si512((void *)(o + 192), _mm512_permutexvar_epi8(v3, tab)); + __m512i v4 = _mm512_and_si512(r1, _mm512_set1_epi8((char)0x03)); + _mm512_storeu_si512((void *)(o + 256), _mm512_permutexvar_epi8(v4, tab)); + __m512i v5 = _mm512_and_si512(_mm512_srli_epi16(r1, 2), _mm512_set1_epi8((char)0x03)); + _mm512_storeu_si512((void *)(o + 320), _mm512_permutexvar_epi8(v5, tab)); + __m512i v6 = _mm512_and_si512(_mm512_srli_epi16(r1, 4), _mm512_set1_epi8((char)0x03)); + _mm512_storeu_si512((void *)(o + 384), _mm512_permutexvar_epi8(v6, tab)); + __m512i v7 = _mm512_and_si512(_mm512_srli_epi16(r1, 6), _mm512_set1_epi8((char)0x03)); + _mm512_storeu_si512((void *)(o + 448), _mm512_permutexvar_epi8(v7, tab)); + } +} + +static void vert512_merge_zmm_d3(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m512i tab = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i *)c2s)); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 3; + uint8_t *o = out + ((size_t)b << 9); + __m512i r0 = _mm512_loadu_si512((const void *)(blk + 0)); + __m512i r1 = _mm512_loadu_si512((const void *)(blk + 64)); + __m512i r2 = _mm512_loadu_si512((const void *)(blk + 128)); + __m512i v0 = _mm512_and_si512(r0, _mm512_set1_epi8((char)0x07)); + _mm512_storeu_si512((void *)(o + 0), _mm512_permutexvar_epi8(v0, tab)); + __m512i v1 = _mm512_and_si512(_mm512_srli_epi16(r0, 3), _mm512_set1_epi8((char)0x07)); + _mm512_storeu_si512((void *)(o + 64), _mm512_permutexvar_epi8(v1, tab)); + __m512i v2 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r0, 6), _mm512_set1_epi8((char)0x03)), + _mm512_and_si512(_mm512_slli_epi16(r1, 2), _mm512_set1_epi8((char)0xFC))), + _mm512_set1_epi8((char)0x07)); + _mm512_storeu_si512((void *)(o + 128), _mm512_permutexvar_epi8(v2, tab)); + __m512i v3 = _mm512_and_si512(_mm512_srli_epi16(r1, 1), _mm512_set1_epi8((char)0x07)); + _mm512_storeu_si512((void *)(o + 192), _mm512_permutexvar_epi8(v3, tab)); + __m512i v4 = _mm512_and_si512(_mm512_srli_epi16(r1, 4), _mm512_set1_epi8((char)0x07)); + _mm512_storeu_si512((void *)(o + 256), _mm512_permutexvar_epi8(v4, tab)); + __m512i v5 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r1, 7), _mm512_set1_epi8((char)0x01)), + _mm512_and_si512(_mm512_slli_epi16(r2, 1), _mm512_set1_epi8((char)0xFE))), + _mm512_set1_epi8((char)0x07)); + _mm512_storeu_si512((void *)(o + 320), _mm512_permutexvar_epi8(v5, tab)); + __m512i v6 = _mm512_and_si512(_mm512_srli_epi16(r2, 2), _mm512_set1_epi8((char)0x07)); + _mm512_storeu_si512((void *)(o + 384), _mm512_permutexvar_epi8(v6, tab)); + __m512i v7 = _mm512_and_si512(_mm512_srli_epi16(r2, 5), _mm512_set1_epi8((char)0x07)); + _mm512_storeu_si512((void *)(o + 448), _mm512_permutexvar_epi8(v7, tab)); + } +} + +static void vert512_merge_zmm_d4(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m512i tab = _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i *)c2s)); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 4; + uint8_t *o = out + ((size_t)b << 9); + __m512i r0 = _mm512_loadu_si512((const void *)(blk + 0)); + __m512i r1 = _mm512_loadu_si512((const void *)(blk + 64)); + __m512i r2 = _mm512_loadu_si512((const void *)(blk + 128)); + __m512i r3 = _mm512_loadu_si512((const void *)(blk + 192)); + __m512i v0 = _mm512_and_si512(r0, _mm512_set1_epi8((char)0x0F)); + _mm512_storeu_si512((void *)(o + 0), _mm512_permutexvar_epi8(v0, tab)); + __m512i v1 = _mm512_and_si512(_mm512_srli_epi16(r0, 4), _mm512_set1_epi8((char)0x0F)); + _mm512_storeu_si512((void *)(o + 64), _mm512_permutexvar_epi8(v1, tab)); + __m512i v2 = _mm512_and_si512(r1, _mm512_set1_epi8((char)0x0F)); + _mm512_storeu_si512((void *)(o + 128), _mm512_permutexvar_epi8(v2, tab)); + __m512i v3 = _mm512_and_si512(_mm512_srli_epi16(r1, 4), _mm512_set1_epi8((char)0x0F)); + _mm512_storeu_si512((void *)(o + 192), _mm512_permutexvar_epi8(v3, tab)); + __m512i v4 = _mm512_and_si512(r2, _mm512_set1_epi8((char)0x0F)); + _mm512_storeu_si512((void *)(o + 256), _mm512_permutexvar_epi8(v4, tab)); + __m512i v5 = _mm512_and_si512(_mm512_srli_epi16(r2, 4), _mm512_set1_epi8((char)0x0F)); + _mm512_storeu_si512((void *)(o + 320), _mm512_permutexvar_epi8(v5, tab)); + __m512i v6 = _mm512_and_si512(r3, _mm512_set1_epi8((char)0x0F)); + _mm512_storeu_si512((void *)(o + 384), _mm512_permutexvar_epi8(v6, tab)); + __m512i v7 = _mm512_and_si512(_mm512_srli_epi16(r3, 4), _mm512_set1_epi8((char)0x0F)); + _mm512_storeu_si512((void *)(o + 448), _mm512_permutexvar_epi8(v7, tab)); + } +} + +static void vert512_merge_zmm_d5(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m512i tab = _mm512_broadcast_i64x4(_mm256_loadu_si256((const __m256i *)c2s)); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 5; + uint8_t *o = out + ((size_t)b << 9); + __m512i r0 = _mm512_loadu_si512((const void *)(blk + 0)); + __m512i r1 = _mm512_loadu_si512((const void *)(blk + 64)); + __m512i r2 = _mm512_loadu_si512((const void *)(blk + 128)); + __m512i r3 = _mm512_loadu_si512((const void *)(blk + 192)); + __m512i r4 = _mm512_loadu_si512((const void *)(blk + 256)); + __m512i v0 = _mm512_and_si512(r0, _mm512_set1_epi8((char)0x1F)); + _mm512_storeu_si512((void *)(o + 0), _mm512_permutexvar_epi8(v0, tab)); + __m512i v1 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r0, 5), _mm512_set1_epi8((char)0x07)), + _mm512_and_si512(_mm512_slli_epi16(r1, 3), _mm512_set1_epi8((char)0xF8))), + _mm512_set1_epi8((char)0x1F)); + _mm512_storeu_si512((void *)(o + 64), _mm512_permutexvar_epi8(v1, tab)); + __m512i v2 = _mm512_and_si512(_mm512_srli_epi16(r1, 2), _mm512_set1_epi8((char)0x1F)); + _mm512_storeu_si512((void *)(o + 128), _mm512_permutexvar_epi8(v2, tab)); + __m512i v3 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r1, 7), _mm512_set1_epi8((char)0x01)), + _mm512_and_si512(_mm512_slli_epi16(r2, 1), _mm512_set1_epi8((char)0xFE))), + _mm512_set1_epi8((char)0x1F)); + _mm512_storeu_si512((void *)(o + 192), _mm512_permutexvar_epi8(v3, tab)); + __m512i v4 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r2, 4), _mm512_set1_epi8((char)0x0F)), + _mm512_and_si512(_mm512_slli_epi16(r3, 4), _mm512_set1_epi8((char)0xF0))), + _mm512_set1_epi8((char)0x1F)); + _mm512_storeu_si512((void *)(o + 256), _mm512_permutexvar_epi8(v4, tab)); + __m512i v5 = _mm512_and_si512(_mm512_srli_epi16(r3, 1), _mm512_set1_epi8((char)0x1F)); + _mm512_storeu_si512((void *)(o + 320), _mm512_permutexvar_epi8(v5, tab)); + __m512i v6 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r3, 6), _mm512_set1_epi8((char)0x03)), + _mm512_and_si512(_mm512_slli_epi16(r4, 2), _mm512_set1_epi8((char)0xFC))), + _mm512_set1_epi8((char)0x1F)); + _mm512_storeu_si512((void *)(o + 384), _mm512_permutexvar_epi8(v6, tab)); + __m512i v7 = _mm512_and_si512(_mm512_srli_epi16(r4, 3), _mm512_set1_epi8((char)0x1F)); + _mm512_storeu_si512((void *)(o + 448), _mm512_permutexvar_epi8(v7, tab)); + } +} + +static void vert512_merge_zmm_d6(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m512i tab = _mm512_loadu_si512((const void *)c2s); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 6; + uint8_t *o = out + ((size_t)b << 9); + __m512i r0 = _mm512_loadu_si512((const void *)(blk + 0)); + __m512i r1 = _mm512_loadu_si512((const void *)(blk + 64)); + __m512i r2 = _mm512_loadu_si512((const void *)(blk + 128)); + __m512i r3 = _mm512_loadu_si512((const void *)(blk + 192)); + __m512i r4 = _mm512_loadu_si512((const void *)(blk + 256)); + __m512i r5 = _mm512_loadu_si512((const void *)(blk + 320)); + __m512i v0 = _mm512_and_si512(r0, _mm512_set1_epi8((char)0x3F)); + _mm512_storeu_si512((void *)(o + 0), _mm512_permutexvar_epi8(v0, tab)); + __m512i v1 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r0, 6), _mm512_set1_epi8((char)0x03)), + _mm512_and_si512(_mm512_slli_epi16(r1, 2), _mm512_set1_epi8((char)0xFC))), + _mm512_set1_epi8((char)0x3F)); + _mm512_storeu_si512((void *)(o + 64), _mm512_permutexvar_epi8(v1, tab)); + __m512i v2 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r1, 4), _mm512_set1_epi8((char)0x0F)), + _mm512_and_si512(_mm512_slli_epi16(r2, 4), _mm512_set1_epi8((char)0xF0))), + _mm512_set1_epi8((char)0x3F)); + _mm512_storeu_si512((void *)(o + 128), _mm512_permutexvar_epi8(v2, tab)); + __m512i v3 = _mm512_and_si512(_mm512_srli_epi16(r2, 2), _mm512_set1_epi8((char)0x3F)); + _mm512_storeu_si512((void *)(o + 192), _mm512_permutexvar_epi8(v3, tab)); + __m512i v4 = _mm512_and_si512(r3, _mm512_set1_epi8((char)0x3F)); + _mm512_storeu_si512((void *)(o + 256), _mm512_permutexvar_epi8(v4, tab)); + __m512i v5 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r3, 6), _mm512_set1_epi8((char)0x03)), + _mm512_and_si512(_mm512_slli_epi16(r4, 2), _mm512_set1_epi8((char)0xFC))), + _mm512_set1_epi8((char)0x3F)); + _mm512_storeu_si512((void *)(o + 320), _mm512_permutexvar_epi8(v5, tab)); + __m512i v6 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r4, 4), _mm512_set1_epi8((char)0x0F)), + _mm512_and_si512(_mm512_slli_epi16(r5, 4), _mm512_set1_epi8((char)0xF0))), + _mm512_set1_epi8((char)0x3F)); + _mm512_storeu_si512((void *)(o + 384), _mm512_permutexvar_epi8(v6, tab)); + __m512i v7 = _mm512_and_si512(_mm512_srli_epi16(r5, 2), _mm512_set1_epi8((char)0x3F)); + _mm512_storeu_si512((void *)(o + 448), _mm512_permutexvar_epi8(v7, tab)); + } +} + +static void vert512_merge_zmm_d7(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m512i tlo = _mm512_loadu_si512((const void *)c2s); + const __m512i thi = _mm512_loadu_si512((const void *)(c2s + 64)); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 7; + uint8_t *o = out + ((size_t)b << 9); + __m512i r0 = _mm512_loadu_si512((const void *)(blk + 0)); + __m512i r1 = _mm512_loadu_si512((const void *)(blk + 64)); + __m512i r2 = _mm512_loadu_si512((const void *)(blk + 128)); + __m512i r3 = _mm512_loadu_si512((const void *)(blk + 192)); + __m512i r4 = _mm512_loadu_si512((const void *)(blk + 256)); + __m512i r5 = _mm512_loadu_si512((const void *)(blk + 320)); + __m512i r6 = _mm512_loadu_si512((const void *)(blk + 384)); + __m512i v0 = _mm512_and_si512(r0, _mm512_set1_epi8((char)0x7F)); + _mm512_storeu_si512((void *)(o + 0), _mm512_permutex2var_epi8(tlo, v0, thi)); + __m512i v1 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r0, 7), _mm512_set1_epi8((char)0x01)), + _mm512_and_si512(_mm512_slli_epi16(r1, 1), _mm512_set1_epi8((char)0xFE))), + _mm512_set1_epi8((char)0x7F)); + _mm512_storeu_si512((void *)(o + 64), _mm512_permutex2var_epi8(tlo, v1, thi)); + __m512i v2 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r1, 6), _mm512_set1_epi8((char)0x03)), + _mm512_and_si512(_mm512_slli_epi16(r2, 2), _mm512_set1_epi8((char)0xFC))), + _mm512_set1_epi8((char)0x7F)); + _mm512_storeu_si512((void *)(o + 128), _mm512_permutex2var_epi8(tlo, v2, thi)); + __m512i v3 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r2, 5), _mm512_set1_epi8((char)0x07)), + _mm512_and_si512(_mm512_slli_epi16(r3, 3), _mm512_set1_epi8((char)0xF8))), + _mm512_set1_epi8((char)0x7F)); + _mm512_storeu_si512((void *)(o + 192), _mm512_permutex2var_epi8(tlo, v3, thi)); + __m512i v4 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r3, 4), _mm512_set1_epi8((char)0x0F)), + _mm512_and_si512(_mm512_slli_epi16(r4, 4), _mm512_set1_epi8((char)0xF0))), + _mm512_set1_epi8((char)0x7F)); + _mm512_storeu_si512((void *)(o + 256), _mm512_permutex2var_epi8(tlo, v4, thi)); + __m512i v5 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r4, 3), _mm512_set1_epi8((char)0x1F)), + _mm512_and_si512(_mm512_slli_epi16(r5, 5), _mm512_set1_epi8((char)0xE0))), + _mm512_set1_epi8((char)0x7F)); + _mm512_storeu_si512((void *)(o + 320), _mm512_permutex2var_epi8(tlo, v5, thi)); + __m512i v6 = _mm512_and_si512(_mm512_or_si512( + _mm512_and_si512(_mm512_srli_epi16(r5, 2), _mm512_set1_epi8((char)0x3F)), + _mm512_and_si512(_mm512_slli_epi16(r6, 6), _mm512_set1_epi8((char)0xC0))), + _mm512_set1_epi8((char)0x7F)); + _mm512_storeu_si512((void *)(o + 384), _mm512_permutex2var_epi8(tlo, v6, thi)); + __m512i v7 = _mm512_and_si512(_mm512_srli_epi16(r6, 1), _mm512_set1_epi8((char)0x7F)); + _mm512_storeu_si512((void *)(o + 448), _mm512_permutex2var_epi8(tlo, v7, thi)); + } +} + +static inline void vert512_merge_zmm(uint8_t *out, int n_v, const uint8_t *bm, int D, const uint8_t *c2s) +{ + switch (D) { + case 2: vert512_merge_zmm_d2(out, n_v, bm, c2s); break; + case 3: vert512_merge_zmm_d3(out, n_v, bm, c2s); break; + case 4: vert512_merge_zmm_d4(out, n_v, bm, c2s); break; + case 5: vert512_merge_zmm_d5(out, n_v, bm, c2s); break; + case 6: vert512_merge_zmm_d6(out, n_v, bm, c2s); break; + default: vert512_merge_zmm_d7(out, n_v, bm, c2s); break; + } +} + +static void vert512_pack_zmm_d2(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m512i basev = _mm512_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 2; + const uint8_t *r = ranks + ((size_t)b << 9); + __m512i c0, c1; + __m512i v0 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 0)), basev); + c0 = v0; + __m512i v1 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 64)), basev); + c0 = _mm512_or_si512(c0, _mm512_and_si512(_mm512_slli_epi16(v1, 2), _mm512_set1_epi8((char)0xFC))); + __m512i v2 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 128)), basev); + c0 = _mm512_or_si512(c0, _mm512_and_si512(_mm512_slli_epi16(v2, 4), _mm512_set1_epi8((char)0xF0))); + __m512i v3 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 192)), basev); + c0 = _mm512_or_si512(c0, _mm512_and_si512(_mm512_slli_epi16(v3, 6), _mm512_set1_epi8((char)0xC0))); + __m512i v4 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 256)), basev); + c1 = v4; + __m512i v5 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 320)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v5, 2), _mm512_set1_epi8((char)0xFC))); + __m512i v6 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 384)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v6, 4), _mm512_set1_epi8((char)0xF0))); + __m512i v7 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 448)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v7, 6), _mm512_set1_epi8((char)0xC0))); + _mm512_storeu_si512((void *)(blk + 0), c0); + _mm512_storeu_si512((void *)(blk + 64), c1); + } +} + +static void vert512_pack_zmm_d3(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m512i basev = _mm512_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 3; + const uint8_t *r = ranks + ((size_t)b << 9); + __m512i c0, c1, c2; + __m512i v0 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 0)), basev); + c0 = v0; + __m512i v1 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 64)), basev); + c0 = _mm512_or_si512(c0, _mm512_and_si512(_mm512_slli_epi16(v1, 3), _mm512_set1_epi8((char)0xF8))); + __m512i v2 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 128)), basev); + c0 = _mm512_or_si512(c0, _mm512_and_si512(_mm512_slli_epi16(v2, 6), _mm512_set1_epi8((char)0xC0))); + c1 = _mm512_and_si512(_mm512_srli_epi16(v2, 2), _mm512_set1_epi8((char)0x3F)); + __m512i v3 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 192)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v3, 1), _mm512_set1_epi8((char)0xFE))); + __m512i v4 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 256)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v4, 4), _mm512_set1_epi8((char)0xF0))); + __m512i v5 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 320)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v5, 7), _mm512_set1_epi8((char)0x80))); + c2 = _mm512_and_si512(_mm512_srli_epi16(v5, 1), _mm512_set1_epi8((char)0x7F)); + __m512i v6 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 384)), basev); + c2 = _mm512_or_si512(c2, _mm512_and_si512(_mm512_slli_epi16(v6, 2), _mm512_set1_epi8((char)0xFC))); + __m512i v7 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 448)), basev); + c2 = _mm512_or_si512(c2, _mm512_and_si512(_mm512_slli_epi16(v7, 5), _mm512_set1_epi8((char)0xE0))); + _mm512_storeu_si512((void *)(blk + 0), c0); + _mm512_storeu_si512((void *)(blk + 64), c1); + _mm512_storeu_si512((void *)(blk + 128), c2); + } +} + +static void vert512_pack_zmm_d4(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m512i basev = _mm512_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 4; + const uint8_t *r = ranks + ((size_t)b << 9); + __m512i c0, c1, c2, c3; + __m512i v0 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 0)), basev); + c0 = v0; + __m512i v1 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 64)), basev); + c0 = _mm512_or_si512(c0, _mm512_and_si512(_mm512_slli_epi16(v1, 4), _mm512_set1_epi8((char)0xF0))); + __m512i v2 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 128)), basev); + c1 = v2; + __m512i v3 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 192)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v3, 4), _mm512_set1_epi8((char)0xF0))); + __m512i v4 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 256)), basev); + c2 = v4; + __m512i v5 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 320)), basev); + c2 = _mm512_or_si512(c2, _mm512_and_si512(_mm512_slli_epi16(v5, 4), _mm512_set1_epi8((char)0xF0))); + __m512i v6 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 384)), basev); + c3 = v6; + __m512i v7 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 448)), basev); + c3 = _mm512_or_si512(c3, _mm512_and_si512(_mm512_slli_epi16(v7, 4), _mm512_set1_epi8((char)0xF0))); + _mm512_storeu_si512((void *)(blk + 0), c0); + _mm512_storeu_si512((void *)(blk + 64), c1); + _mm512_storeu_si512((void *)(blk + 128), c2); + _mm512_storeu_si512((void *)(blk + 192), c3); + } +} + +static void vert512_pack_zmm_d5(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m512i basev = _mm512_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 5; + const uint8_t *r = ranks + ((size_t)b << 9); + __m512i c0, c1, c2, c3, c4; + __m512i v0 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 0)), basev); + c0 = v0; + __m512i v1 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 64)), basev); + c0 = _mm512_or_si512(c0, _mm512_and_si512(_mm512_slli_epi16(v1, 5), _mm512_set1_epi8((char)0xE0))); + c1 = _mm512_and_si512(_mm512_srli_epi16(v1, 3), _mm512_set1_epi8((char)0x1F)); + __m512i v2 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 128)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v2, 2), _mm512_set1_epi8((char)0xFC))); + __m512i v3 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 192)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v3, 7), _mm512_set1_epi8((char)0x80))); + c2 = _mm512_and_si512(_mm512_srli_epi16(v3, 1), _mm512_set1_epi8((char)0x7F)); + __m512i v4 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 256)), basev); + c2 = _mm512_or_si512(c2, _mm512_and_si512(_mm512_slli_epi16(v4, 4), _mm512_set1_epi8((char)0xF0))); + c3 = _mm512_and_si512(_mm512_srli_epi16(v4, 4), _mm512_set1_epi8((char)0x0F)); + __m512i v5 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 320)), basev); + c3 = _mm512_or_si512(c3, _mm512_and_si512(_mm512_slli_epi16(v5, 1), _mm512_set1_epi8((char)0xFE))); + __m512i v6 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 384)), basev); + c3 = _mm512_or_si512(c3, _mm512_and_si512(_mm512_slli_epi16(v6, 6), _mm512_set1_epi8((char)0xC0))); + c4 = _mm512_and_si512(_mm512_srli_epi16(v6, 2), _mm512_set1_epi8((char)0x3F)); + __m512i v7 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 448)), basev); + c4 = _mm512_or_si512(c4, _mm512_and_si512(_mm512_slli_epi16(v7, 3), _mm512_set1_epi8((char)0xF8))); + _mm512_storeu_si512((void *)(blk + 0), c0); + _mm512_storeu_si512((void *)(blk + 64), c1); + _mm512_storeu_si512((void *)(blk + 128), c2); + _mm512_storeu_si512((void *)(blk + 192), c3); + _mm512_storeu_si512((void *)(blk + 256), c4); + } +} + +static void vert512_pack_zmm_d6(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m512i basev = _mm512_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 6; + const uint8_t *r = ranks + ((size_t)b << 9); + __m512i c0, c1, c2, c3, c4, c5; + __m512i v0 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 0)), basev); + c0 = v0; + __m512i v1 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 64)), basev); + c0 = _mm512_or_si512(c0, _mm512_and_si512(_mm512_slli_epi16(v1, 6), _mm512_set1_epi8((char)0xC0))); + c1 = _mm512_and_si512(_mm512_srli_epi16(v1, 2), _mm512_set1_epi8((char)0x3F)); + __m512i v2 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 128)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v2, 4), _mm512_set1_epi8((char)0xF0))); + c2 = _mm512_and_si512(_mm512_srli_epi16(v2, 4), _mm512_set1_epi8((char)0x0F)); + __m512i v3 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 192)), basev); + c2 = _mm512_or_si512(c2, _mm512_and_si512(_mm512_slli_epi16(v3, 2), _mm512_set1_epi8((char)0xFC))); + __m512i v4 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 256)), basev); + c3 = v4; + __m512i v5 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 320)), basev); + c3 = _mm512_or_si512(c3, _mm512_and_si512(_mm512_slli_epi16(v5, 6), _mm512_set1_epi8((char)0xC0))); + c4 = _mm512_and_si512(_mm512_srli_epi16(v5, 2), _mm512_set1_epi8((char)0x3F)); + __m512i v6 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 384)), basev); + c4 = _mm512_or_si512(c4, _mm512_and_si512(_mm512_slli_epi16(v6, 4), _mm512_set1_epi8((char)0xF0))); + c5 = _mm512_and_si512(_mm512_srli_epi16(v6, 4), _mm512_set1_epi8((char)0x0F)); + __m512i v7 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 448)), basev); + c5 = _mm512_or_si512(c5, _mm512_and_si512(_mm512_slli_epi16(v7, 2), _mm512_set1_epi8((char)0xFC))); + _mm512_storeu_si512((void *)(blk + 0), c0); + _mm512_storeu_si512((void *)(blk + 64), c1); + _mm512_storeu_si512((void *)(blk + 128), c2); + _mm512_storeu_si512((void *)(blk + 192), c3); + _mm512_storeu_si512((void *)(blk + 256), c4); + _mm512_storeu_si512((void *)(blk + 320), c5); + } +} + +static void vert512_pack_zmm_d7(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m512i basev = _mm512_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 7; + const uint8_t *r = ranks + ((size_t)b << 9); + __m512i c0, c1, c2, c3, c4, c5, c6; + __m512i v0 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 0)), basev); + c0 = v0; + __m512i v1 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 64)), basev); + c0 = _mm512_or_si512(c0, _mm512_and_si512(_mm512_slli_epi16(v1, 7), _mm512_set1_epi8((char)0x80))); + c1 = _mm512_and_si512(_mm512_srli_epi16(v1, 1), _mm512_set1_epi8((char)0x7F)); + __m512i v2 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 128)), basev); + c1 = _mm512_or_si512(c1, _mm512_and_si512(_mm512_slli_epi16(v2, 6), _mm512_set1_epi8((char)0xC0))); + c2 = _mm512_and_si512(_mm512_srli_epi16(v2, 2), _mm512_set1_epi8((char)0x3F)); + __m512i v3 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 192)), basev); + c2 = _mm512_or_si512(c2, _mm512_and_si512(_mm512_slli_epi16(v3, 5), _mm512_set1_epi8((char)0xE0))); + c3 = _mm512_and_si512(_mm512_srli_epi16(v3, 3), _mm512_set1_epi8((char)0x1F)); + __m512i v4 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 256)), basev); + c3 = _mm512_or_si512(c3, _mm512_and_si512(_mm512_slli_epi16(v4, 4), _mm512_set1_epi8((char)0xF0))); + c4 = _mm512_and_si512(_mm512_srli_epi16(v4, 4), _mm512_set1_epi8((char)0x0F)); + __m512i v5 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 320)), basev); + c4 = _mm512_or_si512(c4, _mm512_and_si512(_mm512_slli_epi16(v5, 3), _mm512_set1_epi8((char)0xF8))); + c5 = _mm512_and_si512(_mm512_srli_epi16(v5, 5), _mm512_set1_epi8((char)0x07)); + __m512i v6 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 384)), basev); + c5 = _mm512_or_si512(c5, _mm512_and_si512(_mm512_slli_epi16(v6, 2), _mm512_set1_epi8((char)0xFC))); + c6 = _mm512_and_si512(_mm512_srli_epi16(v6, 6), _mm512_set1_epi8((char)0x03)); + __m512i v7 = _mm512_sub_epi8(_mm512_loadu_si512((const void *)(r + 448)), basev); + c6 = _mm512_or_si512(c6, _mm512_and_si512(_mm512_slli_epi16(v7, 1), _mm512_set1_epi8((char)0xFE))); + _mm512_storeu_si512((void *)(blk + 0), c0); + _mm512_storeu_si512((void *)(blk + 64), c1); + _mm512_storeu_si512((void *)(blk + 128), c2); + _mm512_storeu_si512((void *)(blk + 192), c3); + _mm512_storeu_si512((void *)(blk + 256), c4); + _mm512_storeu_si512((void *)(blk + 320), c5); + _mm512_storeu_si512((void *)(blk + 384), c6); + } +} + +static inline void vert512_pack_zmm(uint8_t *out, const uint8_t *ranks, int n_v, int D, uint8_t base) +{ + switch (D) { + case 2: vert512_pack_zmm_d2(out, ranks, n_v, base); break; + case 3: vert512_pack_zmm_d3(out, ranks, n_v, base); break; + case 4: vert512_pack_zmm_d4(out, ranks, n_v, base); break; + case 5: vert512_pack_zmm_d5(out, ranks, n_v, base); break; + case 6: vert512_pack_zmm_d6(out, ranks, n_v, base); break; + default: vert512_pack_zmm_d7(out, ranks, n_v, base); break; + } +} + +#endif /* AVX512BW + VBMI + VL */ + + +#if defined(__AVX2__) + +static void vert512_merge_ymm_d2(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m256i t0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)c2s)); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 2; + uint8_t *o = out + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + const uint8_t *bh = blk + 32 * h; + uint8_t *oh = o + 32 * h; + __m256i r0 = _mm256_loadu_si256((const __m256i *)(bh + 0)); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(bh + 64)); + __m256i v0 = _mm256_and_si256(r0, _mm256_set1_epi8((char)0x03)); + _mm256_storeu_si256((__m256i *)(oh + 0), + _mm256_shuffle_epi8(t0, v0)); + __m256i v1 = _mm256_and_si256(_mm256_srli_epi16(r0, 2), _mm256_set1_epi8((char)0x03)); + _mm256_storeu_si256((__m256i *)(oh + 64), + _mm256_shuffle_epi8(t0, v1)); + __m256i v2 = _mm256_and_si256(_mm256_srli_epi16(r0, 4), _mm256_set1_epi8((char)0x03)); + _mm256_storeu_si256((__m256i *)(oh + 128), + _mm256_shuffle_epi8(t0, v2)); + __m256i v3 = _mm256_and_si256(_mm256_srli_epi16(r0, 6), _mm256_set1_epi8((char)0x03)); + _mm256_storeu_si256((__m256i *)(oh + 192), + _mm256_shuffle_epi8(t0, v3)); + __m256i v4 = _mm256_and_si256(r1, _mm256_set1_epi8((char)0x03)); + _mm256_storeu_si256((__m256i *)(oh + 256), + _mm256_shuffle_epi8(t0, v4)); + __m256i v5 = _mm256_and_si256(_mm256_srli_epi16(r1, 2), _mm256_set1_epi8((char)0x03)); + _mm256_storeu_si256((__m256i *)(oh + 320), + _mm256_shuffle_epi8(t0, v5)); + __m256i v6 = _mm256_and_si256(_mm256_srli_epi16(r1, 4), _mm256_set1_epi8((char)0x03)); + _mm256_storeu_si256((__m256i *)(oh + 384), + _mm256_shuffle_epi8(t0, v6)); + __m256i v7 = _mm256_and_si256(_mm256_srli_epi16(r1, 6), _mm256_set1_epi8((char)0x03)); + _mm256_storeu_si256((__m256i *)(oh + 448), + _mm256_shuffle_epi8(t0, v7)); + } + } +} + +static void vert512_merge_ymm_d3(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m256i t0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)c2s)); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 3; + uint8_t *o = out + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + const uint8_t *bh = blk + 32 * h; + uint8_t *oh = o + 32 * h; + __m256i r0 = _mm256_loadu_si256((const __m256i *)(bh + 0)); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(bh + 64)); + __m256i r2 = _mm256_loadu_si256((const __m256i *)(bh + 128)); + __m256i v0 = _mm256_and_si256(r0, _mm256_set1_epi8((char)0x07)); + _mm256_storeu_si256((__m256i *)(oh + 0), + _mm256_shuffle_epi8(t0, v0)); + __m256i v1 = _mm256_and_si256(_mm256_srli_epi16(r0, 3), _mm256_set1_epi8((char)0x07)); + _mm256_storeu_si256((__m256i *)(oh + 64), + _mm256_shuffle_epi8(t0, v1)); + __m256i v2 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r0, 6), _mm256_set1_epi8((char)0x03)), + _mm256_and_si256(_mm256_slli_epi16(r1, 2), _mm256_set1_epi8((char)0xFC))), + _mm256_set1_epi8((char)0x07)); + _mm256_storeu_si256((__m256i *)(oh + 128), + _mm256_shuffle_epi8(t0, v2)); + __m256i v3 = _mm256_and_si256(_mm256_srli_epi16(r1, 1), _mm256_set1_epi8((char)0x07)); + _mm256_storeu_si256((__m256i *)(oh + 192), + _mm256_shuffle_epi8(t0, v3)); + __m256i v4 = _mm256_and_si256(_mm256_srli_epi16(r1, 4), _mm256_set1_epi8((char)0x07)); + _mm256_storeu_si256((__m256i *)(oh + 256), + _mm256_shuffle_epi8(t0, v4)); + __m256i v5 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r1, 7), _mm256_set1_epi8((char)0x01)), + _mm256_and_si256(_mm256_slli_epi16(r2, 1), _mm256_set1_epi8((char)0xFE))), + _mm256_set1_epi8((char)0x07)); + _mm256_storeu_si256((__m256i *)(oh + 320), + _mm256_shuffle_epi8(t0, v5)); + __m256i v6 = _mm256_and_si256(_mm256_srli_epi16(r2, 2), _mm256_set1_epi8((char)0x07)); + _mm256_storeu_si256((__m256i *)(oh + 384), + _mm256_shuffle_epi8(t0, v6)); + __m256i v7 = _mm256_and_si256(_mm256_srli_epi16(r2, 5), _mm256_set1_epi8((char)0x07)); + _mm256_storeu_si256((__m256i *)(oh + 448), + _mm256_shuffle_epi8(t0, v7)); + } + } +} + +static void vert512_merge_ymm_d4(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m256i t0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)c2s)); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 4; + uint8_t *o = out + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + const uint8_t *bh = blk + 32 * h; + uint8_t *oh = o + 32 * h; + __m256i r0 = _mm256_loadu_si256((const __m256i *)(bh + 0)); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(bh + 64)); + __m256i r2 = _mm256_loadu_si256((const __m256i *)(bh + 128)); + __m256i r3 = _mm256_loadu_si256((const __m256i *)(bh + 192)); + __m256i v0 = _mm256_and_si256(r0, _mm256_set1_epi8((char)0x0F)); + _mm256_storeu_si256((__m256i *)(oh + 0), + _mm256_shuffle_epi8(t0, v0)); + __m256i v1 = _mm256_and_si256(_mm256_srli_epi16(r0, 4), _mm256_set1_epi8((char)0x0F)); + _mm256_storeu_si256((__m256i *)(oh + 64), + _mm256_shuffle_epi8(t0, v1)); + __m256i v2 = _mm256_and_si256(r1, _mm256_set1_epi8((char)0x0F)); + _mm256_storeu_si256((__m256i *)(oh + 128), + _mm256_shuffle_epi8(t0, v2)); + __m256i v3 = _mm256_and_si256(_mm256_srli_epi16(r1, 4), _mm256_set1_epi8((char)0x0F)); + _mm256_storeu_si256((__m256i *)(oh + 192), + _mm256_shuffle_epi8(t0, v3)); + __m256i v4 = _mm256_and_si256(r2, _mm256_set1_epi8((char)0x0F)); + _mm256_storeu_si256((__m256i *)(oh + 256), + _mm256_shuffle_epi8(t0, v4)); + __m256i v5 = _mm256_and_si256(_mm256_srli_epi16(r2, 4), _mm256_set1_epi8((char)0x0F)); + _mm256_storeu_si256((__m256i *)(oh + 320), + _mm256_shuffle_epi8(t0, v5)); + __m256i v6 = _mm256_and_si256(r3, _mm256_set1_epi8((char)0x0F)); + _mm256_storeu_si256((__m256i *)(oh + 384), + _mm256_shuffle_epi8(t0, v6)); + __m256i v7 = _mm256_and_si256(_mm256_srli_epi16(r3, 4), _mm256_set1_epi8((char)0x0F)); + _mm256_storeu_si256((__m256i *)(oh + 448), + _mm256_shuffle_epi8(t0, v7)); + } + } +} + +static void vert512_merge_ymm_d5(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m256i t0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 0))); + const __m256i t1 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 16))); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 5; + uint8_t *o = out + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + const uint8_t *bh = blk + 32 * h; + uint8_t *oh = o + 32 * h; + __m256i r0 = _mm256_loadu_si256((const __m256i *)(bh + 0)); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(bh + 64)); + __m256i r2 = _mm256_loadu_si256((const __m256i *)(bh + 128)); + __m256i r3 = _mm256_loadu_si256((const __m256i *)(bh + 192)); + __m256i r4 = _mm256_loadu_si256((const __m256i *)(bh + 256)); + __m256i v0 = _mm256_and_si256(r0, _mm256_set1_epi8((char)0x1F)); + _mm256_storeu_si256((__m256i *)(oh + 0), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v0), + _mm256_shuffle_epi8(t1, v0), _mm256_slli_epi16(v0, 3))); + __m256i v1 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r0, 5), _mm256_set1_epi8((char)0x07)), + _mm256_and_si256(_mm256_slli_epi16(r1, 3), _mm256_set1_epi8((char)0xF8))), + _mm256_set1_epi8((char)0x1F)); + _mm256_storeu_si256((__m256i *)(oh + 64), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v1), + _mm256_shuffle_epi8(t1, v1), _mm256_slli_epi16(v1, 3))); + __m256i v2 = _mm256_and_si256(_mm256_srli_epi16(r1, 2), _mm256_set1_epi8((char)0x1F)); + _mm256_storeu_si256((__m256i *)(oh + 128), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v2), + _mm256_shuffle_epi8(t1, v2), _mm256_slli_epi16(v2, 3))); + __m256i v3 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r1, 7), _mm256_set1_epi8((char)0x01)), + _mm256_and_si256(_mm256_slli_epi16(r2, 1), _mm256_set1_epi8((char)0xFE))), + _mm256_set1_epi8((char)0x1F)); + _mm256_storeu_si256((__m256i *)(oh + 192), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v3), + _mm256_shuffle_epi8(t1, v3), _mm256_slli_epi16(v3, 3))); + __m256i v4 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r2, 4), _mm256_set1_epi8((char)0x0F)), + _mm256_and_si256(_mm256_slli_epi16(r3, 4), _mm256_set1_epi8((char)0xF0))), + _mm256_set1_epi8((char)0x1F)); + _mm256_storeu_si256((__m256i *)(oh + 256), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v4), + _mm256_shuffle_epi8(t1, v4), _mm256_slli_epi16(v4, 3))); + __m256i v5 = _mm256_and_si256(_mm256_srli_epi16(r3, 1), _mm256_set1_epi8((char)0x1F)); + _mm256_storeu_si256((__m256i *)(oh + 320), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v5), + _mm256_shuffle_epi8(t1, v5), _mm256_slli_epi16(v5, 3))); + __m256i v6 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r3, 6), _mm256_set1_epi8((char)0x03)), + _mm256_and_si256(_mm256_slli_epi16(r4, 2), _mm256_set1_epi8((char)0xFC))), + _mm256_set1_epi8((char)0x1F)); + _mm256_storeu_si256((__m256i *)(oh + 384), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v6), + _mm256_shuffle_epi8(t1, v6), _mm256_slli_epi16(v6, 3))); + __m256i v7 = _mm256_and_si256(_mm256_srli_epi16(r4, 3), _mm256_set1_epi8((char)0x1F)); + _mm256_storeu_si256((__m256i *)(oh + 448), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v7), + _mm256_shuffle_epi8(t1, v7), _mm256_slli_epi16(v7, 3))); + } + } +} + +static void vert512_merge_ymm_d6(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m256i t0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 0))); + const __m256i t1 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 16))); + const __m256i t2 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 32))); + const __m256i t3 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 48))); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 6; + uint8_t *o = out + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + const uint8_t *bh = blk + 32 * h; + uint8_t *oh = o + 32 * h; + __m256i r0 = _mm256_loadu_si256((const __m256i *)(bh + 0)); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(bh + 64)); + __m256i r2 = _mm256_loadu_si256((const __m256i *)(bh + 128)); + __m256i r3 = _mm256_loadu_si256((const __m256i *)(bh + 192)); + __m256i r4 = _mm256_loadu_si256((const __m256i *)(bh + 256)); + __m256i r5 = _mm256_loadu_si256((const __m256i *)(bh + 320)); + __m256i v0 = _mm256_and_si256(r0, _mm256_set1_epi8((char)0x3F)); + _mm256_storeu_si256((__m256i *)(oh + 0), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v0), + _mm256_shuffle_epi8(t1, v0), _mm256_slli_epi16(v0, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v0), + _mm256_shuffle_epi8(t3, v0), _mm256_slli_epi16(v0, 3)), _mm256_slli_epi16(v0, 2))); + __m256i v1 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r0, 6), _mm256_set1_epi8((char)0x03)), + _mm256_and_si256(_mm256_slli_epi16(r1, 2), _mm256_set1_epi8((char)0xFC))), + _mm256_set1_epi8((char)0x3F)); + _mm256_storeu_si256((__m256i *)(oh + 64), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v1), + _mm256_shuffle_epi8(t1, v1), _mm256_slli_epi16(v1, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v1), + _mm256_shuffle_epi8(t3, v1), _mm256_slli_epi16(v1, 3)), _mm256_slli_epi16(v1, 2))); + __m256i v2 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r1, 4), _mm256_set1_epi8((char)0x0F)), + _mm256_and_si256(_mm256_slli_epi16(r2, 4), _mm256_set1_epi8((char)0xF0))), + _mm256_set1_epi8((char)0x3F)); + _mm256_storeu_si256((__m256i *)(oh + 128), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v2), + _mm256_shuffle_epi8(t1, v2), _mm256_slli_epi16(v2, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v2), + _mm256_shuffle_epi8(t3, v2), _mm256_slli_epi16(v2, 3)), _mm256_slli_epi16(v2, 2))); + __m256i v3 = _mm256_and_si256(_mm256_srli_epi16(r2, 2), _mm256_set1_epi8((char)0x3F)); + _mm256_storeu_si256((__m256i *)(oh + 192), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v3), + _mm256_shuffle_epi8(t1, v3), _mm256_slli_epi16(v3, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v3), + _mm256_shuffle_epi8(t3, v3), _mm256_slli_epi16(v3, 3)), _mm256_slli_epi16(v3, 2))); + __m256i v4 = _mm256_and_si256(r3, _mm256_set1_epi8((char)0x3F)); + _mm256_storeu_si256((__m256i *)(oh + 256), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v4), + _mm256_shuffle_epi8(t1, v4), _mm256_slli_epi16(v4, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v4), + _mm256_shuffle_epi8(t3, v4), _mm256_slli_epi16(v4, 3)), _mm256_slli_epi16(v4, 2))); + __m256i v5 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r3, 6), _mm256_set1_epi8((char)0x03)), + _mm256_and_si256(_mm256_slli_epi16(r4, 2), _mm256_set1_epi8((char)0xFC))), + _mm256_set1_epi8((char)0x3F)); + _mm256_storeu_si256((__m256i *)(oh + 320), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v5), + _mm256_shuffle_epi8(t1, v5), _mm256_slli_epi16(v5, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v5), + _mm256_shuffle_epi8(t3, v5), _mm256_slli_epi16(v5, 3)), _mm256_slli_epi16(v5, 2))); + __m256i v6 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r4, 4), _mm256_set1_epi8((char)0x0F)), + _mm256_and_si256(_mm256_slli_epi16(r5, 4), _mm256_set1_epi8((char)0xF0))), + _mm256_set1_epi8((char)0x3F)); + _mm256_storeu_si256((__m256i *)(oh + 384), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v6), + _mm256_shuffle_epi8(t1, v6), _mm256_slli_epi16(v6, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v6), + _mm256_shuffle_epi8(t3, v6), _mm256_slli_epi16(v6, 3)), _mm256_slli_epi16(v6, 2))); + __m256i v7 = _mm256_and_si256(_mm256_srli_epi16(r5, 2), _mm256_set1_epi8((char)0x3F)); + _mm256_storeu_si256((__m256i *)(oh + 448), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v7), + _mm256_shuffle_epi8(t1, v7), _mm256_slli_epi16(v7, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v7), + _mm256_shuffle_epi8(t3, v7), _mm256_slli_epi16(v7, 3)), _mm256_slli_epi16(v7, 2))); + } + } +} + +static void vert512_merge_ymm_d7(uint8_t *out, int n_v, const uint8_t *bm, + const uint8_t *c2s) +{ + const __m256i t0 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 0))); + const __m256i t1 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 16))); + const __m256i t2 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 32))); + const __m256i t3 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 48))); + const __m256i t4 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 64))); + const __m256i t5 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 80))); + const __m256i t6 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 96))); + const __m256i t7 = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)(c2s + 112))); + for (int b = 0; b < n_v >> 9; b++) { + const uint8_t *blk = bm + (size_t)b * 64 * 7; + uint8_t *o = out + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + const uint8_t *bh = blk + 32 * h; + uint8_t *oh = o + 32 * h; + __m256i r0 = _mm256_loadu_si256((const __m256i *)(bh + 0)); + __m256i r1 = _mm256_loadu_si256((const __m256i *)(bh + 64)); + __m256i r2 = _mm256_loadu_si256((const __m256i *)(bh + 128)); + __m256i r3 = _mm256_loadu_si256((const __m256i *)(bh + 192)); + __m256i r4 = _mm256_loadu_si256((const __m256i *)(bh + 256)); + __m256i r5 = _mm256_loadu_si256((const __m256i *)(bh + 320)); + __m256i r6 = _mm256_loadu_si256((const __m256i *)(bh + 384)); + __m256i v0 = _mm256_and_si256(r0, _mm256_set1_epi8((char)0x7F)); + _mm256_storeu_si256((__m256i *)(oh + 0), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v0), + _mm256_shuffle_epi8(t1, v0), _mm256_slli_epi16(v0, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v0), + _mm256_shuffle_epi8(t3, v0), _mm256_slli_epi16(v0, 3)), _mm256_slli_epi16(v0, 2)), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t4, v0), + _mm256_shuffle_epi8(t5, v0), _mm256_slli_epi16(v0, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t6, v0), + _mm256_shuffle_epi8(t7, v0), _mm256_slli_epi16(v0, 3)), _mm256_slli_epi16(v0, 2)), _mm256_slli_epi16(v0, 1))); + __m256i v1 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r0, 7), _mm256_set1_epi8((char)0x01)), + _mm256_and_si256(_mm256_slli_epi16(r1, 1), _mm256_set1_epi8((char)0xFE))), + _mm256_set1_epi8((char)0x7F)); + _mm256_storeu_si256((__m256i *)(oh + 64), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v1), + _mm256_shuffle_epi8(t1, v1), _mm256_slli_epi16(v1, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v1), + _mm256_shuffle_epi8(t3, v1), _mm256_slli_epi16(v1, 3)), _mm256_slli_epi16(v1, 2)), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t4, v1), + _mm256_shuffle_epi8(t5, v1), _mm256_slli_epi16(v1, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t6, v1), + _mm256_shuffle_epi8(t7, v1), _mm256_slli_epi16(v1, 3)), _mm256_slli_epi16(v1, 2)), _mm256_slli_epi16(v1, 1))); + __m256i v2 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r1, 6), _mm256_set1_epi8((char)0x03)), + _mm256_and_si256(_mm256_slli_epi16(r2, 2), _mm256_set1_epi8((char)0xFC))), + _mm256_set1_epi8((char)0x7F)); + _mm256_storeu_si256((__m256i *)(oh + 128), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v2), + _mm256_shuffle_epi8(t1, v2), _mm256_slli_epi16(v2, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v2), + _mm256_shuffle_epi8(t3, v2), _mm256_slli_epi16(v2, 3)), _mm256_slli_epi16(v2, 2)), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t4, v2), + _mm256_shuffle_epi8(t5, v2), _mm256_slli_epi16(v2, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t6, v2), + _mm256_shuffle_epi8(t7, v2), _mm256_slli_epi16(v2, 3)), _mm256_slli_epi16(v2, 2)), _mm256_slli_epi16(v2, 1))); + __m256i v3 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r2, 5), _mm256_set1_epi8((char)0x07)), + _mm256_and_si256(_mm256_slli_epi16(r3, 3), _mm256_set1_epi8((char)0xF8))), + _mm256_set1_epi8((char)0x7F)); + _mm256_storeu_si256((__m256i *)(oh + 192), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v3), + _mm256_shuffle_epi8(t1, v3), _mm256_slli_epi16(v3, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v3), + _mm256_shuffle_epi8(t3, v3), _mm256_slli_epi16(v3, 3)), _mm256_slli_epi16(v3, 2)), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t4, v3), + _mm256_shuffle_epi8(t5, v3), _mm256_slli_epi16(v3, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t6, v3), + _mm256_shuffle_epi8(t7, v3), _mm256_slli_epi16(v3, 3)), _mm256_slli_epi16(v3, 2)), _mm256_slli_epi16(v3, 1))); + __m256i v4 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r3, 4), _mm256_set1_epi8((char)0x0F)), + _mm256_and_si256(_mm256_slli_epi16(r4, 4), _mm256_set1_epi8((char)0xF0))), + _mm256_set1_epi8((char)0x7F)); + _mm256_storeu_si256((__m256i *)(oh + 256), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v4), + _mm256_shuffle_epi8(t1, v4), _mm256_slli_epi16(v4, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v4), + _mm256_shuffle_epi8(t3, v4), _mm256_slli_epi16(v4, 3)), _mm256_slli_epi16(v4, 2)), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t4, v4), + _mm256_shuffle_epi8(t5, v4), _mm256_slli_epi16(v4, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t6, v4), + _mm256_shuffle_epi8(t7, v4), _mm256_slli_epi16(v4, 3)), _mm256_slli_epi16(v4, 2)), _mm256_slli_epi16(v4, 1))); + __m256i v5 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r4, 3), _mm256_set1_epi8((char)0x1F)), + _mm256_and_si256(_mm256_slli_epi16(r5, 5), _mm256_set1_epi8((char)0xE0))), + _mm256_set1_epi8((char)0x7F)); + _mm256_storeu_si256((__m256i *)(oh + 320), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v5), + _mm256_shuffle_epi8(t1, v5), _mm256_slli_epi16(v5, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v5), + _mm256_shuffle_epi8(t3, v5), _mm256_slli_epi16(v5, 3)), _mm256_slli_epi16(v5, 2)), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t4, v5), + _mm256_shuffle_epi8(t5, v5), _mm256_slli_epi16(v5, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t6, v5), + _mm256_shuffle_epi8(t7, v5), _mm256_slli_epi16(v5, 3)), _mm256_slli_epi16(v5, 2)), _mm256_slli_epi16(v5, 1))); + __m256i v6 = _mm256_and_si256(_mm256_or_si256( + _mm256_and_si256(_mm256_srli_epi16(r5, 2), _mm256_set1_epi8((char)0x3F)), + _mm256_and_si256(_mm256_slli_epi16(r6, 6), _mm256_set1_epi8((char)0xC0))), + _mm256_set1_epi8((char)0x7F)); + _mm256_storeu_si256((__m256i *)(oh + 384), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v6), + _mm256_shuffle_epi8(t1, v6), _mm256_slli_epi16(v6, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v6), + _mm256_shuffle_epi8(t3, v6), _mm256_slli_epi16(v6, 3)), _mm256_slli_epi16(v6, 2)), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t4, v6), + _mm256_shuffle_epi8(t5, v6), _mm256_slli_epi16(v6, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t6, v6), + _mm256_shuffle_epi8(t7, v6), _mm256_slli_epi16(v6, 3)), _mm256_slli_epi16(v6, 2)), _mm256_slli_epi16(v6, 1))); + __m256i v7 = _mm256_and_si256(_mm256_srli_epi16(r6, 1), _mm256_set1_epi8((char)0x7F)); + _mm256_storeu_si256((__m256i *)(oh + 448), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t0, v7), + _mm256_shuffle_epi8(t1, v7), _mm256_slli_epi16(v7, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t2, v7), + _mm256_shuffle_epi8(t3, v7), _mm256_slli_epi16(v7, 3)), _mm256_slli_epi16(v7, 2)), + _mm256_blendv_epi8( + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t4, v7), + _mm256_shuffle_epi8(t5, v7), _mm256_slli_epi16(v7, 3)), + _mm256_blendv_epi8( + _mm256_shuffle_epi8(t6, v7), + _mm256_shuffle_epi8(t7, v7), _mm256_slli_epi16(v7, 3)), _mm256_slli_epi16(v7, 2)), _mm256_slli_epi16(v7, 1))); + } + } +} + +static inline void vert512_merge_ymm(uint8_t *out, int n_v, const uint8_t *bm, int D, const uint8_t *c2s) +{ + switch (D) { + case 2: vert512_merge_ymm_d2(out, n_v, bm, c2s); break; + case 3: vert512_merge_ymm_d3(out, n_v, bm, c2s); break; + case 4: vert512_merge_ymm_d4(out, n_v, bm, c2s); break; + case 5: vert512_merge_ymm_d5(out, n_v, bm, c2s); break; + case 6: vert512_merge_ymm_d6(out, n_v, bm, c2s); break; + default: vert512_merge_ymm_d7(out, n_v, bm, c2s); break; + } +} + +static void vert512_pack_ymm_d2(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m256i basev = _mm256_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 2; + const uint8_t *r = ranks + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + uint8_t *bh = blk + 32 * h; + const uint8_t *rh = r + 32 * h; + __m256i c0, c1; + __m256i v0 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 0)), basev); + c0 = v0; + __m256i v1 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 64)), basev); + c0 = _mm256_or_si256(c0, _mm256_and_si256(_mm256_slli_epi16(v1, 2), _mm256_set1_epi8((char)0xFC))); + __m256i v2 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 128)), basev); + c0 = _mm256_or_si256(c0, _mm256_and_si256(_mm256_slli_epi16(v2, 4), _mm256_set1_epi8((char)0xF0))); + __m256i v3 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 192)), basev); + c0 = _mm256_or_si256(c0, _mm256_and_si256(_mm256_slli_epi16(v3, 6), _mm256_set1_epi8((char)0xC0))); + __m256i v4 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 256)), basev); + c1 = v4; + __m256i v5 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 320)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v5, 2), _mm256_set1_epi8((char)0xFC))); + __m256i v6 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 384)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v6, 4), _mm256_set1_epi8((char)0xF0))); + __m256i v7 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 448)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v7, 6), _mm256_set1_epi8((char)0xC0))); + _mm256_storeu_si256((__m256i *)(bh + 0), c0); + _mm256_storeu_si256((__m256i *)(bh + 64), c1); + } + } +} + +static void vert512_pack_ymm_d3(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m256i basev = _mm256_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 3; + const uint8_t *r = ranks + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + uint8_t *bh = blk + 32 * h; + const uint8_t *rh = r + 32 * h; + __m256i c0, c1, c2; + __m256i v0 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 0)), basev); + c0 = v0; + __m256i v1 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 64)), basev); + c0 = _mm256_or_si256(c0, _mm256_and_si256(_mm256_slli_epi16(v1, 3), _mm256_set1_epi8((char)0xF8))); + __m256i v2 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 128)), basev); + c0 = _mm256_or_si256(c0, _mm256_and_si256(_mm256_slli_epi16(v2, 6), _mm256_set1_epi8((char)0xC0))); + c1 = _mm256_and_si256(_mm256_srli_epi16(v2, 2), _mm256_set1_epi8((char)0x3F)); + __m256i v3 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 192)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v3, 1), _mm256_set1_epi8((char)0xFE))); + __m256i v4 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 256)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v4, 4), _mm256_set1_epi8((char)0xF0))); + __m256i v5 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 320)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v5, 7), _mm256_set1_epi8((char)0x80))); + c2 = _mm256_and_si256(_mm256_srli_epi16(v5, 1), _mm256_set1_epi8((char)0x7F)); + __m256i v6 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 384)), basev); + c2 = _mm256_or_si256(c2, _mm256_and_si256(_mm256_slli_epi16(v6, 2), _mm256_set1_epi8((char)0xFC))); + __m256i v7 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 448)), basev); + c2 = _mm256_or_si256(c2, _mm256_and_si256(_mm256_slli_epi16(v7, 5), _mm256_set1_epi8((char)0xE0))); + _mm256_storeu_si256((__m256i *)(bh + 0), c0); + _mm256_storeu_si256((__m256i *)(bh + 64), c1); + _mm256_storeu_si256((__m256i *)(bh + 128), c2); + } + } +} + +static void vert512_pack_ymm_d4(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m256i basev = _mm256_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 4; + const uint8_t *r = ranks + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + uint8_t *bh = blk + 32 * h; + const uint8_t *rh = r + 32 * h; + __m256i c0, c1, c2, c3; + __m256i v0 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 0)), basev); + c0 = v0; + __m256i v1 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 64)), basev); + c0 = _mm256_or_si256(c0, _mm256_and_si256(_mm256_slli_epi16(v1, 4), _mm256_set1_epi8((char)0xF0))); + __m256i v2 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 128)), basev); + c1 = v2; + __m256i v3 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 192)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v3, 4), _mm256_set1_epi8((char)0xF0))); + __m256i v4 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 256)), basev); + c2 = v4; + __m256i v5 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 320)), basev); + c2 = _mm256_or_si256(c2, _mm256_and_si256(_mm256_slli_epi16(v5, 4), _mm256_set1_epi8((char)0xF0))); + __m256i v6 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 384)), basev); + c3 = v6; + __m256i v7 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 448)), basev); + c3 = _mm256_or_si256(c3, _mm256_and_si256(_mm256_slli_epi16(v7, 4), _mm256_set1_epi8((char)0xF0))); + _mm256_storeu_si256((__m256i *)(bh + 0), c0); + _mm256_storeu_si256((__m256i *)(bh + 64), c1); + _mm256_storeu_si256((__m256i *)(bh + 128), c2); + _mm256_storeu_si256((__m256i *)(bh + 192), c3); + } + } +} + +static void vert512_pack_ymm_d5(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m256i basev = _mm256_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 5; + const uint8_t *r = ranks + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + uint8_t *bh = blk + 32 * h; + const uint8_t *rh = r + 32 * h; + __m256i c0, c1, c2, c3, c4; + __m256i v0 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 0)), basev); + c0 = v0; + __m256i v1 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 64)), basev); + c0 = _mm256_or_si256(c0, _mm256_and_si256(_mm256_slli_epi16(v1, 5), _mm256_set1_epi8((char)0xE0))); + c1 = _mm256_and_si256(_mm256_srli_epi16(v1, 3), _mm256_set1_epi8((char)0x1F)); + __m256i v2 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 128)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v2, 2), _mm256_set1_epi8((char)0xFC))); + __m256i v3 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 192)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v3, 7), _mm256_set1_epi8((char)0x80))); + c2 = _mm256_and_si256(_mm256_srli_epi16(v3, 1), _mm256_set1_epi8((char)0x7F)); + __m256i v4 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 256)), basev); + c2 = _mm256_or_si256(c2, _mm256_and_si256(_mm256_slli_epi16(v4, 4), _mm256_set1_epi8((char)0xF0))); + c3 = _mm256_and_si256(_mm256_srli_epi16(v4, 4), _mm256_set1_epi8((char)0x0F)); + __m256i v5 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 320)), basev); + c3 = _mm256_or_si256(c3, _mm256_and_si256(_mm256_slli_epi16(v5, 1), _mm256_set1_epi8((char)0xFE))); + __m256i v6 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 384)), basev); + c3 = _mm256_or_si256(c3, _mm256_and_si256(_mm256_slli_epi16(v6, 6), _mm256_set1_epi8((char)0xC0))); + c4 = _mm256_and_si256(_mm256_srli_epi16(v6, 2), _mm256_set1_epi8((char)0x3F)); + __m256i v7 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 448)), basev); + c4 = _mm256_or_si256(c4, _mm256_and_si256(_mm256_slli_epi16(v7, 3), _mm256_set1_epi8((char)0xF8))); + _mm256_storeu_si256((__m256i *)(bh + 0), c0); + _mm256_storeu_si256((__m256i *)(bh + 64), c1); + _mm256_storeu_si256((__m256i *)(bh + 128), c2); + _mm256_storeu_si256((__m256i *)(bh + 192), c3); + _mm256_storeu_si256((__m256i *)(bh + 256), c4); + } + } +} + +static void vert512_pack_ymm_d6(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m256i basev = _mm256_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 6; + const uint8_t *r = ranks + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + uint8_t *bh = blk + 32 * h; + const uint8_t *rh = r + 32 * h; + __m256i c0, c1, c2, c3, c4, c5; + __m256i v0 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 0)), basev); + c0 = v0; + __m256i v1 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 64)), basev); + c0 = _mm256_or_si256(c0, _mm256_and_si256(_mm256_slli_epi16(v1, 6), _mm256_set1_epi8((char)0xC0))); + c1 = _mm256_and_si256(_mm256_srli_epi16(v1, 2), _mm256_set1_epi8((char)0x3F)); + __m256i v2 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 128)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v2, 4), _mm256_set1_epi8((char)0xF0))); + c2 = _mm256_and_si256(_mm256_srli_epi16(v2, 4), _mm256_set1_epi8((char)0x0F)); + __m256i v3 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 192)), basev); + c2 = _mm256_or_si256(c2, _mm256_and_si256(_mm256_slli_epi16(v3, 2), _mm256_set1_epi8((char)0xFC))); + __m256i v4 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 256)), basev); + c3 = v4; + __m256i v5 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 320)), basev); + c3 = _mm256_or_si256(c3, _mm256_and_si256(_mm256_slli_epi16(v5, 6), _mm256_set1_epi8((char)0xC0))); + c4 = _mm256_and_si256(_mm256_srli_epi16(v5, 2), _mm256_set1_epi8((char)0x3F)); + __m256i v6 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 384)), basev); + c4 = _mm256_or_si256(c4, _mm256_and_si256(_mm256_slli_epi16(v6, 4), _mm256_set1_epi8((char)0xF0))); + c5 = _mm256_and_si256(_mm256_srli_epi16(v6, 4), _mm256_set1_epi8((char)0x0F)); + __m256i v7 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 448)), basev); + c5 = _mm256_or_si256(c5, _mm256_and_si256(_mm256_slli_epi16(v7, 2), _mm256_set1_epi8((char)0xFC))); + _mm256_storeu_si256((__m256i *)(bh + 0), c0); + _mm256_storeu_si256((__m256i *)(bh + 64), c1); + _mm256_storeu_si256((__m256i *)(bh + 128), c2); + _mm256_storeu_si256((__m256i *)(bh + 192), c3); + _mm256_storeu_si256((__m256i *)(bh + 256), c4); + _mm256_storeu_si256((__m256i *)(bh + 320), c5); + } + } +} + +static void vert512_pack_ymm_d7(uint8_t *out, const uint8_t *ranks, + int n_v, uint8_t base) +{ + const __m256i basev = _mm256_set1_epi8((char)base); + for (int b = 0; b < n_v >> 9; b++) { + uint8_t *blk = out + (size_t)b * 64 * 7; + const uint8_t *r = ranks + ((size_t)b << 9); + for (int h = 0; h < 2; h++) { + uint8_t *bh = blk + 32 * h; + const uint8_t *rh = r + 32 * h; + __m256i c0, c1, c2, c3, c4, c5, c6; + __m256i v0 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 0)), basev); + c0 = v0; + __m256i v1 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 64)), basev); + c0 = _mm256_or_si256(c0, _mm256_and_si256(_mm256_slli_epi16(v1, 7), _mm256_set1_epi8((char)0x80))); + c1 = _mm256_and_si256(_mm256_srli_epi16(v1, 1), _mm256_set1_epi8((char)0x7F)); + __m256i v2 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 128)), basev); + c1 = _mm256_or_si256(c1, _mm256_and_si256(_mm256_slli_epi16(v2, 6), _mm256_set1_epi8((char)0xC0))); + c2 = _mm256_and_si256(_mm256_srli_epi16(v2, 2), _mm256_set1_epi8((char)0x3F)); + __m256i v3 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 192)), basev); + c2 = _mm256_or_si256(c2, _mm256_and_si256(_mm256_slli_epi16(v3, 5), _mm256_set1_epi8((char)0xE0))); + c3 = _mm256_and_si256(_mm256_srli_epi16(v3, 3), _mm256_set1_epi8((char)0x1F)); + __m256i v4 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 256)), basev); + c3 = _mm256_or_si256(c3, _mm256_and_si256(_mm256_slli_epi16(v4, 4), _mm256_set1_epi8((char)0xF0))); + c4 = _mm256_and_si256(_mm256_srli_epi16(v4, 4), _mm256_set1_epi8((char)0x0F)); + __m256i v5 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 320)), basev); + c4 = _mm256_or_si256(c4, _mm256_and_si256(_mm256_slli_epi16(v5, 3), _mm256_set1_epi8((char)0xF8))); + c5 = _mm256_and_si256(_mm256_srli_epi16(v5, 5), _mm256_set1_epi8((char)0x07)); + __m256i v6 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 384)), basev); + c5 = _mm256_or_si256(c5, _mm256_and_si256(_mm256_slli_epi16(v6, 2), _mm256_set1_epi8((char)0xFC))); + c6 = _mm256_and_si256(_mm256_srli_epi16(v6, 6), _mm256_set1_epi8((char)0x03)); + __m256i v7 = _mm256_sub_epi8(_mm256_loadu_si256((const __m256i *)(rh + 448)), basev); + c6 = _mm256_or_si256(c6, _mm256_and_si256(_mm256_slli_epi16(v7, 1), _mm256_set1_epi8((char)0xFE))); + _mm256_storeu_si256((__m256i *)(bh + 0), c0); + _mm256_storeu_si256((__m256i *)(bh + 64), c1); + _mm256_storeu_si256((__m256i *)(bh + 128), c2); + _mm256_storeu_si256((__m256i *)(bh + 192), c3); + _mm256_storeu_si256((__m256i *)(bh + 256), c4); + _mm256_storeu_si256((__m256i *)(bh + 320), c5); + _mm256_storeu_si256((__m256i *)(bh + 384), c6); + } + } +} + +static inline void vert512_pack_ymm(uint8_t *out, const uint8_t *ranks, int n_v, int D, uint8_t base) +{ + switch (D) { + case 2: vert512_pack_ymm_d2(out, ranks, n_v, base); break; + case 3: vert512_pack_ymm_d3(out, ranks, n_v, base); break; + case 4: vert512_pack_ymm_d4(out, ranks, n_v, base); break; + case 5: vert512_pack_ymm_d5(out, ranks, n_v, base); break; + case 6: vert512_pack_ymm_d6(out, ranks, n_v, base); break; +#if defined(PIVCO_X86_INTEL) + default: vert512_pack_ymm_d7(out, ranks, n_v, base); break; +#else + /* Zen 3 splits 256-bit ops in two; the 16-lane quarter walk wins + * there (1.12 vs 0.89 on c6a) while Intel AVX2 prefers ymm (1.54 + * vs 1.22 on c5). */ + default: vert512_pack_x86_d7(out, ranks, n_v, base); break; +#endif + } +} + +#endif /* __AVX2__ */ + + +/* Best-available 512-block kernels. */ +static inline void vert512_merge_x86_best(uint8_t *out, int n_v, const uint8_t *bm, + int D, const uint8_t *c2s) +{ +#if defined(__AVX512BW__) && defined(__AVX512VBMI__) && defined(__AVX512VL__) + vert512_merge_zmm(out, n_v, bm, D, c2s); +#elif defined(__AVX2__) + vert512_merge_ymm(out, n_v, bm, D, c2s); +#else + vert512_merge_x86v(out, n_v, bm, D, c2s); +#endif +} + +static inline void vert512_pack_x86_best(uint8_t *out, const uint8_t *ranks, + int n_v, int D, uint8_t base) +{ +#if defined(__AVX512BW__) && defined(__AVX512VBMI__) && defined(__AVX512VL__) + vert512_pack_zmm(out, ranks, n_v, D, base); +#elif defined(__AVX2__) + vert512_pack_ymm(out, ranks, n_v, D, base); +#else + vert512_pack_x86v(out, ranks, n_v, D, base); +#endif +} + +#endif /* PIVCO_HUFFMAN_X86_VERTICAL_H */ diff --git a/third_party/pivco/src/pivcohuf_file.c b/third_party/pivco/src/pivcohuf_file.c new file mode 100644 index 0000000..febfae7 --- /dev/null +++ b/third_party/pivco/src/pivcohuf_file.c @@ -0,0 +1,512 @@ +/* pivcohuf file format codec. See include/pivcohuf_file.h for the + * wire-format specification. */ + +#include "pivcohuf_file.h" +#include "pivco_huffman.h" +#include "pivco_prof.h" + +#include +#include +#include +#include + +/* Monotonic wall-clock in nanoseconds, for the *_timed phase breakdown. + * Coarse (phase-level, never inside hot inner loops), so always-on cost is + * a handful of clock_gettime calls per compress/decompress. */ +static double now_ns(void) { + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (double)t.tv_sec * 1e9 + (double)t.tv_nsec; +} +#define TIC(t) ((t) ? now_ns() : 0.0) +#define TOC(t, fld, s) do { if (t) (t)->fld += now_ns() - (s); } while (0) + +/* ============================================================ + * XXH32 -- 32-bit xxHash, tiny self-contained implementation. + * Used for header + body integrity (not crypto). Seed 0. + * Algorithm reference: github.com/Cyan4973/xxHash (BSD-2). + * ============================================================ */ + +#define XXH_PRIME32_1 0x9E3779B1U +#define XXH_PRIME32_2 0x85EBCA77U +#define XXH_PRIME32_3 0xC2B2AE3DU +#define XXH_PRIME32_4 0x27D4EB2FU +#define XXH_PRIME32_5 0x165667B1U + +static inline uint32_t rotl32(uint32_t x, int r) { + return (x << r) | (x >> (32 - r)); +} + +__attribute__((unused)) +static uint32_t xxh32(const void *data, size_t len) +{ + const uint8_t *p = (const uint8_t *)data; + const uint8_t *end = p + len; + uint32_t h; + + if (len >= 16) { + uint32_t v1 = 0 + XXH_PRIME32_1 + XXH_PRIME32_2; + uint32_t v2 = 0 + XXH_PRIME32_2; + uint32_t v3 = 0; + uint32_t v4 = 0 - XXH_PRIME32_1; + const uint8_t *limit = end - 16; + while (p <= limit) { + uint32_t k; + memcpy(&k, p, 4); p += 4; + v1 = rotl32(v1 + k * XXH_PRIME32_2, 13) * XXH_PRIME32_1; + memcpy(&k, p, 4); p += 4; + v2 = rotl32(v2 + k * XXH_PRIME32_2, 13) * XXH_PRIME32_1; + memcpy(&k, p, 4); p += 4; + v3 = rotl32(v3 + k * XXH_PRIME32_2, 13) * XXH_PRIME32_1; + memcpy(&k, p, 4); p += 4; + v4 = rotl32(v4 + k * XXH_PRIME32_2, 13) * XXH_PRIME32_1; + } + h = rotl32(v1, 1) + rotl32(v2, 7) + rotl32(v3, 12) + rotl32(v4, 18); + } else { + h = 0 + XXH_PRIME32_5; + } + h += (uint32_t)len; + + while (p + 4 <= end) { + uint32_t k; + memcpy(&k, p, 4); p += 4; + h += k * XXH_PRIME32_3; + h = rotl32(h, 17) * XXH_PRIME32_4; + } + while (p < end) { + h += (uint32_t)(*p++) * XXH_PRIME32_5; + h = rotl32(h, 11) * XXH_PRIME32_1; + } + h ^= h >> 15; h *= XXH_PRIME32_2; + h ^= h >> 13; h *= XXH_PRIME32_3; + h ^= h >> 16; + return h; +} + +/* ============================================================ + * Little-endian field readers/writers. + * ============================================================ */ +__attribute__((unused)) static inline void put_u8 (uint8_t *p, uint8_t v) { p[0] = v; } +static inline void put_u16(uint8_t *p, uint16_t v) { p[0] = v & 0xff; p[1] = (v >> 8) & 0xff; } +static inline void put_u32(uint8_t *p, uint32_t v) { + p[0] = v & 0xff; p[1] = (v>>8) & 0xff; + p[2] = (v>>16) & 0xff; p[3] = (v>>24) & 0xff; +} +static inline void put_u64(uint8_t *p, uint64_t v) { + put_u32(p, (uint32_t)v); + put_u32(p + 4, (uint32_t)(v >> 32)); +} +__attribute__((unused)) static inline uint8_t get_u8 (const uint8_t *p) { return p[0]; } +static inline uint16_t get_u16(const uint8_t *p) { return (uint16_t)p[0] | ((uint16_t)p[1] << 8); } +static inline uint32_t get_u32(const uint8_t *p) { + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) + | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} +static inline uint64_t get_u64(const uint8_t *p) { + return (uint64_t)get_u32(p) | ((uint64_t)get_u32(p + 4) << 32); +} + +/* ============================================================ + * compress_bound + compress + * ============================================================ */ + +size_t pivcohuf_compress_bound_blk(size_t in_len, size_t block_size) +{ + /* Per-block worst case is the full block size (no compression) plus + * a small overhead for the encoded format. We bound generously at + * 2x block size; the K_right header adds <1%. Smaller blocks mean + * more per-block overhead, so the bound must use the actual block + * size the caller will compress with. */ + if (block_size < 1) block_size = 1; + if (block_size > PIVCO_WIRE_MAX_N) block_size = PIVCO_WIRE_MAX_N; + const size_t B = block_size; + size_t nblocks = (in_len + B - 1) / B; + if (nblocks == 0) nblocks = 1; /* zero-byte input still produces one header */ + size_t worst_per_block = 4 /* length prefix */ + 2 * B + 64; + return PIVCOHUF_HEADER_SIZE /* header */ + + 8 + 2 + 1 + 128 /* body header: usize + blk + flags + code-len nibbles */ + + nblocks * worst_per_block; +} + +size_t pivcohuf_compress_bound(size_t in_len) +{ + return pivcohuf_compress_bound_blk(in_len, PIVCO_BLOCK_SIZE); +} + + +static int pivcohuf_compress_impl(pivco_encoder_t *enc_ctx, + const pivco_cfg_t *cfg, + const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + size_t block_size, + pivcohuf_timing_t *tm) +{ + if (!in && in_len > 0) return PIVCOHUF_ERR_NULL; + if (!out || !out_len) return PIVCOHUF_ERR_NULL; + if (block_size < 1 || block_size > PIVCO_WIRE_MAX_N) + return PIVCOHUF_ERR_BAD_BLOCK_SIZE; + if (*out_len < pivcohuf_compress_bound_blk(in_len, block_size)) + return PIVCOHUF_ERR_OUTPUT_TOO_SMALL; + + const size_t B = block_size; + + /* Build histogram over real input via file_histogram above -- + * prim_histogram_chunk under a chunked u32 -> u64 wrapper. */ + uint64_t real_freq[256] = {0}; + { double _t = TIC(tm); + if (pivco_histogram(enc_ctx, in, in_len, real_freq) != PIVCO_OK) + return PIVCOHUF_ERR_INTERNAL; + if (in_len == 0) real_freq[0] = 1; + TOC(tm, freq_ns, _t); } + + pivco_table_t real_table; + { PROF_TIC(); double _t = TIC(tm); + if (pivco_build_table(cfg, real_freq, &real_table) != PIVCO_OK) + return PIVCOHUF_ERR_INTERNAL; + PROF_TOC(PROF_FILE_BUILD_TABLE_REAL, 1); TOC(tm, build_ns, _t); } + + /* Rebuild the encode-time table via the code-lens builder, so encode + * uses the exact table the decoder reconstructs from the wire. The tree + * is fully determined by the code lengths (within-tier order is symbol- + * value), so nothing beyond the lengths is transmitted. */ + pivco_table_t table; + { PROF_TIC(); double _t = TIC(tm); + if (pivco_build_table_from_code_lens(cfg, real_table.code_len, + &table) != PIVCO_OK) + return PIVCOHUF_ERR_INTERNAL; + PROF_TOC(PROF_FILE_BUILD_TABLE_SYN, 1); TOC(tm, build_ns, _t); } + + /* Pad with the most-frequent symbol (sorted_symbols[0] -- always has + * the shortest code). Padding with arbitrary bytes can hit pathological + * deep-recursion paths in the encoder when blk_in << B. */ + const uint8_t pad_byte = table.sorted_symbols[0]; + + uint8_t *p = out; + /* === Reserve HEADER bytes; fill at end. === */ + uint8_t *hdr = p; + p += PIVCOHUF_HEADER_SIZE; + + /* === BODY start. === */ + uint8_t *body = p; + + /* UNCOMPRESSED_SIZE */ + put_u64(p, (uint64_t)in_len); p += 8; + + /* BLOCK_SIZE (uint16, 1024..65535). */ + put_u16(p, (uint16_t)B); p += 2; + + /* FLAGS (v0.9): bits0-1 = flat-region layout, from the build cfg + * (already validated by the table build above). */ + put_u8(p, (uint8_t)cfg->flat_layout); p += 1; + + /* CODE_LENGTHS packed as 4-bit nibbles, sym 2i in low nibble. */ + for (int i = 0; i < 128; i++) { + uint8_t lo = table.code_len[2*i] & 0x0F; + uint8_t hi = table.code_len[2*i + 1] & 0x0F; + p[i] = (uint8_t)(lo | (hi << 4)); + } + p += 128; + + /* === Encode block-by-block. === */ + size_t off = 0; + double _tm = TIC(tm); + uint8_t *block_buf = (uint8_t *)malloc(B); + TOC(tm, malloc_ns, _tm); + if (!block_buf) return PIVCOHUF_ERR_INTERNAL; + double _te = TIC(tm); + while (off < in_len) { + size_t blk_in = in_len - off; + size_t this_n; + const uint8_t *blk_src; + uint8_t *len_field; + { PROF_TIC(); + if (blk_in >= B) { + blk_src = in + off; + this_n = B; + off += B; + } else { + /* Final (short) block: encode it at its actual size. The + * codec writes a 2-byte N header at the start of the encoded + * stream so the decoder recovers the count without any + * out-of-band channel. */ + blk_src = in + off; + this_n = blk_in; + off = in_len; + } + (void)pad_byte; (void)block_buf; /* padding path retired */ + len_field = p; p += 4; + PROF_TOC(PROF_FILE_BLOCK_PROLOGUE, (uint64_t)this_n); } + + { PROF_TIC(); + size_t enc_len = 0; + if (pivco_encode(enc_ctx, &table, blk_src, this_n, p, &enc_len) != PIVCO_OK) { + free(block_buf); + return PIVCOHUF_ERR_INTERNAL; + } + put_u32(len_field, (uint32_t)enc_len); + p += enc_len; + PROF_TOC(PROF_FILE_BLOCK_ENCODE, (uint64_t)this_n); } + } + TOC(tm, codec_ns, _te); + free(block_buf); + + size_t body_len = (size_t)(p - body); + + /* Write HEADER (positions are fixed). Checksums temporarily disabled + * -- always zero (2026-05-12). Format byte positions preserved so a + * later commit can turn them back on without a wire-format break. */ + memcpy(hdr + 0, PIVCOHUF_MAGIC, 8); + hdr[8] = PIVCOHUF_VERSION_MAJOR; + hdr[9] = PIVCOHUF_VERSION_MINOR; + put_u64(hdr + 10, (uint64_t)body_len); + put_u32(hdr + 18, 0); /* BODY_CHECKSUM = 0 (disabled) */ + put_u32(hdr + 22, 0); /* HEADER_CHECKSUM = 0 (disabled) */ + + *out_len = (size_t)(p - out); + return PIVCOHUF_OK; +} + +/* pha (#PHA): same wire/decoder, but per-block bitmaps may be ANS(FSE)-coded. + * The FSE path is selected per build via pivco_cfg_t.fse_enabled (baked + * into the table). Decompress needs no flag — it auto-detects FSE + * markers per block. */ +static int compress_dispatch(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + const pivco_cfg_t *cfg_in, int use_ans, + size_t block_size, pivcohuf_timing_t *tm) +{ + if (tm) memset(tm, 0, sizeof(*tm)); + pivco_cfg_t cfg = cfg_in ? *cfg_in : pivco_cfg_default; + cfg.fse_enabled = use_ans; + /* FASTEST_COMPRESS is the one effort mode the bare table build + * cannot resolve (it needs the input size): below 256 KiB plain + * Huffman lengths encode fastest; above, a flatter tree ENCODES + * faster than the BALANCED shaping solve costs, and the solve's + * cost keeps shrinking as 1/n. */ + if (cfg.effort == PIVCO_EFFORT_FASTEST_COMPRESS) + cfg.effort = in_len < (size_t)262144 ? PIVCO_EFFORT_PLAIN + : PIVCO_EFFORT_BALANCED; + pivco_encoder_t *enc_ctx = pivco_encoder_create(); + if (!enc_ctx) return PIVCOHUF_ERR_INTERNAL; + int r = pivcohuf_compress_impl(enc_ctx, &cfg, in, in_len, out, out_len, + block_size, tm); + pivco_encoder_free(enc_ctx); + return r; +} + +int pivcohuf_compress_blk(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + int use_ans, size_t block_size, + pivcohuf_timing_t *timing) +{ + return compress_dispatch(in, in_len, out, out_len, NULL, use_ans, + block_size, timing); +} + +int pivcohuf_compress_cfg(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + const pivco_cfg_t *cfg, size_t block_size, + pivcohuf_timing_t *timing) +{ + int use_ans = cfg ? cfg->fse_enabled : 0; + return compress_dispatch(in, in_len, out, out_len, cfg, use_ans, + block_size, timing); +} + +int pivcohuf_compress_ex(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, int use_ans) +{ + return compress_dispatch(in, in_len, out, out_len, NULL, use_ans, + PIVCO_BLOCK_SIZE, NULL); +} + +int pivcohuf_compress(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len) +{ + return compress_dispatch(in, in_len, out, out_len, NULL, 0, + PIVCO_BLOCK_SIZE, NULL); +} + +int pivcohuf_compress_timed(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + int use_ans, pivcohuf_timing_t *timing) +{ + return compress_dispatch(in, in_len, out, out_len, NULL, use_ans, + PIVCO_BLOCK_SIZE, timing); +} + +/* ============================================================ + * peek + decompress + * ============================================================ */ + +/* Oldest minor this decoder still reads. v0.8 bodies have no FLAGS + * byte and natural flat regions; anything older is a hard break. */ +#define PIVCOHUF_MIN_DECODE_MINOR 8 + +static int parse_header(const uint8_t *in, size_t in_len, uint64_t *body_len, + uint8_t *minor) +{ + if (in_len < PIVCOHUF_HEADER_SIZE) return PIVCOHUF_ERR_TOO_SHORT; + if (memcmp(in, PIVCOHUF_MAGIC, 8) != 0) return PIVCOHUF_ERR_BAD_MAGIC; + if (in[8] != PIVCOHUF_VERSION_MAJOR + || in[9] < PIVCOHUF_MIN_DECODE_MINOR + || in[9] > PIVCOHUF_VERSION_MINOR) + return PIVCOHUF_ERR_BAD_VERSION; + /* HEADER_CHECKSUM verification disabled (2026-05-12) -- bytes are + * still in the format at offset 22..25, currently always zero. */ + *body_len = get_u64(in + 10); + *minor = in[9]; + return PIVCOHUF_OK; +} + +int pivcohuf_peek_uncompressed_size(const uint8_t *in, size_t in_len, + size_t *uncompressed_size) +{ + if (!in || !uncompressed_size) return PIVCOHUF_ERR_NULL; + uint64_t body_len; + uint8_t minor; + int rc = parse_header(in, in_len, &body_len, &minor); + if (rc != PIVCOHUF_OK) return rc; + if (in_len < PIVCOHUF_HEADER_SIZE + 8) return PIVCOHUF_ERR_TOO_SHORT; + *uncompressed_size = (size_t)get_u64(in + PIVCOHUF_HEADER_SIZE); + return PIVCOHUF_OK; +} + +static int pivcohuf_decompress_impl(pivco_decoder_t *dec_ctx, + const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + pivcohuf_timing_t *tm) +{ + if (!in || !out || !out_len) return PIVCOHUF_ERR_NULL; + uint64_t body_len_u64; + uint8_t minor; + int rc = parse_header(in, in_len, &body_len_u64, &minor); + if (rc != PIVCOHUF_OK) return rc; + if (in_len < PIVCOHUF_HEADER_SIZE + body_len_u64) + return PIVCOHUF_ERR_TOO_SHORT; + size_t body_len = (size_t)body_len_u64; + const uint8_t *body = in + PIVCOHUF_HEADER_SIZE; + /* BODY_CHECKSUM verification disabled (2026-05-12). */ + + /* Parse body header: UNCOMPRESSED_SIZE(8) + BLOCK_SIZE(2) + + * FLAGS(1, v0.9+) + nibbles(128). */ + size_t flags_size = (minor >= 9) ? 1 : 0; + if (body_len < 8 + 2 + flags_size + 128) return PIVCOHUF_ERR_TOO_SHORT; + size_t uncomp_size = (size_t)get_u64(body); + uint16_t file_blk = get_u16(body + 8); + uint8_t flags = 0; + if (flags_size) { + flags = get_u8(body + 10); + /* Strict: any layout value or set bit this decoder does not + * implement (including the reserved QUAD_NODES bit) is a + * refusal, not a guess. */ + if ((flags & PIVCOHUF_FLAGS_LAYOUT_MASK) > PIVCO_FLAT_VERTICAL_128 + || (flags & (uint8_t)~PIVCOHUF_FLAGS_LAYOUT_MASK)) + return PIVCOHUF_ERR_BAD_VERSION; + } + /* The block size is read from the file, not fixed at compile time: the + * codec sizes its scratch dynamically off the per-block wire N header, + * so any block size the encoder could write is decodable here. Only a + * zero block size (impossible from a valid encoder) is rejected. */ + if (file_blk == 0) return PIVCOHUF_ERR_BAD_BLOCK_SIZE; + const size_t B = (size_t)file_blk; + + if (*out_len < uncomp_size) return PIVCOHUF_ERR_OUTPUT_TOO_SMALL; + + /* Reconstruct Huffman table from code lengths. The build cfg follows + * the stream, not this build's defaults: v0.8 streams and v0.9 streams + * without the flag have natural flat regions. */ + uint8_t code_lens[256]; + const uint8_t *nibbles = body + 10 + flags_size; + for (int i = 0; i < 128; i++) { + code_lens[2*i] = nibbles[i] & 0x0F; + code_lens[2*i + 1] = (nibbles[i] >> 4) & 0x0F; + } + + pivco_cfg_t cfg = pivco_cfg_default; + cfg.flat_layout = (pivco_flat_layout_t)(flags & PIVCOHUF_FLAGS_LAYOUT_MASK); + + pivco_table_t table; + { PROF_TIC(); double _t = TIC(tm); + if (pivco_build_table_from_code_lens(&cfg, code_lens, &table) != PIVCO_OK) + return PIVCOHUF_ERR_INTERNAL; + PROF_TOC(PROF_FILE_BUILD_TABLE_SYN, 1); TOC(tm, build_ns, _t); } + /* Sanity check: rebuilt code lengths must match. */ + for (int s = 0; s < 256; s++) { + if (table.code_len[s] != code_lens[s]) { + return PIVCOHUF_ERR_INTERNAL; + } + } + + /* Decode blocks. block_buf is on heap (avoids large stack frames; also + * sized B which is read from the file). */ + double _tm = TIC(tm); + uint8_t *block_buf = (uint8_t *)malloc(B); + TOC(tm, malloc_ns, _tm); + if (!block_buf) return PIVCOHUF_ERR_INTERNAL; + const uint8_t *p = body + 10 + flags_size + 128; + const uint8_t *body_end = body + body_len; + size_t written = 0; + int err = 0; + double _td = TIC(tm); + while (p < body_end && written < uncomp_size) { + uint32_t blk_enc_len; + uint8_t *blk_out; + size_t blk_remaining; + { PROF_TIC(); + if (p + 4 > body_end) { err = PIVCOHUF_ERR_TOO_SHORT; break; } + blk_enc_len = get_u32(p); p += 4; + if (p + blk_enc_len > body_end) { err = PIVCOHUF_ERR_TOO_SHORT; break; } + blk_remaining = uncomp_size - written; + blk_out = (blk_remaining >= B) ? (out + written) : block_buf; + PROF_TOC(PROF_FILE_BLOCK_PROLOGUE, (uint64_t)B); } + + { PROF_TIC(); + size_t consumed = 0; + if (pivco_decode(dec_ctx, &table, p, blk_enc_len, + blk_out, &consumed) != PIVCO_OK) { + err = PIVCOHUF_ERR_INTERNAL; break; + } + PROF_TOC(PROF_FILE_BLOCK_DECODE, (uint64_t)B); } + if (blk_remaining < B) { + memcpy(out + written, block_buf, blk_remaining); + written = uncomp_size; + } else { + written += B; + } + p += blk_enc_len; + } + TOC(tm, codec_ns, _td); + free(block_buf); + if (err) return err; + + if (written != uncomp_size) return PIVCOHUF_ERR_INTERNAL; + *out_len = uncomp_size; + return PIVCOHUF_OK; +} + +int pivcohuf_decompress(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len) +{ + { + pivco_decoder_t *dec_ctx = pivco_decoder_create(); + if (!dec_ctx) return PIVCOHUF_ERR_INTERNAL; + int r = pivcohuf_decompress_impl(dec_ctx, in, in_len, out, out_len, NULL); + pivco_decoder_free(dec_ctx); + return r; + } +} + +int pivcohuf_decompress_timed(const uint8_t *in, size_t in_len, + uint8_t *out, size_t *out_len, + pivcohuf_timing_t *timing) +{ + if (timing) memset(timing, 0, sizeof(*timing)); + { + pivco_decoder_t *dec_ctx = pivco_decoder_create(); + if (!dec_ctx) return PIVCOHUF_ERR_INTERNAL; + int r = pivcohuf_decompress_impl(dec_ctx, in, in_len, out, out_len, timing); + pivco_decoder_free(dec_ctx); + return r; + } +}